diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 429ec2523..d923ab72a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,11 +15,11 @@ env: jobs: checkout: name: "Checkout" - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - name: Cache checkout - uses: actions/cache@v3 + uses: actions/cache@v5 id: cache-checkout with: path: ${{ env.wc }} @@ -27,7 +27,7 @@ jobs: - name: Checkout if: steps.cache-checkout.outputs.cache-hit != 'true' - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: recursive path: ${{ env.wc }} @@ -44,12 +44,15 @@ jobs: pcre_suite: name: "Import PCRE suite" - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ ubuntu-22.04 ] needs: [ build ] # for cvtpcre steps: - name: Cache PCRE suite - uses: actions/cache@v3 + uses: actions/cache@v5 id: cache-pcre with: path: pcre-suite/${{ env.pcre2 }} @@ -67,19 +70,20 @@ jobs: chmod -R ug-w pcre-suite - name: Cache converted PCRE tests - uses: actions/cache@v3 + uses: actions/cache@v5 id: cache-cvtpcre with: path: ${{ env.cvtpcre }} - key: cvtpcre-bmake-ubuntu-gcc-DEBUG-AUSAN-${{ github.sha }}-${{ env.pcre2 }} + key: cvtpcre-bmake-${{ matrix.os }}-gcc-DEBUG-AUSAN-${{ github.sha }}-${{ env.pcre2 }} - - name: Fetch build + - name: Restore build if: steps.cache-cvtpcre.outputs.cache-hit != 'true' - uses: actions/cache@v3 + uses: actions/cache/restore@v5 id: cache-build with: path: ${{ env.build }} - key: build-bmake-ubuntu-gcc-DEBUG-AUSAN-${{ github.sha }} # arbitary build, just for cvtpcre + key: build-bmake-${{ matrix.os }}-gcc-DEBUG-AUSAN-${{ github.sha }} # arbitrary build, just for cvtpcre + fail-on-cache-miss: true - name: Convert PCRE suite if: steps.cache-cvtpcre.outputs.cache-hit != 'true' @@ -130,14 +134,14 @@ jobs: build: name: "Build ${{ matrix.san }} ${{ matrix.cc }} ${{ matrix.os }} ${{ matrix.debug }}" - runs-on: ${{ matrix.os }}-latest + runs-on: ${{ matrix.os }} needs: [ checkout ] strategy: fail-fast: true matrix: san: [ NO_SANITIZER, AUSAN, MSAN, EFENCE, FUZZER ] # NO_SANITIZER=1 is a no-op - os: [ ubuntu ] + os: [ ubuntu-22.04 ] cc: [ clang, gcc ] make: [ bmake ] # we test makefiles separately debug: [ DEBUG, RELEASE ] # RELEASE=1 is a no-op @@ -154,22 +158,23 @@ jobs: cc: gcc # -fsanitize=fuzzer is clang-only steps: - - name: Fetch checkout - uses: actions/cache@v3 + - name: Restore checkout + uses: actions/cache/restore@v5 id: cache-checkout with: path: ${{ env.wc }} key: checkout-${{ github.sha }} + fail-on-cache-miss: true - name: Cache build - uses: actions/cache@v3 + uses: actions/cache@v5 id: cache-build with: path: ${{ env.build }} key: build-${{ matrix.make }}-${{ matrix.os }}-${{ matrix.cc }}-${{ matrix.debug }}-${{ matrix.san }}-${{ github.sha }} - name: Dependencies (Ubuntu) - if: matrix.os == 'ubuntu' && steps.cache-build.outputs.cache-hit != 'true' + if: matrix.os == 'ubuntu-22.04' && steps.cache-build.outputs.cache-hit != 'true' run: | uname -a sudo apt-get install bmake electric-fence @@ -185,7 +190,7 @@ jobs: - name: Get number of CPU cores if: steps.cache-build.outputs.cache-hit != 'true' - uses: SimenB/github-actions-cpu-cores@v1 + uses: SimenB/github-actions-cpu-cores@v2 id: cpu-cores - name: Make @@ -216,14 +221,14 @@ jobs: # of the build during CI, even if we don't run that during tests. test_makefiles: name: "Test (Makefiles) ${{ matrix.make }} ${{ matrix.os }} ${{ matrix.debug }}" - runs-on: ${{ matrix.os }}-latest + runs-on: ${{ matrix.os }} needs: [ checkout, build ] strategy: fail-fast: false matrix: san: [ NO_SANITIZER ] # NO_SANITIZER=1 is a no-op - os: [ ubuntu ] + os: [ ubuntu-22.04 ] cc: [ clang ] make: [ bmake, pmake ] debug: [ EXPENSIVE_CHECKS, DEBUG, RELEASE ] # RELEASE=1 is a no-op @@ -232,20 +237,26 @@ jobs: make: pmake # not packaged steps: - - name: Fetch checkout - uses: actions/cache@v3 + - name: Restore checkout + uses: actions/cache/restore@v5 id: cache-checkout with: path: ${{ env.wc }} key: checkout-${{ github.sha }} + fail-on-cache-miss: true # An arbitary build. - - name: Fetch build - uses: actions/cache@v3 + # Failing to fetch this is not fatal, we're testing Makefiles here. + # Some combinations of our options (pmake, EXPENSIVE_CHECKS, whatever) + # won't exist in cache because we didn't build those. That's okay for + # the purposes of this step, building those is harmless. + - name: Restore build + uses: actions/cache/restore@v5 id: cache-build with: path: ${{ env.build }} key: build-${{ matrix.make }}-${{ matrix.os }}-${{ matrix.cc }}-${{ matrix.debug }}-${{ matrix.san }}-${{ github.sha }} + fail-on-cache-miss: false # We don't need to build the entire repo to know that the makefiles work, # I'm just deleting a couple of .o files and rebuilding those instead. @@ -258,7 +269,7 @@ jobs: run: find ${{ env.wc }} -type f -name '*.c' | sort -r | head -5 | xargs touch - name: Dependencies (Ubuntu) - if: matrix.os == 'ubuntu' + if: matrix.os == 'ubuntu-22.04' run: | uname -a sudo apt-get install pmake bmake pcregrep @@ -273,7 +284,7 @@ jobs: ${{ matrix.cc }} --version - name: Get number of CPU cores - uses: SimenB/github-actions-cpu-cores@v1 + uses: SimenB/github-actions-cpu-cores@v2 id: cpu-cores - name: Make @@ -292,7 +303,7 @@ jobs: # kmkf duplicate install targets, it's not interesting for libfsm's CI, # so I'm retrying on error here. # github.com/katef/kmkf/issues/14 - name: Install - uses: nick-fields/retry@v2.8.3 + uses: nick-fields/retry@v3 with: timeout_seconds: 10 # required, but not a problem for the kmkf bug max_attempts: 3 @@ -301,14 +312,14 @@ jobs: test_san: name: "Test (Sanitizers) ${{ matrix.san }} ${{ matrix.cc }} ${{ matrix.os }} ${{ matrix.debug }}" - runs-on: ${{ matrix.os }}-latest + runs-on: ${{ matrix.os }} needs: [ build ] strategy: fail-fast: false matrix: san: [ AUSAN, MSAN, EFENCE ] - os: [ ubuntu ] + os: [ ubuntu-22.04 ] cc: [ clang, gcc ] make: [ bmake ] debug: [ DEBUG, RELEASE ] # RELEASE=1 is a no-op @@ -321,15 +332,16 @@ jobs: san: MSAN # not supported steps: - - name: Fetch checkout - uses: actions/cache@v3 + - name: Restore checkout + uses: actions/cache/restore@v5 id: cache-checkout with: path: ${{ env.wc }} key: checkout-${{ github.sha }} + fail-on-cache-miss: true - name: Dependencies (Ubuntu) - if: matrix.os == 'ubuntu' + if: matrix.os == 'ubuntu-22.04' run: | uname -a sudo apt-get install bmake pcregrep electric-fence @@ -343,15 +355,16 @@ jobs: brew install bmake pcre ${{ matrix.cc }} --version - - name: Fetch build - uses: actions/cache@v3 + - name: Restore build + uses: actions/cache/restore@v5 id: cache-build with: path: ${{ env.build }} key: build-${{ matrix.make }}-${{ matrix.os }}-${{ matrix.cc }}-${{ matrix.debug }}-${{ matrix.san }}-${{ github.sha }} + fail-on-cache-miss: true - name: Get number of CPU cores - uses: SimenB/github-actions-cpu-cores@v1 + uses: SimenB/github-actions-cpu-cores@v2 id: cpu-cores - name: Test @@ -362,7 +375,7 @@ jobs: test_fuzz: name: "Fuzz (mode ${{ matrix.mode }}) ${{ matrix.cc }} ${{ matrix.os }} ${{ matrix.debug }}" - runs-on: ${{ matrix.os }}-latest + runs-on: ${{ matrix.os }} timeout-minutes: 5 # this should never be reached, it's a safeguard for bugs in the fuzzer itself needs: [ build ] @@ -370,25 +383,26 @@ jobs: fail-fast: false matrix: san: [ FUZZER ] - os: [ ubuntu ] + os: [ ubuntu-22.04 ] cc: [ clang ] make: [ bmake ] debug: [ DEBUG, RELEASE ] # RELEASE=1 is a no-op - mode: [ m, p, d ] + mode: [ r, s, m, i, M, p ] exclude: - os: macos cc: gcc # it's clang anyway steps: - - name: Fetch checkout - uses: actions/cache@v3 + - name: Restore checkout + uses: actions/cache/restore@v5 id: cache-checkout with: path: ${{ env.wc }} key: checkout-${{ github.sha }} + fail-on-cache-miss: true - name: Dependencies (Ubuntu) - if: matrix.os == 'ubuntu' + if: matrix.os == 'ubuntu-22.04' run: | uname -a sudo apt-get install bmake @@ -402,16 +416,24 @@ jobs: brew install bmake ${{ matrix.cc }} --version - - name: Fetch build - uses: actions/cache@v3 + - name: Restore build + uses: actions/cache/restore@v5 id: cache-build with: path: ${{ env.build }} key: build-${{ matrix.make }}-${{ matrix.os }}-${{ matrix.cc }}-${{ matrix.debug }}-${{ matrix.san }}-${{ github.sha }} - - # note we do the fuzzing unconditionally; each run adds to the corpus + fail-on-cache-miss: true + + # note we do the fuzzing unconditionally; each run adds to the corpus. + # + # We only run fuzzing for PRs in the base repo, this prevents attempting + # to purge the seed cache from a PR syncing a forked repo, which fails + # due to a permissions error (I'm unsure why, I think PRs from clones can't + # purge a cache in CI presumably for security/DoS reasons). PRs from clones + # still run fuzzing, just from empty, and do not save their seeds. - name: Restore seeds (mode ${{ matrix.mode }}) - uses: actions/cache/restore@v3 + if: github.repository == 'katef/libfsm' + uses: actions/cache/restore@v5 id: cache-seeds with: path: ${{ env.seeds }}-${{ matrix.mode }} @@ -422,7 +444,7 @@ jobs: run: mkdir -p ${{ env.seeds }}-${{ matrix.mode }} - name: Get number of CPU cores - uses: SimenB/github-actions-cpu-cores@v1 + uses: SimenB/github-actions-cpu-cores@v2 id: cpu-cores - name: Fuzz @@ -448,7 +470,7 @@ jobs: # the same seeds for a given bug. # The explicit cache/restore and cache/save actions are just for that. - name: Save seeds (mode ${{ matrix.mode }}-${{ matrix.debug }}) - uses: actions/cache/save@v3 + uses: actions/cache/save@v5 if: always() with: path: ${{ env.seeds }}-${{ matrix.mode }} @@ -456,25 +478,25 @@ jobs: # nothing to do with the caching, I'm uploading the seeds so a developer can grab them to fuzz locally - name: Upload seeds (mode ${{ matrix.mode }}-${{ matrix.debug }}) - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: seeds-${{ matrix.mode }}-${{ matrix.debug }} path: ${{ env.seeds }}-${{ matrix.mode }} test_pcre: name: "Test (PCRE suite) ${{ matrix.lang }} ${{ matrix.san }} ${{ matrix.cc }} ${{ matrix.os }} ${{ matrix.debug }}" - runs-on: ${{ matrix.os }}-latest + runs-on: ${{ matrix.os }} needs: [ pcre_suite ] # and also build, but pcre_suite gives us that strategy: fail-fast: false matrix: san: [ AUSAN, MSAN, EFENCE ] - os: [ ubuntu ] + os: [ ubuntu-22.04 ] cc: [ clang, gcc ] make: [ bmake ] debug: [ DEBUG, RELEASE ] # RELEASE=1 is a no-op - lang: [ "vm -x v1", "vm -x v2", asm, c, rust, vmc, vmops, go, goasm ] + lang: [ "vm -x v1", "vm -x v2", asm, c, rust, vmc, vmops, go, goasm, llvm ] exclude: - os: macos cc: gcc # it's clang anyway @@ -485,7 +507,7 @@ jobs: steps: - name: Dependencies (Ubuntu) - if: matrix.os == 'ubuntu' && matrix.san == 'EFENCE' + if: matrix.os == 'ubuntu-22.04' && matrix.san == 'EFENCE' run: | uname -a sudo apt-get install electric-fence @@ -499,32 +521,33 @@ jobs: ${{ matrix.cc }} --version - name: Dependencies (Ubuntu/Go) - if: matrix.os == 'ubuntu' && (matrix.lang == 'go' || matrix.lang == 'goasm') + if: matrix.os == 'ubuntu-22.04' && (matrix.lang == 'go' || matrix.lang == 'goasm') run: | uname -a sudo apt-get install golang go version - - name: Fetch build - uses: actions/cache@v3 + - name: Restore build + uses: actions/cache/restore@v5 id: cache-build with: path: ${{ env.build }} key: build-${{ matrix.make }}-${{ matrix.os }}-${{ matrix.cc }}-${{ matrix.debug }}-${{ matrix.san }}-${{ github.sha }} + fail-on-cache-miss: true - name: Fetch converted PCRE tests - uses: actions/cache@v3 + uses: actions/cache@v5 id: cache-cvtpcre with: path: ${{ env.cvtpcre }} - key: cvtpcre-bmake-ubuntu-gcc-DEBUG-AUSAN-${{ github.sha }}-${{ env.pcre2 }} + key: cvtpcre-bmake-${{ matrix.os }}-gcc-DEBUG-AUSAN-${{ github.sha }}-${{ env.pcre2 }} - name: Run PCRE suite (${{ matrix.lang }}) run: CC=${{ matrix.cc }} ./${{ env.build }}/bin/retest -O1 -l ${{ matrix.lang }} ${{ env.cvtpcre }}/*.tst docs: name: Documentation - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 needs: [ checkout ] env: @@ -532,7 +555,7 @@ jobs: steps: - name: Cache docs - uses: actions/cache@v3 + uses: actions/cache@v5 id: cache-docs with: path: ${{ env.build }} @@ -545,17 +568,18 @@ jobs: sudo apt-get update sudo apt-get install bmake libxml2-utils xsltproc docbook-xml docbook-xsl - - name: Fetch checkout + - name: Restore checkout if: steps.cache-docs.outputs.cache-hit != 'true' - uses: actions/cache@v3 + uses: actions/cache/restore@v5 id: cache-checkout with: path: ${{ env.wc }} key: checkout-${{ github.sha }} + fail-on-cache-miss: true - name: Get number of CPU cores if: steps.cache-docs.outputs.cache-hit != 'true' - uses: SimenB/github-actions-cpu-cores@v1 + uses: SimenB/github-actions-cpu-cores@v2 id: cpu-cores - name: Test docs @@ -575,57 +599,60 @@ jobs: install: name: Install - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 needs: [ build, docs ] env: san: NO_SANITIZER # NO_SANITIZER=1 is a no-op - os: ubuntu + os: ubuntu-22.04 cc: clang make: bmake debug: RELEASE # RELEASE=1 is a no-op steps: - name: Cache prefix - uses: actions/cache@v3 + uses: actions/cache@v5 id: cache-prefix with: path: ${{ env.prefix }} key: prefix-${{ env.make }}-${{ env.os }}-${{ env.cc }}-${{ env.debug }}-${{ env.san }}-${{ github.sha }} - name: Dependencies (Ubuntu) - if: env.os == 'ubuntu' && steps.cache-prefix.outputs.cache-hit != 'true' + if: env.os == 'ubuntu-22.04' && steps.cache-prefix.outputs.cache-hit != 'true' run: | uname -a sudo apt-get install bmake - - name: Fetch checkout + - name: Restore checkout if: steps.cache-prefix.outputs.cache-hit != 'true' - uses: actions/cache@v3 + uses: actions/cache/restore@v5 id: cache-checkout with: path: ${{ env.wc }} key: checkout-${{ github.sha }} + fail-on-cache-miss: true - - name: Fetch build + - name: Restore build if: steps.cache-prefix.outputs.cache-hit != 'true' - uses: actions/cache@v3 + uses: actions/cache/restore@v5 id: cache-build with: path: ${{ env.build }} key: build-${{ env.make }}-${{ env.os }}-${{ env.cc }}-${{ env.debug }}-${{ env.san }}-${{ github.sha }} + fail-on-cache-miss: true - - name: Fetch docs + - name: Restore docs if: steps.cache-prefix.outputs.cache-hit != 'true' - uses: actions/cache@v3 + uses: actions/cache/restore@v5 id: cache-docs with: path: ${{ env.build }} key: docs-${{ github.sha }} + fail-on-cache-miss: true - name: Get number of CPU cores if: steps.cache-prefix.outputs.cache-hit != 'true' - uses: SimenB/github-actions-cpu-cores@v1 + uses: SimenB/github-actions-cpu-cores@v2 id: cpu-cores - name: Install @@ -636,12 +663,12 @@ jobs: fpm: name: Package ${{ matrix.pkg }} - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 needs: [ install ] env: san: NO_SANITIZER # NO_SANITIZER=1 is a no-op - os: ubuntu + os: ubuntu-22.04 cc: clang make: bmake debug: RELEASE # RELEASE=1 is a no-op @@ -654,18 +681,19 @@ jobs: steps: - name: Dependencies (Ubuntu) - if: env.os == 'ubuntu' + if: env.os == 'ubuntu-22.04' run: | uname -a sudo gem install --no-document fpm fpm -v - - name: Fetch prefix - uses: actions/cache@v3 + - name: Restore prefix + uses: actions/cache/restore@v5 id: cache-prefix with: path: ${{ env.prefix }} key: prefix-${{ env.make }}-${{ env.os }}-${{ env.cc }}-${{ env.debug }}-${{ env.san }}-${{ github.sha }} + fail-on-cache-miss: true - name: Find version # TODO: would get a tag or branch name here @@ -684,7 +712,7 @@ jobs: printf "package_file=%s\n" $(basename pkg/*) >> $GITHUB_ENV - name: Upload - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: ${{ env.package_file }} path: pkg/${{ env.package_file }} diff --git a/Makefile b/Makefile index 8d742883e..683e1d906 100644 --- a/Makefile +++ b/Makefile @@ -71,6 +71,7 @@ PKG += libtheft .if !defined(NODOC) SUBDIR += man/fsm.1 SUBDIR += man/re.1 +SUBDIR += man/rx.1 SUBDIR += man/lx.1 SUBDIR += man/fsm_print.3 SUBDIR += man/libfsm.3 @@ -101,6 +102,7 @@ SUBDIR += src/libre/print SUBDIR += src/libre SUBDIR += src/fsm SUBDIR += src/re +SUBDIR += src/rx SUBDIR += src/retest SUBDIR += src/lx/print SUBDIR += src/lx @@ -108,19 +110,22 @@ SUBDIR += src SUBDIR += tests/capture SUBDIR += tests/complement SUBDIR += tests/gen +SUBDIR += tests/idmap SUBDIR += tests/intersect #SUBDIR += tests/ir # XXX: fragile due to state numbering SUBDIR += tests/eclosure SUBDIR += tests/equals SUBDIR += tests/subtract +SUBDIR += tests/detect_required SUBDIR += tests/determinise +SUBDIR += tests/eager_output SUBDIR += tests/endids SUBDIR += tests/epsilons +SUBDIR += tests/fsm SUBDIR += tests/glob SUBDIR += tests/like SUBDIR += tests/literal -# FIXME: commenting this out for now due to Makefile error -#SUBDIR += tests/lxpos +SUBDIR += tests/lxpos SUBDIR += tests/minimise SUBDIR += tests/native SUBDIR += tests/pcre @@ -130,6 +135,8 @@ SUBDIR += tests/pcre-flags SUBDIR += tests/pcre-repeat SUBDIR += tests/pred SUBDIR += tests/re_literal +SUBDIR += tests/re_strings +SUBDIR += tests/regressions SUBDIR += tests/reverse SUBDIR += tests/trim SUBDIR += tests/union @@ -140,6 +147,7 @@ SUBDIR += tests/sql SUBDIR += tests/queue SUBDIR += tests/aho_corasick SUBDIR += tests/retest +SUBDIR += tests/re_interpolate_groups SUBDIR += tests .if make(theft) || make(${BUILD}/theft/theft) SUBDIR += theft @@ -183,6 +191,6 @@ STAGE_BUILD := ${STAGE_BUILD:Nbin/cvtpcre} .if make(test) .END:: - grep FAIL ${BUILD}/tests/*/res*; [ $$? -ne 0 ] + grep -I FAIL ${BUILD}/tests/*/*res*; [ $$? -ne 0 ] .endif diff --git a/README.md b/README.md index 48ec80949..d30e405b1 100644 --- a/README.md +++ b/README.md @@ -4,19 +4,23 @@ ; re -cb -pl dot '[Ll]ibf+(sm)*' '[Ll]ibre' | dot ![libfsm.svg](doc/tutorial/libfsm.svg) +libfsm is not a drop-in replacement for other regex engines, and it only supports patterns that can be compiled to deterministic FSMs. In return, supported patterns run in linear time. + Getting started: * See the [tutorial introduction](doc/tutorial/re.md) for a quick overview of the re(1) command line interface. * [Compilation phases](doc/tutorial/phases.md) for typical applications which compile regular expressions to code. + * [Advice on using libfsm](doc/advice.md) for suggestions around compilation time, unsupported features, common usage patterns, and examples. You get: * libfsm — library for manipulating FSM (NFA and DFA) * libre — library for compiling regular expressions to NFA * fsm(1) — command line interface for FSM - * re(1) — command line interface for executing regular expressions + * re(1) — command line interface for regular expressions + * rx(1) — command line interface for compiling sets of regular expressions * lx(1) — lexer generator lx is an attempt to produce a simple, expressive, and unobtrusive diff --git a/doc/advice.md b/doc/advice.md new file mode 100644 index 000000000..0eb429f94 --- /dev/null +++ b/doc/advice.md @@ -0,0 +1,276 @@ +# Advice on using libfsm for high-performance pattern matching + +libfsm compiles regular expressions to deterministic finite state machines (FSMs) and generates executable code. FSM-based matching runs in **linear time O(n)** with **no backtracking**. + +Regex engines like PCRE use backtracking to explore multiple possible match paths at **runtime**. +This means the same pattern can have different execution costs depending on the input. + +libfsm instead resolves all match decisions at **compile time** by constructing a Deterministic Finite Automaton (DFA). +At runtime, matching is a single linear pass over the input with no alternative paths to explore. + +As a result, libfsm avoids input-dependent slowdowns and is not susceptible to regular expression–based denial-of-service (ReDoS) attacks. + +**libfsm is not a drop-in replacement for traditional regex engines.** It only supports patterns that can be compiled to FSMs. + +### **Topics** + +- [What libfsm Cannot Do](#what-libfsm-cannot-do) +- [Quick Start](#quick-start) +- [Supported Code Generation Targets](#supported-code-generation-targets) +- [Workflow Overview](#workflow-overview) +- [Writing Effective libfsm Patterns](#writing-effective-libfsm-patterns) +- [Byte Search Optimization](#byte-search-optimization-optional) +- [Troubleshooting](#troubleshooting) +- [Pattern Matches Empty String Unintentionally](#pattern-matches-empty-string-unintentionally) + +## What libfsm Cannot Do + +These PCRE features will not compile: + +* Word boundaries (`\b`) +* Non-greedy quantifiers (`*?`, `+?`, `??`) +* Group capture (coming soon!) and backreferences +* Lookahead/lookbehind assertions (`(?=`, `(?!`, `(?<=`, `(? user_detector.go +``` + +This produces a standalone matcher function. + +## Supported Code Generation Targets + +libfsm provides stable, “first-class” code generation for: +- High-level languages: C (via `-l vmc`), Go, Rust +- LLVM IR +- Native WebAssembly + +Adding code generation for new languages is straightforward and is defined in [src/libfsm/print/](../src/libfsm/print/). + +## Workflow Overview + +libfsm provides two main tools for pattern matching: + - **`re`** takes patterns from the command line + - **`rx`** takes patterns from a file + +A recommended workflow when using libfsm is: + +1. Validate the regex + + Test behavior using any PCRE-compatible tool (e.g., [pcregrep(1)](https://man7.org/linux/man-pages/man1/pcregrep.1.html) on the CLI or [https://regex101.com/](https://regex101.com/) in the browser). + +2. Verify libfsm compatibility + + If unsupported constructs exist, libfsm reports the failing location: + ```sh + re -r pcre -l ast 'x*?' + # Output: /x*?/:3: Unsupported operator + ``` + In this example, `:3` indicates that the character at byte offset three in the pattern is an unsupported feature. + + ```sh + # patterns with unsupported operators are output to declined.txt + rx -r pcre -l ast -d declined.txt 'x*?' + ``` + + +3. Generate code + + ```sh + re -p -r pcre -l rust -k str '^item-[A-Z]{3}\z' > item_detector.rs + ``` + +4. Use multiple patterns + + Execution complexity for the generated code is proportional to the length of the text being matched, not to the number of patterns. + Assuming your generated code isn't too large to compile, this means you can have as many patterns as you want, + for the same time it takes to execute a single pattern. + + Take advantage of this. + + ```sh + # re - patterns from command line: + re -p -r pcre -l go -k str '^x?a b+c$' '^x*def?$' '^x$' + + # rx - patterns from file: + rx -p -r pcre -l vmc -k str -d skipped.txt patterns.txt > detectors.c + ``` + +5. Call the generated code from your program somehow + + You're on your own for this. `-k` controls the API for the generated code to read in data to match. Try different options for the language you're using and see which suits you. + + The generated API can also vary depending on how you want libfsm to handle ambiguities between different patterns. See the `AMBIG_*` flags in [include/fsm/options.h](../include/fsm/options.h) for different approaches there. + +Both tools: +* Combine all patterns into one function (like using `|` to join them) +* Generate code that can return `(bool, int)` for the match status and pattern ID +* Pattern ID is argument position for `re`, line number for `rx` +* When encountering unsupported patterns: `rx` can decline them to `-d` file and generates code with working patterns only; `re` fails completely + +### Common Flags + +| Flag | Purpose | Common Options | Notes | +|:----:|:---------------------------- |:------------------------------------------ |:---------------------------------------------------------------- | +| `-r` | Regex dialect | `pcre`, `literal`, `glob`, `native`, `sql` | `pcre` supports the widest set of features | +| `-l` | Output language for printing | `go`, `rust`, `vmc`, `llvm`, `wasm`, `dot` | Use `vmc` for `C` code. Pipe `dot` into `idot` for visualization | +| `-k` | Generated function I/O API | `str`, `getc`, `pair` | `str` takes string, `pair` takes byte array, `getc` uses callback for streaming | +| `-p` | Print mode | *(no value)* | Abbrv. of `-l fsm`. Print the constructed fsm, rather than executing it. | +| `-d` | Declined patterns | filename | Only applies to `rx` (batch mode) | + +This is not an exhaustive list. For full flag details, see [include/fsm/options.h](../include/fsm/options.h) and the [man pages](../man). +The man pages can be built by running `bmake -r doc`, then view with `man build/man/re.1/re.1`. + +## Writing Effective libfsm Patterns + +Generally, to keep generated code compact, stick to the least expressive subset of features. + +libfsm has no way to know in advance what text you'll be passing to its generated code. +For example, are you matching a string that you know will never contain a newline? +libfsm doesn't know that. +It has to generate code that's capable of handling any input. +You can help it out by making your patterns precise. + +Think about what you intend your pattern to match, and what it's actually capable of matching given arbitrary text. +This helps restrict the scope of your pattern from arbitrary text to exactly what you mean. +The following bits of advice illustrate various specific ways to bring down this scope. + +1. Replace broad wildcards + + Avoid `.*` and `.+` when possible. Wildcards match “anything,” which is often imprecise. And although they look compact, libfsm must enumerate every possible byte and continuation. This quickly leads to large DFAs. + + For example, a double-quoted string should not use `".*"` because the content cannot contain an unescaped quote. Using `.*` forces libfsm to consider all characters -- including both the presence and absence of the closing `"` at every step. This greatly increases the number of states. + + Instead, restrict it to the actual valid characters `"[^"\r\n]*"`, which matches only what is allowed and will keep the DFA more compact. + + Use negated character classes to match only the allowed content: + + | Avoid | Better | + | ---------- | -------------- | + | `<.*>` | `<[^>]*>` | + | `\((.*)\)` | `\([^)]*\)`| + | `price=.+` | `price=[0-9]+` | + | `var\s.+=` | `var\s[^=]+=` | + + The overlap between `.*` or `.+` and strings that follow is often the cause of an “explosion” in the size of the generated FSM. So when compilation is slow or generated output is large, look for `.*` and `.+` first and replace them with a narrower character class. + +2. Take care with bounded repetition + + If you have the pattern `^x{3,5}$`, libfsm's resulting DFA will be structured like "match an x, then match an x, then match an x, then match an x or skip it, then match an x or skip it, then report an overall match if at the end of input". It has to repeat the pattern, noting each time whether it's required or optional (beyond the lower count in `{min,max}`), because DFA execution doesn't have a counter, just the current state within the overall DFA. + + When the subexpression (represented by `x`) unintentionally matches too many things, they all have to be spelled out every time. + So pay especially close attention to tightening up subexpressions in bounded repetition clauses. + +3. Anchor when matching full string + + When the intention is to match an entire string, use anchors. + Use `^` at the beginning and `\z` for the true end of the string. + + ```regex + # Correct: matches only this exact hostname + # Matches "web12.example.com" + # Does not match "foo-web12.example.com-bar" + ^web\d+\.example\.com\z  + + # Incorrect: would match inside a larger string + # Matches "web12.example.com" + # Also matches "foo-web12.example.com-bar" + web\d+\.example\.com + ``` + +4. Prefer `\z` over `$` for End-of-String + + `\z` always matches the end of the string. + `$` will also match a trailing newline at the end of the string, + so if you use this in combination with capturing groups, you may not be capturing what you expect. + Also, `\z` produces a smaller FSM, so it is better to use it in places where `\n` cannot appear. + + ```regex + # Preferred: matches only if the string ends with "bar" + # Matches "/foo/bar" + # Does NOT match "/foo/bar\n" + /bar\z + + # Incorrect: allows a trailing newline, + # which is usually unintended and adds unnecessary complexity + # Matches "/foo/bar" + # Also matches "/foo/bar\n" + /bar$ + ``` + +5. Escape special characters when used as literals + + Many characters have special meaning in regex (for example `.`, `+`, `*`, `?`, `[`, `(`). + If you mean to match them literally, escape them: + + | Literal You Want | Correct Regex | Explanation | + |----------------------------|-----------------------------|--------------------------------------------| + | `example.com` | `example\.com` | `.` matches any character unless escaped | + | `a+b` | `a\+b` | `+` means “one or more” | + | `price?` | `price\?` | `?` means “optional” | + | `[value]` | `\[value\]` | `[` and `]` start/end a character class | + | `(test)` | `\(test\)` | `(` and `)` begin/end a group | + | Markdown link `[t](u)` | `(\[[^]]*\]\([^)]*\))` | Matches `[text](url)` without crossing `]` or `)` | + + The `.` wildcard in particular is often mistakenly left unescaped in practice. + On testing, it will match a literal `.` as intended. But it will also match any other character. + This means that not only is your pattern incorrect (write negative test cases!), + but also this part of your FSM is 256 times larger than it should be. + +6. Use non-capturing groups + + Capture groups are _currently_ not supported (coming soon!). + + If you don't need to capture things, don't use capture. + If you need grouping for alternation or precedence, use PCRE's non-capturing syntax `(?:...)`: + + ```regex + # Correct + (?:private|no-store) + + # Not what's intended + (private|no-store) + ``` + +## Byte Search Optimization + +Patterns that start with an uncommon character can be accelerated using an initial byte scan before running the FSM. +This quickly jumps to likely match positions instead of scanning every byte. + +Good candidates are patterns that start with uncommon prefix characters, for example: + +```regex +#tag-[a-z]+ +@user-[0-9]+ +\[section\] +{"key": +"name='[^']+'" +``` + +These prefixes (`#`, `@`, `[`, `{`, `'`, `"`) are rare in normal text, so a byte search can skip ahead before running the matcher. + +We found using `strings.IndexByte` before calling the generated matcher in Go code significantly improved performance when matching strings with a large (>5k) leading prefix. + +## Pattern Matches Empty String Unintentionally + +Pattern: + +```regex +\s* +``` + +Will compile to code that always returns true. + +This is only an issue if that is not what you intend. + +**Fix options:** + +* Require at least one match: `\s+` +* Anchor context: `^\s+$` or alternatively, use `-Fb` flag diff --git a/examples/bm/Makefile b/examples/bm/Makefile index c0689f362..65c1b1ce2 100644 --- a/examples/bm/Makefile +++ b/examples/bm/Makefile @@ -10,3 +10,6 @@ pcre: pcre.c libfsm: libfsm.c gcc -o libfsm -O3 -Wall -std=c99 ${BM_CFLAGS} libfsm.c -I ../../include ../../build/lib/libre.a ../../build/lib/libfsm.a +clean: + rm -f pcre libfsm + diff --git a/examples/bm/libfsm.c b/examples/bm/libfsm.c index 27bedbb8e..a7581bd59 100644 --- a/examples/bm/libfsm.c +++ b/examples/bm/libfsm.c @@ -61,7 +61,7 @@ main(int argc, char *argv[]) opt.io = FSM_IO_STR; p = argv[0]; - fsm = re_comp(RE_PCRE, fsm_sgetc, &p, &opt, flags, &e); + fsm = re_comp(RE_PCRE, fsm_sgetc, &p, NULL, flags, &e); if (fsm == NULL) { re_perror(RE_LITERAL, &e, NULL, s); return 1; @@ -80,7 +80,7 @@ main(int argc, char *argv[]) printf("#include \n"); printf("\n"); - fsm_print_c(stdout, fsm); + fsm_print(stdout, fsm, &opt, NULL, FSM_PRINT_C); printf("int\n"); printf("main(void)\n"); diff --git a/examples/glob/main.c b/examples/glob/main.c index 38970e559..65703352c 100644 --- a/examples/glob/main.c +++ b/examples/glob/main.c @@ -55,7 +55,7 @@ match(const struct fsm *fsm, const char *s) assert(fsm_all(fsm, fsm_isdfa)); assert(s != NULL); - if (1 != fsm_exec(fsm, fsm_sgetc, &s, &state)) { + if (1 != fsm_exec(fsm, fsm_sgetc, &s, &state, NULL)) { return 0; } @@ -196,7 +196,7 @@ main(int argc, char *argv[]) } if (!quiet) { - fsm_print_fsm(stdout, fsm); + fsm_print(stdout, fsm, NULL, NULL, FSM_PRINT_FSM); } matched = match(fsm, argv[1]); diff --git a/examples/iprange/main.c b/examples/iprange/main.c index d88593eff..ba028351b 100644 --- a/examples/iprange/main.c +++ b/examples/iprange/main.c @@ -49,7 +49,8 @@ #define VALUE_SEP ',' static struct fsm *fsm; -static struct fsm_state *fsmstart; +static fsm_state_t fsm_start; +static fsm_state_t fsm_none = (fsm_state_t) -1; static unsigned char zeroes[16]; static unsigned char ones[16]; @@ -61,10 +62,10 @@ struct record { unsigned id; struct fsm *fsm; - struct fsm_state *start; - struct fsm_state *end; + fsm_state_t start; + fsm_state_t end; struct { - struct fsm_state *s; + fsm_state_t s; unsigned char c; } regs[16]; }; @@ -87,6 +88,24 @@ RB_GENERATE_STATIC(recmap, record, entry, recmap_cmp) static unsigned nrecords; static struct fsm_options opt; +static struct fsm_hooks hooks; + +static struct record * +find_id(unsigned id) +{ + struct record *r; + + /* XXX: this is a crime, we have a tree. + * we should be able to RB_FIND() */ + RB_FOREACH(r, recmap, &recmap) { + if (r->id == id) { + return r; + } + } + + assert(!"unreached"); + abort(); +} static struct record * get_id(char *rec, size_t reclen) @@ -114,27 +133,28 @@ get_id(char *rec, size_t reclen) r->len = reclen; r->id = nrecords++; - r->fsm = fsm_new(&opt); + r->fsm = fsm_new(NULL); if (r->fsm == NULL) { perror("fsm_new"); exit(-1); } - r->start = fsm_addstate(r->fsm); - if (r->start == NULL) { + if (!fsm_addstate(r->fsm, &r->start)) { perror("fsm_addstate"); exit(-1); } fsm_setstart(r->fsm, r->start); - r->end = fsm_addstate(r->fsm); - if (r->end == NULL) { + if (!fsm_addstate(r->fsm, &r->end)) { perror("fsm_addstate"); exit(-1); } fsm_setend(r->fsm, r->end, 1); - memset(r->regs, 0, sizeof r->regs); + for (size_t i = 0; i < sizeof r->regs / sizeof *r->regs; i++) { + r->regs[i].c = '\0'; + r->regs[i].s = fsm_none; + } RB_INSERT(recmap, &recmap, r); @@ -162,7 +182,7 @@ get_id(char *rec, size_t reclen) static void usage(void) { - fprintf(stderr, "ip2fsm -[46] [-f ] -l fmt\n" + fprintf(stderr, "iprange -[46] [-f ] -l fmt\n" "\t-4\t\tIPv4\n" "\t-6\t\tIPv6\n" "\t-f \tuse as input\n" @@ -170,7 +190,7 @@ usage(void) exit(-1); } -struct fsm_state * +static fsm_state_t get_from(struct record *r, unsigned oct) { if (oct == 0) { @@ -180,15 +200,16 @@ get_from(struct record *r, unsigned oct) return r->regs[oct - 1].s; } -struct fsm_state * +static fsm_state_t get_to(struct record *r, unsigned oct, unsigned noct, const unsigned char *octs) { - if (r->regs[oct].s != NULL && r->regs[oct].c == octs[oct]) { + fsm_state_t to; + + if (r->regs[oct].s != fsm_none && r->regs[oct].c == octs[oct]) { return r->regs[oct].s; } - struct fsm_state *to = fsm_addstate(r->fsm); - if (to == NULL) { + if (!fsm_addstate(r->fsm, &to)) { perror("fsm_addstate"); exit(-1); } @@ -197,7 +218,7 @@ get_to(struct record *r, unsigned oct, unsigned noct, const unsigned char *octs) r->regs[oct].c = octs[oct]; for (unsigned i = oct + 1; i < noct; i++) { - r->regs[i].s = NULL; + r->regs[i].s = fsm_none; } return to; @@ -207,8 +228,7 @@ static void gen_edge_range(struct record *r, unsigned oct, const unsigned char *octs, unsigned range_start, unsigned range_end, unsigned noct, unsigned final) { - struct fsm_state *from, *to; - struct fsm_edge *e; + fsm_state_t from, to; from = get_from(r, oct); if (oct > 0) { @@ -225,8 +245,7 @@ gen_edge_range(struct record *r, unsigned oct, const unsigned char *octs, } for (unsigned i = range_start; i <= range_end; i++) { - e = fsm_addedge_literal(r->fsm, from, to, (unsigned char) i); - if (e == NULL) { + if (!fsm_addedge_literal(r->fsm, from, to, (unsigned char) i)) { perror("fsm_addedge_literal"); exit(-1); } @@ -377,6 +396,9 @@ handle_line(unsigned char *socts, unsigned char *eocts, unsigned noct, } socts[spos] = eocts[spos]; + + // XXX: not sure about this + break; } else { gen_range(r, noct, spos - 1, socts[spos], 255, socts); @@ -420,72 +442,105 @@ important(unsigned n) } } -static void -carryopaque(const struct fsm *src_fsm, const fsm_state_t *src_set, size_t n, - struct fsm *dst_fsm, struct fsm_state *dst_state) +static int +conflict(FILE *f, const struct fsm_options *opt, + const fsm_end_id_t *ids, size_t count, + const char *example, void *hook_opaque) { - void *o = NULL; size_t i; - assert(src_fsm != NULL); - assert(src_set != NULL); - assert(n > 0); - assert(dst_fsm != NULL); - assert(fsm_isend(dst_fsm, dst_state)); - assert(fsm_getopaque(dst_fsm, dst_state) == NULL); + (void) f; + (void) hook_opaque; + (void) opt; - for (i = 0; i < n; i++) { - /* - * The opaque data is attached to end states only, so we skip - * non-end states here. - */ - if (!fsm_isend(src_fsm, src_set[i])) { - continue; - } + fprintf(stderr, "ambiguous matches for "); - assert(fsm_getopaque(src_fsm, src_set[i]) != NULL); + for (i = 0; i < count; i++) { + const struct record *r; - if (o == NULL) { - o = fsm_getopaque(src_fsm, src_set[i]); - fsm_setopaque(dst_fsm, dst_state, o); - continue; + r = (const void *) (intptr_t) ids[i]; /* XXX */ + + fprintf(stderr, "%s", r->rec); + + if (i + 1 < count) { + fprintf(stderr, ", "); } + } - assert(o == fsm_getopaque(src_fsm, src_set[i])); + if (example != NULL) { + fprintf(stderr, "; for example on input '%s'", example); } + + fprintf(stderr, "\n"); + + return 0; } static int -leaf(FILE *f, const void *state_opaque, const void *leaf_opaque) +accept_dot(FILE *f, const struct fsm_options *opt, + const fsm_end_id_t *ids, size_t count, + void *lang_opaque, void *hook_opaque) { - const struct record *r = state_opaque; + fsm_state_t s; - (void) leaf_opaque; + assert(f != NULL); - if (r == NULL) { - fprintf(f, "return -1;"); - return 0; + (void) hook_opaque; + + s = * (fsm_state_t *) lang_opaque; + + fprintf(f, "label = <"); + + if (!opt->anonymous_states) { + fprintf(f, "%u", s); + + if (count > 0) { + fprintf(f, "
"); + } + } + + for (size_t i = 0; i < count; i++) { + const struct record *r; + + r = find_id(ids[i]); + + fprintf(f, "%s", r->rec); /* XXX: escape */ + + if (i + 1 < count) { + fprintf(f, ", "); + } } - fprintf(f, "return 0x%u; /* %s */", r->id, r->rec); + fprintf(f, ">"); return 0; } static int -endleaf_dot(FILE *f, const void *state_opaque, const void *endleaf_opaque) +comment_c(FILE *f, const struct fsm_options *opt, + const fsm_end_id_t *ids, size_t count, + void *hook_opaque) { - const struct record *r; - assert(f != NULL); - assert(state_opaque != NULL); - assert(endleaf_opaque == NULL); - (void) endleaf_opaque; + (void) opt; + (void) hook_opaque; + + fprintf(f, "/* "); + + for (size_t i = 0; i < count; i++) { + const struct record *r; - r = state_opaque; + r = find_id(ids[i]); - fprintf(f, "label = <%s>", r->rec); /* XXX: escape */ + fprintf(f, "%s", r->rec); /* XXX: escape */ + + if (i + 1 < count) { + fprintf(f, ", "); + } + } + + fprintf(f, " */\n"); return 0; } @@ -500,12 +555,15 @@ main(int argc, char **argv) int oc = 0; int c; + opt.ambig = AMBIG_ERROR; opt.prefix = NULL; opt.always_hex = 1; opt.anonymous_states = 1; opt.consolidate_edges = 1; opt.case_ranges = 1; + hooks.conflict = conflict; + while (c = getopt(argc, argv, "46f:l:Q"), c != -1) { switch (c) { case '4': ipv = IPV4; break; @@ -550,14 +608,13 @@ main(int argc, char **argv) memset(ones, 0xff, sizeof ones); - fsm = fsm_new(&opt); + fsm = fsm_new(NULL); if (fsm == NULL) { perror("fsm_new"); return -1; } - fsmstart = fsm_addstate(fsm); - if (fsmstart == NULL) { + if (!fsm_addstate(fsm, &fsm_start)) { perror("fsm_addstate"); return -1; } @@ -654,24 +711,32 @@ main(int argc, char **argv) struct record *r; RB_FOREACH(r, recmap, &recmap) { + struct fsm_combine_info ci; fsm_state_t start; - if (fsm_minimise(r->fsm) == 0) { + if (!fsm_determinise(r->fsm)) { + perror("fsm_determinse"); + exit(-1); + } + + if (!fsm_minimise(r->fsm)) { perror("fsm_minimise"); exit(-1); } - fsm_setendopaque(r->fsm, r); + fsm_setendid(r->fsm, r->id); (void) fsm_getstart(r->fsm, &start); - fsm = fsm_merge(fsm, r->fsm); + fsm = fsm_merge(fsm, r->fsm, &ci); if (fsm == NULL) { perror("fsm_merge"); exit(-1); } - if (!fsm_addedge_epsilon(fsm, fsmstart, start)) { + (void) ci; + + if (!fsm_addedge_epsilon(fsm, fsm_start, start)) { perror("fsm_addedge_epsilon"); exit(-1); } @@ -686,7 +751,7 @@ main(int argc, char **argv) } } - fsm_setstart(fsm, fsmstart); + fsm_setstart(fsm, fsm_start); if (progress) { tend = time(NULL); @@ -696,15 +761,9 @@ main(int argc, char **argv) tstart = time(NULL); } - { - opt.carryopaque = carryopaque; - - if (!fsm_determinise(fsm) == 0) { - perror("fsm_determinise"); - exit(-1); - } - - opt.carryopaque = NULL; + if (!fsm_determinise(fsm)) { + perror("fsm_determinise"); + exit(-1); } if (progress) { @@ -713,15 +772,13 @@ main(int argc, char **argv) } if (oc) { - opt.fragment = 1; - opt.cp = "c"; - opt.leaf = leaf; - opt.leaf_opaque = NULL; - fsm_print_c(stdout, fsm); + opt.fragment = 1; + opt.comments = 1; + hooks.comment = comment_c; + fsm_print(stdout, fsm, &opt, &hooks, FSM_PRINT_C); } else if (odot) { - opt.endleaf = endleaf_dot; - opt.endleaf_opaque = NULL; - fsm_print_dot(stdout, fsm); + hooks.accept = accept_dot; + fsm_print(stdout, fsm, &opt, &hooks, FSM_PRINT_DOT); } } diff --git a/examples/rpn/Makefile b/examples/rpn/Makefile index 698ee63d6..16018c35c 100644 --- a/examples/rpn/Makefile +++ b/examples/rpn/Makefile @@ -19,13 +19,13 @@ lexer.h: lexer.lx ${LX} -g fgetc -l h < lexer.lx > lexer.h lexer.o: lexer.c - ${CC} -c -DLX_HEADER='"lexer.h"' ${CFLAGS} ${.ALLSRC} + ${CC} -c -DLX_HEADER='"lexer.h"' ${CFLAGS} $> main.o: main.c - ${CC} -c ${CFLAGS} ${.ALLSRC} + ${CC} -c ${CFLAGS} $> rpn: lexer.o main.o - ${CC} -o ${.TARGET} ${.ALLSRC} + ${CC} -o ${.TARGET} $> lexer.c: lexer.h diff --git a/examples/utf8dfa/main.c b/examples/utf8dfa/main.c index fa26bbddd..811cd76c0 100644 --- a/examples/utf8dfa/main.c +++ b/examples/utf8dfa/main.c @@ -120,7 +120,7 @@ main(int argc, char *argv[]) { struct fsm *fsm; fsm_state_t start; - fsm_print *print; + enum fsm_print_lang lang; opt.anonymous_states = 1; opt.consolidate_edges = 1; @@ -128,7 +128,7 @@ main(int argc, char *argv[]) opt.comments = 0; opt.case_ranges = 0; - print = fsm_print_api; + lang = FSM_PRINT_API; { int c; @@ -139,11 +139,11 @@ main(int argc, char *argv[]) case 'l': if (strcmp(optarg, "dot") == 0) { - print = fsm_print_dot; + lang = FSM_PRINT_DOT; } else if (strcmp(optarg, "api") == 0) { - print = fsm_print_api; + lang = FSM_PRINT_API; } else if (strcmp(optarg, "c") == 0) { - print = fsm_print_c; + lang = FSM_PRINT_C; } else { fprintf(stderr, "Invalid language '%s'", optarg); exit(1); @@ -164,7 +164,7 @@ main(int argc, char *argv[]) return EXIT_FAILURE; } - fsm = fsm_new(&opt); + fsm = fsm_new(NULL); if (fsm == NULL) { perror("fsm_new"); exit(1); @@ -215,12 +215,17 @@ main(int argc, char *argv[]) } } + if (!fsm_determinise(fsm)) { + perror("fsm_determinise"); + exit(1); + } + if (!fsm_minimise(fsm)) { perror("fsm_minimise"); exit(1); } - print(stdout, fsm); + fsm_print(stdout, fsm, NULL, NULL, lang); fsm_free(fsm); diff --git a/examples/words/main.c b/examples/words/main.c index 5aebcf8cc..078848b78 100644 --- a/examples/words/main.c +++ b/examples/words/main.c @@ -28,7 +28,7 @@ int main(int argc, char *argv[]) { fsm_state_t start; char s[BUFSIZ]; int (*dmf)(struct fsm *); - fsm_print *print; + enum fsm_print_lang lang; unsigned long ms, mt; int timing; int native = 0; @@ -40,7 +40,7 @@ int main(int argc, char *argv[]) { opt.consolidate_edges = 1; dmf = NULL; - print = NULL; + lang = FSM_PRINT_NONE; timing = 0; { @@ -55,8 +55,8 @@ int main(int argc, char *argv[]) { case 'n': native = 1; break; case 'N': unanchored = 1; break; - case 'c': print = fsm_print_dot; break; - case 'f': print = fsm_print_fsm; break; + case 'c': lang = FSM_PRINT_DOT; break; + case 'f': lang = FSM_PRINT_FSM; break; case 't': timing = 1; @@ -86,7 +86,7 @@ int main(int argc, char *argv[]) { exit(EXIT_FAILURE); } } else { - fsm = fsm_new(&opt); + fsm = fsm_new(NULL); if (fsm == NULL) { perror("fsm_new"); return 1; @@ -116,7 +116,7 @@ int main(int argc, char *argv[]) { } if (ahocorasick) { - if (!re_strings_add_str(g, s)) { + if (!re_strings_add_str(g, s, NULL)) { perror("re_strings_add_str"); exit(EXIT_FAILURE); } @@ -127,7 +127,7 @@ int main(int argc, char *argv[]) { struct fsm *r; struct fsm_combine_info ci; - r = re_comp(native ? RE_NATIVE : RE_LITERAL, fsm_sgetc, &p, &opt, 0, &e); + r = re_comp(native ? RE_NATIVE : RE_LITERAL, fsm_sgetc, &p, NULL, 0, &e); if (r == NULL) { re_perror(native ? RE_NATIVE : RE_LITERAL, &e, NULL, s); return 1; @@ -168,7 +168,7 @@ int main(int argc, char *argv[]) { } fsm = re_strings_build(g, - &opt, unanchored ? 0 : (RE_STRINGS_ANCHOR_LEFT | RE_STRINGS_ANCHOR_RIGHT)); + NULL, unanchored ? 0 : (RE_STRINGS_ANCHOR_LEFT | RE_STRINGS_ANCHOR_RIGHT)); if (fsm == NULL) { perror("re_strings_builder_build"); exit(EXIT_FAILURE); @@ -211,9 +211,7 @@ int main(int argc, char *argv[]) { + ((long) post.tv_nsec - (long) pre.tv_nsec) / 1000000; } - if (print != NULL) { - print(stdout, fsm); - } + fsm_print(stdout, fsm, &opt, NULL, lang); if (timing) { printf("construction, reduction, total: %lu, %lu, %lu\n", ms, mt, ms + mt); diff --git a/fuzz/Makefile b/fuzz/Makefile index 1d418cd97..2327a811b 100644 --- a/fuzz/Makefile +++ b/fuzz/Makefile @@ -7,6 +7,15 @@ ${BUILD}/fuzz/: ${BUILD} DIR += ${BUILD}/fuzz +# Uncomment to enable capture fuzzing using PCRE as a test oracle. +#PCRE_CMP=1 + +.if PCRE_CMP +PKG += libpcre2-8 +LFLAGS.fuzzer += ${LIBS.libpcre2-8} +CFLAGS.${SRC:Mfuzz/target.c} += -DCMP_PCRE=1 +.endif + .for src in ${SRC:Mfuzz/*.c} CFLAGS.${src} += -std=c99 .endfor @@ -15,7 +24,7 @@ CFLAGS.${src} += -std=c99 fuzz:: ${BUILD}/fuzz/fuzzer ${BUILD}/fuzz/fuzzer: mkdir - ${CC} -o $@ ${LFLAGS} ${.ALLSRC:M*.o} ${.ALLSRC:M*.a} + ${CC} -o $@ ${LFLAGS} ${LFLAGS.fuzzer} ${.ALLSRC:M*.o} ${.ALLSRC:M*.a} .for lib in ${LIB:Mlibfsm} ${LIB:Mlibre} ${BUILD}/fuzz/fuzzer: ${BUILD}/lib/${lib:R}.a diff --git a/fuzz/run_fuzzer b/fuzz/run_fuzzer index be8ba1d95..429ffa961 100755 --- a/fuzz/run_fuzzer +++ b/fuzz/run_fuzzer @@ -4,6 +4,8 @@ BUILD=../build FUZZER=${BUILD}/fuzz/fuzzer SEEDS=${BUILD}/fuzz/fuzzer_seeds +ARG=$1 + SECONDS=${SECONDS:-60} WORKERS=${WORKERS:-4} SEEDS=${SEEDS:-seeds} @@ -25,5 +27,9 @@ if [ ! -d "${SEEDS}" ]; then mkdir -p "${SEEDS}" fi -echo "\n==== ${FUZZER}" -${FUZZER} -jobs=${WORKERS} -workers=${WORKERS} -max_total_time=${SECONDS} ${SEEDS} +if [ -z "${ARG}" ]; then + echo "\n==== ${FUZZER}" + exec ${FUZZER} -jobs=${WORKERS} -workers=${WORKERS} -max_total_time=${SECONDS} ${SEEDS} +else + exec ${FUZZER} ${ARG} +fi \ No newline at end of file diff --git a/fuzz/target.c b/fuzz/target.c index 87086b929..66f25f5a6 100644 --- a/fuzz/target.c +++ b/fuzz/target.c @@ -12,6 +12,7 @@ #include #include +#include #include #include #include @@ -23,17 +24,59 @@ #include "../src/libfsm/minimise_test_oracle.h" +/* for fsm_capture_dump */ +/* FIXME: should this be a public interface? */ +#include "../src/libfsm/capture.h" + +/* Buffer for sanitized fuzzer input */ +#define MAX_FUZZER_DATA (64 * 1024) +static uint8_t data_buf[MAX_FUZZER_DATA + 1]; + +/* Should fuzzer harness code be built that compares behavior + * with PCRE? (Obviously, this depends on PCRE.) */ +#ifndef CMP_PCRE +#define CMP_PCRE 0 +#endif + +#if CMP_PCRE +#define PCRE2_CODE_UNIT_WIDTH 8 +#include + +static int +compare_with_pcre(const char *pattern, struct fsm *fsm); +#endif + /* 10 seconds */ #define TIMEOUT_USEC (10ULL * 1000 * 1000) +/* for TRACK_TIMES and EXPENSIVE_CHECKS */ +#include "../src/libfsm/internal.h" + +static bool verbosity_checked = false; +static bool verbose = false; + +#define LOG(...) \ + do { \ + if (verbose) { \ + fprintf(stderr, __VA_ARGS__); \ + } \ + } while (0) \ + enum run_mode { MODE_DEFAULT, + MODE_REGEX, + MODE_REGEX_SINGLE_ONLY, + MODE_REGEX_MULTI_ONLY, + MODE_IDEMPOTENT_DET_MIN, MODE_SHUFFLE_MINIMISE, MODE_ALL_PRINT_FUNCTIONS, + MODE_EAGER_OUTPUT, }; +static size_t +get_env_config(size_t default_value, const char *env_var_name); -/* This stuff will already exist elsewhere once other branches are merged. */ +/* TODO: These could be moved to a common file for test utils. */ #if 1 static void time_get(struct timeval *tv) @@ -75,7 +118,105 @@ scanner_next(void *opaque) } #endif -static const struct fsm_options opt; +/* This is used to track allocation during each fuzzer + * run. Note that hwm is not reduced when memory is + * free'd or realloc'd, because the size info is not + * passed to those calls. */ +#define MB(X) ((size_t)X * 1000 * 1000) +#define FH_ALLOCATOR_HWM_LIMIT (MB(50)) +struct fh_allocator_stats { + size_t hwm; /* high water mark */ +}; + +static void +fh_memory_hwm_limit_hook(const char *caller_name) +{ + /* It doesn't really help to exit here because libfuzzer will + * still treat it as a failure, but at least we can print a + * message about hitting the allocator limit and exit so we + * don't need to spend time investigating timeouts or ooms + * that are due to obvious resource exhaustion. */ + fprintf(stderr, "%s: hit FH_ALLOCATOR_HWM_LIMIT (%zu), exiting\n", + caller_name, FH_ALLOCATOR_HWM_LIMIT); + exit(EXIT_SUCCESS); +} + +static void +fh_free(void *opaque, void *p) +{ + (void)opaque; + free(p); +} + +static void * +fh_calloc(void *opaque, size_t n, size_t sz) +{ + struct fh_allocator_stats *stats = opaque; + stats->hwm += sz; + if (stats->hwm > FH_ALLOCATOR_HWM_LIMIT) { + fh_memory_hwm_limit_hook(__func__); + return NULL; + } + + (void)opaque; + return calloc(n, sz); +} + +static void * +fh_malloc(void *opaque, size_t sz) +{ + struct fh_allocator_stats *stats = opaque; + stats->hwm += sz; + if (stats->hwm > FH_ALLOCATOR_HWM_LIMIT) { + fh_memory_hwm_limit_hook(__func__); + return NULL; + } + + return malloc(sz); +} + +static void * +fh_realloc(void *opaque, void *p, size_t sz) +{ + struct fh_allocator_stats *stats = opaque; + stats->hwm += sz; + if (stats->hwm > FH_ALLOCATOR_HWM_LIMIT) { + fh_memory_hwm_limit_hook(__func__); + return NULL; + } + + return realloc(p, sz); +} + +static struct fh_allocator_stats allocator_stats; + +/* fuzzer harness allocators */ +static struct fsm_alloc custom_allocators = { + .free = fh_free, + .calloc = fh_calloc, + .malloc = fh_malloc, + .realloc = fh_realloc, + .opaque = &allocator_stats, +}; + +static void +dump_pattern(const char *pattern) +{ + const size_t pattern_length = strlen(pattern); + fprintf(stderr, "-- Pattern: %zu bytes\n", pattern_length); + for (size_t i = 0; i < pattern_length; i++) { + fprintf(stderr, " %02x", (uint8_t)pattern[i]); + if ((i & 31) == 31) { fprintf(stderr, "\n"); } + } + if ((pattern_length & 31) != 31) { + fprintf(stderr, "\n"); + } + for (size_t i = 0; i < pattern_length; i++) { + fprintf(stderr, "%c", isprint(pattern[i]) ? pattern[i] : '.'); + if ((i & 63) == 63) { fprintf(stderr, "\n"); } + } + fprintf(stderr, "\n"); +} static struct fsm * build(const char *pattern) @@ -95,7 +236,8 @@ build(const char *pattern) }; time_get(&pre); - fsm = re_comp(RE_PCRE, scanner_next, &s, &opt, RE_MULTI, &err); + + fsm = re_comp(RE_PCRE, scanner_next, &s, &custom_allocators, RE_MULTI, &err); time_get(&post); delta_usec = time_diff_usec(&pre, &post); total_usec += delta_usec; @@ -122,8 +264,10 @@ build(const char *pattern) if (total_usec > TIMEOUT_USEC) { #ifndef EXPENSIVE_CHECKS + dump_pattern(pattern); assert(!"timeout"); #else + (void)dump_pattern; fprintf(stderr, "exiting zero due to timeout under EXPENSIVE_CHECKS\n"); exit(0); #endif @@ -132,31 +276,1115 @@ build(const char *pattern) return fsm; } +static size_t +get_env_config(size_t default_value, const char *env_var_name) +{ + const char *s = getenv(env_var_name); + if (s == NULL) { + return default_value; + } else { + return strtoul(s, NULL, 10); + } +} + static int -codegen(const struct fsm *fsm) +codegen(const struct fsm *fsm, enum fsm_io io_mode) { + const struct fsm_options opt = { + .io = io_mode, + }; + FILE *dev_null = fopen("/dev/null", "w"); assert(dev_null != NULL); - fsm_print_c(dev_null, fsm); + fsm_print(dev_null, fsm, &opt, NULL, FSM_PRINT_C); fclose(dev_null); return 1; } -static int -build_and_codegen(const char *pattern) -{ - struct fsm *fsm = build(pattern); - if (fsm == NULL) { - return EXIT_SUCCESS; +static int +build_and_check_single(const char *pattern) +{ + const int verbosity = get_env_config(0, "VERBOSITY"); + if (verbosity > 1) { + fprintf(stderr, "pattern: \"%s\"\n", pattern); + } + + INIT_TIMERS(); + TIME(&pre); + struct fsm *fsm = build(pattern); + if (fsm == NULL) { + return EXIT_SUCCESS; + } + TIME(&post); + DIFF_MSEC("build", pre, post, NULL); + + if (getenv("DUMP")) { + fprintf(stderr,"==================================================\n"); + fsm_print(stderr, fsm, NULL, NULL, FSM_PRINT_FSM); + fprintf(stderr,"==================================================\n"); + fsm_capture_dump(stderr, "CAPTURE", fsm); + fprintf(stderr,"==================================================\n"); + } + +#if CMP_PCRE + TIME(&pre); + const int cmp_res = compare_with_pcre(pattern, fsm); + TIME(&post); + DIFF_MSEC("cmp", pre, post, NULL); + if (!cmp_res) { + fsm_free(fsm); + return EXIT_SUCCESS; + } +#endif + + TIME(&pre); + const int codegen_res = codegen(fsm, FSM_IO_STR); + TIME(&post); + DIFF_MSEC("codegen", pre, post, NULL); + if (!codegen_res) { + return EXIT_SUCCESS; + } + + fsm_free(fsm); + return EXIT_SUCCESS; +} + +static int +build_and_codegen(const char *pattern, enum fsm_io io_mode) +{ + const int verbosity = get_env_config(0, "VERBOSITY"); + if (verbosity > 1) { + fprintf(stderr, "pattern: \"%s\"\n", pattern); + } + + INIT_TIMERS(); + TIME(&pre); + struct fsm *fsm = build(pattern); + if (fsm == NULL) { + return EXIT_SUCCESS; + } + TIME(&post); + DIFF_MSEC("build", pre, post, NULL); + + if (!codegen(fsm, io_mode)) { + return EXIT_SUCCESS; + } + + fsm_free(fsm); + return EXIT_SUCCESS; +} + + +#define DEF_MAX_DEPTH 20 +#define DEF_MAX_LENGTH 10 +#define DEF_MAX_STEPS 10000 +#define DEF_MAX_MATCH_COUNT 1000 + +#if CMP_PCRE +/* These two are only used with PCRE2 */ +#define ANCHORED_PCRE 0 +#define FUZZ_RE_MATCH_LIMIT 10000 +#define FUZZ_RE_RECURSION_LIMIT 200 +#define MAX_OVEC_SIZE 512 + +static pcre2_match_context *pcre2_mc = NULL; + +struct cmp_pcre_env { + int verbosity; + const char *pattern; + const struct fsm *fsm; + pcre2_match_data *md; + pcre2_code *p; + + struct fsm_capture *captures; + size_t captures_length; + + size_t max_depth; + size_t max_steps; + size_t max_match_count; +}; + +struct test_pcre_match_info { + int res; + int pcre_error; + size_t ovector[MAX_OVEC_SIZE]; +}; + +static pcre2_code * +build_pcre2(const char *pattern, int verbosity) +{ + const uint32_t options = ANCHORED_PCRE ? PCRE2_ANCHORED : 0; + int errorcode; + PCRE2_SIZE erroffset = 0; + pcre2_compile_context *cctx = NULL; + + /* Set match limits */ + if (pcre2_mc == NULL) { + pcre2_mc = pcre2_match_context_create(NULL); + assert(pcre2_mc != NULL); + + pcre2_set_match_limit(pcre2_mc, FUZZ_RE_MATCH_LIMIT); + pcre2_set_recursion_limit(pcre2_mc, FUZZ_RE_RECURSION_LIMIT); + } + + pcre2_code *p = pcre2_compile((const unsigned char *)pattern, + PCRE2_ZERO_TERMINATED, + options, &errorcode, &erroffset, cctx); + if (verbosity > 0 && p == NULL && errorcode != 0) { +#define ERRSIZE 4096 + unsigned char errbuf[ERRSIZE] = {0}; + if (!pcre2_get_error_message(errorcode, + errbuf, ERRSIZE)) { + fprintf(stderr, "pcre2_get_error_message: failed\n"); + } + fprintf(stderr, "pcre2_compile: error: %s\n", errbuf); + } + return p; +} + +enum do_pcre_match_res { + DO_PCRE_MATCH_HIT, + DO_PCRE_MATCH_MISS, + DO_PCRE_MATCH_SKIP, /* an exceptional case we don't care about */ + DO_PCRE_MATCH_ERROR = -1, +}; +enum do_pcre_match_res +do_pcre_match(FILE *f, const pcre2_code *p, pcre2_match_data *md, int verbosity, + const char *input, struct test_pcre_match_info *match_info) +{ +#define MAX_BUF (64*1024) + const size_t input_len = strlen(input); + enum do_pcre_match_res mres; + + /* turn off the JIT because it can give inconsistent results while fuzzing */ + const uint32_t options = (ANCHORED_PCRE ? PCRE2_ANCHORED : 0) + | PCRE2_NO_JIT; + + assert(pcre2_mc != NULL); + + /* The value returned by pcre2_match() is one more than the + * highest numbered pair that has been set. */ + int res = pcre2_match(p, (const unsigned char *)input, input_len, + 0, options, md, pcre2_mc); + + if (res == PCRE2_ERROR_NOMATCH || res == PCRE2_ERROR_PARTIAL) { + if (f != NULL && verbosity > 1) { + fprintf(f, " -- no match (%s)\n", + res == PCRE2_ERROR_NOMATCH ? "NOMATCH" + : res == PCRE2_ERROR_PARTIAL ? "PARTIAL" + : ""); + } + mres = DO_PCRE_MATCH_MISS; + goto cleanup; + } else if (res == PCRE2_ERROR_MATCHLIMIT || res == PCRE2_ERROR_DEPTHLIMIT) { + /* It's possible to exhaust PCRE's internal limits with pathologically + * nested regexes like "(((((((((^.)?)*)?)?)?)*)+)+)*$" and + * "((((((((akbzaabdcOaa)|((((b*))))?|.|.|.*|.|.))+)+)+$)*)?)" , but + * as long as they don't cause it to block for excessively long or + * exhaust resources that's fine. */ + if (f != NULL) { + fprintf(f, " -- PCRE2_ERROR_MATCHLIMIT (returning SKIP)\n"); + } + mres = DO_PCRE_MATCH_SKIP; + } else if (res <= 0) { + if (f != NULL) { +#define ERR_MAX 4096 + unsigned char err_buf[ERR_MAX]; + if (pcre2_get_error_message(res, err_buf, ERR_MAX)) { + fprintf(f, " -- error %d: %s\n", res, err_buf); + } else { + fprintf(f, " -- error %d\n", res); + } +#undef ERR_MAX + } + if (match_info != NULL) { + match_info->pcre_error = res; + } + mres = DO_PCRE_MATCH_ERROR; + goto cleanup; + } else { + const uint32_t ovc = pcre2_get_ovector_count(md); + PCRE2_SIZE *ovector = pcre2_get_ovector_pointer(md); + assert(res >= 0); + size_t ures = (size_t)res; + assert(ovc > ures); + + assert(ovector[1] >= ovector[0]); + const size_t mlen = ovector[1] - ovector[0]; + if (ANCHORED_PCRE && (ovector[0] != 0 || mlen != input_len)) { + mres = DO_PCRE_MATCH_MISS; + goto cleanup; + } + mres = DO_PCRE_MATCH_HIT; + + if (f != NULL && verbosity > 1) { + for (size_t i = 0; i < ures; i++) { + char buf[MAX_BUF] = { 0 }; + memcpy(buf, &input[ovector[2*i]], + ovector[2*i + 1U] - ovector[2*i]); + fprintf(f, " -- %zu: \"%s\"\n", i, buf); + } + } + + if (match_info != NULL && res < MAX_OVEC_SIZE) { + match_info->res = res; + assert(res >= 0); + const size_t ures = (size_t)res; + + for (size_t i = 0; i < 2*ures; i++) { + match_info->ovector[i] = ovector[i]; + } + } + } + +cleanup: + return mres; +#undef MAX_BUF +} + +static bool +exec_and_compare_captures(struct cmp_pcre_env *env, + const char *input, size_t input_size, + const struct test_pcre_match_info *match_info) +{ + bool matching = true; + fsm_state_t end_state; + const uint8_t *u8_input = (const uint8_t *)input; + int res = fsm_exec_with_captures(env->fsm, u8_input, input_size, + &end_state, env->captures, env->captures_length); + + if (res < 0) { + if (env->verbosity > 1) { + fprintf(stderr, "got res of %d\n", res); + } + + return false; + } + + if (res > 0) { + assert(match_info->res >= 0); + const size_t ures = (size_t)match_info->res; + + if (env->verbosity > 1) { + fprintf(stderr, "ures %zu\n", ures); + } + + for (size_t i = 0; i < ures; i++) { + if (env->verbosity > 1) { + fprintf(stderr, "%zu/%zu: pcre [%ld, %ld] <-> libfsm [%ld, %ld]\n", + i, ures, + match_info->ovector[2*i], match_info->ovector[2*i + 1], + env->captures[i].pos[0], env->captures[i].pos[1]); + } + if ((match_info->ovector[2*i] != env->captures[i].pos[0]) + || (match_info->ovector[2*i + 1] != env->captures[i].pos[1])) { + matching = false; + } + } + + if (!matching) { + for (size_t i = 0; i < ures; i++) { + fprintf(stderr, "%zu/%zu: pcre [%ld, %ld] <-> libfsm [%ld, %ld]\n", + i, ures, + match_info->ovector[2*i], match_info->ovector[2*i + 1], + env->captures[i].pos[0], env->captures[i].pos[1]); + } + } + } + + return matching; +} + +static void +dump_pattern_and_input(const char *pattern, const char *input, size_t input_length) +{ + dump_pattern(pattern); + + fprintf(stderr, "-- Input: %zu bytes\n", input_length); + for (size_t i = 0; i < input_length; i++) { + fprintf(stderr, " %02x", (uint8_t)input[i]); + if ((i & 31) == 31) { fprintf(stderr, "\n"); } + } + if ((input_length & 31) != 31) { + fprintf(stderr, "\n"); + } + for (size_t i = 0; i < input_length; i++) { + fprintf(stderr, "%c", isprint(input[i]) ? input[i] : '.'); + if ((i & 63) == 63) { fprintf(stderr, "\n"); } + } + fprintf(stderr, "\n"); +} + +static enum fsm_generate_matches_cb_res +cmp_pcre_gen_cb(const struct fsm *fsm, + size_t depth, size_t match_count, size_t steps, + const char *input, size_t input_length, + fsm_state_t end_state, void *opaque) +{ + struct cmp_pcre_env *env = opaque; + assert(env != NULL); + + (void)fsm; + (void)depth; + (void)end_state; + + const size_t len = strlen(input); + + if (env->verbosity > 4) { + fprintf(stderr, "%s: depth %zu/%zu, match_count %zu/%zu, steps %zu/%zu\n", + __func__, + depth, env->max_depth, + match_count, env->max_match_count, + steps, env->max_steps); + } + + if (steps > env->max_steps) { + return FSM_GENERATE_MATCHES_CB_RES_HALT; + } + + if (match_count > env->max_match_count) { + return FSM_GENERATE_MATCHES_CB_RES_HALT; + } + + if (depth > env->max_depth) { + return FSM_GENERATE_MATCHES_CB_RES_PRUNE; + } + + /* Completely avoid exploring inputs with embedded 0x00 bytes. */ + if (input_length != len) { + return FSM_GENERATE_MATCHES_CB_RES_PRUNE; + } + + if (len > 0 && input[len - 1] == '\n') { + /* These will need to be handled properly, but PCRE has + * special cases for '\n' handling. */ + /* fprintf(stderr, " -- skipping input ending with '\\n'.\n"); */ + return FSM_GENERATE_MATCHES_CB_RES_PRUNE; + } + + struct test_pcre_match_info match_info = { .pcre_error = 0 }; + enum do_pcre_match_res mres = do_pcre_match(stderr, + env->p, env->md, env->verbosity, input, &match_info); + switch (mres) { + case DO_PCRE_MATCH_SKIP: + break; + case DO_PCRE_MATCH_MISS: + dump_pattern_and_input(env->pattern, input, input_length); + assert(!"matches libfsm but not with PCRE"); + return FSM_GENERATE_MATCHES_CB_RES_HALT; + case DO_PCRE_MATCH_ERROR: + fprintf(stderr, "FAIL: PCRE returned ERROR %d: pattern \"%s\"\n", + match_info.pcre_error, env->pattern); + return FSM_GENERATE_MATCHES_CB_RES_HALT; + case DO_PCRE_MATCH_HIT: + break; /* okay; continue below */ + } + + if (env->verbosity > 1) { + fprintf(stderr, "-- comparing captures for pattern \"%s\", input \"%s\" (len %zu)\n", + env->pattern, input, len); + } + + if (!exec_and_compare_captures(env, input, input_length, &match_info)) { + if (env->verbosity > 1 || 1) { + dump_pattern_and_input(env->pattern, input, input_length); + fsm_print(stderr, env->fsm, NULL, NULL, FSM_PRINT_FSM); + fsm_capture_dump(stderr, "fsm", env->fsm); + } + assert(!"captures don't match"); + } + + return FSM_GENERATE_MATCHES_CB_RES_CONTINUE; +} + +static int +compare_fixed_input(struct fsm *fsm, const char *pattern, const char *input, pcre2_match_data *md, pcre2_code *p) +{ + fsm_state_t end_state; + const size_t capture_ceil = fsm_capture_ceiling(fsm); + + struct fsm_capture *captures = malloc(capture_ceil * sizeof(captures[0])); + assert(captures != NULL); + for (size_t i = 0; i < capture_ceil; i++) { + /* clobber with meaningless but visually distinct values */ + captures[i].pos[0] = 88888888; + captures[i].pos[1] = 99999999; + }; + + const uint8_t *u8_input = (const uint8_t *)input; + const size_t input_len = strlen(input); + const int libfsm_res = fsm_exec_with_captures(fsm, u8_input, input_len, + &end_state, captures, capture_ceil); + + const bool libfsm_matching = libfsm_res > 0; + + int res = 1; + + struct test_pcre_match_info match_info = { .pcre_error = 0 }; + enum do_pcre_match_res mres = do_pcre_match(stderr, + p, md, 0, input, &match_info); + switch (mres) { + case DO_PCRE_MATCH_SKIP: + return 1; + case DO_PCRE_MATCH_MISS: + if (!libfsm_matching) { + goto cleanup; + } + dump_pattern_and_input(pattern, input, 0); + assert(!"matches libfsm but not with PCRE"); + return 0; + case DO_PCRE_MATCH_ERROR: + fprintf(stderr, "FAIL: PCRE returned ERROR %d: pattern \"%s\"\n", + match_info.pcre_error, pattern); + return 0; + case DO_PCRE_MATCH_HIT: + if (!libfsm_matching) { + dump_pattern_and_input(pattern, input, input_len); + assert(!"matches PCRE but not libfsm"); + res = 0; + goto cleanup; + } + + const size_t ures = (size_t)match_info.res; + if (ures > capture_ceil) { + dump_pattern_and_input(pattern, input, 0); + fprintf(stderr, "error: capture_ceil: %zu exceeded by ures: %zd\n", + capture_ceil, ures); + assert(!"both PCRE and libfsm match but with different capture counts"); + } + + bool matching = true; + for (size_t i = 0; i < ures; i++) { + if ((match_info.ovector[2*i] != captures[i].pos[0]) + || (match_info.ovector[2*i + 1] != captures[i].pos[1])) { + matching = false; + } + } + for (size_t i = 0; i < ures; i++) { + if (!matching) { + fprintf(stderr, "%zu/%zu: pcre [%ld, %ld] <-> libfsm [%ld, %ld]\n", + i, ures, + match_info.ovector[2*i], match_info.ovector[2*i + 1], + captures[i].pos[0], captures[i].pos[1]); + } + } + + if (!matching) { + dump_pattern_and_input(pattern, input, 0); + assert(!"both PCRE and libfsm match but with different captures"); + } + + goto cleanup; /* ok, both matched */ + } + + assert(!"unreachable"); + +cleanup: + free(captures); + return res; + +} + +static int +compare_with_pcre(const char *pattern, struct fsm *fsm) +{ + size_t verbosity = get_env_config(0, "VERBOSITY"); + size_t max_length = get_env_config(DEF_MAX_LENGTH, "MAX_LENGTH"); + size_t max_steps = get_env_config(DEF_MAX_STEPS, "MAX_STEPS"); + size_t max_depth = get_env_config(DEF_MAX_DEPTH, "MAX_DEPTH"); + size_t max_match_count = get_env_config(DEF_MAX_MATCH_COUNT, "MAX_MATCH_COUNT"); + int res = 1; + + pcre2_match_data *md; + + pcre2_code *p = build_pcre2(pattern, 0); + if (p == NULL) { + return 1; + } + + md = pcre2_match_data_create(MAX_OVEC_SIZE, NULL); + assert(md != NULL); + + /* Check the empty string and "\n", because PCRE has an awkward + * special case for "\n" that has complicated interactions + * with start and end anchoring. */ + if (!compare_fixed_input(fsm, pattern, "", md, p) + || !compare_fixed_input(fsm, pattern, "\n", md, p)) { + pcre2_match_data_free(md); + pcre2_code_free(p); + return res; + } + + struct fsm_capture captures[MAX_OVEC_SIZE/2] = { 0 }; + + const size_t pattern_length = strlen(pattern); + if (pattern_length >= max_length) { + max_length = pattern_length + 1; + static size_t max_max_length; + if (max_length > max_max_length) { + fprintf(stderr, "Note: increasing max_length to %zu\n", + pattern_length + 1); + max_max_length = max_length; + if (max_depth < max_length) { + max_depth = max_length + 1; + } + } + } + + struct cmp_pcre_env env = { + .verbosity = (int)verbosity, + .pattern = pattern, + .fsm = fsm, + .captures = captures, + .captures_length = MAX_OVEC_SIZE/2, + .md = md, + .p = p, + .max_steps = max_steps, + .max_depth = max_depth, + .max_match_count = max_match_count, + }; + + if (!fsm_generate_matches(fsm, max_length, 0, cmp_pcre_gen_cb, &env)) { + res = 0; + } + + pcre2_match_data_free(md); + pcre2_code_free(p); + return res; +} +#endif + +/* Note: combined_fsm and fsms[] are non-const because fsm_generate_matches + * calls fsm_trim on them. */ +static int +compare_separate_and_combined(int verbosity, size_t max_length, size_t count, + struct fsm *combined_fsm, const struct fsm_combined_base_pair *bases, + struct fsm **fsms); + +static enum fsm_generate_matches_cb_res +cmp_separate_and_combined_cb(const struct fsm *fsm, + size_t depth, size_t match_count, size_t steps, + const char *input, size_t input_length, + fsm_state_t end_state, void *opaque); + +static int +build_and_check_multi(const char *input) +{ + int res = EXIT_FAILURE; + const int verbosity = get_env_config(0, "VERBOSITY"); +#define MAX_PATTERNS 8 +#define MAX_PATTERN_LEN 256 + char patterns[MAX_PATTERNS][MAX_PATTERN_LEN] = { 0 }; + size_t count = 0; + const size_t len = strlen(input); + size_t max_length = get_env_config(DEF_MAX_LENGTH, "MAX_LENGTH"); + INIT_TIMERS(); + + /* if nonzero, apply a timeout to the combined FSM det/min below */ + const size_t timeout = get_env_config(0, "TIMEOUT"); + + if (timeout > 0) { + if (TRACK_TIMES == 0) { + fprintf(stderr, "\n\n\n\n\nError: src/libfsm/internal.h:TRACK_TIMES needs to be nonzero for this use case, exiting.\n\n\n\n\n"); + exit(EXIT_FAILURE); + } else { + static bool printed_timeout_msg; + if (!printed_timeout_msg) { + fprintf(stderr, "Using timeout of %zu msec for fsm_determinise/fsm_minimise on combined FSM.\n", + timeout); + printed_timeout_msg = true; + } + } + } + + size_t i, j; + for (i = 0, j = 0; i < len; i++) { + const char c = input[i]; + if (c == '\n' || c == '\r') { + if (j > max_length) { + max_length = j; + } + count++; + if (count == MAX_PATTERNS) { + /* ignore: too many patterns */ + return EXIT_SUCCESS; + } + j = 0; + } else { + patterns[count][j] = c; + j++; + if (j == MAX_PATTERN_LEN) { + /* ignore: pattern too long */ + return EXIT_SUCCESS; + } + } + } + if (j > 0) { count++; } + + if (count == 1) { + /* multi mode with only one pattern is pointless */ + return EXIT_SUCCESS; + } + + struct re_err err; + const enum re_flags flags = 0; + + /* build each regex, combining them and keeping track of capture offsets */ + struct fsm *fsms[count]; + struct fsm *fsms_cp[count]; + struct fsm_combined_base_pair bases[count]; + struct fsm *combined_fsm = NULL; + for (size_t i = 0; i < count; i++) { + fsms[i] = NULL; + fsms_cp[i] = NULL; + + bases[i].state = 0; + bases[i].capture = 0; + } + + /* compile each individually */ + /* FIXME: apply and check endids */ + for (size_t i = 0; i < count; i++) { + if (verbosity > 1) { + fprintf(stderr, "%s: compiling \"%s\"\n", + __func__, patterns[i]); + } + + struct scanner s = { + .str = (const uint8_t *)patterns[i], + .size = strlen(patterns[i]), + }; + + struct fsm *fsm = re_comp(RE_PCRE, scanner_next, &s, &custom_allocators, flags, &err); + if (fsm == NULL) { + res = EXIT_SUCCESS; /* invalid regex, so skip this batch */ + goto cleanup; + } + + /* set endid to associate each FSM with its pattern */ + if (!fsm_setendid(fsm, (fsm_end_id_t)i)) { + goto cleanup; + } + + char label_buf[100]; + snprintf(label_buf, 100, "single_determisise_%zu", i); + + TIME(&pre); + if (!fsm_determinise(fsm)) { + goto cleanup; + } + TIME(&post); + DIFF_MSEC(label_buf, pre, post, NULL); + + snprintf(label_buf, 100, "single_minimise_%zu", i); + TIME(&pre); + if (!fsm_minimise(fsm)) { + goto cleanup; + } + TIME(&post); + DIFF_MSEC(label_buf, pre, post, NULL); + + if (verbosity > 4) { + char tag_buf[16] = { 0 }; + snprintf(tag_buf, sizeof(tag_buf), "fsm[%zu]", i); + + fprintf(stderr, "==== fsm[%zu]\n", i); + fsm_print(stderr, fsm, NULL, NULL, FSM_PRINT_FSM); + fsm_capture_dump(stderr, tag_buf, fsm); + } + + fsms[i] = fsm; + fsms_cp[i] = fsm_clone(fsm); /* save a copy for comparison */ + } + + combined_fsm = fsm_union_array(count, fsms, bases); + assert(combined_fsm != NULL); + if (verbosity > 1) { + fprintf(stderr, "%s: combined_fsm: %d states after fsm_union_array\n", + __func__, fsm_countstates(combined_fsm)); + } + if (verbosity > 1) { + for (size_t i = 0; i < count; i++) { + fprintf(stderr, "%s: base[%zu]: state %d, capture %u\n", + __func__, i, bases[i].state, bases[i].capture); + } + } + + TIME(&pre); + if (!fsm_determinise(combined_fsm)) { + goto cleanup; + } + TIME(&post); + size_t timeout_accum = 0; + if (timeout != 0) { + if (verbosity > 1) { + DIFF_MSEC_ALWAYS("combined_determinise", pre, post, &timeout_accum); + } else { + DIFF_MSEC("combined_determinise", pre, post, &timeout_accum); + } + assert(timeout_accum < timeout); + timeout_accum = 0; + } + + const unsigned states_after_determinise = fsm_countstates(combined_fsm); + if (verbosity > 1) { + fprintf(stderr, "%s: combined_fsm: %d states after determinise\n", + __func__, states_after_determinise); + } + + TIME(&pre); + if (!fsm_minimise(combined_fsm)) { + goto cleanup; + } + TIME(&post); + if (timeout != 0) { + if (verbosity > 1) { + DIFF_MSEC_ALWAYS("combined_minimise", pre, post, &timeout_accum); + } else { + DIFF_MSEC("combined_minimise", pre, post, &timeout_accum); + } + assert(timeout_accum < timeout); + timeout_accum = 0; + } + + const unsigned states_after_minimise = fsm_countstates(combined_fsm); + if (verbosity > 1) { + fprintf(stderr, "%s: combined_fsm: %d states after minimise\n", + __func__, states_after_minimise); + } + + if (verbosity > 4) { + fprintf(stderr, "==== combined\n"); + fsm_print(stderr, combined_fsm, NULL, NULL, FSM_PRINT_FSM); + fsm_capture_dump(stderr, "combined", combined_fsm); + } + + res = compare_separate_and_combined(verbosity, max_length, + count, combined_fsm, bases, (struct fsm **)fsms_cp); + + for (i = 0; i < count; i++) { + fsm_free(fsms_cp[i]); + } + fsm_free(combined_fsm); + + if (res == EXIT_SUCCESS) { + static size_t pass_count; + if (verbosity == 1) { + fprintf(stderr, "%s: pass: %zu, %zu patterns\n", + __func__, ++pass_count, count); + } else if (verbosity > 1) { + fprintf(stderr, "%s: pass: %zu, %zu patterns\n", + __func__, ++pass_count, count); + for (i = 0; i < count; i++) { + fprintf(stderr, " -- %zu: \"%s\"\n", + i, patterns[i]); + } + } + } + + return res; + +cleanup: + for (i = 0; i < count; i++) { + if (fsms[i] != NULL) { + fsm_free(fsms[i]); + } + if (fsms_cp[i] != NULL) { + fsm_free(fsms_cp[i]); + } + } + if (combined_fsm != NULL) { + fsm_free(combined_fsm); + } + return res; +} + +struct cmp_combined_env { + bool ok; + int verbosity; + size_t count; + struct fsm *combined_fsm; + const struct fsm_combined_base_pair *bases; + size_t current_i; + struct fsm **fsms; + size_t max_depth; + size_t max_steps; + size_t max_match_count; +}; + +static enum fsm_generate_matches_cb_res +cmp_combined_with_separate_cb(const struct fsm *fsm, + size_t depth, size_t match_count, size_t steps, + const char *input, size_t input_length, + fsm_state_t end_state, void *opaque); + +static int +compare_separate_and_combined(int verbosity, size_t max_length, size_t count, + struct fsm *combined_fsm, const struct fsm_combined_base_pair *bases, + struct fsm **fsms) +{ + const size_t max_steps = get_env_config(DEF_MAX_STEPS, "MAX_STEPS"); + const size_t max_depth = get_env_config(DEF_MAX_DEPTH, "MAX_DEPTH"); + const size_t max_match_count = get_env_config(DEF_MAX_MATCH_COUNT, "MAX_MATCH_COUNT"); + + struct cmp_combined_env env = { + .ok = true, + .verbosity = verbosity, + .count = count, + .combined_fsm = combined_fsm, + .bases = bases, + .fsms = fsms, + .max_steps = max_steps, + .max_depth = max_depth, + .max_match_count = max_match_count, + }; + + /* For each individual fsm, generate matching inputs and check that + * they match with the same captures in the combined fsm. */ + for (env.current_i = 0; env.current_i < count; env.current_i++) { + if (!fsm_generate_matches(env.fsms[env.current_i], max_length, + 0, cmp_separate_and_combined_cb, &env)) { + env.ok = false; + } + if (!env.ok) { + break; + } + } + env.current_i = (size_t)-1; + + /* Also go in the other direction, generating matches with + * combined and check the individual ones match as expected. */ + if (env.ok) { + if (!fsm_generate_matches(env.combined_fsm, max_length, + 0, cmp_combined_with_separate_cb, &env)) { + env.ok = false; + } + } + + return env.ok ? EXIT_SUCCESS : EXIT_FAILURE; +} +#undef DEF_MAX_STEPS + +static enum fsm_generate_matches_cb_res +cmp_separate_and_combined_cb(const struct fsm *fsm, + size_t depth, size_t match_count, size_t steps, + const char *input, size_t input_length, + fsm_state_t end_state, void *opaque) +{ + struct cmp_combined_env *env = opaque; + (void)end_state; + + if (steps > env->max_steps) { + return FSM_GENERATE_MATCHES_CB_RES_HALT; + } + + if (depth > env->max_depth) { + return FSM_GENERATE_MATCHES_CB_RES_PRUNE; + } + + if (match_count > env->max_match_count) { + return FSM_GENERATE_MATCHES_CB_RES_HALT; + } + +#define MAX_CAPTURES 256 + struct fsm_capture captures_single[MAX_CAPTURES]; + struct fsm_capture captures_combined[MAX_CAPTURES]; + + const fsm_end_id_t expected_end_id = (fsm_end_id_t)env->current_i; + + const uint8_t *u8_input = (const uint8_t *)input; + fsm_state_t end_state_combined, end_state_single; + + const int res_combined = fsm_exec_with_captures(env->combined_fsm, u8_input, input_length, + &end_state_combined, captures_combined, MAX_CAPTURES); + const int res_single = fsm_exec_with_captures(fsm, u8_input, input_length, + &end_state_single, captures_single, MAX_CAPTURES); + + if (res_combined != res_single) { + env->ok = false; + if (env->verbosity > 0) { + fprintf(stderr, "%s: res_combined %d != res_single %d\n", + __func__, res_combined, res_single); + } + return FSM_GENERATE_MATCHES_CB_RES_HALT; + } + + fsm_end_id_t id_buf_combined[MAX_PATTERNS]; + size_t exp_written_combined = 0; + if (res_combined > 0) { + exp_written_combined = fsm_endid_count(env->combined_fsm, end_state_combined); + assert(exp_written_combined <= env->count); + const enum fsm_getendids_res gres = fsm_endid_get(env->combined_fsm, + end_state_combined, MAX_PATTERNS, id_buf_combined); + assert(gres == FSM_GETENDIDS_FOUND); + } + + /* we got here, so we have a match */ + assert(res_single > 0); + + if (env->verbosity > 3) { + fprintf(stderr, "%s: res %d (single and combined)\n", __func__, res_single); + } + + /* Check that the end state's endid for the single DFA is among the + * endids for the combined DFA's end state. */ + assert(fsm_endid_count(fsm, end_state_single) == 1); + assert(fsm_endid_count(env->combined_fsm, end_state_combined) <= env->count); + + fsm_end_id_t id_buf_single[1]; + const enum fsm_getendids_res gres = fsm_endid_get(fsm, + end_state_single, 1, id_buf_single); + assert(gres == FSM_GETENDIDS_FOUND); + assert(id_buf_single[0] == expected_end_id); + + bool found_single_id_in_combined = false; + for (size_t i = 0; i < exp_written_combined; i++) { + if (id_buf_combined[i] == expected_end_id) { + found_single_id_in_combined = true; + break; + } + } + assert(found_single_id_in_combined); + + bool matching = true; + const unsigned base = env->bases[env->current_i].capture; + assert(base < MAX_CAPTURES); + for (int i = 0; i < res_single; i++) { + if (env->verbosity > 3) { + fprintf(stderr, "%d/%d: single [%ld, %ld] <-> combined [%ld, %ld]\n", + i, res_single, + captures_single[i].pos[0], captures_single[i].pos[1], + captures_combined[i + base].pos[0], captures_combined[i + base].pos[1]); + } + if ((captures_single[i].pos[0] != captures_combined[i + base].pos[0]) || + (captures_single[i].pos[1] != captures_combined[i + base].pos[1])) { + matching = false; + } + } + + if (!matching) { + for (int i = 0; i < res_single; i++) { + fprintf(stderr, "%d/%d: single [%ld, %ld] <-> combined [%ld, %ld]\n", + i, res_single, + captures_single[i].pos[0], captures_single[i].pos[1], + captures_combined[i + base].pos[0], captures_combined[i + base].pos[1]); + } + env->ok = false; + return FSM_GENERATE_MATCHES_CB_RES_HALT; + } + + return FSM_GENERATE_MATCHES_CB_RES_CONTINUE; +} + +static enum fsm_generate_matches_cb_res +cmp_combined_with_separate_cb(const struct fsm *fsm, + size_t depth, size_t match_count, size_t steps, + const char *input, size_t input_length, + fsm_state_t end_state, void *opaque) +{ + /* We have an input that matched the combined DFA, + * use the set of end IDs to check which of the + * single DFAs it should/should not match, and check + * the endid behavior. */ + + struct cmp_combined_env *env = opaque; + + if (steps > env->max_steps) { + return FSM_GENERATE_MATCHES_CB_RES_HALT; + } + + if (depth > env->max_depth) { + return FSM_GENERATE_MATCHES_CB_RES_PRUNE; + } + + if (match_count > env->max_match_count) { + return FSM_GENERATE_MATCHES_CB_RES_HALT; + } + +#define MAX_CAPTURES 256 + struct fsm_capture captures_single[MAX_CAPTURES]; + struct fsm_capture captures_combined[MAX_CAPTURES]; + + const uint8_t *u8_input = (const uint8_t *)input; + + fsm_state_t end_state_combined; + assert(fsm == env->combined_fsm); + const int res_combined = fsm_exec_with_captures(env->combined_fsm, u8_input, input_length, + &end_state_combined, captures_combined, MAX_CAPTURES); + assert(res_combined > 0); /* we got here, so we have a match */ + assert(end_state_combined == end_state); + + fsm_end_id_t id_buf_combined[MAX_PATTERNS]; + const size_t combined_endid_count = fsm_endid_count(env->combined_fsm, end_state_combined); + { + assert(combined_endid_count <= env->count); + const enum fsm_getendids_res gres = fsm_endid_get(env->combined_fsm, + end_state_combined, MAX_PATTERNS, id_buf_combined); + assert(gres == FSM_GETENDIDS_FOUND); } - if (!codegen(fsm)) { - return EXIT_SUCCESS; + /* For each pattern, check if its endid is in the combined DFA's end state + * endids. If so, it should match, otherwise it should not. */ + for (size_t pattern_i = 0; pattern_i < env->count; pattern_i++) { + const struct fsm *single_fsm = env->fsms[pattern_i]; + bool found = false; + for (size_t endid_i = 0; endid_i < combined_endid_count; endid_i++) { + const fsm_end_id_t endid = id_buf_combined[endid_i]; + if (endid == pattern_i) { + found = true; + break; + } + } + fsm_state_t end_state_single; + + const int res_single = fsm_exec_with_captures(single_fsm, + u8_input, input_length, + &end_state_single, captures_single, MAX_CAPTURES); + + if (found) { + assert(res_single > 0); + fsm_end_id_t id_buf_single[1]; + const enum fsm_getendids_res gres = fsm_endid_get(single_fsm, + end_state_single, 1, id_buf_single); + assert(gres == FSM_GETENDIDS_FOUND); + assert(id_buf_single[0] == pattern_i); + + /* check captures */ + bool matching = true; + const unsigned base = env->bases[pattern_i].capture; + assert(base < MAX_CAPTURES); + for (int i = 0; i < res_single; i++) { + if (env->verbosity > 3) { + fprintf(stderr, "%d/%d: single [%ld, %ld] <-> combined [%ld, %ld]\n", + i, res_single, + captures_single[i].pos[0], captures_single[i].pos[1], + captures_combined[i + base].pos[0], captures_combined[i + base].pos[1]); + } + if ((captures_single[i].pos[0] != captures_combined[i + base].pos[0]) || + (captures_single[i].pos[1] != captures_combined[i + base].pos[1])) { + matching = false; + } + } + + if (!matching) { + for (int i = 0; i < res_single; i++) { + fprintf(stderr, "%d/%d: single [%ld, %ld] <-> combined [%ld, %ld]\n", + i, res_single, + captures_single[i].pos[0], captures_single[i].pos[1], + captures_combined[i + base].pos[0], captures_combined[i + base].pos[1]); + } + env->ok = false; + return FSM_GENERATE_MATCHES_CB_RES_HALT; + } + } else { + assert(res_single == 0); /* no match */ + } } - fsm_free(fsm); - return EXIT_SUCCESS; + return FSM_GENERATE_MATCHES_CB_RES_CONTINUE; } +#undef MAX_PATTERNS + #define DEF_MAX_SHUFFLE 10 #define DEF_MAX_MINIMISE_ORACLE_STATE_COUNT 1000 @@ -176,7 +1404,7 @@ shuffle_minimise(const char *pattern) .offset = 0 }; - fsm = re_comp(RE_PCRE, scanner_next, &s, &opt, RE_MULTI, &err); + fsm = re_comp(RE_PCRE, scanner_next, &s, &custom_allocators, RE_MULTI, &err); if (fsm == NULL) { /* ignore invalid regexp syntax, etc. */ @@ -245,14 +1473,14 @@ shuffle_minimise(const char *pattern) __func__, s_i, pattern, expected_state_count, cp_state_count); fprintf(stderr, "== original input:\n"); - fsm_print_fsm(stderr, fsm); + fsm_print(stderr, fsm, NULL, NULL, FSM_PRINT_FSM); fprintf(stderr, "== expected:\n"); - fsm_print_fsm(stderr, oracle_min); + fsm_print(stderr, oracle_min, NULL, NULL, FSM_PRINT_FSM); fprintf(stderr, "== got:\n"); - fsm_print_fsm(stderr, cp); + fsm_print(stderr, cp, NULL, NULL, FSM_PRINT_FSM); fsm_free(cp); fsm_free(oracle_min); @@ -283,11 +1511,11 @@ fuzz_all_print_functions(FILE *f, const char *pattern, bool det, bool min, const .offset = 0 }; - const struct fsm_options options = { + const struct fsm_options opt = { .io = io_mode, }; - fsm = re_comp(RE_PCRE, scanner_next, &s, &options, RE_MULTI, &err); + fsm = re_comp(RE_PCRE, scanner_next, &s, NULL, RE_MULTI, &err); if (fsm == NULL) { /* ignore invalid regexp syntax, etc. */ return EXIT_SUCCESS; @@ -312,29 +1540,613 @@ fuzz_all_print_functions(FILE *f, const char *pattern, bool det, bool min, const /* see if this triggers any asserts */ int r = 0; - r |= fsm_print_api(f, fsm); - r |= fsm_print_awk(f, fsm); - r |= fsm_print_c(f, fsm); - r |= fsm_print_dot(f, fsm); - r |= fsm_print_fsm(f, fsm); - r |= fsm_print_ir(f, fsm); - r |= fsm_print_irjson(f, fsm); - r |= fsm_print_json(f, fsm); - r |= fsm_print_vmc(f, fsm); - r |= fsm_print_vmdot(f, fsm); - r |= fsm_print_vmasm(f, fsm); - r |= fsm_print_vmasm_amd64_att(f, fsm); - r |= fsm_print_vmasm_amd64_nasm(f, fsm); - r |= fsm_print_vmasm_amd64_go(f, fsm); - r |= fsm_print_sh(f, fsm); - r |= fsm_print_go(f, fsm); - r |= fsm_print_rust(f, fsm); + + r |= fsm_print(f, fsm, &opt, NULL, FSM_PRINT_AMD64_ATT); + r |= fsm_print(f, fsm, &opt, NULL, FSM_PRINT_AMD64_GO); + r |= fsm_print(f, fsm, &opt, NULL, FSM_PRINT_AMD64_NASM); + + r |= fsm_print(f, fsm, &opt, NULL, FSM_PRINT_API); + r |= fsm_print(f, fsm, &opt, NULL, FSM_PRINT_AWK); + r |= fsm_print(f, fsm, &opt, NULL, FSM_PRINT_C); + r |= fsm_print(f, fsm, &opt, NULL, FSM_PRINT_DOT); + r |= fsm_print(f, fsm, &opt, NULL, FSM_PRINT_FSM); + r |= fsm_print(f, fsm, &opt, NULL, FSM_PRINT_GO); + r |= fsm_print(f, fsm, &opt, NULL, FSM_PRINT_IR); + r |= fsm_print(f, fsm, &opt, NULL, FSM_PRINT_IRJSON); + r |= fsm_print(f, fsm, &opt, NULL, FSM_PRINT_JSON); + r |= fsm_print(f, fsm, &opt, NULL, FSM_PRINT_LLVM); + r |= fsm_print(f, fsm, &opt, NULL, FSM_PRINT_RUST); + r |= fsm_print(f, fsm, &opt, NULL, FSM_PRINT_SH); + r |= fsm_print(f, fsm, &opt, NULL, FSM_PRINT_VMC); + r |= fsm_print(f, fsm, &opt, NULL, FSM_PRINT_VMDOT); + + r |= fsm_print(f, fsm, &opt, NULL, FSM_PRINT_VMOPS_C); + r |= fsm_print(f, fsm, &opt, NULL, FSM_PRINT_VMOPS_H); + r |= fsm_print(f, fsm, &opt, NULL, FSM_PRINT_VMOPS_MAIN); + assert(r == 0 || errno != 0); fsm_free(fsm); return EXIT_SUCCESS; } +static int +build_and_test_idempotent_det_and_min(const char *pattern) +{ + const int verbosity = get_env_config(0, "VERBOSITY"); + assert(pattern != NULL); + + struct re_err err; + struct fsm *fsm; + const size_t length = strlen(pattern); + + struct scanner s = { + .str = (const uint8_t *)pattern, + .size = length, + }; + + fsm = re_comp(RE_PCRE, scanner_next, &s, &custom_allocators, RE_MULTI, &err); + if (fsm == NULL) { + return EXIT_SUCCESS; + } + + if (!fsm_determinise(fsm)) { + return EXIT_FAILURE; + } + if (verbosity >= 3) { + fprintf(stderr, "=== post_det_a\n"); + fsm_print(stderr, fsm, NULL, NULL, FSM_PRINT_FSM); + } + const size_t post_det_a = fsm_countstates(fsm); + + if (!fsm_determinise(fsm)) { + return EXIT_FAILURE; + } + if (verbosity >= 3) { + fprintf(stderr, "=== post_det_b\n"); + fsm_print(stderr, fsm, NULL, NULL, FSM_PRINT_FSM); + } + const size_t post_det_b = fsm_countstates(fsm); + assert(post_det_b == post_det_a); + + if (!fsm_minimise(fsm)) { + return EXIT_FAILURE; + } + if (verbosity >= 3) { + fprintf(stderr, "=== post_min_a\n"); + fsm_print(stderr, fsm, NULL, NULL, FSM_PRINT_FSM); + fsm_capture_dump(stderr, "post_a", fsm); + } + const size_t post_min_a = fsm_countstates(fsm); + + if (!fsm_minimise(fsm)) { + return EXIT_FAILURE; + } + if (verbosity >= 3) { + fprintf(stderr, "=== post_min_b\n"); + fsm_print(stderr, fsm, NULL, NULL, FSM_PRINT_FSM); + fsm_capture_dump(stderr, "post_b", fsm); + } + const size_t post_min_b = fsm_countstates(fsm); + assert(post_min_b == post_min_a); + + if (!fsm_determinise(fsm)) { + return EXIT_FAILURE; + } + const size_t post_det_c = fsm_countstates(fsm); + assert(post_det_c == post_min_b); + + if (!fsm_minimise(fsm)) { + return EXIT_FAILURE; + } + const size_t post_min_c = fsm_countstates(fsm); + assert(post_min_c == post_det_c); + + fsm_free(fsm); + return EXIT_SUCCESS; +} + +#define MAX_PATTERNS 4 +struct eager_output_cb_info { + size_t used; + fsm_output_id_t ids[MAX_PATTERNS]; +}; + +static void +reset_eager_output_info(struct eager_output_cb_info *info) +{ + info->used = 0; +} + +struct feo_env { + bool ok; + size_t pattern_count; + size_t fsm_count; + size_t max_match_count; + size_t max_steps; + + char *patterns[MAX_PATTERNS]; + struct fsm *fsms[MAX_PATTERNS]; + struct fsm *combined; + + /* which pattern is being used for generation, (size_t)-1 for combined */ + size_t current_pattern; + + struct eager_output_cb_info outputs; + struct eager_output_cb_info outputs_combined; +}; + +void +append_eager_output_cb(fsm_output_id_t id, void *opaque) +{ + struct eager_output_cb_info *info = (struct eager_output_cb_info *)opaque; + + for (size_t i = 0; i < info->used; i++) { + if (info->ids[i] == id) { + return; /* already present */ + } + } + + assert(info->used < MAX_PATTERNS); + info->ids[info->used++] = id; +} + +static enum fsm_generate_matches_cb_res +gen_combined_check_individual_cb(const struct fsm *fsm, + size_t depth, size_t match_count, size_t steps, + const char *input, size_t input_length, + fsm_state_t end_state, void *opaque); + +static enum fsm_generate_matches_cb_res +gen_individual_check_combined_cb(const struct fsm *fsm, + size_t depth, size_t match_count, size_t steps, + const char *input, size_t input_length, + fsm_state_t end_state, void *opaque); + +#define DEF_MAX_STEPS 100000 +#define DEF_MAX_MATCH_COUNT 1000 + +/* This isn't part of the public interface, per se. */ +void +fsm_eager_output_dump(FILE *f, const struct fsm *fsm); + +static int +fuzz_eager_output(const uint8_t *data, size_t size) +{ + struct feo_env env = { + .ok = true, + .pattern_count = 0, + .max_steps = DEF_MAX_STEPS, + .max_match_count = DEF_MAX_MATCH_COUNT, + }; + + { + const char *steps = getenv("STEPS"); + const char *matches = getenv("MATCHES"); + if (steps != NULL) { + env.max_steps = strtoul(steps, NULL, 10); + assert(env.max_steps > 0); + } + if (matches != NULL) { + env.max_match_count = strtoul(matches, NULL, 10); + assert(env.max_match_count > 0); + } + } + + int ret = 0; + + size_t max_pattern_length = 0; + + const unsigned seed = size == 0 ? 0 : data[0]; + + /* chop data into a series of patterns */ + { + size_t prev = 0; + size_t offset = 0; + + /* Patterns with lots of '.' can take a while to determinise. + * That slows down fuzzer coverage, but isn't interesting here. */ + size_t dots = 0; + + while (offset < size && env.pattern_count < MAX_PATTERNS) { +#define MAX_DOTS 4 + if (data[offset] == '.') { dots++; } + + if (data[offset] == '\0' || data[offset] == '\n' || offset == size - 1) { + size_t len = offset - prev; + + if (dots > MAX_DOTS) { + /* ignored */ + prev = offset; + } else if (len > 0) { + char *pattern = malloc(len + 1); + assert(pattern != NULL); + + memcpy(pattern, &data[prev], len); + if (len > 0 && pattern[len] == '\n') { + len--; /* drop trailing newline */ + } + pattern[len] = '\0'; + bool keep = true; + + if (len > 0) { + for (size_t i = 0; i < len - 1; i++) { + if (pattern[i] == '\\' && pattern[i + 1] == 'x') { + /* ignore unhandled parser errors from "\x", see #386 */ + keep = false; + } + } + } + + if (keep) { + env.patterns[env.pattern_count++] = pattern; + + if (len > max_pattern_length) { + max_pattern_length = len; + } + } else { + free(pattern); + } + prev = offset; + dots = 0; + } + } + + offset++; + } + } + + /* for each pattern, attempt to compile to a DFA */ + for (size_t p_i = 0; p_i < env.pattern_count; p_i++) { + const char *p = env.patterns[p_i]; + + enum re_is_anchored_res a = re_is_anchored(RE_PCRE, fsm_sgetc, &p, 0, NULL); + if (a == RE_IS_ANCHORED_ERROR) { + continue; /* unsupported regex */ + } + + p = env.patterns[p_i]; + struct fsm *fsm = re_comp(RE_PCRE, fsm_sgetc, &p, NULL, 0, NULL); + + LOG("%s: pattern %zd: '%s' => %p\n", __func__, p_i, env.patterns[p_i], (void *)fsm); + + if (fsm == NULL) { + continue; /* invalid regex */ + } + + const fsm_output_id_t endid = (fsm_output_id_t)p_i; + ret = fsm_eager_output_set_on_ends(fsm, endid); + assert(ret == 1); + + if (verbose) { + fprintf(stderr, "==== pattern %zd, pre det\n", p_i); + fsm_dump(stderr, fsm); + fsm_eager_output_dump(stderr, fsm); + fprintf(stderr, "====\n"); + + fsm_state_t c = fsm_countstates(fsm); + for (fsm_state_t i = 0; i < c; i++) { + fprintf(stderr, "-- %d: end? %d\n", i, fsm_isend(fsm, i)); + } + } + + ret = fsm_determinise(fsm); + assert(ret == 1); + + ret = fsm_minimise(fsm); + assert(ret == 1); + + fsm_state_t start; + if (!fsm_getstart(fsm, &start)) { + fsm_free(fsm); + continue; + } + + if (verbose) { + fprintf(stderr, "==== pattern %zd, post det\n", p_i); + fsm_dump(stderr, fsm); + fsm_eager_output_dump(stderr, fsm); + fprintf(stderr, "====\n"); + + fsm_state_t c = fsm_countstates(fsm); + for (fsm_state_t i = 0; i < c; i++) { + fprintf(stderr, "-- %d: end? %d\n", i, fsm_isend(fsm, i)); + } + } + + fsm_eager_output_set_cb(fsm, append_eager_output_cb, &env.outputs); + env.fsms[env.fsm_count++] = fsm; + } + + /* don't bother checking combined behavior unless there's multiple DFAs */ + if (env.fsm_count < 2) { goto cleanup; } + + /* copy and combine fsms into one DFA */ + { + size_t used = 0; + struct fsm *nfas[MAX_PATTERNS] = {0}; + + for (size_t i = 0; i < env.fsm_count; i++) { + /* there can be gaps, fsms[] lines up with patterns[] */ + if (env.fsms[i] == NULL) { continue; } + + fsm_state_t start; + if (!fsm_getstart(env.fsms[i], &start)) { + assert(!"hit"); + } + + struct fsm *cp = fsm_clone(env.fsms[i]); + assert(cp != NULL); + + if (verbose) { + fprintf(stderr, "==== cp %zd\n", i); + fsm_dump(stderr, cp); + fsm_eager_output_dump(stderr, cp); + fprintf(stderr, "====\n"); + + fsm_state_t c = fsm_countstates(cp); + for (fsm_state_t i = 0; i < c; i++) { + fprintf(stderr, "-- %d: end? %d\n", i, fsm_isend(cp, i)); + } + } + + nfas[used] = cp; + used++; + } + + if (used == 0) { + goto cleanup; /* nothing to do */ + } + + /* consumes nfas[] */ + struct fsm *fsm = fsm_union_repeated_pattern_group(used, nfas, NULL, 0); + assert(fsm != NULL); + + if (verbose) { + fprintf(stderr, "==== combined (pre-det)\n"); + fsm_dump(stderr, fsm); + fsm_eager_output_dump(stderr, fsm); + fprintf(stderr, "====\n"); + } + + if (!fsm_determinise(fsm)) { + assert(!"failed to determinise"); + } + + if (!fsm_minimise(fsm)) { + assert(!"failed to minimise"); + } + + LOG("%s: combined state_count %d\n", __func__, fsm_countstates(fsm)); + env.combined = fsm; + /* fsm_eager_output_set_cb(fsm, append_eager_output_cb, &env.outputs_combined); */ + + if (verbose) { + fprintf(stderr, "==== combined\n"); + fsm_dump(stderr, env.combined); + fsm_eager_output_dump(stderr, env.combined); + fprintf(stderr, "====\n"); + } + + } + + /* Use fsm_generate_matches to check for matches that got lost + * and false positives introduced while combining the DFAs. + * Use the combined DFA to generate matches, check that the + * match behavior agrees with the individual DFA copies. */ + env.current_pattern = (size_t)-1; + if (!fsm_generate_matches(env.combined, max_pattern_length, seed, gen_combined_check_individual_cb, &env)) { + goto cleanup; + } + + if (!env.ok) { goto cleanup; } + + /* Likewise, use every individual DFA to generate matches and */ + /* check behavior against the combined DFA. */ + for (size_t i = 0; i < env.pattern_count; i++) { + env.current_pattern = i; + if (!fsm_generate_matches(env.combined, max_pattern_length, seed, gen_individual_check_combined_cb, &env)) { + goto cleanup; + } + } + + ret = env.ok ? EXIT_SUCCESS : EXIT_FAILURE; +cleanup: + for (size_t i = 0; i < MAX_PATTERNS; i++) { + if (env.patterns[i] != NULL) { + free(env.patterns[i]); + env.patterns[i] = NULL; + } + if (env.fsms[i] != NULL) { + fsm_free(env.fsms[i]); + } + } + if (env.combined != NULL) { + fsm_free(env.combined); + } + + return ret; +} + +static int +cmp_output_id(const void *pa, const void *pb) +{ + const fsm_output_id_t a = *(fsm_output_id_t *)pa; + const fsm_output_id_t b = *(fsm_output_id_t *)pb; + return a < b ? -1 : a > b ? 1 : 0; +} + +static bool +match_input_get_eager_outputs(struct fsm *fsm, const char *input, size_t input_length, + struct eager_output_cb_info *dst) +{ + (void)input_length; + fsm_state_t end; + + reset_eager_output_info(dst); + + fsm_eager_output_set_cb(fsm, append_eager_output_cb, dst); + const int ret = fsm_exec(fsm, fsm_sgetc, &input, &end); + if (ret == 0) { + return false; /* no match */ + } else { + assert(ret == 1); /* match */ + } + + /* sort the IDs, to make comparison cheaper */ + qsort(dst->ids, dst->used, sizeof(dst->ids[0]), cmp_output_id); + return true; /* match */ +} + +/* For a given matching input generated by the combined DFA, check that + * only the expected individual source DFAs match. */ +static enum fsm_generate_matches_cb_res +gen_combined_check_individual_cb(const struct fsm *fsm, + size_t depth, size_t match_count, size_t steps, + const char *input, size_t input_length, + fsm_state_t end_state, void *opaque) +{ + (void)fsm; + (void)depth; + (void)end_state; + + struct feo_env *env = opaque; + assert(env->current_pattern == (size_t)-1); + + if (match_count > env->max_match_count) { return FSM_GENERATE_MATCHES_CB_RES_HALT; } + if (steps > env->max_steps) { return FSM_GENERATE_MATCHES_CB_RES_HALT; } + + /* execute, to set eager outputs */ + if (!match_input_get_eager_outputs(env->combined, input, input_length, &env->outputs_combined)) { + env->ok = false; + return FSM_GENERATE_MATCHES_CB_RES_HALT; + } + + size_t individual_outputs_used = 0; + fsm_output_id_t individual_outputs[MAX_PATTERNS]; + + for (size_t i = 0; i < env->pattern_count; i++) { + struct fsm *fsm = env->fsms[i]; + if (fsm == NULL) { continue; } + + if (!match_input_get_eager_outputs(fsm, input, input_length, &env->outputs)) { + env->ok = false; + return FSM_GENERATE_MATCHES_CB_RES_HALT; + } + + if (env->outputs.used > 0) { + assert(env->outputs.used == 1); + individual_outputs[individual_outputs_used++] = env->outputs.ids[0]; + } + } + + bool match = true; + if (env->outputs_combined.used != individual_outputs_used) { + match = false; + } + + for (size_t cmb_i = 0; cmb_i < env->outputs_combined.used; cmb_i++) { + const fsm_output_id_t cur = env->outputs_combined.ids[cmb_i]; + assert(env->fsms[cmb_i] != NULL); + bool found = false; + for (size_t i = 0; i < individual_outputs_used; i++) { + if (individual_outputs[i] == cur) { + found = true; + break; + } + } + if (!found) { + match = false; + break; + } + } + + if (!match) { + fprintf(stderr, "%s: combined <-> individual mismatch for input '%s'(%zd)!\n", __func__, input, input_length); + + fprintf(stderr, "-- combined: %zu IDs:", env->outputs_combined.used); + for (size_t cmb_i = 0; cmb_i < env->outputs_combined.used; cmb_i++) { + fprintf(stderr, " %d", env->outputs_combined.ids[cmb_i]); + } + fprintf(stderr, "\n"); + fprintf(stderr, "-- individiual: %zu IDs:", individual_outputs_used); + for (size_t i = 0; i < individual_outputs_used; i++) { + fprintf(stderr, " %d", individual_outputs[i]); + } + fprintf(stderr, "\n"); + goto fail; + } + + return FSM_GENERATE_MATCHES_CB_RES_CONTINUE; + +fail: + env->ok = false; + return FSM_GENERATE_MATCHES_CB_RES_HALT; +} + +/* For a given matching input generated by one of the source DFAs, check that + * the combined DFA also matches, and that the only other source DFAs that match + * are ones that should according to the combined DFA. */ +static enum fsm_generate_matches_cb_res +gen_individual_check_combined_cb(const struct fsm *fsm, + size_t depth, size_t match_count, size_t steps, + const char *input, size_t input_length, + fsm_state_t end_state, void *opaque) +{ + (void)fsm; + (void)depth; + (void)end_state; + + struct feo_env *env = opaque; + assert(env->current_pattern < env->pattern_count); + if (match_count > env->max_match_count) { return FSM_GENERATE_MATCHES_CB_RES_HALT; } + if (steps > env->max_steps) { return FSM_GENERATE_MATCHES_CB_RES_HALT; } + + struct fsm *cur_fsm = env->fsms[env->current_pattern]; + if (cur_fsm == NULL) { return FSM_GENERATE_MATCHES_CB_RES_CONTINUE; } + + /* execute, to set eager outputs */ + if (!match_input_get_eager_outputs(cur_fsm, input, input_length, &env->outputs)) { + goto fail; + } + if (!match_input_get_eager_outputs(env->combined, input, input_length, &env->outputs_combined)) { + goto fail; + } + + assert(env->outputs.used == 1); + + bool found = false; + for (size_t i = 0; i < env->outputs_combined.used; i++) { + if (env->outputs_combined.ids[i] == env->outputs.ids[0]) { + found = true; + break; + } + } + + if (!found) { + fprintf(stderr, "%s: combined <-> individual mismatch for input '%s'(%zd)!\n", __func__, input, input_length); + + fprintf(stderr, "-- combined: %zu IDs:", env->outputs_combined.used); + for (size_t cmb_i = 0; cmb_i < env->outputs_combined.used; cmb_i++) { + fprintf(stderr, " %d", env->outputs_combined.ids[cmb_i]); + } + fprintf(stderr, "\n"); + fprintf(stderr, "-- pattern %zd: %zu IDs:", env->current_pattern, env->outputs.used); + for (size_t i = 0; i < env->outputs.used; i++) { + fprintf(stderr, " %d", env->outputs.ids[i]); + } + fprintf(stderr, "\n"); + goto fail; + } + + return FSM_GENERATE_MATCHES_CB_RES_CONTINUE; + +fail: + env->ok = false; + return FSM_GENERATE_MATCHES_CB_RES_HALT; +} +#undef MAX_PATTERNS + #define MAX_FUZZER_DATA (64 * 1024) static uint8_t data_buf[MAX_FUZZER_DATA + 1]; @@ -343,15 +2155,30 @@ get_run_mode(void) { const char *mode = getenv("MODE"); if (mode == NULL) { - return MODE_DEFAULT; + return MODE_REGEX; /* default */ } switch (mode[0]) { - case 'm': return MODE_SHUFFLE_MINIMISE; + case '\0': return MODE_REGEX; /* default */ + case 'r': return MODE_REGEX; + case 's': return MODE_REGEX_SINGLE_ONLY; + case 'm': return MODE_REGEX_MULTI_ONLY; + case 'i': return MODE_IDEMPOTENT_DET_MIN; + case 'M': return MODE_SHUFFLE_MINIMISE; case 'p': return MODE_ALL_PRINT_FUNCTIONS; + case 'E': return MODE_EAGER_OUTPUT; case 'd': default: - return MODE_DEFAULT; + fprintf(stderr, "Unrecognized mode '%c', expect one of:\n", mode[0]); + fprintf(stderr, " - r.egex (default)\n"); + fprintf(stderr, " - s.ingle regex only\n"); + fprintf(stderr, " - m.ulti regex only\n"); + fprintf(stderr, " - M.inimisation shuffling\n"); + fprintf(stderr, " - i.dempotent determinise/minimise\n"); + fprintf(stderr, " - p.rint functions\n"); + fprintf(stderr, " - E.ager output\n"); + exit(EXIT_FAILURE); + break; } } @@ -364,21 +2191,110 @@ harness_fuzzer_target(const uint8_t *data, size_t size) return EXIT_SUCCESS; } + if (!verbosity_checked) { + verbosity_checked = true; + verbose = getenv("VERBOSE") != NULL; + } + /* Ensure that input is '\0'-terminated. */ if (size > MAX_FUZZER_DATA) { size = MAX_FUZZER_DATA; } memcpy(data_buf, data, size); + /* ensure the buffer is 0-terminated */ + data_buf[size] = 0; + + /* truncate to a valid c string */ + size = strlen((const char *)data_buf); + data_buf[size] = 0; + + /* reset for each run */ + allocator_stats.hwm = 0; + + size_t dot_count = 0; + bool has_newline = false; + size_t first_newline; + + for (size_t i = 0; i < size; i++) { + const uint8_t c = data_buf[i]; + if (c == '.') { + dot_count++; + if (dot_count >= 4) { + /* Too many '.'s can lead to a regex that is + * very slow to determinise/minimise, but that + * failure mode is not interesting to this + * particular fuzzer. */ + return EXIT_SUCCESS; + } + } + + if (c == '(') { + /* This triggers an "unreached" assertion in the parser. + * It's already been reported (issue #386), but once the + * fuzzer finds it, it will report it over and over. + * Exit here so that the fuzzer considers it uninteresting. */ + if (size - i >= 3 && 0 == memcmp("(*:", &data_buf[i], 3)) { + return EXIT_SUCCESS; + } + } + + if (c == '\\') { + /* Not supported yet. */ + return EXIT_SUCCESS; + } + + if (c == '\r' || c == '\n') { + if (!has_newline) { + first_newline = i; + } + has_newline = true; + } + } const char *pattern = (const char *)data_buf; switch (get_run_mode()) { - case MODE_DEFAULT: - return build_and_codegen(pattern); + case MODE_REGEX: + if (has_newline) { + return build_and_check_multi(pattern); + } else { + return build_and_check_single(pattern); + } + + case MODE_REGEX_SINGLE_ONLY: + if (has_newline) { + return EXIT_SUCCESS; /* ignore */ + } else { + return build_and_check_single(pattern); + } + + case MODE_REGEX_MULTI_ONLY: + if (has_newline) { + return build_and_check_multi(pattern); + } else { + return EXIT_SUCCESS; /* ignore */ + } + + case MODE_IDEMPOTENT_DET_MIN: + if (has_newline) { + assert(data_buf[first_newline] == '\n' + || data_buf[first_newline] == '\r'); + data_buf[first_newline] = '\0'; + } + return build_and_test_idempotent_det_and_min(pattern); + case MODE_DEFAULT: { + const uint8_t b0 = data_buf[0]; + const enum fsm_io io_mode = (b0 >> 2) % 3; + + return build_and_codegen(pattern, io_mode); + } case MODE_SHUFFLE_MINIMISE: return shuffle_minimise(pattern); + case MODE_EAGER_OUTPUT: + return fuzz_eager_output(data, size); + case MODE_ALL_PRINT_FUNCTIONS: { if (dev_null == NULL) { @@ -390,11 +2306,14 @@ harness_fuzzer_target(const uint8_t *data, size_t size) const bool det = b0 & 0x1; const bool min = b0 & 0x2; const enum fsm_io io_mode = (b0 >> 2) % 3; - + const char *shifted_pattern = (const char *)&data_buf[1]; int res = fuzz_all_print_functions(dev_null, shifted_pattern, det, min, io_mode); return res; } + +default: + assert(!"match fail"); } assert(!"unreached"); diff --git a/include/adt/bitmap.h b/include/adt/bitmap.h index ac4a94995..61c4f9a57 100644 --- a/include/adt/bitmap.h +++ b/include/adt/bitmap.h @@ -8,6 +8,9 @@ #define ADT_BITMAP_H #include +#include + +#include #include "print/esc.h" struct fsm_state; @@ -23,6 +26,9 @@ bm_get(const struct bm *bm, size_t i); void bm_set(struct bm *bm, size_t i); +void +bm_unset(struct bm *bm, size_t i); + /* Get a writeable pointer to the Nth word of the char set bitmap, * or NULL if out of bounds. */ uint64_t * @@ -51,5 +57,17 @@ bm_snprint(const struct bm *bm, const struct fsm_options *opt, int boxed, escputc *escputc); +void +bm_copy(struct bm *dst, const struct bm *src); + +void +bm_intersect(struct bm *dst, const struct bm *src); + +void +bm_union(struct bm *dst, const struct bm *src); + +int +bm_any(const struct bm *bm); + #endif diff --git a/include/adt/common.h b/include/adt/common.h index 8604edacd..a83604c97 100644 --- a/include/adt/common.h +++ b/include/adt/common.h @@ -36,7 +36,7 @@ /* If non-zero, expand the timer macros defined below, otherwise * they compile away. */ #ifndef TRACK_TIMES -#define TRACK_TIMES 0 +#define TRACK_TIMES (0 && !BUILD_FOR_FUZZER) #endif #if EXPENSIVE_CHECKS && TRACK_TIMES diff --git a/include/adt/hash.h b/include/adt/hash.h index d4ed1227d..65c8dc897 100644 --- a/include/adt/hash.h +++ b/include/adt/hash.h @@ -10,53 +10,40 @@ #include #include "adt/common.h" +#include "fsm/fsm.h" -/* 32 and 64-bit approximations of the golden ratio. */ -#define FSM_PHI_32 0x9e3779b9UL -#define FSM_PHI_64 (uint64_t)0x9e3779b97f4a7c15UL +#if EXPENSIVE_CHECKS +#include +#endif -/* A suitable hash function for individual sequentially allocated - * identifiers. See Knuth 6.4, Fibonacci hashing. */ +#define HASH_LOG_PROBES 0 +/* #define HASH_PROBE_LIMIT 100 */ SUPPRESS_EXPECTED_UNSIGNED_INTEGER_OVERFLOW() static __inline__ uint64_t -hash_id(unsigned id) +hash_id(uint64_t id) { - return FSM_PHI_64 * (uint64_t)(id + (unsigned)1); -} - -/* FNV-1a hash function, 32 and 64 bit versions. This is in the public - * domain. For details, see: - * - * http://www.isthe.com/chongo/tech/comp/fnv/index.html - */ - -SUPPRESS_EXPECTED_UNSIGNED_INTEGER_OVERFLOW() -static __inline__ uint32_t -hash_fnv1a_32(const uint8_t *buf, size_t length) -{ -#define FNV1a_32_OFFSET_BASIS 0x811c9dc5UL -#define FNV1a_32_PRIME 0x01000193UL - uint32_t h = FNV1a_32_OFFSET_BASIS; - size_t i; - for (i = 0; i < length; i++) { - h ^= buf[i]; - h *= FNV1a_32_PRIME; - } - return h; + /* xorshift* A1(12,25,27), + * from http://vigna.di.unimi.it/ftp/papers/xorshift.pdf */ + uint64_t x = id + 1; + x ^= x >> 12; // a + x ^= x << 25; // b + x ^= x >> 27; // c + return x * 2685821657736338717LLU; } SUPPRESS_EXPECTED_UNSIGNED_INTEGER_OVERFLOW() static __inline__ uint64_t -hash_fnv1a_64(const uint8_t *buf, size_t length) +hash_ids(size_t count, const fsm_state_t *ids) { -#define FNV1a_64_OFFSET_BASIS 0xcbf29ce484222325UL -#define FNV1a_64_PRIME 0x100000001b3UL - uint64_t h = FNV1a_64_OFFSET_BASIS; - size_t i; - for (i = 0; i < length; i++) { - h ^= buf[i]; - h *= FNV1a_64_PRIME; + uint64_t h = 0; + for (size_t i = 0; i < count; i++) { + h = hash_id(h ^ ids[i]); +#if EXPENSIVE_CHECKS + if (i > 0) { + assert(ids[i-1] <= ids[i]); + } +#endif } return h; } diff --git a/include/adt/hashrec.h b/include/adt/hashrec.h index 54816286e..545a20960 100644 --- a/include/adt/hashrec.h +++ b/include/adt/hashrec.h @@ -7,7 +7,7 @@ #ifndef ADT_HASHREC_H #define ADT_HASHREC_H -unsigned long +uint64_t hashrec(const void *p, size_t n); #endif diff --git a/include/adt/idmap.h b/include/adt/idmap.h new file mode 100644 index 000000000..504fd382b --- /dev/null +++ b/include/adt/idmap.h @@ -0,0 +1,59 @@ +#ifndef IDMAP_H +#define IDMAP_H + +/* Mapping between one fsm_state_t and a set of + * unsigned IDs. The implementation assumes that both + * IDs are sequentially assigned and don't need a sparse + * mapping -- it will handle 10 -> [1, 3, 47] well, but + * not 1000000 -> [14, 524288, 1073741823]. */ + +#include + +#include "fsm/fsm.h" +#include "fsm/alloc.h" + +struct idmap; /* Opaque handle. */ + +struct idmap * +idmap_new(const struct fsm_alloc *alloc); + +void +idmap_free(struct idmap *m); + +/* Associate a value with a state (if not already present.) + * Returns 1 on success, or 0 on allocation failure. */ +int +idmap_set(struct idmap *m, fsm_state_t state_id, unsigned value); + +/* How many values are associated with an ID? */ +size_t +idmap_get_value_count(const struct idmap *m, fsm_state_t state_id); + +/* Get the values associated with an ID. + * + * Returns 1 on success and writes them into the buffer, in ascending + * order, with the count in *written (if non-NULL). + * + * Returns 0 on error (insufficient buffer space). */ +int +idmap_get(const struct idmap *m, fsm_state_t state_id, + size_t buf_size, unsigned *buf, size_t *written); + +/* Iterator callback. + * Return status indicates whether to continue. */ +typedef int +idmap_iter_fun(fsm_state_t state_id, unsigned value, void *opaque); + +/* Iterate over the ID map. State IDs may be yielded out of order, + * values will be in ascending order. */ +void +idmap_iter(const struct idmap *m, + idmap_iter_fun *cb, void *opaque); + +/* Iterate over the values associated with a single state + * (in ascending order). */ +void +idmap_iter_for_state(const struct idmap *m, fsm_state_t state_id, + idmap_iter_fun *cb, void *opaque); + +#endif diff --git a/include/adt/mappingset.h b/include/adt/mappingset.h deleted file mode 100644 index d4f91105e..000000000 --- a/include/adt/mappingset.h +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright 2019 Shannon F. Stewman - * - * See LICENCE for the full copyright terms. - */ - -#ifndef ADT_MAPPINGSET_H -#define ADT_MAPPINGSET_H - -struct fsm_alloc; -struct mapping_set; -struct mapping; - -struct mapping_iter { - struct hashset_iter iter; -}; - -struct mapping_set * -mapping_set_create(const struct fsm_alloc *a, - unsigned long (*hash)(const struct mapping *a), - int (*cmp)(const void *a, const void *b)); - -void -mapping_set_free(struct mapping_set *set); - -struct mapping * -mapping_set_add(struct mapping_set *set, struct mapping *item); - -struct mapping * -mapping_set_contains(const struct mapping_set *set, const struct mapping *item); - -void -mapping_set_clear(struct mapping_set *set); - -struct mapping * -mapping_set_first(const struct mapping_set *set, struct mapping_iter *it); - -struct mapping * -mapping_set_next(struct mapping_iter *it); - -#endif - diff --git a/include/adt/path.h b/include/adt/path.h index 63ea760fb..b9c2a4d5e 100644 --- a/include/adt/path.h +++ b/include/adt/path.h @@ -12,8 +12,9 @@ struct fsm_state; struct path { fsm_state_t state; - struct path *next; char c; + + struct path *next; }; struct path * diff --git a/include/adt/stateset.h b/include/adt/stateset.h index 83e835467..5afc5c3e1 100644 --- a/include/adt/stateset.h +++ b/include/adt/stateset.h @@ -7,6 +7,8 @@ #ifndef ADT_STATESET_H #define ADT_STATESET_H +#include + struct set; struct fsm_alloc; struct state_set; @@ -72,8 +74,5 @@ state_set_rebase(struct state_set **set, fsm_state_t base); void state_set_replace(struct state_set **set, fsm_state_t old, fsm_state_t new); -unsigned long -state_set_hash(const struct state_set *set); - #endif diff --git a/include/adt/xalloc.h b/include/adt/xalloc.h index 373e72d46..cf7fd7b11 100644 --- a/include/adt/xalloc.h +++ b/include/adt/xalloc.h @@ -1,5 +1,5 @@ /* - * Copyright 2008-2017 Katherine Flavel + * Copyright 2008-2024 Katherine Flavel * * See LICENCE for the full copyright terms. */ @@ -8,14 +8,14 @@ #define XALLOC_H /* - * Duplicate a string. Returns NULL on error. + * These functions are for CLI use only; they exit on error */ + char * xstrdup(const char *s); -/* - * The following functions are for CLI use only; they exit on error - */ +char * +xstrndup(const char *s, size_t n); void * xmalloc(size_t sz); @@ -23,6 +23,7 @@ xmalloc(size_t sz); void * xcalloc(size_t count, size_t sz); +/* equivalent to free(p) when sz is 0, returns NULL */ void * xrealloc(void *p, size_t sz); diff --git a/include/fsm/alloc.h b/include/fsm/alloc.h index ea9c9a885..4388629a0 100644 --- a/include/fsm/alloc.h +++ b/include/fsm/alloc.h @@ -7,6 +7,8 @@ #ifndef FSM_ALLOC_H #define FSM_ALLOC_H +struct fsm; + struct fsm_alloc { void (*free)(void *opaque, void *p); void *(*calloc)(void *opaque, size_t n, size_t sz); @@ -15,5 +17,8 @@ struct fsm_alloc { void *opaque; }; +void +fsm_set_alloc(struct fsm *fsm, const struct fsm_alloc *alloc); + #endif diff --git a/include/fsm/bool.h b/include/fsm/bool.h index f9807f9f9..150d63f5d 100644 --- a/include/fsm/bool.h +++ b/include/fsm/bool.h @@ -44,7 +44,7 @@ fsm_union(struct fsm *a, struct fsm *b, * starting states and (if used) capture bases. * * bases[] is expected to have the same count as fsms[], and - * will be initialized by the function. + * will be initialized by the function. May be NULL. * * On success, returns the unioned fsm and updates bases[]. * On failure, returns NULL; all fsms are freed. */ @@ -52,9 +52,38 @@ struct fsm * fsm_union_array(size_t fsm_count, struct fsm **fsms, struct fsm_combined_base_pair *bases); +/* Combine an array of NFAs into a single NFA that attempts to match them + * all in one pass, with an extra loop so that more than one pattern with + * eager outputs can match. Ownership of the NFAs is transferred, they will + * be combined (or freed, if they don't have a start state). + * + * This must be called with NFAs constructed via re_comp, using its + * RE_SAVE_LINKAGE_INFO flag. That saves details during construction + * that are necessary to correctly handle anchoring while linking + * them into the combined NFA. If any of the NFAs do not have that + * information populated, the whole set will be rejected and it + * will return NULL. + * + * This will set end IDs and/or output IDs representing matching each + * of the original NFAs on the combined result, where nfas[i] will + * get ID of (id_base + i). + * + * Returns NULL on error. */ +struct fsm * +fsm_union_repeated_pattern_group(size_t nfa_count, + struct fsm **nfas, struct fsm_combined_base_pair *bases, size_t id_base); + struct fsm * fsm_intersect(struct fsm *a, struct fsm *b); +/* + * A convenience to intersect against a character set, rather than + * a pre-existing FSM. Unlike fsm_intersect(), the FSM is required + * to be a DFA. + */ +struct fsm * +fsm_intersect_charset(struct fsm *a, size_t n, const char *charset); + /* * Subtract b from a. This is not commutative. */ diff --git a/include/fsm/capture.h b/include/fsm/capture.h index d3c1aaa54..0fd418925 100644 --- a/include/fsm/capture.h +++ b/include/fsm/capture.h @@ -23,41 +23,32 @@ struct fsm_capture { size_t pos[2]; }; -/* How many captures does the FSM use? */ +/* What is the max capture ID an FSM uses? */ unsigned -fsm_countcaptures(const struct fsm *fsm); +fsm_capture_ceiling(const struct fsm *fsm); /* Does a specific state have any capture actions? */ int fsm_capture_has_capture_actions(const struct fsm *fsm, fsm_state_t state); -/* Set a capture path on an FSM. This means that during matching, the - * portion of a match between the path's START and END states will be - * captured. As the FSM is transformed (determinisation, minimisation, - * unioning, etc.), the path will be converted to refer to the pair(s) - * of new states instead. If the path's END state is no longer reachable - * from its START state, then the capture path will be ignored. - * Multiple instances of the same capture_id and path are ignored. */ -int -fsm_capture_set_path(struct fsm *fsm, unsigned capture_id, - fsm_state_t start, fsm_state_t end); - -/* Increase the base capture ID for all captures in an fsm. - * This could be used before combining multiple FSMs -- for - * example, before unioning a and b, where a has 3 captures - * and b has 2, b may be rebase'd to 3 -- so a has captures - * 0-2 and b has 3-4. */ -void -fsm_capture_rebase_capture_id(struct fsm *fsm, unsigned base); +/* Allocate a capture buffer with enough space for + * the current FSM's captures. + * + * This is provided for convenience -- the necessary array + * count can be checked with fsm_capture_ceiling, and then + * the buffer can be allocated directly. */ +struct fsm_capture * +fsm_capture_alloc_capture_buffer(const struct fsm *fsm); -/* Same, but for capture action states. */ +/* Free a capture buffer. */ void -fsm_capture_rebase_capture_action_states(struct fsm *fsm, fsm_state_t base); +fsm_capture_free_capture_buffer(const struct fsm *fsm, struct fsm_capture *capture_buffer); -/* Allocate a capture buffer with enough space for - * the current FSM's captures. */ -struct fsm_capture * -fsm_capture_alloc(const struct fsm *fsm); +/* Note that a capture is active for a particular end state. + * Using this for a non-end state is an unchecked error. */ +int +fsm_capture_set_active_for_end(struct fsm *fsm, + unsigned capture_id, fsm_state_t end_state); #ifndef NDEBUG #include diff --git a/include/fsm/fsm.h b/include/fsm/fsm.h index 7c3883749..0e419378c 100644 --- a/include/fsm/fsm.h +++ b/include/fsm/fsm.h @@ -7,16 +7,11 @@ #ifndef FSM_H #define FSM_H -/* - * TODO: This API needs quite some refactoring. Mostly we ought to operate - * in-place, else the user would only free() everything. Having an explicit - * clone interface leaves the option for duplicating, if they wish. However be - * careful about leaving things in an indeterminate state; everything ought to - * be atomic. - */ +#include +#include struct fsm; -struct fsm_options; +struct fsm_alloc; struct path; /* XXX */ struct fsm_capture; struct fsm_combine_info; @@ -33,30 +28,34 @@ typedef unsigned int fsm_state_t; * original FSM(s) matched when executing a combined FSM. */ typedef unsigned int fsm_end_id_t; -#define FSM_END_ID_MAX UINT_MAX +/* Eager output ID. */ +typedef unsigned int fsm_output_id_t; -/* struct used to return a collection of end IDs. */ -struct fsm_end_ids { - unsigned count; - fsm_end_id_t ids[1]; -}; +#define FSM_END_ID_MAX UINT_MAX /* * Create a new FSM. This is to be freed with fsm_free(). A structure allocated * from fsm_new() is expected to be passed as the "fsm" argument to the * functions in this API. * - * An options pointer may be passed for control over various details of - * FSM construction and output. This may be NULL, in which case default - * options are used. + * An fsm_alloc pointer may be passed for callbacks on memory allocation. + * This may be NULL, in which case the default libc functions are used. * When non-NULL, the storage pointed to must remain extant until fsm_free(). * * Returns NULL on error; see errno. * TODO: perhaps automatically create a start state, and never have an empty FSM - * TODO: also fsm_parse should create an FSM, not add into an existing one */ struct fsm * -fsm_new(const struct fsm_options *opt); +fsm_new(const struct fsm_alloc *alloc); + +/* + * As fsm_new(), but with an explicit number of pre-allocated states. + * This is intended to save on internal reallocations for situations + * where it's known in advance exactly how many states an FSM will have. + * You can still add more states per usual. + */ +struct fsm * +fsm_new_statealloc(const struct fsm_alloc *alloc, size_t statealloc); /* * Free a structure created by fsm_new(), and all of its contents. @@ -71,14 +70,6 @@ fsm_free(struct fsm *fsm); struct fsm * fsm_clone(const struct fsm *fsm); -/* Returns the options of an FSM */ -const struct fsm_options * -fsm_getoptions(const struct fsm *fsm); - -/* Sets the options of an FSM */ -void -fsm_setoptions(struct fsm *fsm, const struct fsm_options *opts); - /* * Copy the contents of src over dst, and free src. * @@ -192,6 +183,10 @@ fsm_findmode(const struct fsm *fsm, fsm_state_t state, unsigned int *freq); void fsm_setend(struct fsm *fsm, fsm_state_t state, int end); +/* Associate a numeric ID with a single end state in an FSM. */ +int +fsm_setendid_state(struct fsm *fsm, fsm_state_t s, fsm_end_id_t id); + /* Associate a numeric ID with the end states in an fsm. * This can be used to track which of the original fsms matched * input when multiple fsms are combined. @@ -207,27 +202,42 @@ fsm_setend(struct fsm *fsm, fsm_state_t state, int end); int fsm_setendid(struct fsm *fsm, fsm_end_id_t id); +/* Associate a numeric ID with a specific end state in an fsm. + * Returns an enum indicating whether it was set, already present, + * or errored (alloc failure). + * */ +enum fsm_endid_set_res { + FSM_ENDID_SET_ADDED, + FSM_ENDID_SET_ALREADY_PRESENT, + FSM_ENDID_SET_ERROR_ALLOC_FAIL = -1 +}; +enum fsm_endid_set_res +fsm_endid_set(struct fsm *fsm, fsm_state_t end_state, fsm_end_id_t id); + /* Get the end IDs associated with an end state, if any. - * If id_buf has enough cells to store all the end IDs (according - * to id_buf_count) then they are written into id_buf[] and - * *ids_written is set to the number of IDs. The end IDs in the - * buffer may appear in any order, but should not have duplicates. + * id_buf is expected to have enough cells (according to id_buf_count) + * to store all the end IDs. You can find this with fsm_endid_count(). * - * Returns 0 if there is not enough space in id_buf for the - * end IDs, or 1 if zero or more end IDs were returned. */ + * The end IDs in the buffer are sorted and do not have duplicates. + * + * A state with no end IDs set is considered equivalent to a state + * that has the empty set, this API does not distinguish these cases. + * This is not an error. + * + * It is an error to attempt to get end IDs associated with a state + * that is not marked as an end state. */ enum fsm_getendids_res { FSM_GETENDIDS_NOT_FOUND, FSM_GETENDIDS_FOUND, FSM_GETENDIDS_ERROR_INSUFFICIENT_SPACE = -1 }; enum fsm_getendids_res -fsm_getendids(const struct fsm *fsm, fsm_state_t end_state, - size_t id_buf_count, fsm_end_id_t *id_buf, - size_t *ids_written); +fsm_endid_get(const struct fsm *fsm, fsm_state_t end_state, + size_t id_buf_count, fsm_end_id_t *id_buf); /* Get the number of end IDs associated with an end state. */ size_t -fsm_getendidcount(const struct fsm *fsm, fsm_state_t end_state); +fsm_endid_count(const struct fsm *fsm, fsm_state_t end_state); /* Callback function to remap the end ids of a state. This function can * remap to fewer end ids, but cannot add additional end ids, and cannot @@ -272,6 +282,70 @@ fsm_mapendids(struct fsm * fsm, fsm_endid_remap_fun remap, void *opaque); void fsm_increndids(struct fsm * fsm, int delta); +/* Set an eager output ID to emit every time the state is entered. + * This is similar to fsm_setendid, but has different performance + * trade-offs for determinisation, and can be applied to + * non-end states. + * + * During DFA execution, states with eager outputs will output their + * ID when output reaches them. With fsm_exec, this happens via a + * callback (see fsm_eager_output_set_cb). Some print languages + * will eventually support eager outputs. + * + * One use case for eager outputs is combining multiple unanchored + * regexes into a single DFA and detecting when input matches more than + * one of them. With endids, determinisation has to represent every + * possible reachable combination of endids as a distinct copy of the + * DFA subgraph, leading to a combinatorial explosion that makes + * combining more than a 8 or so regexes (even very simple ones) + * prohibitively expensive. With eager outputs, the graph no longer + * needs a separate subgraph copy for each combination of IDs, so it is + * possible to combine several dozen or even hundreds of FSMs into a + * single DFA. See fsm_union_repeated_pattern_group for more details. */ +int +fsm_eager_output_set(struct fsm *fsm, fsm_state_t state, fsm_output_id_t id); + +/* Set an eager output ID on all current end states. */ +int +fsm_eager_output_set_on_ends(struct fsm *fsm, fsm_output_id_t id); + +/* Callback for eager output processing. + * If set (using fsm_eager_output_set_cb), this may be called while fsm_exec runs. */ +typedef void +fsm_eager_output_cb(fsm_output_id_t id, void *opaque); + +/* Set a callback and opaque argument on an FSM for eager outputs encountered + * while fsm_exec is running. Rather than adding another pair of arguments to + * fsm_exec, this is called as a separate step -- most DFAs will not use eager + * outputs, or use them with code generation rather than fsm_exec. + * + * See fsm_eager_output_set for more details about eager output functionality. */ +void +fsm_eager_output_set_cb(struct fsm *fsm, fsm_eager_output_cb *cb, void *opaque); + +/* Get the eager output callback set on a FSM and its opaque pointer, if any. */ +void +fsm_eager_output_get_cb(const struct fsm *fsm, fsm_eager_output_cb **cb, void **opaque); + +/* Get the number of eager output IDs associated with a state. */ +size_t +fsm_eager_output_count(const struct fsm *fsm, fsm_state_t state); + +/* Get eager output IDs associated with a state, if any. + * id_buf is expected to have enough cells (according to id_buf_count) + * to store all the end IDs. You can find this with fsm_eager_output_count(). + * + * The IDs in the buffer are sorted and do not have duplicates. + * + * Unlike end IDs, eager outputs can appear on states that are + * not marked as end states. + * + * Returns 0 if there is not enough space in id_buf for the + * eager output IDs, or 1 if zero more IDs were returned. */ +int +fsm_eager_output_get(const struct fsm *fsm, fsm_state_t state, + size_t buf_count, fsm_output_id_t *id_buf); + /* * Find the state (if there is just one), or add epsilon edges from all states, * for which the given predicate is true. @@ -391,6 +465,21 @@ fsm_remove_epsilons(struct fsm *fsm); int fsm_determinise(struct fsm *fsm); +/* Determinise, with a passed in configuration + * and a distinct return value for reaching + * the state limit. */ +struct fsm_determinise_config { + size_t state_limit; /* 0: no limit */ +}; +enum fsm_determinise_with_config_res { + FSM_DETERMINISE_WITH_CONFIG_OK, + FSM_DETERMINISE_WITH_CONFIG_STATE_LIMIT_REACHED, + FSM_DETERMINISE_WITH_CONFIG_ERRNO, +}; +enum fsm_determinise_with_config_res +fsm_determinise_with_config(struct fsm *fsm, + const struct fsm_determinise_config *config); + /* * Make a DFA complete, as per fsm_iscomplete. */ @@ -462,8 +551,21 @@ fsm_shortest(const struct fsm *fsm, * The given FSM is expected to be a DFA. */ int -fsm_exec(const struct fsm *fsm, int (*fsm_getc)(void *opaque), void *opaque, - fsm_state_t *end, struct fsm_capture *captures); +fsm_exec(const struct fsm *fsm, + int (*fsm_getc)(void *opaque), void *opaque, fsm_state_t *end); + +/* Same as fsm_exec, but also populate information about captures if + * *captures is non-NULL and capture metadata is available for the DFA. + * Captures is expected to be large enough to fit captures from the FSM. + * To check, use `fsm_capture_ceiling`. + * + * The current implementation requires all input to be buffered ahead of + * time, so this takes a pointer to an input array rather than a + * character iterator. */ +int +fsm_exec_with_captures(const struct fsm *fsm, const unsigned char *input, + size_t input_length, fsm_state_t *end, + struct fsm_capture *captures, size_t capture_buf_length); /* * Callbacks which may be passed to fsm_exec(). These are conveniences for @@ -487,5 +589,11 @@ int fsm_fgetc(void *opaque); /* expects opaque to be FILE * */ int fsm_shuffle(struct fsm *fsm, unsigned seed); +/* Attempt to reclaim memory if an FSM has significantly reduced its + * state count. Returns true if the vacuuming attempt succeeded (including + * deciding not to do anything), false if realloc failed. */ +bool +fsm_vacuum(struct fsm *fsm); + #endif diff --git a/include/fsm/options.h b/include/fsm/options.h index b4193cc78..cd162e9e3 100644 --- a/include/fsm/options.h +++ b/include/fsm/options.h @@ -7,17 +7,46 @@ #ifndef FSM_OPTIONS_H #define FSM_OPTIONS_H -#include - -struct fsm; -struct fsm_state; - enum fsm_io { FSM_IO_GETC, FSM_IO_STR, FSM_IO_PAIR }; +/* + * Ambiguity mode. + * + * The ambiguity mode controls how endids are emitted into generated code, + * and passed to the hooks.accept() callback for an accepting state. + * + * This also causes fsm_print() to resolve endid conflicts in various ways + * for accepting states that have multiple endids present: + * + * AMBIG_NONE - Do not emit endids. + * AMBIG_ERROR - Emit a single endid. + * hook.conflict() is called for accepting states + * which have multiple endids, and printing errors. + * AMBIG_EARLIEST - Emit the lowest numbered endid only. + * AMBIG_MULTIPLE - Emit all endids. + * + * What it means to emit multiple or single endids is particular to each + * output language; not every language supports both. + */ +enum fsm_ambig { + AMBIG_NONE = 0, /* default */ + AMBIG_ERROR = 1 << 0, + AMBIG_EARLIEST = 1 << 1, + AMBIG_MULTIPLE = 1 << 2, + + AMBIG_SINGLE = AMBIG_ERROR | AMBIG_EARLIEST, + AMBIG_ANY = ~0 +}; + +/* + * Print options. + * + * This is separate to because we also borrow it for libre. + */ struct fsm_options { /* boolean: true indicates to omit names for states in output */ unsigned int anonymous_states:1; @@ -52,30 +81,17 @@ struct fsm_options { */ unsigned int group_edges:1; - /* for generated code, what kind of I/O API to generate */ - enum fsm_io io; - /* a prefix for namespacing generated identifiers. NULL if not required. */ const char *prefix; /* the name of the enclosing package; NULL to use `prefix` (default). */ const char *package_prefix; - /* character pointer, for C code fragment output. NULL for the default. */ - const char *cp; - - /* TODO: explain. for C code fragment output */ - int (*leaf)(FILE *, const struct fsm_end_ids *ids, - const void *leaf_opaque); - void *leaf_opaque; - - /* TODO: explain. for C code fragment output */ - int (*endleaf)(FILE *, const struct fsm_end_ids *ids, - const void *endleaf_opaque); - void *endleaf_opaque; + /* for generated code, what kind of I/O API to generate */ + enum fsm_io io; - /* custom allocation functions */ - const struct fsm_alloc *alloc; + /* for generated code, how to handle multiple endids on an accepting state */ + enum fsm_ambig ambig; }; #endif diff --git a/include/fsm/parser.h b/include/fsm/parser.h index 414126926..15e053dd1 100644 --- a/include/fsm/parser.h +++ b/include/fsm/parser.h @@ -1,12 +1,10 @@ #ifndef FSM_PARSER_H #define FSM_PARSER_H -#include - struct fsm; -struct fsm_options; +struct fsm_alloc; struct fsm * -fsm_parse(FILE *, const struct fsm_options *); +fsm_parse(FILE *, const struct fsm_alloc *alloc); #endif diff --git a/include/fsm/print.h b/include/fsm/print.h index 790e620bc..a8aa4db54 100644 --- a/include/fsm/print.h +++ b/include/fsm/print.h @@ -7,60 +7,122 @@ #ifndef FSM_PRINT_H #define FSM_PRINT_H -#include - struct fsm; +struct fsm_options; + +/* a convenience for debugging */ +void fsm_dump(FILE *f, const struct fsm *fsm, + const char *file, unsigned line, const char *name); +#define fsm_dump(f, fsm) (fsm_dump)((f), (fsm), __FILE__, __LINE__, #fsm); + +enum fsm_print_lang { + FSM_PRINT_NONE, /* No output */ + + FSM_PRINT_AMD64_ATT, /* Assembly in various dialects */ + FSM_PRINT_AMD64_GO, + FSM_PRINT_AMD64_NASM, + + FSM_PRINT_API, /* C code which calls the fsm(3) API */ + FSM_PRINT_AWK, /* awk code (gawk dialect) */ + FSM_PRINT_C, /* ISO C90 code */ + FSM_PRINT_DOT, /* Graphviz Dot format, intended for rendering graphically */ + FSM_PRINT_FSM, /* fsm(5) .fsm format, suitable for parsing by fsm(1) */ + FSM_PRINT_GO, /* Go code */ + FSM_PRINT_IR, /* Codegen IR as Dot */ + FSM_PRINT_IRJSON, /* Codegen IR as JSON */ + FSM_PRINT_JSON, /* JavaScript Object Notation */ + FSM_PRINT_LLVM, /* Live Laugh Virtual Machine */ + FSM_PRINT_RUST, /* Rust code */ + FSM_PRINT_SH, /* Shell script (bash dialect) */ + FSM_PRINT_VMC, /* ISO C90 code, VM style */ + FSM_PRINT_VMDOT, /* Graphviz Dot format, showing VM opcodes */ + + FSM_PRINT_VMOPS_C, /* VM opcodes as a datastructure */ + FSM_PRINT_VMOPS_H, + FSM_PRINT_VMOPS_MAIN +}; + +struct fsm_state_metadata { + const fsm_end_id_t *end_ids; + size_t end_id_count; + + const fsm_output_id_t *eager_output_ids; + size_t eager_output_count; +}; + +/* + * Hooks to override generated code. These give an oportunity to + * emit application-specific constructs, especially based on ids + * attached to end states. + * + * These hooks are optional, and default to whatever makes sense + * for the language. + * + * Placement in the output stream depends on the format. + * This replaces an entire "return xyz;" statement for C-like formats, + * but appends extra information for others. + * + * ids[] is sorted and does not have duplicates. + * + * example may be NULL. + */ +struct fsm_hooks { + /* character pointer, for C code fragment output. NULL for the default. */ + const char *cp; + + int (*args)(FILE *, const struct fsm_options *opt, + void *lang_opaque, void *hook_opaque); + + int (*accept)(FILE *, const struct fsm_options *opt, + const struct fsm_state_metadata *state_metadata, + void *lang_opaque, void *hook_opaque); + + int (*reject)(FILE *, const struct fsm_options *opt, + const struct fsm_state_metadata *state_metadata, + void *lang_opaque, void *hook_opaque); + + /* If non-NULL, this will be called to generate code + * in scope immediately after advancing to the + * next character of input. */ + int (*advance)(FILE *, const struct fsm_options *opt, + const char *cur_char_var, void *hook_opaque); + + int (*comment)(FILE *, const struct fsm_options *opt, + const struct fsm_state_metadata *state_metadata, + void *hook_opaque); + + /* only called for AMBIG_ERROR; see opt.ambig */ + int (*conflict)(FILE *, const struct fsm_options *opt, + const struct fsm_state_metadata *state_metadata, + const char *example, + void *hook_opaque); + + void *hook_opaque; +}; /* * Print an FSM to the given file stream. The output is written in the format - * specified. The available formats are: + * specified by the lang enum. * - * fsm_print_api - C code which calls the fsm(3) API - * fsm_print_awk - awk code (gawk dialect) - * fsm_print_c - ISO C90 code - * fsm_print_dot - Graphviz Dot format, intended for rendering graphically - * fsm_print_fsm - fsm(5) .fsm format, suitable for parsing by fsm(1) - * fsm_print_ir - Codegen IR as Dot - * fsm_print_irjson - Codegen IR as JSON - * fsm_print_json - JavaScript Object Notation - * fsm_print_vmc - ISO C90 code, VM style - * fsm_print_vmdot - Graphviz Dot format, showing VM opcodes - * fsm_print_rust - Rust code - * fsm_print_sh - Shell script (bash dialect) - * fsm_print_go - Go code - * fsm_print_goasm/vmasm_* - Assembly in various dialects - * fsm_print_vmops_* - VM opcodes as a datastructure + * Not all languages support all options, and fsm_print will ENOTSUP where + * these are not possible. This is different to when an option is possible + * but simply not yet implemented, where fsm_print() will print a message + * to stderr and exit. * - * The output options may be NULL, indicating to use defaults. + * The code generation for the typical case of matching input requires the FSM + * to be a DFA, and will EINVAL if the FSM is not a DFA. As opposed to e.g. + * FSM_PRINT_API, which generates code for other purposes, and does not place + * particular expecations on the FSM. * - * TODO: explain constraints + * The output options may be NULL, indicating to use defaults. * * Returns 0, or -1 on error and errno will be set. An errno of ENOTSUP means - * the requested IO API is not implemented for this output format. + * the requested IO API is not implemented for this output language. */ - -typedef int (fsm_print)(FILE *f, const struct fsm *fsm); - -fsm_print fsm_print_api; -fsm_print fsm_print_awk; -fsm_print fsm_print_c; -fsm_print fsm_print_dot; -fsm_print fsm_print_fsm; -fsm_print fsm_print_ir; -fsm_print fsm_print_irjson; -fsm_print fsm_print_json; -fsm_print fsm_print_vmc; -fsm_print fsm_print_vmdot; -fsm_print fsm_print_vmasm; -fsm_print fsm_print_vmasm_amd64_att; /* output amd64 assembler in AT&T format */ -fsm_print fsm_print_vmasm_amd64_nasm; /* output amd64 assembler in NASM format */ -fsm_print fsm_print_vmasm_amd64_go; /* output amd64 assembler in Go format */ -fsm_print fsm_print_sh; -fsm_print fsm_print_go; -fsm_print fsm_print_rust; -fsm_print fsm_print_vmops_c; -fsm_print fsm_print_vmops_h; -fsm_print fsm_print_vmops_main; +int fsm_print(FILE *f, const struct fsm *fsm, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + enum fsm_print_lang lang); #endif diff --git a/include/fsm/vm.h b/include/fsm/vm.h index b45741f6a..6fab2c1a4 100644 --- a/include/fsm/vm.h +++ b/include/fsm/vm.h @@ -5,6 +5,7 @@ #include struct fsm; +struct fsm_options; struct fsm_dfavm; enum fsm_vm_compile_flags { @@ -34,7 +35,8 @@ struct fsm_dfavm * fsm_vm_compile(const struct fsm *fsm); struct fsm_dfavm * -fsm_vm_compile_with_options(const struct fsm *fsm, struct fsm_vm_compile_opts opts); +fsm_vm_compile_with_options(const struct fsm *fsm, + const struct fsm_options *opt, struct fsm_vm_compile_opts opts); struct fsm_dfavm * fsm_vm_read(FILE *f); diff --git a/include/fsm/walk.h b/include/fsm/walk.h index b433380d5..cb97e989a 100644 --- a/include/fsm/walk.h +++ b/include/fsm/walk.h @@ -7,6 +7,8 @@ #ifndef FSM_WALK_H #define FSM_WALK_H +#include + struct fsm; struct fsm_state; @@ -88,6 +90,11 @@ fsm_walk_edges(const struct fsm *fsm, void *opaque, * functionally equivalent cases makes testing dramatically faster, * but exploring every edge could be added later. * + * If randomized is zero then it will generate the first label in the + * label set, otherwise a label from the set will be chosen using rand() + * (favoring printable characters). The caller can use srand() + * beforehand to set a PRNG seed. + * * Note: fsm is non-const because it calls fsm_trim on the FSM * internally. This records the shortest distance from each state to an * end state, which is used to prune branches that would not produce @@ -112,7 +119,7 @@ fsm_generate_matches_cb(const struct fsm *fsm, const char *input, size_t input_length, fsm_state_t end_state, void *opaque); int -fsm_generate_matches(struct fsm *fsm, size_t max_length, +fsm_generate_matches(struct fsm *fsm, size_t max_length, int randomized, fsm_generate_matches_cb *cb, void *opaque); /* Callback provided for the most basic use case for @@ -128,5 +135,34 @@ fsm_generate_matches_cb fsm_generate_cb_printf; * to escape all characters or just nonprintable ones. */ fsm_generate_matches_cb fsm_generate_cb_printf_escaped; +/* Walk a DFA and detect which characters MUST appear in the input for a + * match to be possible. For example, if input for the DFA corresponding + * to /^(abc|dbe)$/ does not contain 'b' at all, there's no way it can + * ever match, so executing the regex is unnecessary. This does not detect + * which characters must appear before/after others or how many times, just + * which must be present. + * + * The input must be a DFA. When run with EXPENSIVE_CHECKS this will + * check and return ERROR_MISUSE if it is not, otherwise this is an + * unchecked error. + * + * The character map will be cleared before populating. If *count is + * non-NULL it will be updated with how many required characters were + * found. + * + * There is an optional step_limit -- if this is reached, then it will + * return FSM_DETECT_REQUIRED_CHARACTERS_STEP_LIMIT_REACHED and a + * cleared bitmap, because any partial information could still have been + * contradicted later. If the step_limit is 0 it will be ignored. */ +enum fsm_detect_required_characters_res { + FSM_DETECT_REQUIRED_CHARACTERS_WRITTEN, + FSM_DETECT_REQUIRED_CHARACTERS_STEP_LIMIT_REACHED, + FSM_DETECT_REQUIRED_CHARACTERS_ERROR_MISUSE = -1, + FSM_DETECT_REQUIRED_CHARACTERS_ERROR_ALLOC = -2, +}; +enum fsm_detect_required_characters_res +fsm_detect_required_characters(const struct fsm *dfa, size_t step_limit, + uint64_t charmap[4], size_t *count); + #endif diff --git a/include/print/esc.h b/include/print/esc.h index 6cf52d395..7d7dd63fd 100644 --- a/include/print/esc.h +++ b/include/print/esc.h @@ -22,6 +22,7 @@ escputc json_escputc; escputc pcre_escputc; escputc rust_escputc_char; escputc rust_escputc_str; +escputc llvm_escputc_char; int awk_escputcharlit(FILE *f, const struct fsm_options *opt, char c); @@ -29,6 +30,9 @@ awk_escputcharlit(FILE *f, const struct fsm_options *opt, char c); int c_escputcharlit(FILE *f, const struct fsm_options *opt, char c); +void +llvm_escputcharlit(FILE *f, const struct fsm_options *opt, char c); + void rust_escputcharlit(FILE *f, const struct fsm_options *opt, char c); diff --git a/include/re/Makefile b/include/re/Makefile index 9c216528d..9e649a526 100644 --- a/include/re/Makefile +++ b/include/re/Makefile @@ -3,4 +3,5 @@ STAGE_COPY += include/re/re.h STAGE_COPY += include/re/strings.h STAGE_COPY += include/re/literal.h +STAGE_COPY += include/re/groups.h diff --git a/include/re/groups.h b/include/re/groups.h new file mode 100644 index 000000000..cddccf6fc --- /dev/null +++ b/include/re/groups.h @@ -0,0 +1,58 @@ +/* + * Copyright 2026 Katherine Flavel + * + * See LICENCE for the full copyright terms. + */ + +#ifndef RE_GROUPS_H +#define RE_GROUPS_H + +struct re_pos; + +/* + * esc is the character for escaping group references, + * typically '\\' or '$'. + * + * group0 is passed separately for caller convenience, + * so you don't have to construct a single array for + * all groups. It's supposed to be the entire string + * that matched. group0 may not be NULL. + * + * groupv is 0-indexed meaning group $1 onwards. + * groupc is the count of elements in groupv. + * + * nonexistent is what to do about references to groups + * that are outside the bounds of the array. NULL means + * to error, otherwise the string value will be used. + * Typically this would be passed as "". + * + * start,end are only populated on error. + * + * You can distinguish compile-time errors (that is, + * syntax errors in the format string) vs. runtime errors + * (that is, nonexistent groups) by calling + * re_interpolate_groups() ahead of time with groupc = 0 + * and passing a non-NULL nonexistent value. + * + * The output string will always be less than or equal in + * length to the format string when all interpolated + * values are the empty string. That is, when groupc is 0 + * and nonexistent is the empty string, or when all groups + * used from groupv[] are the empty string. + * + * The output is \0-terminated. outn includes the \0. + * + * outs may be NULL in which case outn must be 0, and no + * output is made. + * + * On error the function returns false and the output + * buffer is indeterminate. + */ +bool +re_interpolate_groups(const char *fmt, char esc, + const char *group0, unsigned groupc, const char *groupv[], const char *nonexistent, + char *outs, size_t outn, + struct re_pos *start, struct re_pos *end); + +#endif + diff --git a/include/re/literal.h b/include/re/literal.h index 23d445b2c..a76ceaa07 100644 --- a/include/re/literal.h +++ b/include/re/literal.h @@ -7,7 +7,6 @@ #ifndef RE_LITERAL_H #define RE_LITERAL_H -struct fsm_options; struct re_err; enum re_literal_category { @@ -20,7 +19,6 @@ enum re_literal_category { int re_is_literal(enum re_dialect dialect, int (*getc)(void *opaque), void *opaque, - const struct fsm_options *opt, enum re_flags flags, struct re_err *err, enum re_literal_category *category, char **s, size_t *n); diff --git a/include/re/re.h b/include/re/re.h index deab6caed..aa08e3522 100644 --- a/include/re/re.h +++ b/include/re/re.h @@ -8,7 +8,7 @@ #define RE_H struct fsm; -struct fsm_options; +struct fsm_alloc; enum re_dialect { RE_LIKE, @@ -20,16 +20,21 @@ enum re_dialect { }; enum re_flags { - RE_ICASE = 1 << 0, - RE_TEXT = 1 << 1, - RE_MULTI = 1 << 2, - RE_REVERSE = 1 << 3, - RE_SINGLE = 1 << 4, /* aka PCRE_DOTALL */ - RE_ZONE = 1 << 5, - RE_ANCHORED = 1 << 6, - RE_EXTENDED = 1 << 7, /* PCRE extended mode */ - RE_END_NL = 1 << 8, /* end anchor matches '\n' */ - RE_FLAGS_NONE = 0 + RE_ICASE = 1 << 0, + RE_TEXT = 1 << 1, + RE_MULTI = 1 << 2, + RE_REVERSE = 1 << 3, + RE_SINGLE = 1 << 4, /* aka PCRE_DOTALL */ + RE_ZONE = 1 << 5, + RE_ANCHORED = 1 << 6, + RE_EXTENDED = 1 << 7, /* PCRE extended mode */ + RE_NOCAPTURE = 1 << 8, /* disable captures */ + RE_END_NL = 1 << 9, /* end anchor matches '\n' */ + RE_END_NL_DISABLE = 1 << 10, /* disable end anchor matching '\n' */ + /* save info about linkage at construction time, to inform + * later operations -- see fsm_union_repeated_pattern_group */ + RE_SAVE_LINKAGE_INFO = 1 << 11, + RE_FLAGS_NONE = 0 }; #define RE_ANCHOR (RE_TEXT | RE_MULTI | RE_ZONE) @@ -133,7 +138,23 @@ re_getchar_fun(void *opaque); struct fsm * re_comp(enum re_dialect dialect, re_getchar_fun *f, void *opaque, - const struct fsm_options *opt, + const struct fsm_alloc *alloc, + enum re_flags flags, struct re_err *err); + +/* Parse and analyze the regex enough to determine whether it is + * anchored at the start and/or end. + * + * As long as the result is checked for RE_IS_ANCHORED_ERROR first, + * the result can be used like a bitset. */ +enum re_is_anchored_res { + RE_IS_ANCHORED_NONE = 0x00, + RE_IS_ANCHORED_START = 0x01, + RE_IS_ANCHORED_END = 0x02, + RE_IS_ANCHORED_BOTH = 0x03, + RE_IS_ANCHORED_ERROR = 0xFFFF, +}; +enum re_is_anchored_res +re_is_anchored(enum re_dialect dialect, re_getchar_fun *f, void *opaque, enum re_flags flags, struct re_err *err); /* diff --git a/include/re/strings.h b/include/re/strings.h index 06387a54c..551ab2eb1 100644 --- a/include/re/strings.h +++ b/include/re/strings.h @@ -7,8 +7,9 @@ #ifndef RE_STRINGS_H #define RE_STRINGS_H -struct fsm; -struct fsm_options; +#include + +struct fsm_alloc; struct re_strings; @@ -32,7 +33,8 @@ enum re_strings_flags { }; struct fsm * -re_strings(const struct fsm_options *opt, const char *a[], size_t n, +re_strings(const struct fsm_alloc *alloc, + const char *a[], size_t n, enum re_strings_flags flags); struct re_strings * @@ -42,14 +44,15 @@ void re_strings_free(struct re_strings *g); int -re_strings_add_raw(struct re_strings *g, const void *p, size_t n); +re_strings_add_raw(struct re_strings *g, const void *p, size_t n, const fsm_end_id_t *endid); int -re_strings_add_str(struct re_strings *g, const char *s); +re_strings_add_str(struct re_strings *g, const char *s, const fsm_end_id_t *endid); struct fsm * re_strings_build(struct re_strings *g, - const struct fsm_options *opt, enum re_strings_flags flags); + const struct fsm_alloc *alloc, + enum re_strings_flags flags); #endif diff --git a/man/fsm.1/fsm.1.xml b/man/fsm.1/fsm.1.xml index 117e8966f..caec5309f 100644 --- a/man/fsm.1/fsm.1.xml +++ b/man/fsm.1/fsm.1.xml @@ -22,6 +22,8 @@ io"> iterations"> length"> + limit"> + charset"> -a"> -w"> @@ -32,6 +34,8 @@ -G &length.arg;"> -k &io.arg;"> -i &iterations.arg;"> + -S &limit.arg;"> + -U &charset.arg;"> -X"> -d"> @@ -75,6 +79,7 @@ fsm &x.opt; + &U.opt; &str.arg; @@ -83,6 +88,7 @@ fsm &p.opt; + &U.opt; &l.opt; &a.opt; @@ -103,6 +109,7 @@ &r.opt; &t.opt; + &U.opt; &i.opt; @@ -111,6 +118,7 @@ fsm &q.opt; + &U.opt; &file.arg; @@ -190,6 +198,21 @@ + + &U.opt; + + + Intersect the resulting state machine down to a + given character set. + This is done after any transformations are applied + (e.g. by ). + + The default character set is a byte. + It is not possible to specify a particular character set + that includes a literal \0. + + + &w.opt; @@ -304,6 +327,14 @@ + + &S.opt; + + + Set a step limit for long-running operations. + + + &t.opt; @@ -466,6 +497,13 @@ of each state in the &fsm;. Printed to &stdout.lit;; exit status is always true. + + requiredchars + – + Determine characters that must appear in any + inputs that could match the &fsm;. Exit status is true + unless the step limit was reached. + diff --git a/man/lx.1/lx.1.xml b/man/lx.1/lx.1.xml index 19423e67b..b9af035c1 100644 --- a/man/lx.1/lx.1.xml +++ b/man/lx.1/lx.1.xml @@ -18,12 +18,14 @@ tokbuf"> getc"> + io"> -n"> -Q"> -X"> -b &api_tokbuf.arg;"> -g &api_getc.arg;"> + -k &api_io.arg;"> -l &lang.arg;"> -e &prefix.arg;"> -t &prefix.arg;"> @@ -62,6 +64,7 @@ &X.opt; &b.opt; &g.opt; + &k.opt; &l.opt; &e.opt; &t.opt; @@ -223,6 +226,16 @@ Dot output is not subject to checks for semantic errors + + &k.opt; + + + Set the IO &api; for output, + per the io option for &fsm_print.3;. + The default is getc. + + + &e.opt; diff --git a/man/re.1/re.1.xml b/man/re.1/re.1.xml index ef2112500..ad5254348 100644 --- a/man/re.1/re.1.xml +++ b/man/re.1/re.1.xml @@ -13,8 +13,10 @@ stdout"> + io"> re"> file"> + flags"> dialect"> prefix"> length"> @@ -33,7 +35,9 @@ -X"> -e &prefix.arg;"> -E &prefix.arg;"> + -F &flags.arg;"> -G &length.arg;"> + -k &io.arg;"> -a"> -d"> @@ -49,7 +53,6 @@ -x"> -h"> - -v"> ]> @@ -97,6 +100,7 @@ query &a.opt; &c.opt; + &k.opt; &w.opt; &X.opt; &e.opt; @@ -169,9 +173,6 @@ query &h.opt; - @@ -184,6 +185,8 @@ query which are “executed” to match against input strings, or used to provide example strings which would match when given as input. + This tool provides an interface to the &re.3; &api;. + The &re.1; tool constructs a Finite State Machine from Regular Expression syntax. - -XXX: when executing, -strings are single or multiple with --. -can't we feed a string to stdin, too? -XXX: now with -x instead: Unlike &grep.1;, strings are matched from -command line arguments rather than &stdin.lit;. -TODO: with no strings to match, will just validate fsm instead. -validate is the wrong word - - + When executing, text to match is given either as strings + from command line arguments, or is given as filenames with &x.opt;. + Multiple arguments may be given using --. + Unlike &grep.1;, &re.1; does not read text from &stdin.lit;. + + With no text to match, &re.1; will just validate + + the pattern syntax instead. + + @@ -299,6 +306,60 @@ group (for dialects which have them) (rather than a "whole" regexp) + + &F.opt; + + + Set RE_* flags per &re.3;. + Note not all enum re_flags constants are meaningful + to expose to the &cli;. + The default is no flags set. + The available flags are: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
FlagValueMeaning
b (&b.opt;)RE_ANCHOREDAnchor patterns at both start and end, + as if by ^ and $ + (\Z for PCRE).
i (&i.opt;)RE_ICASEPatterns are case insensitive.
sRE_SINGLESingle-line mode. + Equivalent to PCRE_DOTALL.
xRE_EXTENDED + Eextended mode. + Equivalent to PCRE_EXTENDED.
+ + + &G.opt; @@ -307,6 +368,16 @@ group (for dialects which have them) (rather than a "whole" regexp) + + &k.opt; + + + Set the IO &api; for output, + per the io option for &fsm_print.3;. + The default is getc. + + + &q.opt; @@ -467,17 +538,6 @@ group (for dialects which have them) (rather than a "whole" regexp) - - &h.opt; diff --git a/man/rx.1/Makefile b/man/rx.1/Makefile new file mode 100644 index 000000000..54c2fe984 --- /dev/null +++ b/man/rx.1/Makefile @@ -0,0 +1,4 @@ +.include "../../share/mk/top.mk" + +MAN += man/rx.1/rx.1.xml + diff --git a/man/rx.1/rx.1.xml b/man/rx.1/rx.1.xml new file mode 100644 index 000000000..01956b47a --- /dev/null +++ b/man/rx.1/rx.1.xml @@ -0,0 +1,485 @@ + + + + + + stdin"> + stdout"> + stderr"> + + + charset"> + file"> + flags"> + dialect"> + io"> + prefix"> + limit"> + + -a"> + -b"> + -C &charset.arg;"> + -c"> + -d &file.arg;"> + -E &prefix.arg;"> + -e &prefix.arg;"> + -F &flags.arg;"> + -i"> + -k &io.arg;"> + -Q"> + -q"> + -R &charset.arg;"> + -r &dialect.arg;"> + -s"> + -t"> + -u"> + -l &lang.arg;"> + -n &limit.arg;"> + -v"> + -w"> + -X"> + -x"> + + -h"> +]> + + + + rx + + + + rx + 1 + + + + rx + Compile sets of Regular Expressions + + + + + rx + + + &a.opt; + &c.opt; + &i.opt; + &Q.opt; + &q.opt; + &s.opt; + &t.opt; + &u.opt; + &v.opt; + &w.opt; + &X.opt; + &x.opt; + + + &C.opt; + &k.opt; + &l.opt; + &r.opt; + &R.opt; + &d.opt; + &E.opt; + &e.opt; + + &file.arg; + + + + + Description + + &rx.1; is a command line tool to compile sets of + regular expressions to executable code. + Patterns are read from one or more files, with multiple patterns per file. + Patterns that &rx.1; declines to handle are written out + to a file given by , if present. + Generated code is printed out to &stdout.lit;. + + Input files have one pattern per line. + Each pattern has an associated id. + ids are assigned depending on the number of input files. + For a single file, ids are assigned per pattern + (that is, the id is the line number within the file). + For multiple files, ids are assigned per file + (that is, the same id is shared by all patterns within a file). + + Pattern ids are made available to the generated code + when successfully matching a set of one or more patterns. + You can see these with output. + It is possible for a given text string to match patterns + associated with different ids. + There are several ways to deal with this, + which of these is appropriate depends on the application: + + + + Error about it at compile time. + This is the default for &rx.1; + To use this mode, ensure your patterns don't overlap. + In particular you can use rx -q as a lint to find conflicts. + + + + Give conflicting patterns the same id in the first place. + This would be the case for a lexer, + where you might have multiple spellings that + produce the same token. + + + + Allow ambiguous patterns, + and the generated &api; returns a set of ids. + See &u.opt;. + + + + Earliest line number (lower id) wins. + This would suit a firewall-like application where it doesn't matter which + See &t.opt;. + + + + Longest match or most specific regex wins. + This doesn't work for &dfa; and so is not provided by &rx.1;. + + + + + &rx.1; places patterns into one of several categories: + literal strings, general regexps, parse errors, + unsupported syntax, unsatisfiable expressions, + expressions rejected due to + or character set violations. + These categories are internal, and do not appear in the + generated code. However they may be seen with the &v.opt; output, + and apply for the &n.opt; limit on the number of general regexps. + + + + Options + + + + &a.opt; + + + Number states, + per the anonymous_states option for &fsm_print.3;. + The default is to anonymise states, + which is the opposite to the equivalent &a.opt; option for &fsm.1;. + + + + + &C.opt; + + + Intersect the constructed &fsm; down to a given + character set. + This effectively cuts regex constructions like + .* down to [[:charset:]]*. + The default is to allow all character values. + + + + + &c.opt; + + + Use RFC 9110 HTTP -field-name tokens + as the intersection character set + (as if by ). + A valid HTTP header also has at least one character, + but &rx.1; does not enforce this. + + + + + &d.opt; + + + Output declined patterns to a file, one per line. + Patterns are declined by &rx.1; either because they're unsupported, + or (in the absence of ) because of an error + compiling the pattern. + + + + + + &E.opt; + + + Set a package prefix for output, + per the package_prefix option for &fsm_print.3;. + The default is to output with the value of prefix. + + + + + &e.opt; + + + Set a prefix for output, + per the prefix option for &fsm_print.3;. + The default is to output with no prefix. + + + + + &i.opt; + + + Strict mode. + This causes &rx.1; to error on parse errors and for unsatisfiable + expressions (such as $x). + + Strict mode does not error on regexps that are parsed correctly + but unsupported, these go to the declined list (see &d.opt;). + + + + + &k.opt; + + + Set the IO &api; for output, + per the io option for &fsm_print.3;. + The default is getc. + + + + + &l.opt; + + + Set the output language for generated code. + See &fsm_lang.5fsm; for the supported language identifiers. + The default is vmc. + + Compiled code is printed is to &stdout.lit;, + &q.opt; disables output. + + &rx.1; does not provide a way to disable + consolidate_edges, + or to disable disable_comments. + See &fsm_print.3; for information on these. + + + + + &F.opt; + + + Set RE_* flags per &re.3;. + Note not all enum re_flags constants are meaningful + to expose to the &cli;. + Flags apply to all patterns. + The default is no flags set. + The available flags are: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
FlagValueMeaning
b (&b.opt;)RE_ANCHOREDAnchor patterns at both start and end, + as if by ^ and $ + (\Z for PCRE).
i (&i.opt;)RE_ICASEPatterns are case insensitive.
sRE_SINGLESingle-line mode. + Equivalent to PCRE_DOTALL.
xRE_EXTENDED + Extended mode. + Equivalent to PCRE_EXTENDED.
+ + + + + &n.opt; + + + &rx.1; categorizes patterns internally when constructing + intermediate state machines. + allows a limit on the general category of patterns. + The default is no limit. + Categorization is visible per-pattern by &v.opt;. + + + + + &R.opt; + + + Reject character set. + When a pattern contains a character in + The default is to accept all characters. + + + + + &r.opt; + + + Set the default &re_dialect.5re; for patterns. + This applies to all files unless overridden by &s.opt;. + The default is pcre. + + + + + &s.opt; + + + Override default dialect per-file by file extension. + + + + + &Q.opt; + + + Print resource consumption stats to &stderr.lit;. + + + + + &q.opt; + + + Quiet (do not print generated code). + The default is to print generated code to &stdout.lit;. + + + + + &t.opt; + + + Resolve ambiguities by picking the earliest (lowest) id + for a set of conflicts. + The default is to error for conflicts. + + + + + &u.opt; + + + Allow ambiguities between patterns. + This means patterns with different ids may match the same text. + + The default is to error for ambiguities, + requiring all regexps unioned to be non-overlapping. + Formally, the requirement is that they are disjoint languages. + Erroring for ambiguities applies after multiple regexps are joined, + either by union or by concatenation (&s.opt;). + + It's possible to have multiple patterns with the same id + (i.e. by being in the same file when using multi-file mode), + and these are not considered a conflict because they key + to the same id. + + &u.opt; is implied by &n.opt;. + + + + + &v.opt; + + + Verbose mode. + This prints information about every pattern and associated &fsm; + to &stderr.lit; during compilation. + In particular this shows the internal categorization + per pattern, which is relevant for the limit. + + + + + &w.opt; + + + Output a fragment, + per the fragment option for &fsm_print.3;. + The default is to output a complete file. + + + + + &X.opt; + + + Always encode byte values in hexadecimal, + per the always_hex option for &fsm_print.3;. + The default is to only use hex for non-printable characters. + + + + + &x.opt; + + + Literals are unanchored. + This applies to all literals; for finer control use a regex dialect. + The default is that literals are anchored, + as if written ^abc$ in regex syntax. + + + + + &h.opt; + + + Print command-line usage help. + + + + + + + See Also + + &re.1;, &fsm.3;, + &native.5re;, &literal.5re;, &lx.5;. + + + diff --git a/share/dtd/ent-ref.dtd b/share/dtd/ent-ref.dtd index fb91fa88c..0e71f6eba 100644 --- a/share/dtd/ent-ref.dtd +++ b/share/dtd/ent-ref.dtd @@ -31,6 +31,7 @@ lx %vol.1;"> re %vol.1;"> +rx %vol.1;"> fsm %vol.1;"> libre %vol.3;"> diff --git a/share/ucd/CaseFolding.txt b/share/ucd/CaseFolding.txt index 65aa0fcd6..a0b0f07fd 100644 --- a/share/ucd/CaseFolding.txt +++ b/share/ucd/CaseFolding.txt @@ -1,8 +1,8 @@ -# CaseFolding-15.0.0.txt -# Date: 2022-02-02, 23:35:35 GMT -# © 2022 Unicode®, Inc. +# CaseFolding-17.0.0.txt +# Date: 2025-07-30, 23:54:36 GMT +# © 2025 Unicode®, Inc. # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. -# For terms of use, see https://www.unicode.org/terms_of_use.html +# For terms of use and license, see https://www.unicode.org/terms_of_use.html # # Unicode Character Database # For documentation, see https://www.unicode.org/reports/tr44/ @@ -18,15 +18,15 @@ # The data supports both implementations that require simple case foldings # (where string lengths don't change), and implementations that allow full case folding # (where string lengths may grow). Note that where they can be supported, the -# full case foldings are superior: for example, they allow "MASSE" and "Maße" to match. +# full case foldings are superior: for example, they allow "FUSS" and "Fuß" to match. # # All code points not listed in this file map to themselves. # # NOTE: case folding does not preserve normalization formats! # # For information on case folding, including how to have case folding -# preserve normalization formats, see Section 3.13 Default Case Algorithms in -# The Unicode Standard. +# preserve normalization formats, see the +# "Conformance" / "Default Case Algorithms" section of the core specification. # # ================================================================================ # Format @@ -603,6 +603,7 @@ 1C86; C; 044A; # CYRILLIC SMALL LETTER TALL HARD SIGN 1C87; C; 0463; # CYRILLIC SMALL LETTER TALL YAT 1C88; C; A64B; # CYRILLIC SMALL LETTER UNBLENDED UK +1C89; C; 1C8A; # CYRILLIC CAPITAL LETTER TJE 1C90; C; 10D0; # GEORGIAN MTAVRULI CAPITAL LETTER AN 1C91; C; 10D1; # GEORGIAN MTAVRULI CAPITAL LETTER BAN 1C92; C; 10D2; # GEORGIAN MTAVRULI CAPITAL LETTER GAN @@ -929,6 +930,7 @@ 1FCC; S; 1FC3; # GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI 1FD2; F; 03B9 0308 0300; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA 1FD3; F; 03B9 0308 0301; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA +1FD3; S; 0390; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA 1FD6; F; 03B9 0342; # GREEK SMALL LETTER IOTA WITH PERISPOMENI 1FD7; F; 03B9 0308 0342; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI 1FD8; C; 1FD0; # GREEK CAPITAL LETTER IOTA WITH VRACHY @@ -937,6 +939,7 @@ 1FDB; C; 1F77; # GREEK CAPITAL LETTER IOTA WITH OXIA 1FE2; F; 03C5 0308 0300; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA 1FE3; F; 03C5 0308 0301; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA +1FE3; S; 03B0; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA 1FE4; F; 03C1 0313; # GREEK SMALL LETTER RHO WITH PSILI 1FE6; F; 03C5 0342; # GREEK SMALL LETTER UPSILON WITH PERISPOMENI 1FE7; F; 03C5 0308 0342; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI @@ -1238,9 +1241,16 @@ A7C5; C; 0282; # LATIN CAPITAL LETTER S WITH HOOK A7C6; C; 1D8E; # LATIN CAPITAL LETTER Z WITH PALATAL HOOK A7C7; C; A7C8; # LATIN CAPITAL LETTER D WITH SHORT STROKE OVERLAY A7C9; C; A7CA; # LATIN CAPITAL LETTER S WITH SHORT STROKE OVERLAY +A7CB; C; 0264; # LATIN CAPITAL LETTER RAMS HORN +A7CC; C; A7CD; # LATIN CAPITAL LETTER S WITH DIAGONAL STROKE +A7CE; C; A7CF; # LATIN CAPITAL LETTER PHARYNGEAL VOICED FRICATIVE A7D0; C; A7D1; # LATIN CAPITAL LETTER CLOSED INSULAR G +A7D2; C; A7D3; # LATIN CAPITAL LETTER DOUBLE THORN +A7D4; C; A7D5; # LATIN CAPITAL LETTER DOUBLE WYNN A7D6; C; A7D7; # LATIN CAPITAL LETTER MIDDLE SCOTS S A7D8; C; A7D9; # LATIN CAPITAL LETTER SIGMOID S +A7DA; C; A7DB; # LATIN CAPITAL LETTER LAMBDA +A7DC; C; 019B; # LATIN CAPITAL LETTER LAMBDA WITH STROKE A7F5; C; A7F6; # LATIN CAPITAL LETTER REVERSED HALF H AB70; C; 13A0; # CHEROKEE SMALL LETTER A AB71; C; 13A1; # CHEROKEE SMALL LETTER E @@ -1328,6 +1338,7 @@ FB02; F; 0066 006C; # LATIN SMALL LIGATURE FL FB03; F; 0066 0066 0069; # LATIN SMALL LIGATURE FFI FB04; F; 0066 0066 006C; # LATIN SMALL LIGATURE FFL FB05; F; 0073 0074; # LATIN SMALL LIGATURE LONG S T +FB05; S; FB06; # LATIN SMALL LIGATURE LONG S T FB06; F; 0073 0074; # LATIN SMALL LIGATURE ST FB13; F; 0574 0576; # ARMENIAN SMALL LIGATURE MEN NOW FB14; F; 0574 0565; # ARMENIAN SMALL LIGATURE MEN ECH @@ -1522,6 +1533,28 @@ FF3A; C; FF5A; # FULLWIDTH LATIN CAPITAL LETTER Z 10CB0; C; 10CF0; # OLD HUNGARIAN CAPITAL LETTER EZS 10CB1; C; 10CF1; # OLD HUNGARIAN CAPITAL LETTER ENT-SHAPED SIGN 10CB2; C; 10CF2; # OLD HUNGARIAN CAPITAL LETTER US +10D50; C; 10D70; # GARAY CAPITAL LETTER A +10D51; C; 10D71; # GARAY CAPITAL LETTER CA +10D52; C; 10D72; # GARAY CAPITAL LETTER MA +10D53; C; 10D73; # GARAY CAPITAL LETTER KA +10D54; C; 10D74; # GARAY CAPITAL LETTER BA +10D55; C; 10D75; # GARAY CAPITAL LETTER JA +10D56; C; 10D76; # GARAY CAPITAL LETTER SA +10D57; C; 10D77; # GARAY CAPITAL LETTER WA +10D58; C; 10D78; # GARAY CAPITAL LETTER LA +10D59; C; 10D79; # GARAY CAPITAL LETTER GA +10D5A; C; 10D7A; # GARAY CAPITAL LETTER DA +10D5B; C; 10D7B; # GARAY CAPITAL LETTER XA +10D5C; C; 10D7C; # GARAY CAPITAL LETTER YA +10D5D; C; 10D7D; # GARAY CAPITAL LETTER TA +10D5E; C; 10D7E; # GARAY CAPITAL LETTER RA +10D5F; C; 10D7F; # GARAY CAPITAL LETTER NYA +10D60; C; 10D80; # GARAY CAPITAL LETTER FA +10D61; C; 10D81; # GARAY CAPITAL LETTER NA +10D62; C; 10D82; # GARAY CAPITAL LETTER PA +10D63; C; 10D83; # GARAY CAPITAL LETTER HA +10D64; C; 10D84; # GARAY CAPITAL LETTER OLD KA +10D65; C; 10D85; # GARAY CAPITAL LETTER OLD NA 118A0; C; 118C0; # WARANG CITI CAPITAL LETTER NGAA 118A1; C; 118C1; # WARANG CITI CAPITAL LETTER A 118A2; C; 118C2; # WARANG CITI CAPITAL LETTER WI @@ -1586,6 +1619,31 @@ FF3A; C; FF5A; # FULLWIDTH LATIN CAPITAL LETTER Z 16E5D; C; 16E7D; # MEDEFAIDRIN CAPITAL LETTER O 16E5E; C; 16E7E; # MEDEFAIDRIN CAPITAL LETTER AI 16E5F; C; 16E7F; # MEDEFAIDRIN CAPITAL LETTER Y +16EA0; C; 16EBB; # BERIA ERFE CAPITAL LETTER ARKAB +16EA1; C; 16EBC; # BERIA ERFE CAPITAL LETTER BASIGNA +16EA2; C; 16EBD; # BERIA ERFE CAPITAL LETTER DARBAI +16EA3; C; 16EBE; # BERIA ERFE CAPITAL LETTER EH +16EA4; C; 16EBF; # BERIA ERFE CAPITAL LETTER FITKO +16EA5; C; 16EC0; # BERIA ERFE CAPITAL LETTER GOWAY +16EA6; C; 16EC1; # BERIA ERFE CAPITAL LETTER HIRDEABO +16EA7; C; 16EC2; # BERIA ERFE CAPITAL LETTER I +16EA8; C; 16EC3; # BERIA ERFE CAPITAL LETTER DJAI +16EA9; C; 16EC4; # BERIA ERFE CAPITAL LETTER KOBO +16EAA; C; 16EC5; # BERIA ERFE CAPITAL LETTER LAKKO +16EAB; C; 16EC6; # BERIA ERFE CAPITAL LETTER MERI +16EAC; C; 16EC7; # BERIA ERFE CAPITAL LETTER NINI +16EAD; C; 16EC8; # BERIA ERFE CAPITAL LETTER GNA +16EAE; C; 16EC9; # BERIA ERFE CAPITAL LETTER NGAY +16EAF; C; 16ECA; # BERIA ERFE CAPITAL LETTER OI +16EB0; C; 16ECB; # BERIA ERFE CAPITAL LETTER PI +16EB1; C; 16ECC; # BERIA ERFE CAPITAL LETTER ERIGO +16EB2; C; 16ECD; # BERIA ERFE CAPITAL LETTER ERIGO TAMURA +16EB3; C; 16ECE; # BERIA ERFE CAPITAL LETTER SERI +16EB4; C; 16ECF; # BERIA ERFE CAPITAL LETTER SHEP +16EB5; C; 16ED0; # BERIA ERFE CAPITAL LETTER TATASOUE +16EB6; C; 16ED1; # BERIA ERFE CAPITAL LETTER UI +16EB7; C; 16ED2; # BERIA ERFE CAPITAL LETTER WASSE +16EB8; C; 16ED3; # BERIA ERFE CAPITAL LETTER AY 1E900; C; 1E922; # ADLAM CAPITAL LETTER ALIF 1E901; C; 1E923; # ADLAM CAPITAL LETTER DAALI 1E902; C; 1E924; # ADLAM CAPITAL LETTER LAAM diff --git a/share/ucd/Makefile b/share/ucd/Makefile index 8af79f1b4..22aa0a869 100644 --- a/share/ucd/Makefile +++ b/share/ucd/Makefile @@ -1,4 +1,4 @@ -UCD_URL ?= https://www.unicode.org/Public/15.0.0/ucd/ +UCD_URL ?= https://www.unicode.org/Public/17.0.0/ucd/ WGET ?= wget diff --git a/share/ucd/Scripts.txt b/share/ucd/Scripts.txt index 2b138bffb..5574fdd6a 100644 --- a/share/ucd/Scripts.txt +++ b/share/ucd/Scripts.txt @@ -1,8 +1,8 @@ -# Scripts-15.0.0.txt -# Date: 2022-04-26, 23:15:02 GMT -# © 2022 Unicode®, Inc. +# Scripts-17.0.0.txt +# Date: 2025-07-24, 13:28:55 GMT +# © 2025 Unicode®, Inc. # Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries. -# For terms of use, see https://www.unicode.org/terms_of_use.html +# For terms of use and license, see https://www.unicode.org/terms_of_use.html # # Unicode Character Database # For documentation, see https://www.unicode.org/reports/tr44/ @@ -154,7 +154,7 @@ 208A..208C ; Common # Sm [3] SUBSCRIPT PLUS SIGN..SUBSCRIPT EQUALS SIGN 208D ; Common # Ps SUBSCRIPT LEFT PARENTHESIS 208E ; Common # Pe SUBSCRIPT RIGHT PARENTHESIS -20A0..20C0 ; Common # Sc [33] EURO-CURRENCY SIGN..SOM SIGN +20A0..20C1 ; Common # Sc [34] EURO-CURRENCY SIGN..SAUDI RIYAL SIGN 2100..2101 ; Common # So [2] ACCOUNT OF..ADDRESSED TO THE SUBJECT 2102 ; Common # L& DOUBLE-STRUCK CAPITAL C 2103..2106 ; Common # So [4] DEGREE CELSIUS..CADA UNA @@ -224,7 +224,7 @@ 239B..23B3 ; Common # Sm [25] LEFT PARENTHESIS UPPER HOOK..SUMMATION BOTTOM 23B4..23DB ; Common # So [40] TOP SQUARE BRACKET..FUSE 23DC..23E1 ; Common # Sm [6] TOP PARENTHESIS..BOTTOM TORTOISE SHELL BRACKET -23E2..2426 ; Common # So [69] WHITE TRAPEZIUM..SYMBOL FOR SUBSTITUTE FORM TWO +23E2..2429 ; Common # So [72] WHITE TRAPEZIUM..SYMBOL FOR DELETE MEDIUM SHADE FORM 2440..244A ; Common # So [11] OCR HOOK..OCR DOUBLE BACKSLASH 2460..249B ; Common # No [60] CIRCLED DIGIT ONE..NUMBER TWENTY FULL STOP 249C..24E9 ; Common # So [78] PARENTHESIZED LATIN SMALL LETTER A..CIRCLED LATIN SMALL LETTER Z @@ -306,8 +306,7 @@ 2B45..2B46 ; Common # So [2] LEFTWARDS QUADRUPLE ARROW..RIGHTWARDS QUADRUPLE ARROW 2B47..2B4C ; Common # Sm [6] REVERSE TILDE OPERATOR ABOVE RIGHTWARDS ARROW..RIGHTWARDS ARROW ABOVE REVERSE TILDE OPERATOR 2B4D..2B73 ; Common # So [39] DOWNWARDS TRIANGLE-HEADED ZIGZAG ARROW..DOWNWARDS TRIANGLE-HEADED ARROW TO BAR -2B76..2B95 ; Common # So [32] NORTH WEST TRIANGLE-HEADED ARROW TO BAR..RIGHTWARDS BLACK ARROW -2B97..2BFF ; Common # So [105] SYMBOL FOR TYPE A ELECTRONICS..HELLSCHREIBER PAUSE SYMBOL +2B76..2BFF ; Common # So [138] NORTH WEST TRIANGLE-HEADED ARROW TO BAR..HELLSCHREIBER PAUSE SYMBOL 2E00..2E01 ; Common # Po [2] RIGHT ANGLE SUBSTITUTION MARKER..RIGHT ANGLE DOTTED SUBSTITUTION MARKER 2E02 ; Common # Pi LEFT SUBSTITUTION BRACKET 2E03 ; Common # Pf RIGHT SUBSTITUTION BRACKET @@ -357,7 +356,7 @@ 2E5B ; Common # Ps BOTTOM HALF LEFT PARENTHESIS 2E5C ; Common # Pe BOTTOM HALF RIGHT PARENTHESIS 2E5D ; Common # Pd OBLIQUE HYPHEN -2FF0..2FFB ; Common # So [12] IDEOGRAPHIC DESCRIPTION CHARACTER LEFT TO RIGHT..IDEOGRAPHIC DESCRIPTION CHARACTER OVERLAID +2FF0..2FFF ; Common # So [16] IDEOGRAPHIC DESCRIPTION CHARACTER LEFT TO RIGHT..IDEOGRAPHIC DESCRIPTION CHARACTER ROTATION 3000 ; Common # Zs IDEOGRAPHIC SPACE 3001..3003 ; Common # Po [3] IDEOGRAPHIC COMMA..DITTO MARK 3004 ; Common # So JAPANESE INDUSTRIAL STANDARD SYMBOL @@ -398,7 +397,8 @@ 3190..3191 ; Common # So [2] IDEOGRAPHIC ANNOTATION LINKING MARK..IDEOGRAPHIC ANNOTATION REVERSE MARK 3192..3195 ; Common # No [4] IDEOGRAPHIC ANNOTATION ONE MARK..IDEOGRAPHIC ANNOTATION FOUR MARK 3196..319F ; Common # So [10] IDEOGRAPHIC ANNOTATION TOP MARK..IDEOGRAPHIC ANNOTATION MAN MARK -31C0..31E3 ; Common # So [36] CJK STROKE T..CJK STROKE Q +31C0..31E5 ; Common # So [38] CJK STROKE T..CJK STROKE SZP +31EF ; Common # So IDEOGRAPHIC DESCRIPTION CHARACTER SUBTRACTION 3220..3229 ; Common # No [10] PARENTHESIZED IDEOGRAPH ONE..PARENTHESIZED IDEOGRAPH TEN 322A..3247 ; Common # So [30] PARENTHESIZED IDEOGRAPH MOON..CIRCLED IDEOGRAPH KOTO 3248..324F ; Common # No [8] CIRCLED NUMBER TEN ON BLACK SQUARE..CIRCLED NUMBER EIGHTY ON BLACK SQUARE @@ -521,6 +521,13 @@ FFFC..FFFD ; Common # So [2] OBJECT REPLACEMENT CHARACTER..REPLACEMENT CHAR 101D0..101FC ; Common # So [45] PHAISTOS DISC SIGN PEDESTRIAN..PHAISTOS DISC SIGN WAVY BAND 102E1..102FB ; Common # No [27] COPTIC EPACT DIGIT ONE..COPTIC EPACT NUMBER NINE HUNDRED 1BCA0..1BCA3 ; Common # Cf [4] SHORTHAND FORMAT LETTER OVERLAP..SHORTHAND FORMAT UP STEP +1CC00..1CCEF ; Common # So [240] UP-POINTING GO-KART..OUTLINED LATIN CAPITAL LETTER Z +1CCF0..1CCF9 ; Common # Nd [10] OUTLINED DIGIT ZERO..OUTLINED DIGIT NINE +1CCFA..1CCFC ; Common # So [3] SNAKE SYMBOL..NOSE SYMBOL +1CD00..1CEB3 ; Common # So [436] BLOCK OCTANT-3..BLACK RIGHT TRIANGLE CARET +1CEBA..1CED0 ; Common # So [23] FRAGILE SYMBOL..LEUKOTHEA +1CEE0..1CEEF ; Common # So [16] GEOMANTIC FIGURE POPULUS..GEOMANTIC FIGURE VIA +1CEF0 ; Common # Sm MEDIUM SMALL WHITE CIRCLE WITH HORIZONTAL BAR 1CF50..1CFC3 ; Common # So [116] ZNAMENNY NEUME KRYUK..ZNAMENNY NEUME PAUK 1D000..1D0F5 ; Common # So [246] BYZANTINE MUSICAL SYMBOL PSILI..BYZANTINE MUSICAL SYMBOL GORGON NEO KATO 1D100..1D126 ; Common # So [39] MUSICAL SYMBOL SINGLE BARLINE..MUSICAL SYMBOL DRUM CLEF-2 @@ -601,11 +608,10 @@ FFFC..FFFD ; Common # So [2] OBJECT REPLACEMENT CHARACTER..REPLACEMENT CHAR 1F260..1F265 ; Common # So [6] ROUNDED SYMBOL FOR FU..ROUNDED SYMBOL FOR CAI 1F300..1F3FA ; Common # So [251] CYCLONE..AMPHORA 1F3FB..1F3FF ; Common # Sk [5] EMOJI MODIFIER FITZPATRICK TYPE-1-2..EMOJI MODIFIER FITZPATRICK TYPE-6 -1F400..1F6D7 ; Common # So [728] RAT..ELEVATOR +1F400..1F6D8 ; Common # So [729] RAT..LANDSLIDE 1F6DC..1F6EC ; Common # So [17] WIRELESS..AIRPLANE ARRIVING 1F6F0..1F6FC ; Common # So [13] SATELLITE..ROLLER SKATE -1F700..1F776 ; Common # So [119] ALCHEMICAL SYMBOL FOR QUINTESSENCE..LUNAR ECLIPSE -1F77B..1F7D9 ; Common # So [95] HAUMEA..NINE POINTED WHITE STAR +1F700..1F7D9 ; Common # So [218] ALCHEMICAL SYMBOL FOR QUINTESSENCE..NINE POINTED WHITE STAR 1F7E0..1F7EB ; Common # So [12] LARGE ORANGE CIRCLE..LARGE BROWN SQUARE 1F7F0 ; Common # So HEAVY EQUALS SIGN 1F800..1F80B ; Common # So [12] LEFTWARDS ARROW WITH SMALL TRIANGLE ARROWHEAD..DOWNWARDS ARROW WITH LARGE TRIANGLE ARROWHEAD @@ -613,23 +619,26 @@ FFFC..FFFD ; Common # So [2] OBJECT REPLACEMENT CHARACTER..REPLACEMENT CHAR 1F850..1F859 ; Common # So [10] LEFTWARDS SANS-SERIF ARROW..UP DOWN SANS-SERIF ARROW 1F860..1F887 ; Common # So [40] WIDE-HEADED LEFTWARDS LIGHT BARB ARROW..WIDE-HEADED SOUTH WEST VERY HEAVY BARB ARROW 1F890..1F8AD ; Common # So [30] LEFTWARDS TRIANGLE ARROWHEAD..WHITE ARROW SHAFT WIDTH TWO THIRDS -1F8B0..1F8B1 ; Common # So [2] ARROW POINTING UPWARDS THEN NORTH WEST..ARROW POINTING RIGHTWARDS THEN CURVING SOUTH WEST -1F900..1FA53 ; Common # So [340] CIRCLED CROSS FORMEE WITH FOUR DOTS..BLACK CHESS KNIGHT-BISHOP +1F8B0..1F8BB ; Common # So [12] ARROW POINTING UPWARDS THEN NORTH WEST..SOUTH WEST ARROW FROM BAR +1F8C0..1F8C1 ; Common # So [2] LEFTWARDS ARROW FROM DOWNWARDS ARROW..RIGHTWARDS ARROW FROM DOWNWARDS ARROW +1F8D0..1F8D8 ; Common # Sm [9] LONG RIGHTWARDS ARROW OVER LONG LEFTWARDS ARROW..LONG LEFT RIGHT ARROW WITH DEPENDENT LOBE +1F900..1FA57 ; Common # So [344] CIRCLED CROSS FORMEE WITH FOUR DOTS..BLACK CHESS ALFIL 1FA60..1FA6D ; Common # So [14] XIANGQI RED GENERAL..XIANGQI BLACK SOLDIER 1FA70..1FA7C ; Common # So [13] BALLET SHOES..CRUTCH -1FA80..1FA88 ; Common # So [9] YO-YO..FLUTE -1FA90..1FABD ; Common # So [46] RINGED PLANET..WING -1FABF..1FAC5 ; Common # So [7] GOOSE..PERSON WITH CROWN -1FACE..1FADB ; Common # So [14] MOOSE..PEA POD -1FAE0..1FAE8 ; Common # So [9] MELTING FACE..SHAKING FACE -1FAF0..1FAF8 ; Common # So [9] HAND WITH INDEX FINGER AND THUMB CROSSED..RIGHTWARDS PUSHING HAND +1FA80..1FA8A ; Common # So [11] YO-YO..TROMBONE +1FA8E..1FAC6 ; Common # So [57] TREASURE CHEST..FINGERPRINT +1FAC8 ; Common # So HAIRY CREATURE +1FACD..1FADC ; Common # So [16] ORCA..ROOT VEGETABLE +1FADF..1FAEA ; Common # So [12] SPLATTER..DISTORTED FACE +1FAEF..1FAF8 ; Common # So [10] FIGHT CLOUD..RIGHTWARDS PUSHING HAND 1FB00..1FB92 ; Common # So [147] BLOCK SEXTANT-1..UPPER HALF INVERSE MEDIUM SHADE AND LOWER HALF BLOCK -1FB94..1FBCA ; Common # So [55] LEFT HALF INVERSE MEDIUM SHADE AND RIGHT HALF BLOCK..WHITE UP-POINTING CHEVRON +1FB94..1FBEF ; Common # So [92] LEFT HALF INVERSE MEDIUM SHADE AND RIGHT HALF BLOCK..TOP LEFT JUSTIFIED LOWER RIGHT QUARTER BLACK CIRCLE 1FBF0..1FBF9 ; Common # Nd [10] SEGMENTED DIGIT ZERO..SEGMENTED DIGIT NINE +1FBFA ; Common # So ALARM BELL SYMBOL E0001 ; Common # Cf LANGUAGE TAG E0020..E007F ; Common # Cf [96] TAG SPACE..CANCEL TAG -# Total code points: 8301 +# Total code points: 9123 # ================================================ @@ -644,8 +653,8 @@ E0020..E007F ; Common # Cf [96] TAG SPACE..CANCEL TAG 01BC..01BF ; Latin # L& [4] LATIN CAPITAL LETTER TONE FIVE..LATIN LETTER WYNN 01C0..01C3 ; Latin # Lo [4] LATIN LETTER DENTAL CLICK..LATIN LETTER RETROFLEX CLICK 01C4..0293 ; Latin # L& [208] LATIN CAPITAL LETTER DZ WITH CARON..LATIN SMALL LETTER EZH WITH CURL -0294 ; Latin # Lo LATIN LETTER GLOTTAL STOP -0295..02AF ; Latin # L& [27] LATIN LETTER PHARYNGEAL VOICED FRICATIVE..LATIN SMALL LETTER TURNED H WITH FISHHOOK AND TAIL +0294..0295 ; Latin # Lo [2] LATIN LETTER GLOTTAL STOP..LATIN LETTER PHARYNGEAL VOICED FRICATIVE +0296..02AF ; Latin # L& [26] LATIN LETTER INVERTED GLOTTAL STOP..LATIN SMALL LETTER TURNED H WITH FISHHOOK AND TAIL 02B0..02B8 ; Latin # Lm [9] MODIFIER LETTER SMALL H..MODIFIER LETTER SMALL Y 02E0..02E4 ; Latin # Lm [5] MODIFIER LETTER SMALL GAMMA..MODIFIER LETTER SMALL REVERSED GLOTTAL STOP 1D00..1D25 ; Latin # L& [38] LATIN LETTER SMALL CAPITAL A..LATIN LETTER AIN @@ -672,11 +681,8 @@ A770 ; Latin # Lm MODIFIER LETTER US A771..A787 ; Latin # L& [23] LATIN SMALL LETTER DUM..LATIN SMALL LETTER INSULAR T A78B..A78E ; Latin # L& [4] LATIN CAPITAL LETTER SALTILLO..LATIN SMALL LETTER L WITH RETROFLEX HOOK AND BELT A78F ; Latin # Lo LATIN LETTER SINOLOGICAL DOT -A790..A7CA ; Latin # L& [59] LATIN CAPITAL LETTER N WITH DESCENDER..LATIN SMALL LETTER S WITH SHORT STROKE OVERLAY -A7D0..A7D1 ; Latin # L& [2] LATIN CAPITAL LETTER CLOSED INSULAR G..LATIN SMALL LETTER CLOSED INSULAR G -A7D3 ; Latin # L& LATIN SMALL LETTER DOUBLE THORN -A7D5..A7D9 ; Latin # L& [5] LATIN SMALL LETTER DOUBLE WYNN..LATIN SMALL LETTER SIGMOID S -A7F2..A7F4 ; Latin # Lm [3] MODIFIER LETTER CAPITAL C..MODIFIER LETTER CAPITAL Q +A790..A7DC ; Latin # L& [77] LATIN CAPITAL LETTER N WITH DESCENDER..LATIN CAPITAL LETTER LAMBDA WITH STROKE +A7F1..A7F4 ; Latin # Lm [4] MODIFIER LETTER CAPITAL S..MODIFIER LETTER CAPITAL Q A7F5..A7F6 ; Latin # L& [2] LATIN CAPITAL LETTER REVERSED HALF H..LATIN SMALL LETTER REVERSED HALF H A7F7 ; Latin # Lo LATIN EPIGRAPHIC LETTER SIDEWAYS I A7F8..A7F9 ; Latin # Lm [2] MODIFIER LETTER CAPITAL H WITH STROKE..MODIFIER LETTER SMALL LIGATURE OE @@ -698,7 +704,7 @@ FF41..FF5A ; Latin # L& [26] FULLWIDTH LATIN SMALL LETTER A..FULLWIDTH LATIN 1DF0B..1DF1E ; Latin # L& [20] LATIN SMALL LETTER ESH WITH DOUBLE BAR..LATIN SMALL LETTER S WITH CURL 1DF25..1DF2A ; Latin # L& [6] LATIN SMALL LETTER D WITH MID-HEIGHT LEFT HOOK..LATIN SMALL LETTER T WITH MID-HEIGHT LEFT HOOK -# Total code points: 1481 +# Total code points: 1492 # ================================================ @@ -768,7 +774,7 @@ AB65 ; Greek # L& GREEK LETTER SMALL CAPITAL OMEGA 0487 ; Cyrillic # Mn COMBINING CYRILLIC POKRYTIE 0488..0489 ; Cyrillic # Me [2] COMBINING CYRILLIC HUNDRED THOUSANDS SIGN..COMBINING CYRILLIC MILLIONS SIGN 048A..052F ; Cyrillic # L& [166] CYRILLIC CAPITAL LETTER SHORT I WITH TAIL..CYRILLIC SMALL LETTER EL WITH DESCENDER -1C80..1C88 ; Cyrillic # L& [9] CYRILLIC SMALL LETTER ROUNDED VE..CYRILLIC SMALL LETTER UNBLENDED UK +1C80..1C8A ; Cyrillic # L& [11] CYRILLIC SMALL LETTER ROUNDED VE..CYRILLIC SMALL LETTER TJE 1D2B ; Cyrillic # L& CYRILLIC LETTER SMALL CAPITAL EL 1D78 ; Cyrillic # Lm MODIFIER LETTER CYRILLIC EN 2DE0..2DFF ; Cyrillic # Mn [32] COMBINING CYRILLIC LETTER BE..COMBINING CYRILLIC LETTER IOTIFIED BIG YUS @@ -787,7 +793,7 @@ FE2E..FE2F ; Cyrillic # Mn [2] COMBINING CYRILLIC TITLO LEFT HALF..COMBININ 1E030..1E06D ; Cyrillic # Lm [62] MODIFIER LETTER CYRILLIC SMALL A..MODIFIER LETTER CYRILLIC SMALL STRAIGHT U WITH STROKE 1E08F ; Cyrillic # Mn COMBINING CYRILLIC SMALL LETTER BYELORUSSIAN-UKRAINIAN I -# Total code points: 506 +# Total code points: 508 # ================================================ @@ -865,27 +871,34 @@ FB46..FB4F ; Hebrew # Lo [10] HEBREW LETTER TSADI WITH DAGESH..HEBREW LIGATU 0750..077F ; Arabic # Lo [48] ARABIC LETTER BEH WITH THREE DOTS HORIZONTALLY BELOW..ARABIC LETTER KAF WITH TWO DOTS ABOVE 0870..0887 ; Arabic # Lo [24] ARABIC LETTER ALEF WITH ATTACHED FATHA..ARABIC BASELINE ROUND DOT 0888 ; Arabic # Sk ARABIC RAISED ROUND DOT -0889..088E ; Arabic # Lo [6] ARABIC LETTER NOON WITH INVERTED SMALL V..ARABIC VERTICAL TAIL +0889..088F ; Arabic # Lo [7] ARABIC LETTER NOON WITH INVERTED SMALL V..ARABIC LETTER NOON WITH RING ABOVE 0890..0891 ; Arabic # Cf [2] ARABIC POUND MARK ABOVE..ARABIC PIASTRE MARK ABOVE -0898..089F ; Arabic # Mn [8] ARABIC SMALL HIGH WORD AL-JUZ..ARABIC HALF MADDA OVER MADDA +0897..089F ; Arabic # Mn [9] ARABIC PEPET..ARABIC HALF MADDA OVER MADDA 08A0..08C8 ; Arabic # Lo [41] ARABIC LETTER BEH WITH SMALL V BELOW..ARABIC LETTER GRAF 08C9 ; Arabic # Lm ARABIC SMALL FARSI YEH 08CA..08E1 ; Arabic # Mn [24] ARABIC SMALL HIGH FARSI YEH..ARABIC SMALL HIGH SIGN SAFHA 08E3..08FF ; Arabic # Mn [29] ARABIC TURNED DAMMA BELOW..ARABIC MARK SIDEWAYS NOON GHUNNA FB50..FBB1 ; Arabic # Lo [98] ARABIC LETTER ALEF WASLA ISOLATED FORM..ARABIC LETTER YEH BARREE WITH HAMZA ABOVE FINAL FORM FBB2..FBC2 ; Arabic # Sk [17] ARABIC SYMBOL DOT ABOVE..ARABIC SYMBOL WASLA ABOVE +FBC3..FBD2 ; Arabic # So [16] ARABIC LIGATURE JALLA WA-ALAA..ARABIC LIGATURE ALAYHI AR-RAHMAH FBD3..FD3D ; Arabic # Lo [363] ARABIC LETTER NG ISOLATED FORM..ARABIC LIGATURE ALEF WITH FATHATAN ISOLATED FORM FD40..FD4F ; Arabic # So [16] ARABIC LIGATURE RAHIMAHU ALLAAH..ARABIC LIGATURE RAHIMAHUM ALLAAH FD50..FD8F ; Arabic # Lo [64] ARABIC LIGATURE TEH WITH JEEM WITH MEEM INITIAL FORM..ARABIC LIGATURE MEEM WITH KHAH WITH MEEM INITIAL FORM +FD90..FD91 ; Arabic # So [2] ARABIC LIGATURE RAHMATU ALLAAHI ALAYH..ARABIC LIGATURE RAHMATU ALLAAHI ALAYHAA FD92..FDC7 ; Arabic # Lo [54] ARABIC LIGATURE MEEM WITH JEEM WITH KHAH INITIAL FORM..ARABIC LIGATURE NOON WITH JEEM WITH YEH FINAL FORM -FDCF ; Arabic # So ARABIC LIGATURE SALAAMUHU ALAYNAA +FDC8..FDCF ; Arabic # So [8] ARABIC LIGATURE RAHIMAHU ALLAAH TAAALAA..ARABIC LIGATURE SALAAMUHU ALAYNAA FDF0..FDFB ; Arabic # Lo [12] ARABIC LIGATURE SALLA USED AS KORANIC STOP SIGN ISOLATED FORM..ARABIC LIGATURE JALLAJALALOUHOU FDFC ; Arabic # Sc RIAL SIGN FDFD..FDFF ; Arabic # So [3] ARABIC LIGATURE BISMILLAH AR-RAHMAN AR-RAHEEM..ARABIC LIGATURE AZZA WA JALL FE70..FE74 ; Arabic # Lo [5] ARABIC FATHATAN ISOLATED FORM..ARABIC KASRATAN ISOLATED FORM FE76..FEFC ; Arabic # Lo [135] ARABIC FATHA ISOLATED FORM..ARABIC LIGATURE LAM WITH ALEF FINAL FORM 10E60..10E7E ; Arabic # No [31] RUMI DIGIT ONE..RUMI FRACTION TWO THIRDS -10EFD..10EFF ; Arabic # Mn [3] ARABIC SMALL LOW WORD SAKTA..ARABIC SMALL LOW WORD MADDA +10EC2..10EC4 ; Arabic # Lo [3] ARABIC LETTER DAL WITH TWO DOTS VERTICALLY BELOW..ARABIC LETTER KAF WITH TWO DOTS VERTICALLY BELOW +10EC5 ; Arabic # Lm ARABIC SMALL YEH BARREE WITH TWO DOTS BELOW +10EC6..10EC7 ; Arabic # Lo [2] ARABIC LETTER THIN NOON..ARABIC LETTER YEH WITH FOUR DOTS BELOW +10ED0 ; Arabic # Po ARABIC BIBLICAL END OF VERSE +10ED1..10ED8 ; Arabic # So [8] ARABIC LIGATURE ALAYHAA AS-SALAATU WAS-SALAAM..ARABIC LIGATURE NAWWARA ALLAAHU MARQADAH +10EFA..10EFF ; Arabic # Mn [6] ARABIC DOUBLE VERTICAL BAR BELOW..ARABIC SMALL LOW WORD MADDA 1EE00..1EE03 ; Arabic # Lo [4] ARABIC MATHEMATICAL ALEF..ARABIC MATHEMATICAL DAL 1EE05..1EE1F ; Arabic # Lo [27] ARABIC MATHEMATICAL WAW..ARABIC MATHEMATICAL DOTLESS QAF 1EE21..1EE22 ; Arabic # Lo [2] ARABIC MATHEMATICAL INITIAL BEH..ARABIC MATHEMATICAL INITIAL JEEM @@ -921,7 +934,7 @@ FE76..FEFC ; Arabic # Lo [135] ARABIC FATHA ISOLATED FORM..ARABIC LIGATURE LA 1EEAB..1EEBB ; Arabic # Lo [17] ARABIC MATHEMATICAL DOUBLE-STRUCK LAM..ARABIC MATHEMATICAL DOUBLE-STRUCK GHAIN 1EEF0..1EEF1 ; Arabic # Sm [2] ARABIC MATHEMATICAL OPERATOR MEEM WITH HAH WITH TATWEEL..ARABIC MATHEMATICAL OPERATOR HAH WITH DAL -# Total code points: 1368 +# Total code points: 1413 # ================================================ @@ -1150,7 +1163,7 @@ A8FF ; Devanagari # Mn DEVANAGARI VOWEL SIGN AY 0C4A..0C4D ; Telugu # Mn [4] TELUGU VOWEL SIGN O..TELUGU SIGN VIRAMA 0C55..0C56 ; Telugu # Mn [2] TELUGU LENGTH MARK..TELUGU AI LENGTH MARK 0C58..0C5A ; Telugu # Lo [3] TELUGU LETTER TSA..TELUGU LETTER RRRA -0C5D ; Telugu # Lo TELUGU LETTER NAKAARA POLLU +0C5C..0C5D ; Telugu # Lo [2] TELUGU ARCHAIC SHRII..TELUGU LETTER NAKAARA POLLU 0C60..0C61 ; Telugu # Lo [2] TELUGU LETTER VOCALIC RR..TELUGU LETTER VOCALIC LL 0C62..0C63 ; Telugu # Mn [2] TELUGU VOWEL SIGN VOCALIC L..TELUGU VOWEL SIGN VOCALIC LL 0C66..0C6F ; Telugu # Nd [10] TELUGU DIGIT ZERO..TELUGU DIGIT NINE @@ -1158,7 +1171,7 @@ A8FF ; Devanagari # Mn DEVANAGARI VOWEL SIGN AY 0C78..0C7E ; Telugu # No [7] TELUGU FRACTION DIGIT ZERO FOR ODD POWERS OF FOUR..TELUGU FRACTION DIGIT THREE FOR EVEN POWERS OF FOUR 0C7F ; Telugu # So TELUGU SIGN TUUMU -# Total code points: 100 +# Total code points: 101 # ================================================ @@ -1181,14 +1194,14 @@ A8FF ; Devanagari # Mn DEVANAGARI VOWEL SIGN AY 0CCA..0CCB ; Kannada # Mc [2] KANNADA VOWEL SIGN O..KANNADA VOWEL SIGN OO 0CCC..0CCD ; Kannada # Mn [2] KANNADA VOWEL SIGN AU..KANNADA SIGN VIRAMA 0CD5..0CD6 ; Kannada # Mc [2] KANNADA LENGTH MARK..KANNADA AI LENGTH MARK -0CDD..0CDE ; Kannada # Lo [2] KANNADA LETTER NAKAARA POLLU..KANNADA LETTER FA +0CDC..0CDE ; Kannada # Lo [3] KANNADA ARCHAIC SHRII..KANNADA LETTER FA 0CE0..0CE1 ; Kannada # Lo [2] KANNADA LETTER VOCALIC RR..KANNADA LETTER VOCALIC LL 0CE2..0CE3 ; Kannada # Mn [2] KANNADA VOWEL SIGN VOCALIC L..KANNADA VOWEL SIGN VOCALIC LL 0CE6..0CEF ; Kannada # Nd [10] KANNADA DIGIT ZERO..KANNADA DIGIT NINE 0CF1..0CF2 ; Kannada # Lo [2] KANNADA SIGN JIHVAMULIYA..KANNADA SIGN UPADHMANIYA 0CF3 ; Kannada # Mc KANNADA SIGN COMBINING ANUSVARA ABOVE RIGHT -# Total code points: 91 +# Total code points: 92 # ================================================ @@ -1368,8 +1381,9 @@ AA7B ; Myanmar # Mc MYANMAR SIGN PAO KAREN TONE AA7C ; Myanmar # Mn MYANMAR SIGN TAI LAING TONE-2 AA7D ; Myanmar # Mc MYANMAR SIGN TAI LAING TONE-5 AA7E..AA7F ; Myanmar # Lo [2] MYANMAR LETTER SHWE PALAUNG CHA..MYANMAR LETTER SHWE PALAUNG SHA +116D0..116E3 ; Myanmar # Nd [20] MYANMAR PAO DIGIT ZERO..MYANMAR EASTERN PWO KAREN DIGIT NINE -# Total code points: 223 +# Total code points: 243 # ================================================ @@ -1588,16 +1602,18 @@ FA70..FAD9 ; Han # Lo [106] CJK COMPATIBILITY IDEOGRAPH-FA70..CJK COMPATIBILI 16FE2 ; Han # Po OLD CHINESE HOOK MARK 16FE3 ; Han # Lm OLD CHINESE ITERATION MARK 16FF0..16FF1 ; Han # Mc [2] VIETNAMESE ALTERNATE READING MARK CA..VIETNAMESE ALTERNATE READING MARK NHAY +16FF2..16FF3 ; Han # Lm [2] CHINESE SMALL SIMPLIFIED ER..CHINESE SMALL TRADITIONAL ER +16FF4..16FF6 ; Han # Nl [3] YANGQIN SIGN SLOW ONE BEAT..YANGQIN SIGN SLOW TWO BEATS 20000..2A6DF ; Han # Lo [42720] CJK UNIFIED IDEOGRAPH-20000..CJK UNIFIED IDEOGRAPH-2A6DF -2A700..2B739 ; Han # Lo [4154] CJK UNIFIED IDEOGRAPH-2A700..CJK UNIFIED IDEOGRAPH-2B739 -2B740..2B81D ; Han # Lo [222] CJK UNIFIED IDEOGRAPH-2B740..CJK UNIFIED IDEOGRAPH-2B81D -2B820..2CEA1 ; Han # Lo [5762] CJK UNIFIED IDEOGRAPH-2B820..CJK UNIFIED IDEOGRAPH-2CEA1 +2A700..2B81D ; Han # Lo [4382] CJK UNIFIED IDEOGRAPH-2A700..CJK UNIFIED IDEOGRAPH-2B81D +2B820..2CEAD ; Han # Lo [5774] CJK UNIFIED IDEOGRAPH-2B820..CJK UNIFIED IDEOGRAPH-2CEAD 2CEB0..2EBE0 ; Han # Lo [7473] CJK UNIFIED IDEOGRAPH-2CEB0..CJK UNIFIED IDEOGRAPH-2EBE0 +2EBF0..2EE5D ; Han # Lo [622] CJK UNIFIED IDEOGRAPH-2EBF0..CJK UNIFIED IDEOGRAPH-2EE5D 2F800..2FA1D ; Han # Lo [542] CJK COMPATIBILITY IDEOGRAPH-2F800..CJK COMPATIBILITY IDEOGRAPH-2FA1D 30000..3134A ; Han # Lo [4939] CJK UNIFIED IDEOGRAPH-30000..CJK UNIFIED IDEOGRAPH-3134A -31350..323AF ; Han # Lo [4192] CJK UNIFIED IDEOGRAPH-31350..CJK UNIFIED IDEOGRAPH-323AF +31350..33479 ; Han # Lo [8490] CJK UNIFIED IDEOGRAPH-31350..CJK UNIFIED IDEOGRAPH-33479 -# Total code points: 98408 +# Total code points: 103351 # ================================================ @@ -1640,7 +1656,8 @@ A490..A4C6 ; Yi # So [55] YI RADICAL QOT..YI RADICAL KE 0951..0954 ; Inherited # Mn [4] DEVANAGARI STRESS SIGN UDATTA..DEVANAGARI ACUTE ACCENT 1AB0..1ABD ; Inherited # Mn [14] COMBINING DOUBLED CIRCUMFLEX ACCENT..COMBINING PARENTHESES BELOW 1ABE ; Inherited # Me COMBINING PARENTHESES OVERLAY -1ABF..1ACE ; Inherited # Mn [16] COMBINING LATIN SMALL LETTER W BELOW..COMBINING LATIN SMALL LETTER INSULAR T +1ABF..1ADD ; Inherited # Mn [31] COMBINING LATIN SMALL LETTER W BELOW..COMBINING DOT-AND-RING BELOW +1AE0..1AEB ; Inherited # Mn [12] COMBINING LEFT TACK ABOVE..COMBINING DOUBLE RIGHTWARDS ARROW ABOVE 1CD0..1CD2 ; Inherited # Mn [3] VEDIC TONE KARSHANA..VEDIC TONE PRENKHA 1CD4..1CE0 ; Inherited # Mn [13] VEDIC SIGN YAJURVEDIC MIDLINE SVARITA..VEDIC TONE RIGVEDIC KASHMIRI INDEPENDENT SVARITA 1CE2..1CE8 ; Inherited # Mn [7] VEDIC SIGN VISARGA SVARITA..VEDIC SIGN VISARGA ANUDATTA WITH TAIL @@ -1669,7 +1686,7 @@ FE20..FE2D ; Inherited # Mn [14] COMBINING LIGATURE LEFT HALF..COMBINING CON 1D1AA..1D1AD ; Inherited # Mn [4] MUSICAL SYMBOL COMBINING DOWN BOW..MUSICAL SYMBOL COMBINING SNAP PIZZICATO E0100..E01EF ; Inherited # Mn [240] VARIATION SELECTOR-17..VARIATION SELECTOR-256 -# Total code points: 657 +# Total code points: 684 # ================================================ @@ -1886,14 +1903,15 @@ A82C ; Syloti_Nagri # Mn SYLOTI NAGRI SIGN ALTERNATE HASANTA 1B42 ; Balinese # Mn BALINESE VOWEL SIGN PEPET 1B43..1B44 ; Balinese # Mc [2] BALINESE VOWEL SIGN PEPET TEDUNG..BALINESE ADEG ADEG 1B45..1B4C ; Balinese # Lo [8] BALINESE LETTER KAF SASAK..BALINESE LETTER ARCHAIC JNYA +1B4E..1B4F ; Balinese # Po [2] BALINESE INVERTED CARIK SIKI..BALINESE INVERTED CARIK PAREREN 1B50..1B59 ; Balinese # Nd [10] BALINESE DIGIT ZERO..BALINESE DIGIT NINE 1B5A..1B60 ; Balinese # Po [7] BALINESE PANTI..BALINESE PAMENENG 1B61..1B6A ; Balinese # So [10] BALINESE MUSICAL SYMBOL DONG..BALINESE MUSICAL SYMBOL DANG GEDE 1B6B..1B73 ; Balinese # Mn [9] BALINESE MUSICAL SYMBOL COMBINING TEGEH..BALINESE MUSICAL SYMBOL COMBINING GONG 1B74..1B7C ; Balinese # So [9] BALINESE MUSICAL SYMBOL RIGHT-HAND OPEN DUG..BALINESE MUSICAL SYMBOL LEFT-HAND OPEN PING -1B7D..1B7E ; Balinese # Po [2] BALINESE PANTI LANTANG..BALINESE PAMADA LANTANG +1B7D..1B7F ; Balinese # Po [3] BALINESE PANTI LANTANG..BALINESE PANTI BAWAK -# Total code points: 124 +# Total code points: 127 # ================================================ @@ -2106,8 +2124,9 @@ AADE..AADF ; Tai_Viet # Po [2] TAI VIET SYMBOL HO HOI..TAI VIET SYMBOL KOI 13440 ; Egyptian_Hieroglyphs # Mn EGYPTIAN HIEROGLYPH MIRROR HORIZONTALLY 13441..13446 ; Egyptian_Hieroglyphs # Lo [6] EGYPTIAN HIEROGLYPH FULL BLANK..EGYPTIAN HIEROGLYPH WIDE LOST SIGN 13447..13455 ; Egyptian_Hieroglyphs # Mn [15] EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT TOP START..EGYPTIAN HIEROGLYPH MODIFIER DAMAGED +13460..143FA ; Egyptian_Hieroglyphs # Lo [3995] EGYPTIAN HIEROGLYPH-13460..EGYPTIAN HIEROGLYPH-143FA -# Total code points: 1110 +# Total code points: 5105 # ================================================ @@ -2338,8 +2357,14 @@ ABF0..ABF9 ; Meetei_Mayek # Nd [10] MEETEI MAYEK DIGIT ZERO..MEETEI MAYEK DI 111DB ; Sharada # Po SHARADA SIGN SIDDHAM 111DC ; Sharada # Lo SHARADA HEADSTROKE 111DD..111DF ; Sharada # Po [3] SHARADA CONTINUATION SIGN..SHARADA SECTION MARK-2 +11B60 ; Sharada # Mn SHARADA VOWEL SIGN OE +11B61 ; Sharada # Mc SHARADA VOWEL SIGN OOE +11B62..11B64 ; Sharada # Mn [3] SHARADA VOWEL SIGN UE..SHARADA VOWEL SIGN SHORT E +11B65 ; Sharada # Mc SHARADA VOWEL SIGN SHORT O +11B66 ; Sharada # Mn SHARADA VOWEL SIGN CANDRA E +11B67 ; Sharada # Mc SHARADA VOWEL SIGN CANDRA O -# Total code points: 96 +# Total code points: 104 # ================================================ @@ -2613,7 +2638,9 @@ ABF0..ABF9 ; Meetei_Mayek # Nd [10] MEETEI MAYEK DIGIT ZERO..MEETEI MAYEK DI # ================================================ 11700..1171A ; Ahom # Lo [27] AHOM LETTER KA..AHOM LETTER ALTERNATE BA -1171D..1171F ; Ahom # Mn [3] AHOM CONSONANT SIGN MEDIAL LA..AHOM CONSONANT SIGN MEDIAL LIGATING RA +1171D ; Ahom # Mn AHOM CONSONANT SIGN MEDIAL LA +1171E ; Ahom # Mc AHOM CONSONANT SIGN MEDIAL RA +1171F ; Ahom # Mn AHOM CONSONANT SIGN MEDIAL LIGATING RA 11720..11721 ; Ahom # Mc [2] AHOM VOWEL SIGN A..AHOM VOWEL SIGN AA 11722..11725 ; Ahom # Mn [4] AHOM VOWEL SIGN I..AHOM VOWEL SIGN UU 11726 ; Ahom # Mc AHOM VOWEL SIGN E @@ -2745,11 +2772,11 @@ ABF0..ABF9 ; Meetei_Mayek # Nd [10] MEETEI MAYEK DIGIT ZERO..MEETEI MAYEK DI # ================================================ 16FE0 ; Tangut # Lm TANGUT ITERATION MARK -17000..187F7 ; Tangut # Lo [6136] TANGUT IDEOGRAPH-17000..TANGUT IDEOGRAPH-187F7 -18800..18AFF ; Tangut # Lo [768] TANGUT COMPONENT-001..TANGUT COMPONENT-768 -18D00..18D08 ; Tangut # Lo [9] TANGUT IDEOGRAPH-18D00..TANGUT IDEOGRAPH-18D08 +17000..18AFF ; Tangut # Lo [6912] TANGUT IDEOGRAPH-17000..TANGUT COMPONENT-768 +18D00..18D1E ; Tangut # Lo [31] TANGUT IDEOGRAPH-18D00..TANGUT IDEOGRAPH-18D1E +18D80..18DF2 ; Tangut # Lo [115] TANGUT COMPONENT-769..TANGUT COMPONENT-883 -# Total code points: 6914 +# Total code points: 7059 # ================================================ @@ -2947,8 +2974,9 @@ ABF0..ABF9 ; Meetei_Mayek # Nd [10] MEETEI MAYEK DIGIT ZERO..MEETEI MAYEK DI 16FE4 ; Khitan_Small_Script # Mn KHITAN SMALL SCRIPT FILLER 18B00..18CD5 ; Khitan_Small_Script # Lo [470] KHITAN SMALL SCRIPT CHARACTER-18B00..KHITAN SMALL SCRIPT CHARACTER-18CD5 +18CFF ; Khitan_Small_Script # Lo KHITAN SMALL SCRIPT CHARACTER-18CFF -# Total code points: 471 +# Total code points: 472 # ================================================ @@ -3016,8 +3044,9 @@ ABF0..ABF9 ; Meetei_Mayek # Nd [10] MEETEI MAYEK DIGIT ZERO..MEETEI MAYEK DI 11F42 ; Kawi # Mn KAWI CONJOINER 11F43..11F4F ; Kawi # Po [13] KAWI DANDA..KAWI PUNCTUATION CLOSING SPIRAL 11F50..11F59 ; Kawi # Nd [10] KAWI DIGIT ZERO..KAWI DIGIT NINE +11F5A ; Kawi # Mn KAWI SIGN NUKTA -# Total code points: 86 +# Total code points: 87 # ================================================ @@ -3028,4 +3057,126 @@ ABF0..ABF9 ; Meetei_Mayek # Nd [10] MEETEI MAYEK DIGIT ZERO..MEETEI MAYEK DI # Total code points: 42 +# ================================================ + +10D40..10D49 ; Garay # Nd [10] GARAY DIGIT ZERO..GARAY DIGIT NINE +10D4A..10D4D ; Garay # Lo [4] GARAY VOWEL SIGN A..GARAY VOWEL SIGN EE +10D4E ; Garay # Lm GARAY VOWEL LENGTH MARK +10D4F ; Garay # Lo GARAY SUKUN +10D50..10D65 ; Garay # L& [22] GARAY CAPITAL LETTER A..GARAY CAPITAL LETTER OLD NA +10D69..10D6D ; Garay # Mn [5] GARAY VOWEL SIGN E..GARAY CONSONANT NASALIZATION MARK +10D6E ; Garay # Pd GARAY HYPHEN +10D6F ; Garay # Lm GARAY REDUPLICATION MARK +10D70..10D85 ; Garay # L& [22] GARAY SMALL LETTER A..GARAY SMALL LETTER OLD NA +10D8E..10D8F ; Garay # Sm [2] GARAY PLUS SIGN..GARAY MINUS SIGN + +# Total code points: 69 + +# ================================================ + +16100..1611D ; Gurung_Khema # Lo [30] GURUNG KHEMA LETTER A..GURUNG KHEMA LETTER SA +1611E..16129 ; Gurung_Khema # Mn [12] GURUNG KHEMA VOWEL SIGN AA..GURUNG KHEMA VOWEL LENGTH MARK +1612A..1612C ; Gurung_Khema # Mc [3] GURUNG KHEMA CONSONANT SIGN MEDIAL YA..GURUNG KHEMA CONSONANT SIGN MEDIAL HA +1612D..1612F ; Gurung_Khema # Mn [3] GURUNG KHEMA SIGN ANUSVARA..GURUNG KHEMA SIGN THOLHOMA +16130..16139 ; Gurung_Khema # Nd [10] GURUNG KHEMA DIGIT ZERO..GURUNG KHEMA DIGIT NINE + +# Total code points: 58 + +# ================================================ + +16D40..16D42 ; Kirat_Rai # Lm [3] KIRAT RAI SIGN ANUSVARA..KIRAT RAI SIGN VISARGA +16D43..16D6A ; Kirat_Rai # Lo [40] KIRAT RAI LETTER A..KIRAT RAI VOWEL SIGN AU +16D6B..16D6C ; Kirat_Rai # Lm [2] KIRAT RAI SIGN VIRAMA..KIRAT RAI SIGN SAAT +16D6D..16D6F ; Kirat_Rai # Po [3] KIRAT RAI SIGN YUPI..KIRAT RAI DOUBLE DANDA +16D70..16D79 ; Kirat_Rai # Nd [10] KIRAT RAI DIGIT ZERO..KIRAT RAI DIGIT NINE + +# Total code points: 58 + +# ================================================ + +1E5D0..1E5ED ; Ol_Onal # Lo [30] OL ONAL LETTER O..OL ONAL LETTER EG +1E5EE..1E5EF ; Ol_Onal # Mn [2] OL ONAL SIGN MU..OL ONAL SIGN IKIR +1E5F0 ; Ol_Onal # Lo OL ONAL SIGN HODDOND +1E5F1..1E5FA ; Ol_Onal # Nd [10] OL ONAL DIGIT ZERO..OL ONAL DIGIT NINE +1E5FF ; Ol_Onal # Po OL ONAL ABBREVIATION SIGN + +# Total code points: 44 + +# ================================================ + +11BC0..11BE0 ; Sunuwar # Lo [33] SUNUWAR LETTER DEVI..SUNUWAR LETTER KLOKO +11BE1 ; Sunuwar # Po SUNUWAR SIGN PVO +11BF0..11BF9 ; Sunuwar # Nd [10] SUNUWAR DIGIT ZERO..SUNUWAR DIGIT NINE + +# Total code points: 44 + +# ================================================ + +105C0..105F3 ; Todhri # Lo [52] TODHRI LETTER A..TODHRI LETTER OO + +# Total code points: 52 + +# ================================================ + +11380..11389 ; Tulu_Tigalari # Lo [10] TULU-TIGALARI LETTER A..TULU-TIGALARI LETTER VOCALIC LL +1138B ; Tulu_Tigalari # Lo TULU-TIGALARI LETTER EE +1138E ; Tulu_Tigalari # Lo TULU-TIGALARI LETTER AI +11390..113B5 ; Tulu_Tigalari # Lo [38] TULU-TIGALARI LETTER OO..TULU-TIGALARI LETTER LLLA +113B7 ; Tulu_Tigalari # Lo TULU-TIGALARI SIGN AVAGRAHA +113B8..113BA ; Tulu_Tigalari # Mc [3] TULU-TIGALARI VOWEL SIGN AA..TULU-TIGALARI VOWEL SIGN II +113BB..113C0 ; Tulu_Tigalari # Mn [6] TULU-TIGALARI VOWEL SIGN U..TULU-TIGALARI VOWEL SIGN VOCALIC LL +113C2 ; Tulu_Tigalari # Mc TULU-TIGALARI VOWEL SIGN EE +113C5 ; Tulu_Tigalari # Mc TULU-TIGALARI VOWEL SIGN AI +113C7..113CA ; Tulu_Tigalari # Mc [4] TULU-TIGALARI VOWEL SIGN OO..TULU-TIGALARI SIGN CANDRA ANUNASIKA +113CC..113CD ; Tulu_Tigalari # Mc [2] TULU-TIGALARI SIGN ANUSVARA..TULU-TIGALARI SIGN VISARGA +113CE ; Tulu_Tigalari # Mn TULU-TIGALARI SIGN VIRAMA +113CF ; Tulu_Tigalari # Mc TULU-TIGALARI SIGN LOOPED VIRAMA +113D0 ; Tulu_Tigalari # Mn TULU-TIGALARI CONJOINER +113D1 ; Tulu_Tigalari # Lo TULU-TIGALARI REPHA +113D2 ; Tulu_Tigalari # Mn TULU-TIGALARI GEMINATION MARK +113D3 ; Tulu_Tigalari # Lo TULU-TIGALARI SIGN PLUTA +113D4..113D5 ; Tulu_Tigalari # Po [2] TULU-TIGALARI DANDA..TULU-TIGALARI DOUBLE DANDA +113D7..113D8 ; Tulu_Tigalari # Po [2] TULU-TIGALARI SIGN OM PUSHPIKA..TULU-TIGALARI SIGN SHRII PUSHPIKA +113E1..113E2 ; Tulu_Tigalari # Mn [2] TULU-TIGALARI VEDIC TONE SVARITA..TULU-TIGALARI VEDIC TONE ANUDATTA + +# Total code points: 80 + +# ================================================ + +10940..10959 ; Sidetic # Lo [26] SIDETIC LETTER N01..SIDETIC LETTER N26 + +# Total code points: 26 + +# ================================================ + +1E6C0..1E6DE ; Tai_Yo # Lo [31] TAI YO LETTER LOW KO..TAI YO LETTER HIGH KVO +1E6E0..1E6E2 ; Tai_Yo # Lo [3] TAI YO LETTER AA..TAI YO LETTER UE +1E6E3 ; Tai_Yo # Mn TAI YO SIGN UE +1E6E4..1E6E5 ; Tai_Yo # Lo [2] TAI YO LETTER U..TAI YO LETTER AE +1E6E6 ; Tai_Yo # Mn TAI YO SIGN AU +1E6E7..1E6ED ; Tai_Yo # Lo [7] TAI YO LETTER O..TAI YO LETTER AUE +1E6EE..1E6EF ; Tai_Yo # Mn [2] TAI YO SIGN AY..TAI YO SIGN ANG +1E6F0..1E6F4 ; Tai_Yo # Lo [5] TAI YO LETTER AN..TAI YO LETTER AP +1E6F5 ; Tai_Yo # Mn TAI YO SIGN OM +1E6FE ; Tai_Yo # Lo TAI YO SYMBOL MUEANG +1E6FF ; Tai_Yo # Lm TAI YO XAM LAI + +# Total code points: 55 + +# ================================================ + +11DB0..11DD8 ; Tolong_Siki # Lo [41] TOLONG SIKI LETTER I..TOLONG SIKI LETTER RRH +11DD9 ; Tolong_Siki # Lm TOLONG SIKI SIGN SELA +11DDA..11DDB ; Tolong_Siki # Lo [2] TOLONG SIKI SIGN HECAKA..TOLONG SIKI UNGGA +11DE0..11DE9 ; Tolong_Siki # Nd [10] TOLONG SIKI DIGIT ZERO..TOLONG SIKI DIGIT NINE + +# Total code points: 54 + +# ================================================ + +16EA0..16EB8 ; Beria_Erfe # L& [25] BERIA ERFE CAPITAL LETTER ARKAB..BERIA ERFE CAPITAL LETTER AY +16EBB..16ED3 ; Beria_Erfe # L& [25] BERIA ERFE SMALL LETTER ARKAB..BERIA ERFE SMALL LETTER AY + +# Total code points: 50 + # EOF diff --git a/share/ucd/UnicodeData.txt b/share/ucd/UnicodeData.txt index ea963a716..fca68e3e1 100644 --- a/share/ucd/UnicodeData.txt +++ b/share/ucd/UnicodeData.txt @@ -409,7 +409,7 @@ 0198;LATIN CAPITAL LETTER K WITH HOOK;Lu;0;L;;;;;N;LATIN CAPITAL LETTER K HOOK;;;0199; 0199;LATIN SMALL LETTER K WITH HOOK;Ll;0;L;;;;;N;LATIN SMALL LETTER K HOOK;;0198;;0198 019A;LATIN SMALL LETTER L WITH BAR;Ll;0;L;;;;;N;LATIN SMALL LETTER BARRED L;;023D;;023D -019B;LATIN SMALL LETTER LAMBDA WITH STROKE;Ll;0;L;;;;;N;LATIN SMALL LETTER BARRED LAMBDA;;;; +019B;LATIN SMALL LETTER LAMBDA WITH STROKE;Ll;0;L;;;;;N;LATIN SMALL LETTER BARRED LAMBDA;;A7DC;;A7DC 019C;LATIN CAPITAL LETTER TURNED M;Lu;0;L;;;;;N;;;;026F; 019D;LATIN CAPITAL LETTER N WITH LEFT HOOK;Lu;0;L;;;;;N;LATIN CAPITAL LETTER N HOOK;;;0272; 019E;LATIN SMALL LETTER N WITH LONG RIGHT LEG;Ll;0;L;;;;;N;;;0220;;0220 @@ -610,7 +610,7 @@ 0261;LATIN SMALL LETTER SCRIPT G;Ll;0;L;;;;;N;;;A7AC;;A7AC 0262;LATIN LETTER SMALL CAPITAL G;Ll;0;L;;;;;N;;;;; 0263;LATIN SMALL LETTER GAMMA;Ll;0;L;;;;;N;;;0194;;0194 -0264;LATIN SMALL LETTER RAMS HORN;Ll;0;L;;;;;N;LATIN SMALL LETTER BABY GAMMA;;;; +0264;LATIN SMALL LETTER RAMS HORN;Ll;0;L;;;;;N;LATIN SMALL LETTER BABY GAMMA;;A7CB;;A7CB 0265;LATIN SMALL LETTER TURNED H;Ll;0;L;;;;;N;;;A78D;;A78D 0266;LATIN SMALL LETTER H WITH HOOK;Ll;0;L;;;;;N;LATIN SMALL LETTER H HOOK;;A7AA;;A7AA 0267;LATIN SMALL LETTER HENG WITH HOOK;Ll;0;L;;;;;N;LATIN SMALL LETTER HENG HOOK;;;; @@ -659,7 +659,7 @@ 0292;LATIN SMALL LETTER EZH;Ll;0;L;;;;;N;LATIN SMALL LETTER YOGH;;01B7;;01B7 0293;LATIN SMALL LETTER EZH WITH CURL;Ll;0;L;;;;;N;LATIN SMALL LETTER YOGH CURL;;;; 0294;LATIN LETTER GLOTTAL STOP;Lo;0;L;;;;;N;;;;; -0295;LATIN LETTER PHARYNGEAL VOICED FRICATIVE;Ll;0;L;;;;;N;LATIN LETTER REVERSED GLOTTAL STOP;;;; +0295;LATIN LETTER PHARYNGEAL VOICED FRICATIVE;Lo;0;L;;;;;N;LATIN LETTER REVERSED GLOTTAL STOP;;;; 0296;LATIN LETTER INVERTED GLOTTAL STOP;Ll;0;L;;;;;N;;;;; 0297;LATIN LETTER STRETCHED C;Ll;0;L;;;;;N;;;;; 0298;LATIN LETTER BILABIAL CLICK;Ll;0;L;;;;;N;LATIN LETTER BULLSEYE;;;; @@ -2121,8 +2121,10 @@ 088C;ARABIC LETTER TAH WITH THREE DOTS BELOW;Lo;0;AL;;;;;N;;;;; 088D;ARABIC LETTER KEHEH WITH TWO DOTS VERTICALLY BELOW;Lo;0;AL;;;;;N;;;;; 088E;ARABIC VERTICAL TAIL;Lo;0;AL;;;;;N;;;;; +088F;ARABIC LETTER NOON WITH RING ABOVE;Lo;0;AL;;;;;N;;;;; 0890;ARABIC POUND MARK ABOVE;Cf;0;AN;;;;;N;;;;; 0891;ARABIC PIASTRE MARK ABOVE;Cf;0;AN;;;;;N;;;;; +0897;ARABIC PEPET;Mn;230;NSM;;;;;N;;;;; 0898;ARABIC SMALL HIGH WORD AL-JUZ;Mn;230;NSM;;;;;N;;;;; 0899;ARABIC SMALL LOW WORD ISHMAAM;Mn;220;NSM;;;;;N;;;;; 089A;ARABIC SMALL LOW WORD IMAALA;Mn;220;NSM;;;;;N;;;;; @@ -2861,6 +2863,7 @@ 0C58;TELUGU LETTER TSA;Lo;0;L;;;;;N;;;;; 0C59;TELUGU LETTER DZA;Lo;0;L;;;;;N;;;;; 0C5A;TELUGU LETTER RRRA;Lo;0;L;;;;;N;;;;; +0C5C;TELUGU ARCHAIC SHRII;Lo;0;L;;;;;N;;;;; 0C5D;TELUGU LETTER NAKAARA POLLU;Lo;0;L;;;;;N;;;;; 0C60;TELUGU LETTER VOCALIC RR;Lo;0;L;;;;;N;;;;; 0C61;TELUGU LETTER VOCALIC LL;Lo;0;L;;;;;N;;;;; @@ -2957,6 +2960,7 @@ 0CCD;KANNADA SIGN VIRAMA;Mn;9;NSM;;;;;N;;;;; 0CD5;KANNADA LENGTH MARK;Mc;0;L;;;;;N;;;;; 0CD6;KANNADA AI LENGTH MARK;Mc;0;L;;;;;N;;;;; +0CDC;KANNADA ARCHAIC SHRII;Lo;0;L;;;;;N;;;;; 0CDD;KANNADA LETTER NAKAARA POLLU;Lo;0;L;;;;;N;;;;; 0CDE;KANNADA LETTER FA;Lo;0;L;;;;;N;;;;; 0CE0;KANNADA LETTER VOCALIC RR;Lo;0;L;;;;;N;;;;; @@ -6136,6 +6140,33 @@ 1ACC;COMBINING LATIN SMALL LETTER INSULAR G;Mn;230;NSM;;;;;N;;;;; 1ACD;COMBINING LATIN SMALL LETTER INSULAR R;Mn;230;NSM;;;;;N;;;;; 1ACE;COMBINING LATIN SMALL LETTER INSULAR T;Mn;230;NSM;;;;;N;;;;; +1ACF;COMBINING DOUBLE CARON;Mn;230;NSM;;;;;N;;;;; +1AD0;COMBINING VERTICAL-LINE-ACUTE;Mn;230;NSM;;;;;N;;;;; +1AD1;COMBINING GRAVE-VERTICAL-LINE;Mn;230;NSM;;;;;N;;;;; +1AD2;COMBINING VERTICAL-LINE-GRAVE;Mn;230;NSM;;;;;N;;;;; +1AD3;COMBINING ACUTE-VERTICAL-LINE;Mn;230;NSM;;;;;N;;;;; +1AD4;COMBINING VERTICAL-LINE-MACRON;Mn;230;NSM;;;;;N;;;;; +1AD5;COMBINING MACRON-VERTICAL-LINE;Mn;230;NSM;;;;;N;;;;; +1AD6;COMBINING VERTICAL-LINE-ACUTE-GRAVE;Mn;230;NSM;;;;;N;;;;; +1AD7;COMBINING VERTICAL-LINE-GRAVE-ACUTE;Mn;230;NSM;;;;;N;;;;; +1AD8;COMBINING MACRON-ACUTE-GRAVE;Mn;230;NSM;;;;;N;;;;; +1AD9;COMBINING SHARP SIGN;Mn;230;NSM;;;;;N;;;;; +1ADA;COMBINING FLAT SIGN;Mn;230;NSM;;;;;N;;;;; +1ADB;COMBINING DOWN TACK ABOVE;Mn;230;NSM;;;;;N;;;;; +1ADC;COMBINING DIAERESIS WITH RAISED LEFT DOT;Mn;230;NSM;;;;;N;;;;; +1ADD;COMBINING DOT-AND-RING BELOW;Mn;220;NSM;;;;;N;;;;; +1AE0;COMBINING LEFT TACK ABOVE;Mn;230;NSM;;;;;N;;;;; +1AE1;COMBINING RIGHT TACK ABOVE;Mn;230;NSM;;;;;N;;;;; +1AE2;COMBINING MINUS SIGN ABOVE;Mn;230;NSM;;;;;N;;;;; +1AE3;COMBINING INVERTED BRIDGE ABOVE;Mn;230;NSM;;;;;N;;;;; +1AE4;COMBINING SQUARE ABOVE;Mn;230;NSM;;;;;N;;;;; +1AE5;COMBINING SEAGULL ABOVE;Mn;230;NSM;;;;;N;;;;; +1AE6;COMBINING DOUBLE ARCH BELOW;Mn;220;NSM;;;;;N;;;;; +1AE7;COMBINING DOUBLE ARCH ABOVE;Mn;230;NSM;;;;;N;;;;; +1AE8;COMBINING EQUALS SIGN ABOVE;Mn;230;NSM;;;;;N;;;;; +1AE9;COMBINING LEFT ANGLE CENTRED ABOVE;Mn;230;NSM;;;;;N;;;;; +1AEA;COMBINING UPWARDS ARROW ABOVE;Mn;230;NSM;;;;;N;;;;; +1AEB;COMBINING DOUBLE RIGHTWARDS ARROW ABOVE;Mn;234;NSM;;;;;N;;;;; 1B00;BALINESE SIGN ULU RICEM;Mn;0;NSM;;;;;N;;;;; 1B01;BALINESE SIGN ULU CANDRA;Mn;0;NSM;;;;;N;;;;; 1B02;BALINESE SIGN CECEK;Mn;0;NSM;;;;;N;;;;; @@ -6213,6 +6244,8 @@ 1B4A;BALINESE LETTER ZAL SASAK;Lo;0;L;;;;;N;;;;; 1B4B;BALINESE LETTER ASYURA SASAK;Lo;0;L;;;;;N;;;;; 1B4C;BALINESE LETTER ARCHAIC JNYA;Lo;0;L;;;;;N;;;;; +1B4E;BALINESE INVERTED CARIK SIKI;Po;0;L;;;;;N;;;;; +1B4F;BALINESE INVERTED CARIK PAREREN;Po;0;L;;;;;N;;;;; 1B50;BALINESE DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; 1B51;BALINESE DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; 1B52;BALINESE DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; @@ -6260,6 +6293,7 @@ 1B7C;BALINESE MUSICAL SYMBOL LEFT-HAND OPEN PING;So;0;L;;;;;N;;;;; 1B7D;BALINESE PANTI LANTANG;Po;0;L;;;;;N;;;;; 1B7E;BALINESE PAMADA LANTANG;Po;0;L;;;;;N;;;;; +1B7F;BALINESE PANTI BAWAK;Po;0;L;;;;;N;;;;; 1B80;SUNDANESE SIGN PANYECEK;Mn;0;NSM;;;;;N;;;;; 1B81;SUNDANESE SIGN PANGLAYAR;Mn;0;NSM;;;;;N;;;;; 1B82;SUNDANESE SIGN PANGWISAD;Mc;0;L;;;;;N;;;;; @@ -6511,6 +6545,8 @@ 1C86;CYRILLIC SMALL LETTER TALL HARD SIGN;Ll;0;L;;;;;N;;;042A;;042A 1C87;CYRILLIC SMALL LETTER TALL YAT;Ll;0;L;;;;;N;;;0462;;0462 1C88;CYRILLIC SMALL LETTER UNBLENDED UK;Ll;0;L;;;;;N;;;A64A;;A64A +1C89;CYRILLIC CAPITAL LETTER TJE;Lu;0;L;;;;;N;;;;1C8A; +1C8A;CYRILLIC SMALL LETTER TJE;Ll;0;L;;;;;N;;;1C89;;1C89 1C90;GEORGIAN MTAVRULI CAPITAL LETTER AN;Lu;0;L;;;;;N;;;;10D0; 1C91;GEORGIAN MTAVRULI CAPITAL LETTER BAN;Lu;0;L;;;;;N;;;;10D1; 1C92;GEORGIAN MTAVRULI CAPITAL LETTER GAN;Lu;0;L;;;;;N;;;;10D2; @@ -7539,6 +7575,7 @@ 20BE;LARI SIGN;Sc;0;ET;;;;;N;;;;; 20BF;BITCOIN SIGN;Sc;0;ET;;;;;N;;;;; 20C0;SOM SIGN;Sc;0;ET;;;;;N;;;;; +20C1;SAUDI RIYAL SIGN;Sc;0;ET;;;;;N;;;;; 20D0;COMBINING LEFT HARPOON ABOVE;Mn;230;NSM;;;;;N;NON-SPACING LEFT HARPOON ABOVE;;;; 20D1;COMBINING RIGHT HARPOON ABOVE;Mn;230;NSM;;;;;N;NON-SPACING RIGHT HARPOON ABOVE;;;; 20D2;COMBINING LONG VERTICAL LINE OVERLAY;Mn;1;NSM;;;;;N;NON-SPACING LONG VERTICAL BAR OVERLAY;;;; @@ -7933,7 +7970,7 @@ 226A;MUCH LESS-THAN;Sm;0;ON;;;;;Y;MUCH LESS THAN;;;; 226B;MUCH GREATER-THAN;Sm;0;ON;;;;;Y;MUCH GREATER THAN;;;; 226C;BETWEEN;Sm;0;ON;;;;;N;;;;; -226D;NOT EQUIVALENT TO;Sm;0;ON;224D 0338;;;;N;;;;; +226D;NOT EQUIVALENT TO;Sm;0;ON;224D 0338;;;;Y;;;;; 226E;NOT LESS-THAN;Sm;0;ON;003C 0338;;;;Y;NOT LESS THAN;;;; 226F;NOT GREATER-THAN;Sm;0;ON;003E 0338;;;;Y;NOT GREATER THAN;;;; 2270;NEITHER LESS-THAN NOR EQUAL TO;Sm;0;ON;2264 0338;;;;Y;NEITHER LESS THAN NOR EQUAL TO;;;; @@ -8375,6 +8412,9 @@ 2424;SYMBOL FOR NEWLINE;So;0;ON;;;;;N;GRAPHIC FOR NEWLINE;;;; 2425;SYMBOL FOR DELETE FORM TWO;So;0;ON;;;;;N;;;;; 2426;SYMBOL FOR SUBSTITUTE FORM TWO;So;0;ON;;;;;N;;;;; +2427;SYMBOL FOR DELETE SQUARE CHECKER BOARD FORM;So;0;ON;;;;;N;;;;; +2428;SYMBOL FOR DELETE RECTANGULAR CHECKER BOARD FORM;So;0;ON;;;;;N;;;;; +2429;SYMBOL FOR DELETE MEDIUM SHADE FORM;So;0;ON;;;;;N;;;;; 2440;OCR HOOK;So;0;ON;;;;;N;;;;; 2441;OCR CHAIR;So;0;ON;;;;;N;;;;; 2442;OCR FORK;So;0;ON;;;;;N;;;;; @@ -10230,6 +10270,7 @@ 2B93;NEWLINE RIGHT;So;0;ON;;;;;N;;;;; 2B94;FOUR CORNER ARROWS CIRCLING ANTICLOCKWISE;So;0;ON;;;;;N;;;;; 2B95;RIGHTWARDS BLACK ARROW;So;0;ON;;;;;N;;;;; +2B96;EQUALS SIGN WITH INFINITY ABOVE;So;0;ON;;;;;N;;;;; 2B97;SYMBOL FOR TYPE A ELECTRONICS;So;0;ON;;;;;N;;;;; 2B98;THREE-D TOP-LIGHTED LEFTWARDS EQUILATERAL ARROWHEAD;So;0;ON;;;;;N;;;;; 2B99;THREE-D RIGHT-LIGHTED UPWARDS EQUILATERAL ARROWHEAD;So;0;ON;;;;;N;;;;; @@ -11231,6 +11272,10 @@ 2FF9;IDEOGRAPHIC DESCRIPTION CHARACTER SURROUND FROM UPPER RIGHT;So;0;ON;;;;;N;;;;; 2FFA;IDEOGRAPHIC DESCRIPTION CHARACTER SURROUND FROM LOWER LEFT;So;0;ON;;;;;N;;;;; 2FFB;IDEOGRAPHIC DESCRIPTION CHARACTER OVERLAID;So;0;ON;;;;;N;;;;; +2FFC;IDEOGRAPHIC DESCRIPTION CHARACTER SURROUND FROM RIGHT;So;0;ON;;;;;N;;;;; +2FFD;IDEOGRAPHIC DESCRIPTION CHARACTER SURROUND FROM LOWER RIGHT;So;0;ON;;;;;N;;;;; +2FFE;IDEOGRAPHIC DESCRIPTION CHARACTER HORIZONTAL REFLECTION;So;0;ON;;;;;N;;;;; +2FFF;IDEOGRAPHIC DESCRIPTION CHARACTER ROTATION;So;0;ON;;;;;N;;;;; 3000;IDEOGRAPHIC SPACE;Zs;0;WS; 0020;;;;N;;;;; 3001;IDEOGRAPHIC COMMA;Po;0;ON;;;;;N;;;;; 3002;IDEOGRAPHIC FULL STOP;Po;0;ON;;;;;N;IDEOGRAPHIC PERIOD;;;; @@ -11705,6 +11750,9 @@ 31E1;CJK STROKE HZZZG;So;0;ON;;;;;N;;;;; 31E2;CJK STROKE PG;So;0;ON;;;;;N;;;;; 31E3;CJK STROKE Q;So;0;ON;;;;;N;;;;; +31E4;CJK STROKE HXG;So;0;ON;;;;;N;;;;; +31E5;CJK STROKE SZP;So;0;ON;;;;;N;;;;; +31EF;IDEOGRAPHIC DESCRIPTION CHARACTER SUBTRACTION;So;0;ON;;;;;N;;;;; 31F0;KATAKANA LETTER SMALL KU;Lo;0;L;;;;;N;;;;; 31F1;KATAKANA LETTER SMALL SI;Lo;0;L;;;;;N;;;;; 31F2;KATAKANA LETTER SMALL SU;Lo;0;L;;;;;N;;;;; @@ -14255,14 +14303,25 @@ A7C7;LATIN CAPITAL LETTER D WITH SHORT STROKE OVERLAY;Lu;0;L;;;;;N;;;;A7C8; A7C8;LATIN SMALL LETTER D WITH SHORT STROKE OVERLAY;Ll;0;L;;;;;N;;;A7C7;;A7C7 A7C9;LATIN CAPITAL LETTER S WITH SHORT STROKE OVERLAY;Lu;0;L;;;;;N;;;;A7CA; A7CA;LATIN SMALL LETTER S WITH SHORT STROKE OVERLAY;Ll;0;L;;;;;N;;;A7C9;;A7C9 +A7CB;LATIN CAPITAL LETTER RAMS HORN;Lu;0;L;;;;;N;;;;0264; +A7CC;LATIN CAPITAL LETTER S WITH DIAGONAL STROKE;Lu;0;L;;;;;N;;;;A7CD; +A7CD;LATIN SMALL LETTER S WITH DIAGONAL STROKE;Ll;0;L;;;;;N;;;A7CC;;A7CC +A7CE;LATIN CAPITAL LETTER PHARYNGEAL VOICED FRICATIVE;Lu;0;L;;;;;N;;;;A7CF; +A7CF;LATIN SMALL LETTER PHARYNGEAL VOICED FRICATIVE;Ll;0;L;;;;;N;;;A7CE;;A7CE A7D0;LATIN CAPITAL LETTER CLOSED INSULAR G;Lu;0;L;;;;;N;;;;A7D1; A7D1;LATIN SMALL LETTER CLOSED INSULAR G;Ll;0;L;;;;;N;;;A7D0;;A7D0 -A7D3;LATIN SMALL LETTER DOUBLE THORN;Ll;0;L;;;;;N;;;;; -A7D5;LATIN SMALL LETTER DOUBLE WYNN;Ll;0;L;;;;;N;;;;; +A7D2;LATIN CAPITAL LETTER DOUBLE THORN;Lu;0;L;;;;;N;;;;A7D3; +A7D3;LATIN SMALL LETTER DOUBLE THORN;Ll;0;L;;;;;N;;;A7D2;;A7D2 +A7D4;LATIN CAPITAL LETTER DOUBLE WYNN;Lu;0;L;;;;;N;;;;A7D5; +A7D5;LATIN SMALL LETTER DOUBLE WYNN;Ll;0;L;;;;;N;;;A7D4;;A7D4 A7D6;LATIN CAPITAL LETTER MIDDLE SCOTS S;Lu;0;L;;;;;N;;;;A7D7; A7D7;LATIN SMALL LETTER MIDDLE SCOTS S;Ll;0;L;;;;;N;;;A7D6;;A7D6 A7D8;LATIN CAPITAL LETTER SIGMOID S;Lu;0;L;;;;;N;;;;A7D9; A7D9;LATIN SMALL LETTER SIGMOID S;Ll;0;L;;;;;N;;;A7D8;;A7D8 +A7DA;LATIN CAPITAL LETTER LAMBDA;Lu;0;L;;;;;N;;;;A7DB; +A7DB;LATIN SMALL LETTER LAMBDA;Ll;0;L;;;;;N;;;A7DA;;A7DA +A7DC;LATIN CAPITAL LETTER LAMBDA WITH STROKE;Lu;0;L;;;;;N;;;;019B; +A7F1;MODIFIER LETTER CAPITAL S;Lm;0;L; 0053;;;;N;;;;; A7F2;MODIFIER LETTER CAPITAL C;Lm;0;L; 0043;;;;N;;;;; A7F3;MODIFIER LETTER CAPITAL F;Lm;0;L; 0046;;;;N;;;;; A7F4;MODIFIER LETTER CAPITAL Q;Lm;0;L; 0051;;;;N;;;;; @@ -15903,6 +15962,22 @@ FBBF;ARABIC SYMBOL RING;Sk;0;AL;;;;;N;;;;; FBC0;ARABIC SYMBOL SMALL TAH ABOVE;Sk;0;AL;;;;;N;;;;; FBC1;ARABIC SYMBOL SMALL TAH BELOW;Sk;0;AL;;;;;N;;;;; FBC2;ARABIC SYMBOL WASLA ABOVE;Sk;0;AL;;;;;N;;;;; +FBC3;ARABIC LIGATURE JALLA WA-ALAA;So;0;ON;;;;;N;;;;; +FBC4;ARABIC LIGATURE DAAMAT BARAKAATUHUM;So;0;ON;;;;;N;;;;; +FBC5;ARABIC LIGATURE RAHMATU ALLAAHI TAAALAA ALAYH;So;0;ON;;;;;N;;;;; +FBC6;ARABIC LIGATURE RAHMATU ALLAAHI ALAYHIM;So;0;ON;;;;;N;;;;; +FBC7;ARABIC LIGATURE RAHMATU ALLAAHI ALAYHIMAA;So;0;ON;;;;;N;;;;; +FBC8;ARABIC LIGATURE RAHIMAHUM ALLAAHU TAAALAA;So;0;ON;;;;;N;;;;; +FBC9;ARABIC LIGATURE RAHIMAHUMAA ALLAAH;So;0;ON;;;;;N;;;;; +FBCA;ARABIC LIGATURE RAHIMAHUMAA ALLAAHU TAAALAA;So;0;ON;;;;;N;;;;; +FBCB;ARABIC LIGATURE RADI ALLAAHU TAAALAA ANHUM;So;0;ON;;;;;N;;;;; +FBCC;ARABIC LIGATURE HAFIZAHU ALLAAH;So;0;ON;;;;;N;;;;; +FBCD;ARABIC LIGATURE HAFIZAHU ALLAAHU TAAALAA;So;0;ON;;;;;N;;;;; +FBCE;ARABIC LIGATURE HAFIZAHUM ALLAAHU TAAALAA;So;0;ON;;;;;N;;;;; +FBCF;ARABIC LIGATURE HAFIZAHUMAA ALLAAHU TAAALAA;So;0;ON;;;;;N;;;;; +FBD0;ARABIC LIGATURE SALLALLAAHU TAAALAA ALAYHI WA-SALLAM;So;0;ON;;;;;N;;;;; +FBD1;ARABIC LIGATURE AJJAL ALLAAHU FARAJAHU ASH-SHAREEF;So;0;ON;;;;;N;;;;; +FBD2;ARABIC LIGATURE ALAYHI AR-RAHMAH;So;0;ON;;;;;N;;;;; FBD3;ARABIC LETTER NG ISOLATED FORM;Lo;0;AL; 06AD;;;;N;;;;; FBD4;ARABIC LETTER NG FINAL FORM;Lo;0;AL; 06AD;;;;N;;;;; FBD5;ARABIC LETTER NG INITIAL FORM;Lo;0;AL; 06AD;;;;N;;;;; @@ -16348,6 +16423,8 @@ FD8C;ARABIC LIGATURE MEEM WITH JEEM WITH HAH INITIAL FORM;Lo;0;AL; 0645 FD8D;ARABIC LIGATURE MEEM WITH JEEM WITH MEEM INITIAL FORM;Lo;0;AL; 0645 062C 0645;;;;N;;;;; FD8E;ARABIC LIGATURE MEEM WITH KHAH WITH JEEM INITIAL FORM;Lo;0;AL; 0645 062E 062C;;;;N;;;;; FD8F;ARABIC LIGATURE MEEM WITH KHAH WITH MEEM INITIAL FORM;Lo;0;AL; 0645 062E 0645;;;;N;;;;; +FD90;ARABIC LIGATURE RAHMATU ALLAAHI ALAYH;So;0;ON;;;;;N;;;;; +FD91;ARABIC LIGATURE RAHMATU ALLAAHI ALAYHAA;So;0;ON;;;;;N;;;;; FD92;ARABIC LIGATURE MEEM WITH JEEM WITH KHAH INITIAL FORM;Lo;0;AL; 0645 062C 062E;;;;N;;;;; FD93;ARABIC LIGATURE HEH WITH MEEM WITH JEEM INITIAL FORM;Lo;0;AL; 0647 0645 062C;;;;N;;;;; FD94;ARABIC LIGATURE HEH WITH MEEM WITH MEEM INITIAL FORM;Lo;0;AL; 0647 0645 0645;;;;N;;;;; @@ -16402,6 +16479,13 @@ FDC4;ARABIC LIGATURE AIN WITH JEEM WITH MEEM INITIAL FORM;Lo;0;AL; 0639 FDC5;ARABIC LIGATURE SAD WITH MEEM WITH MEEM INITIAL FORM;Lo;0;AL; 0635 0645 0645;;;;N;;;;; FDC6;ARABIC LIGATURE SEEN WITH KHAH WITH YEH FINAL FORM;Lo;0;AL; 0633 062E 064A;;;;N;;;;; FDC7;ARABIC LIGATURE NOON WITH JEEM WITH YEH FINAL FORM;Lo;0;AL; 0646 062C 064A;;;;N;;;;; +FDC8;ARABIC LIGATURE RAHIMAHU ALLAAH TAAALAA;So;0;ON;;;;;N;;;;; +FDC9;ARABIC LIGATURE RADI ALLAAHU TAAALAA ANH;So;0;ON;;;;;N;;;;; +FDCA;ARABIC LIGATURE RADI ALLAAHU TAAALAA ANHAA;So;0;ON;;;;;N;;;;; +FDCB;ARABIC LIGATURE RADI ALLAAHU TAAALAA ANHUMAA;So;0;ON;;;;;N;;;;; +FDCC;ARABIC LIGATURE SALLALLAHU ALAYHI WA-ALAA AALIHEE WA-SALLAM;So;0;ON;;;;;N;;;;; +FDCD;ARABIC LIGATURE AJJAL ALLAAHU TAAALAA FARAJAHU ASH-SHAREEF;So;0;ON;;;;;N;;;;; +FDCE;ARABIC LIGATURE KARRAMA ALLAAHU WAJHAH;So;0;ON;;;;;N;;;;; FDCF;ARABIC LIGATURE SALAAMUHU ALAYNAA;So;0;ON;;;;;N;;;;; FDF0;ARABIC LIGATURE SALLA USED AS KORANIC STOP SIGN ISOLATED FORM;Lo;0;AL; 0635 0644 06D2;;;;N;;;;; FDF1;ARABIC LIGATURE QALA USED AS KORANIC STOP SIGN ISOLATED FORM;Lo;0;AL; 0642 0644 06D2;;;;N;;;;; @@ -17996,6 +18080,58 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 105B9;VITHKUQI SMALL LETTER XE;Ll;0;L;;;;;N;;;10592;;10592 105BB;VITHKUQI SMALL LETTER Y;Ll;0;L;;;;;N;;;10594;;10594 105BC;VITHKUQI SMALL LETTER ZE;Ll;0;L;;;;;N;;;10595;;10595 +105C0;TODHRI LETTER A;Lo;0;L;;;;;N;;;;; +105C1;TODHRI LETTER AS;Lo;0;L;;;;;N;;;;; +105C2;TODHRI LETTER BA;Lo;0;L;;;;;N;;;;; +105C3;TODHRI LETTER MBA;Lo;0;L;;;;;N;;;;; +105C4;TODHRI LETTER CA;Lo;0;L;;;;;N;;;;; +105C5;TODHRI LETTER CHA;Lo;0;L;;;;;N;;;;; +105C6;TODHRI LETTER DA;Lo;0;L;;;;;N;;;;; +105C7;TODHRI LETTER NDA;Lo;0;L;;;;;N;;;;; +105C8;TODHRI LETTER DHA;Lo;0;L;;;;;N;;;;; +105C9;TODHRI LETTER EI;Lo;0;L;105D2 0307;;;;N;;;;; +105CA;TODHRI LETTER E;Lo;0;L;;;;;N;;;;; +105CB;TODHRI LETTER FA;Lo;0;L;;;;;N;;;;; +105CC;TODHRI LETTER GA;Lo;0;L;;;;;N;;;;; +105CD;TODHRI LETTER NGA;Lo;0;L;;;;;N;;;;; +105CE;TODHRI LETTER GJA;Lo;0;L;;;;;N;;;;; +105CF;TODHRI LETTER NGJA;Lo;0;L;;;;;N;;;;; +105D0;TODHRI LETTER HA;Lo;0;L;;;;;N;;;;; +105D1;TODHRI LETTER HJA;Lo;0;L;;;;;N;;;;; +105D2;TODHRI LETTER I;Lo;0;L;;;;;N;;;;; +105D3;TODHRI LETTER JA;Lo;0;L;;;;;N;;;;; +105D4;TODHRI LETTER KA;Lo;0;L;;;;;N;;;;; +105D5;TODHRI LETTER LA;Lo;0;L;;;;;N;;;;; +105D6;TODHRI LETTER LLA;Lo;0;L;;;;;N;;;;; +105D7;TODHRI LETTER MA;Lo;0;L;;;;;N;;;;; +105D8;TODHRI LETTER NA;Lo;0;L;;;;;N;;;;; +105D9;TODHRI LETTER NJAN;Lo;0;L;;;;;N;;;;; +105DA;TODHRI LETTER O;Lo;0;L;;;;;N;;;;; +105DB;TODHRI LETTER PA;Lo;0;L;;;;;N;;;;; +105DC;TODHRI LETTER QA;Lo;0;L;;;;;N;;;;; +105DD;TODHRI LETTER RA;Lo;0;L;;;;;N;;;;; +105DE;TODHRI LETTER RRA;Lo;0;L;;;;;N;;;;; +105DF;TODHRI LETTER SA;Lo;0;L;;;;;N;;;;; +105E0;TODHRI LETTER SHA;Lo;0;L;;;;;N;;;;; +105E1;TODHRI LETTER SHTA;Lo;0;L;;;;;N;;;;; +105E2;TODHRI LETTER TA;Lo;0;L;;;;;N;;;;; +105E3;TODHRI LETTER THA;Lo;0;L;;;;;N;;;;; +105E4;TODHRI LETTER U;Lo;0;L;105DA 0307;;;;N;;;;; +105E5;TODHRI LETTER VA;Lo;0;L;;;;;N;;;;; +105E6;TODHRI LETTER XA;Lo;0;L;;;;;N;;;;; +105E7;TODHRI LETTER NXA;Lo;0;L;;;;;N;;;;; +105E8;TODHRI LETTER XHA;Lo;0;L;;;;;N;;;;; +105E9;TODHRI LETTER NXHA;Lo;0;L;;;;;N;;;;; +105EA;TODHRI LETTER Y;Lo;0;L;;;;;N;;;;; +105EB;TODHRI LETTER JY;Lo;0;L;;;;;N;;;;; +105EC;TODHRI LETTER ZA;Lo;0;L;;;;;N;;;;; +105ED;TODHRI LETTER ZHA;Lo;0;L;;;;;N;;;;; +105EE;TODHRI LETTER GHA;Lo;0;L;;;;;N;;;;; +105EF;TODHRI LETTER STA;Lo;0;L;;;;;N;;;;; +105F0;TODHRI LETTER SKAN;Lo;0;L;;;;;N;;;;; +105F1;TODHRI LETTER KHA;Lo;0;L;;;;;N;;;;; +105F2;TODHRI LETTER PSA;Lo;0;L;;;;;N;;;;; +105F3;TODHRI LETTER OO;Lo;0;L;;;;;N;;;;; 10600;LINEAR A SIGN AB001;Lo;0;L;;;;;N;;;;; 10601;LINEAR A SIGN AB002;Lo;0;L;;;;;N;;;;; 10602;LINEAR A SIGN AB003;Lo;0;L;;;;;N;;;;; @@ -18634,6 +18770,32 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 10938;LYDIAN LETTER NN;Lo;0;R;;;;;N;;;;; 10939;LYDIAN LETTER C;Lo;0;R;;;;;N;;;;; 1093F;LYDIAN TRIANGULAR MARK;Po;0;R;;;;;N;;;;; +10940;SIDETIC LETTER N01;Lo;0;R;;;;;N;;;;; +10941;SIDETIC LETTER N02;Lo;0;R;;;;;N;;;;; +10942;SIDETIC LETTER N03;Lo;0;R;;;;;N;;;;; +10943;SIDETIC LETTER N04;Lo;0;R;;;;;N;;;;; +10944;SIDETIC LETTER N05;Lo;0;R;;;;;N;;;;; +10945;SIDETIC LETTER N06;Lo;0;R;;;;;N;;;;; +10946;SIDETIC LETTER N07;Lo;0;R;;;;;N;;;;; +10947;SIDETIC LETTER N08;Lo;0;R;;;;;N;;;;; +10948;SIDETIC LETTER N09;Lo;0;R;;;;;N;;;;; +10949;SIDETIC LETTER N10;Lo;0;R;;;;;N;;;;; +1094A;SIDETIC LETTER N11;Lo;0;R;;;;;N;;;;; +1094B;SIDETIC LETTER N12;Lo;0;R;;;;;N;;;;; +1094C;SIDETIC LETTER N13;Lo;0;R;;;;;N;;;;; +1094D;SIDETIC LETTER N14;Lo;0;R;;;;;N;;;;; +1094E;SIDETIC LETTER N15;Lo;0;R;;;;;N;;;;; +1094F;SIDETIC LETTER N16;Lo;0;R;;;;;N;;;;; +10950;SIDETIC LETTER N17;Lo;0;R;;;;;N;;;;; +10951;SIDETIC LETTER N18;Lo;0;R;;;;;N;;;;; +10952;SIDETIC LETTER N19;Lo;0;R;;;;;N;;;;; +10953;SIDETIC LETTER N20;Lo;0;R;;;;;N;;;;; +10954;SIDETIC LETTER N21;Lo;0;R;;;;;N;;;;; +10955;SIDETIC LETTER N22;Lo;0;R;;;;;N;;;;; +10956;SIDETIC LETTER N23;Lo;0;R;;;;;N;;;;; +10957;SIDETIC LETTER N24;Lo;0;R;;;;;N;;;;; +10958;SIDETIC LETTER N25;Lo;0;R;;;;;N;;;;; +10959;SIDETIC LETTER N26;Lo;0;R;;;;;N;;;;; 10980;MEROITIC HIEROGLYPHIC LETTER A;Lo;0;R;;;;;N;;;;; 10981;MEROITIC HIEROGLYPHIC LETTER E;Lo;0;R;;;;;N;;;;; 10982;MEROITIC HIEROGLYPHIC LETTER I;Lo;0;R;;;;;N;;;;; @@ -19317,6 +19479,75 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 10D37;HANIFI ROHINGYA DIGIT SEVEN;Nd;0;AN;;7;7;7;N;;;;; 10D38;HANIFI ROHINGYA DIGIT EIGHT;Nd;0;AN;;8;8;8;N;;;;; 10D39;HANIFI ROHINGYA DIGIT NINE;Nd;0;AN;;9;9;9;N;;;;; +10D40;GARAY DIGIT ZERO;Nd;0;AN;;0;0;0;N;;;;; +10D41;GARAY DIGIT ONE;Nd;0;AN;;1;1;1;N;;;;; +10D42;GARAY DIGIT TWO;Nd;0;AN;;2;2;2;N;;;;; +10D43;GARAY DIGIT THREE;Nd;0;AN;;3;3;3;N;;;;; +10D44;GARAY DIGIT FOUR;Nd;0;AN;;4;4;4;N;;;;; +10D45;GARAY DIGIT FIVE;Nd;0;AN;;5;5;5;N;;;;; +10D46;GARAY DIGIT SIX;Nd;0;AN;;6;6;6;N;;;;; +10D47;GARAY DIGIT SEVEN;Nd;0;AN;;7;7;7;N;;;;; +10D48;GARAY DIGIT EIGHT;Nd;0;AN;;8;8;8;N;;;;; +10D49;GARAY DIGIT NINE;Nd;0;AN;;9;9;9;N;;;;; +10D4A;GARAY VOWEL SIGN A;Lo;0;R;;;;;N;;;;; +10D4B;GARAY VOWEL SIGN I;Lo;0;R;;;;;N;;;;; +10D4C;GARAY VOWEL SIGN O;Lo;0;R;;;;;N;;;;; +10D4D;GARAY VOWEL SIGN EE;Lo;0;R;;;;;N;;;;; +10D4E;GARAY VOWEL LENGTH MARK;Lm;0;R;;;;;N;;;;; +10D4F;GARAY SUKUN;Lo;0;R;;;;;N;;;;; +10D50;GARAY CAPITAL LETTER A;Lu;0;R;;;;;N;;;;10D70; +10D51;GARAY CAPITAL LETTER CA;Lu;0;R;;;;;N;;;;10D71; +10D52;GARAY CAPITAL LETTER MA;Lu;0;R;;;;;N;;;;10D72; +10D53;GARAY CAPITAL LETTER KA;Lu;0;R;;;;;N;;;;10D73; +10D54;GARAY CAPITAL LETTER BA;Lu;0;R;;;;;N;;;;10D74; +10D55;GARAY CAPITAL LETTER JA;Lu;0;R;;;;;N;;;;10D75; +10D56;GARAY CAPITAL LETTER SA;Lu;0;R;;;;;N;;;;10D76; +10D57;GARAY CAPITAL LETTER WA;Lu;0;R;;;;;N;;;;10D77; +10D58;GARAY CAPITAL LETTER LA;Lu;0;R;;;;;N;;;;10D78; +10D59;GARAY CAPITAL LETTER GA;Lu;0;R;;;;;N;;;;10D79; +10D5A;GARAY CAPITAL LETTER DA;Lu;0;R;;;;;N;;;;10D7A; +10D5B;GARAY CAPITAL LETTER XA;Lu;0;R;;;;;N;;;;10D7B; +10D5C;GARAY CAPITAL LETTER YA;Lu;0;R;;;;;N;;;;10D7C; +10D5D;GARAY CAPITAL LETTER TA;Lu;0;R;;;;;N;;;;10D7D; +10D5E;GARAY CAPITAL LETTER RA;Lu;0;R;;;;;N;;;;10D7E; +10D5F;GARAY CAPITAL LETTER NYA;Lu;0;R;;;;;N;;;;10D7F; +10D60;GARAY CAPITAL LETTER FA;Lu;0;R;;;;;N;;;;10D80; +10D61;GARAY CAPITAL LETTER NA;Lu;0;R;;;;;N;;;;10D81; +10D62;GARAY CAPITAL LETTER PA;Lu;0;R;;;;;N;;;;10D82; +10D63;GARAY CAPITAL LETTER HA;Lu;0;R;;;;;N;;;;10D83; +10D64;GARAY CAPITAL LETTER OLD KA;Lu;0;R;;;;;N;;;;10D84; +10D65;GARAY CAPITAL LETTER OLD NA;Lu;0;R;;;;;N;;;;10D85; +10D69;GARAY VOWEL SIGN E;Mn;230;NSM;;;;;N;;;;; +10D6A;GARAY CONSONANT GEMINATION MARK;Mn;230;NSM;;;;;N;;;;; +10D6B;GARAY COMBINING DOT ABOVE;Mn;230;NSM;;;;;N;;;;; +10D6C;GARAY COMBINING DOUBLE DOT ABOVE;Mn;230;NSM;;;;;N;;;;; +10D6D;GARAY CONSONANT NASALIZATION MARK;Mn;230;NSM;;;;;N;;;;; +10D6E;GARAY HYPHEN;Pd;0;ON;;;;;N;;;;; +10D6F;GARAY REDUPLICATION MARK;Lm;0;R;;;;;N;;;;; +10D70;GARAY SMALL LETTER A;Ll;0;R;;;;;N;;;10D50;;10D50 +10D71;GARAY SMALL LETTER CA;Ll;0;R;;;;;N;;;10D51;;10D51 +10D72;GARAY SMALL LETTER MA;Ll;0;R;;;;;N;;;10D52;;10D52 +10D73;GARAY SMALL LETTER KA;Ll;0;R;;;;;N;;;10D53;;10D53 +10D74;GARAY SMALL LETTER BA;Ll;0;R;;;;;N;;;10D54;;10D54 +10D75;GARAY SMALL LETTER JA;Ll;0;R;;;;;N;;;10D55;;10D55 +10D76;GARAY SMALL LETTER SA;Ll;0;R;;;;;N;;;10D56;;10D56 +10D77;GARAY SMALL LETTER WA;Ll;0;R;;;;;N;;;10D57;;10D57 +10D78;GARAY SMALL LETTER LA;Ll;0;R;;;;;N;;;10D58;;10D58 +10D79;GARAY SMALL LETTER GA;Ll;0;R;;;;;N;;;10D59;;10D59 +10D7A;GARAY SMALL LETTER DA;Ll;0;R;;;;;N;;;10D5A;;10D5A +10D7B;GARAY SMALL LETTER XA;Ll;0;R;;;;;N;;;10D5B;;10D5B +10D7C;GARAY SMALL LETTER YA;Ll;0;R;;;;;N;;;10D5C;;10D5C +10D7D;GARAY SMALL LETTER TA;Ll;0;R;;;;;N;;;10D5D;;10D5D +10D7E;GARAY SMALL LETTER RA;Ll;0;R;;;;;N;;;10D5E;;10D5E +10D7F;GARAY SMALL LETTER NYA;Ll;0;R;;;;;N;;;10D5F;;10D5F +10D80;GARAY SMALL LETTER FA;Ll;0;R;;;;;N;;;10D60;;10D60 +10D81;GARAY SMALL LETTER NA;Ll;0;R;;;;;N;;;10D61;;10D61 +10D82;GARAY SMALL LETTER PA;Ll;0;R;;;;;N;;;10D62;;10D62 +10D83;GARAY SMALL LETTER HA;Ll;0;R;;;;;N;;;10D63;;10D63 +10D84;GARAY SMALL LETTER OLD KA;Ll;0;R;;;;;N;;;10D64;;10D64 +10D85;GARAY SMALL LETTER OLD NA;Ll;0;R;;;;;N;;;10D65;;10D65 +10D8E;GARAY PLUS SIGN;Sm;0;R;;;;;N;;;;; +10D8F;GARAY MINUS SIGN;Sm;0;R;;;;;N;;;;; 10E60;RUMI DIGIT ONE;No;0;AN;;;1;1;N;;;;; 10E61;RUMI DIGIT TWO;No;0;AN;;;2;2;N;;;;; 10E62;RUMI DIGIT THREE;No;0;AN;;;3;3;N;;;;; @@ -19395,6 +19626,24 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 10EAD;YEZIDI HYPHENATION MARK;Pd;0;R;;;;;N;;;;; 10EB0;YEZIDI LETTER LAM WITH DOT ABOVE;Lo;0;R;;;;;N;;;;; 10EB1;YEZIDI LETTER YOT WITH CIRCUMFLEX ABOVE;Lo;0;R;;;;;N;;;;; +10EC2;ARABIC LETTER DAL WITH TWO DOTS VERTICALLY BELOW;Lo;0;AL;;;;;N;;;;; +10EC3;ARABIC LETTER TAH WITH TWO DOTS VERTICALLY BELOW;Lo;0;AL;;;;;N;;;;; +10EC4;ARABIC LETTER KAF WITH TWO DOTS VERTICALLY BELOW;Lo;0;AL;;;;;N;;;;; +10EC5;ARABIC SMALL YEH BARREE WITH TWO DOTS BELOW;Lm;0;AL;;;;;N;;;;; +10EC6;ARABIC LETTER THIN NOON;Lo;0;AL;;;;;N;;;;; +10EC7;ARABIC LETTER YEH WITH FOUR DOTS BELOW;Lo;0;AL;;;;;N;;;;; +10ED0;ARABIC BIBLICAL END OF VERSE;Po;0;ON;;;;;N;;;;; +10ED1;ARABIC LIGATURE ALAYHAA AS-SALAATU WAS-SALAAM;So;0;ON;;;;;N;;;;; +10ED2;ARABIC LIGATURE ALAYHIM AS-SALAATU WAS-SALAAM;So;0;ON;;;;;N;;;;; +10ED3;ARABIC LIGATURE ALAYHIMAA AS-SALAATU WAS-SALAAM;So;0;ON;;;;;N;;;;; +10ED4;ARABIC LIGATURE QADDASA ALLAAHU SIRRAH;So;0;ON;;;;;N;;;;; +10ED5;ARABIC LIGATURE QUDDISA SIRRUHUM;So;0;ON;;;;;N;;;;; +10ED6;ARABIC LIGATURE QUDDISA SIRRUHUMAA;So;0;ON;;;;;N;;;;; +10ED7;ARABIC LIGATURE QUDDISAT ASRAARUHUM;So;0;ON;;;;;N;;;;; +10ED8;ARABIC LIGATURE NAWWARA ALLAAHU MARQADAH;So;0;ON;;;;;N;;;;; +10EFA;ARABIC DOUBLE VERTICAL BAR BELOW;Mn;220;NSM;;;;;N;;;;; +10EFB;ARABIC SMALL LOW NOON;Mn;220;NSM;;;;;N;;;;; +10EFC;ARABIC COMBINING ALEF OVERLAY;Mn;0;NSM;;;;;N;;;;; 10EFD;ARABIC SMALL LOW WORD SAKTA;Mn;220;NSM;;;;;N;;;;; 10EFE;ARABIC SMALL LOW WORD QASR;Mn;220;NSM;;;;;N;;;;; 10EFF;ARABIC SMALL LOW WORD MADDA;Mn;220;NSM;;;;;N;;;;; @@ -20259,6 +20508,86 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 11372;COMBINING GRANTHA LETTER NA;Mn;230;NSM;;;;;N;;;;; 11373;COMBINING GRANTHA LETTER VI;Mn;230;NSM;;;;;N;;;;; 11374;COMBINING GRANTHA LETTER PA;Mn;230;NSM;;;;;N;;;;; +11380;TULU-TIGALARI LETTER A;Lo;0;L;;;;;N;;;;; +11381;TULU-TIGALARI LETTER AA;Lo;0;L;;;;;N;;;;; +11382;TULU-TIGALARI LETTER I;Lo;0;L;;;;;N;;;;; +11383;TULU-TIGALARI LETTER II;Lo;0;L;11382 113C9;;;;N;;;;; +11384;TULU-TIGALARI LETTER U;Lo;0;L;;;;;N;;;;; +11385;TULU-TIGALARI LETTER UU;Lo;0;L;11384 113BB;;;;N;;;;; +11386;TULU-TIGALARI LETTER VOCALIC R;Lo;0;L;;;;;N;;;;; +11387;TULU-TIGALARI LETTER VOCALIC RR;Lo;0;L;;;;;N;;;;; +11388;TULU-TIGALARI LETTER VOCALIC L;Lo;0;L;;;;;N;;;;; +11389;TULU-TIGALARI LETTER VOCALIC LL;Lo;0;L;;;;;N;;;;; +1138B;TULU-TIGALARI LETTER EE;Lo;0;L;;;;;N;;;;; +1138E;TULU-TIGALARI LETTER AI;Lo;0;L;1138B 113C2;;;;N;;;;; +11390;TULU-TIGALARI LETTER OO;Lo;0;L;;;;;N;;;;; +11391;TULU-TIGALARI LETTER AU;Lo;0;L;11390 113C9;;;;N;;;;; +11392;TULU-TIGALARI LETTER KA;Lo;0;L;;;;;N;;;;; +11393;TULU-TIGALARI LETTER KHA;Lo;0;L;;;;;N;;;;; +11394;TULU-TIGALARI LETTER GA;Lo;0;L;;;;;N;;;;; +11395;TULU-TIGALARI LETTER GHA;Lo;0;L;;;;;N;;;;; +11396;TULU-TIGALARI LETTER NGA;Lo;0;L;;;;;N;;;;; +11397;TULU-TIGALARI LETTER CA;Lo;0;L;;;;;N;;;;; +11398;TULU-TIGALARI LETTER CHA;Lo;0;L;;;;;N;;;;; +11399;TULU-TIGALARI LETTER JA;Lo;0;L;;;;;N;;;;; +1139A;TULU-TIGALARI LETTER JHA;Lo;0;L;;;;;N;;;;; +1139B;TULU-TIGALARI LETTER NYA;Lo;0;L;;;;;N;;;;; +1139C;TULU-TIGALARI LETTER TTA;Lo;0;L;;;;;N;;;;; +1139D;TULU-TIGALARI LETTER TTHA;Lo;0;L;;;;;N;;;;; +1139E;TULU-TIGALARI LETTER DDA;Lo;0;L;;;;;N;;;;; +1139F;TULU-TIGALARI LETTER DDHA;Lo;0;L;;;;;N;;;;; +113A0;TULU-TIGALARI LETTER NNA;Lo;0;L;;;;;N;;;;; +113A1;TULU-TIGALARI LETTER TA;Lo;0;L;;;;;N;;;;; +113A2;TULU-TIGALARI LETTER THA;Lo;0;L;;;;;N;;;;; +113A3;TULU-TIGALARI LETTER DA;Lo;0;L;;;;;N;;;;; +113A4;TULU-TIGALARI LETTER DHA;Lo;0;L;;;;;N;;;;; +113A5;TULU-TIGALARI LETTER NA;Lo;0;L;;;;;N;;;;; +113A6;TULU-TIGALARI LETTER PA;Lo;0;L;;;;;N;;;;; +113A7;TULU-TIGALARI LETTER PHA;Lo;0;L;;;;;N;;;;; +113A8;TULU-TIGALARI LETTER BA;Lo;0;L;;;;;N;;;;; +113A9;TULU-TIGALARI LETTER BHA;Lo;0;L;;;;;N;;;;; +113AA;TULU-TIGALARI LETTER MA;Lo;0;L;;;;;N;;;;; +113AB;TULU-TIGALARI LETTER YA;Lo;0;L;;;;;N;;;;; +113AC;TULU-TIGALARI LETTER RA;Lo;0;L;;;;;N;;;;; +113AD;TULU-TIGALARI LETTER LA;Lo;0;L;;;;;N;;;;; +113AE;TULU-TIGALARI LETTER VA;Lo;0;L;;;;;N;;;;; +113AF;TULU-TIGALARI LETTER SHA;Lo;0;L;;;;;N;;;;; +113B0;TULU-TIGALARI LETTER SSA;Lo;0;L;;;;;N;;;;; +113B1;TULU-TIGALARI LETTER SA;Lo;0;L;;;;;N;;;;; +113B2;TULU-TIGALARI LETTER HA;Lo;0;L;;;;;N;;;;; +113B3;TULU-TIGALARI LETTER LLA;Lo;0;L;;;;;N;;;;; +113B4;TULU-TIGALARI LETTER RRA;Lo;0;L;;;;;N;;;;; +113B5;TULU-TIGALARI LETTER LLLA;Lo;0;L;;;;;N;;;;; +113B7;TULU-TIGALARI SIGN AVAGRAHA;Lo;0;L;;;;;N;;;;; +113B8;TULU-TIGALARI VOWEL SIGN AA;Mc;0;L;;;;;N;;;;; +113B9;TULU-TIGALARI VOWEL SIGN I;Mc;0;L;;;;;N;;;;; +113BA;TULU-TIGALARI VOWEL SIGN II;Mc;0;L;;;;;N;;;;; +113BB;TULU-TIGALARI VOWEL SIGN U;Mn;0;NSM;;;;;N;;;;; +113BC;TULU-TIGALARI VOWEL SIGN UU;Mn;0;NSM;;;;;N;;;;; +113BD;TULU-TIGALARI VOWEL SIGN VOCALIC R;Mn;0;NSM;;;;;N;;;;; +113BE;TULU-TIGALARI VOWEL SIGN VOCALIC RR;Mn;0;NSM;;;;;N;;;;; +113BF;TULU-TIGALARI VOWEL SIGN VOCALIC L;Mn;0;NSM;;;;;N;;;;; +113C0;TULU-TIGALARI VOWEL SIGN VOCALIC LL;Mn;0;NSM;;;;;N;;;;; +113C2;TULU-TIGALARI VOWEL SIGN EE;Mc;0;L;;;;;N;;;;; +113C5;TULU-TIGALARI VOWEL SIGN AI;Mc;0;L;113C2 113C2;;;;N;;;;; +113C7;TULU-TIGALARI VOWEL SIGN OO;Mc;0;L;113C2 113B8;;;;N;;;;; +113C8;TULU-TIGALARI VOWEL SIGN AU;Mc;0;L;113C2 113C9;;;;N;;;;; +113C9;TULU-TIGALARI AU LENGTH MARK;Mc;0;L;;;;;N;;;;; +113CA;TULU-TIGALARI SIGN CANDRA ANUNASIKA;Mc;0;L;;;;;N;;;;; +113CC;TULU-TIGALARI SIGN ANUSVARA;Mc;0;L;;;;;N;;;;; +113CD;TULU-TIGALARI SIGN VISARGA;Mc;0;L;;;;;N;;;;; +113CE;TULU-TIGALARI SIGN VIRAMA;Mn;9;NSM;;;;;N;;;;; +113CF;TULU-TIGALARI SIGN LOOPED VIRAMA;Mc;9;L;;;;;N;;;;; +113D0;TULU-TIGALARI CONJOINER;Mn;9;NSM;;;;;N;;;;; +113D1;TULU-TIGALARI REPHA;Lo;0;L;;;;;N;;;;; +113D2;TULU-TIGALARI GEMINATION MARK;Mn;0;NSM;;;;;N;;;;; +113D3;TULU-TIGALARI SIGN PLUTA;Lo;0;L;;;;;N;;;;; +113D4;TULU-TIGALARI DANDA;Po;0;L;;;;;N;;;;; +113D5;TULU-TIGALARI DOUBLE DANDA;Po;0;L;;;;;N;;;;; +113D7;TULU-TIGALARI SIGN OM PUSHPIKA;Po;0;L;;;;;N;;;;; +113D8;TULU-TIGALARI SIGN SHRII PUSHPIKA;Po;0;L;;;;;N;;;;; +113E1;TULU-TIGALARI VEDIC TONE SVARITA;Mn;0;NSM;;;;;N;;;;; +113E2;TULU-TIGALARI VEDIC TONE ANUDATTA;Mn;0;NSM;;;;;N;;;;; 11400;NEWA LETTER A;Lo;0;L;;;;;N;;;;; 11401;NEWA LETTER AA;Lo;0;L;;;;;N;;;;; 11402;NEWA LETTER I;Lo;0;L;;;;;N;;;;; @@ -20690,6 +21019,26 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 116C7;TAKRI DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; 116C8;TAKRI DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; 116C9;TAKRI DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; +116D0;MYANMAR PAO DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +116D1;MYANMAR PAO DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +116D2;MYANMAR PAO DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +116D3;MYANMAR PAO DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +116D4;MYANMAR PAO DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +116D5;MYANMAR PAO DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +116D6;MYANMAR PAO DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +116D7;MYANMAR PAO DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +116D8;MYANMAR PAO DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +116D9;MYANMAR PAO DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; +116DA;MYANMAR EASTERN PWO KAREN DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +116DB;MYANMAR EASTERN PWO KAREN DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +116DC;MYANMAR EASTERN PWO KAREN DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +116DD;MYANMAR EASTERN PWO KAREN DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +116DE;MYANMAR EASTERN PWO KAREN DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +116DF;MYANMAR EASTERN PWO KAREN DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +116E0;MYANMAR EASTERN PWO KAREN DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +116E1;MYANMAR EASTERN PWO KAREN DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +116E2;MYANMAR EASTERN PWO KAREN DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +116E3;MYANMAR EASTERN PWO KAREN DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; 11700;AHOM LETTER KA;Lo;0;L;;;;;N;;;;; 11701;AHOM LETTER KHA;Lo;0;L;;;;;N;;;;; 11702;AHOM LETTER NGA;Lo;0;L;;;;;N;;;;; @@ -20718,7 +21067,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 11719;AHOM LETTER JHA;Lo;0;L;;;;;N;;;;; 1171A;AHOM LETTER ALTERNATE BA;Lo;0;L;;;;;N;;;;; 1171D;AHOM CONSONANT SIGN MEDIAL LA;Mn;0;NSM;;;;;N;;;;; -1171E;AHOM CONSONANT SIGN MEDIAL RA;Mn;0;NSM;;;;;N;;;;; +1171E;AHOM CONSONANT SIGN MEDIAL RA;Mc;0;L;;;;;N;;;;; 1171F;AHOM CONSONANT SIGN MEDIAL LIGATING RA;Mn;0;NSM;;;;;N;;;;; 11720;AHOM VOWEL SIGN A;Mc;0;L;;;;;N;;;;; 11721;AHOM VOWEL SIGN AA;Mc;0;L;;;;;N;;;;; @@ -21274,6 +21623,58 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 11B07;DEVANAGARI SIGN WESTERN NINE-LIKE BHALE;Po;0;L;;;;;N;;;;; 11B08;DEVANAGARI SIGN REVERSED NINE-LIKE BHALE;Po;0;L;;;;;N;;;;; 11B09;DEVANAGARI SIGN MINDU;Po;0;L;;;;;N;;;;; +11B60;SHARADA VOWEL SIGN OE;Mn;0;NSM;;;;;N;;;;; +11B61;SHARADA VOWEL SIGN OOE;Mc;0;L;;;;;N;;;;; +11B62;SHARADA VOWEL SIGN UE;Mn;0;NSM;;;;;N;;;;; +11B63;SHARADA VOWEL SIGN UUE;Mn;0;NSM;;;;;N;;;;; +11B64;SHARADA VOWEL SIGN SHORT E;Mn;0;NSM;;;;;N;;;;; +11B65;SHARADA VOWEL SIGN SHORT O;Mc;0;L;;;;;N;;;;; +11B66;SHARADA VOWEL SIGN CANDRA E;Mn;0;NSM;;;;;N;;;;; +11B67;SHARADA VOWEL SIGN CANDRA O;Mc;0;L;;;;;N;;;;; +11BC0;SUNUWAR LETTER DEVI;Lo;0;L;;;;;N;;;;; +11BC1;SUNUWAR LETTER TASLA;Lo;0;L;;;;;N;;;;; +11BC2;SUNUWAR LETTER EKO;Lo;0;L;;;;;N;;;;; +11BC3;SUNUWAR LETTER IMAR;Lo;0;L;;;;;N;;;;; +11BC4;SUNUWAR LETTER REU;Lo;0;L;;;;;N;;;;; +11BC5;SUNUWAR LETTER UTTHI;Lo;0;L;;;;;N;;;;; +11BC6;SUNUWAR LETTER KIK;Lo;0;L;;;;;N;;;;; +11BC7;SUNUWAR LETTER MA;Lo;0;L;;;;;N;;;;; +11BC8;SUNUWAR LETTER APPHO;Lo;0;L;;;;;N;;;;; +11BC9;SUNUWAR LETTER PIP;Lo;0;L;;;;;N;;;;; +11BCA;SUNUWAR LETTER GIL;Lo;0;L;;;;;N;;;;; +11BCB;SUNUWAR LETTER HAMSO;Lo;0;L;;;;;N;;;;; +11BCC;SUNUWAR LETTER CARMI;Lo;0;L;;;;;N;;;;; +11BCD;SUNUWAR LETTER NAH;Lo;0;L;;;;;N;;;;; +11BCE;SUNUWAR LETTER BUR;Lo;0;L;;;;;N;;;;; +11BCF;SUNUWAR LETTER JYAH;Lo;0;L;;;;;N;;;;; +11BD0;SUNUWAR LETTER LOACHA;Lo;0;L;;;;;N;;;;; +11BD1;SUNUWAR LETTER OTTHI;Lo;0;L;;;;;N;;;;; +11BD2;SUNUWAR LETTER SHYELE;Lo;0;L;;;;;N;;;;; +11BD3;SUNUWAR LETTER VARCA;Lo;0;L;;;;;N;;;;; +11BD4;SUNUWAR LETTER YAT;Lo;0;L;;;;;N;;;;; +11BD5;SUNUWAR LETTER AVA;Lo;0;L;;;;;N;;;;; +11BD6;SUNUWAR LETTER AAL;Lo;0;L;;;;;N;;;;; +11BD7;SUNUWAR LETTER DONGA;Lo;0;L;;;;;N;;;;; +11BD8;SUNUWAR LETTER THARI;Lo;0;L;;;;;N;;;;; +11BD9;SUNUWAR LETTER PHAR;Lo;0;L;;;;;N;;;;; +11BDA;SUNUWAR LETTER NGAR;Lo;0;L;;;;;N;;;;; +11BDB;SUNUWAR LETTER KHA;Lo;0;L;;;;;N;;;;; +11BDC;SUNUWAR LETTER SHYER;Lo;0;L;;;;;N;;;;; +11BDD;SUNUWAR LETTER CHELAP;Lo;0;L;;;;;N;;;;; +11BDE;SUNUWAR LETTER TENTU;Lo;0;L;;;;;N;;;;; +11BDF;SUNUWAR LETTER THELE;Lo;0;L;;;;;N;;;;; +11BE0;SUNUWAR LETTER KLOKO;Lo;0;L;;;;;N;;;;; +11BE1;SUNUWAR SIGN PVO;Po;0;L;;;;;N;;;;; +11BF0;SUNUWAR DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +11BF1;SUNUWAR DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +11BF2;SUNUWAR DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +11BF3;SUNUWAR DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +11BF4;SUNUWAR DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +11BF5;SUNUWAR DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +11BF6;SUNUWAR DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +11BF7;SUNUWAR DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +11BF8;SUNUWAR DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +11BF9;SUNUWAR DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; 11C00;BHAIKSUKI LETTER A;Lo;0;L;;;;;N;;;;; 11C01;BHAIKSUKI LETTER AA;Lo;0;L;;;;;N;;;;; 11C02;BHAIKSUKI LETTER I;Lo;0;L;;;;;N;;;;; @@ -21577,6 +21978,60 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 11DA7;GUNJALA GONDI DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; 11DA8;GUNJALA GONDI DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; 11DA9;GUNJALA GONDI DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; +11DB0;TOLONG SIKI LETTER I;Lo;0;L;;;;;N;;;;; +11DB1;TOLONG SIKI LETTER E;Lo;0;L;;;;;N;;;;; +11DB2;TOLONG SIKI LETTER U;Lo;0;L;;;;;N;;;;; +11DB3;TOLONG SIKI LETTER O;Lo;0;L;;;;;N;;;;; +11DB4;TOLONG SIKI LETTER A;Lo;0;L;;;;;N;;;;; +11DB5;TOLONG SIKI LETTER AA;Lo;0;L;;;;;N;;;;; +11DB6;TOLONG SIKI LETTER P;Lo;0;L;;;;;N;;;;; +11DB7;TOLONG SIKI LETTER PH;Lo;0;L;;;;;N;;;;; +11DB8;TOLONG SIKI LETTER B;Lo;0;L;;;;;N;;;;; +11DB9;TOLONG SIKI LETTER BH;Lo;0;L;;;;;N;;;;; +11DBA;TOLONG SIKI LETTER M;Lo;0;L;;;;;N;;;;; +11DBB;TOLONG SIKI LETTER T;Lo;0;L;;;;;N;;;;; +11DBC;TOLONG SIKI LETTER TH;Lo;0;L;;;;;N;;;;; +11DBD;TOLONG SIKI LETTER D;Lo;0;L;;;;;N;;;;; +11DBE;TOLONG SIKI LETTER DH;Lo;0;L;;;;;N;;;;; +11DBF;TOLONG SIKI LETTER N;Lo;0;L;;;;;N;;;;; +11DC0;TOLONG SIKI LETTER TT;Lo;0;L;;;;;N;;;;; +11DC1;TOLONG SIKI LETTER TTH;Lo;0;L;;;;;N;;;;; +11DC2;TOLONG SIKI LETTER DD;Lo;0;L;;;;;N;;;;; +11DC3;TOLONG SIKI LETTER DDH;Lo;0;L;;;;;N;;;;; +11DC4;TOLONG SIKI LETTER NN;Lo;0;L;;;;;N;;;;; +11DC5;TOLONG SIKI LETTER C;Lo;0;L;;;;;N;;;;; +11DC6;TOLONG SIKI LETTER CH;Lo;0;L;;;;;N;;;;; +11DC7;TOLONG SIKI LETTER J;Lo;0;L;;;;;N;;;;; +11DC8;TOLONG SIKI LETTER JH;Lo;0;L;;;;;N;;;;; +11DC9;TOLONG SIKI LETTER NY;Lo;0;L;;;;;N;;;;; +11DCA;TOLONG SIKI LETTER K;Lo;0;L;;;;;N;;;;; +11DCB;TOLONG SIKI LETTER KH;Lo;0;L;;;;;N;;;;; +11DCC;TOLONG SIKI LETTER G;Lo;0;L;;;;;N;;;;; +11DCD;TOLONG SIKI LETTER GH;Lo;0;L;;;;;N;;;;; +11DCE;TOLONG SIKI LETTER NG;Lo;0;L;;;;;N;;;;; +11DCF;TOLONG SIKI LETTER Y;Lo;0;L;;;;;N;;;;; +11DD0;TOLONG SIKI LETTER R;Lo;0;L;;;;;N;;;;; +11DD1;TOLONG SIKI LETTER L;Lo;0;L;;;;;N;;;;; +11DD2;TOLONG SIKI LETTER V;Lo;0;L;;;;;N;;;;; +11DD3;TOLONG SIKI LETTER NNY;Lo;0;L;;;;;N;;;;; +11DD4;TOLONG SIKI LETTER S;Lo;0;L;;;;;N;;;;; +11DD5;TOLONG SIKI LETTER H;Lo;0;L;;;;;N;;;;; +11DD6;TOLONG SIKI LETTER X;Lo;0;L;;;;;N;;;;; +11DD7;TOLONG SIKI LETTER RR;Lo;0;L;;;;;N;;;;; +11DD8;TOLONG SIKI LETTER RRH;Lo;0;L;;;;;N;;;;; +11DD9;TOLONG SIKI SIGN SELA;Lm;0;L;;;;;N;;;;; +11DDA;TOLONG SIKI SIGN HECAKA;Lo;0;L;;;;;N;;;;; +11DDB;TOLONG SIKI UNGGA;Lo;0;L;;;;;N;;;;; +11DE0;TOLONG SIKI DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +11DE1;TOLONG SIKI DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +11DE2;TOLONG SIKI DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +11DE3;TOLONG SIKI DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +11DE4;TOLONG SIKI DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +11DE5;TOLONG SIKI DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +11DE6;TOLONG SIKI DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +11DE7;TOLONG SIKI DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +11DE8;TOLONG SIKI DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +11DE9;TOLONG SIKI DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; 11EE0;MAKASAR LETTER KA;Lo;0;L;;;;;N;;;;; 11EE1;MAKASAR LETTER GA;Lo;0;L;;;;;N;;;;; 11EE2;MAKASAR LETTER NGA;Lo;0;L;;;;;N;;;;; @@ -21688,6 +22143,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 11F57;KAWI DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; 11F58;KAWI DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; 11F59;KAWI DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; +11F5A;KAWI SIGN NUKTA;Mn;0;NSM;;;;;N;;;;; 11FB0;LISU LETTER YHA;Lo;0;L;;;;;N;;;;; 11FC0;TAMIL FRACTION ONE THREE-HUNDRED-AND-TWENTIETH;No;0;L;;;;1/320;N;;;;; 11FC1;TAMIL FRACTION ONE ONE-HUNDRED-AND-SIXTIETH;No;0;L;;;;1/160;N;;;;; @@ -21796,8 +22252,8 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 12035;CUNEIFORM SIGN ARAD TIMES KUR;Lo;0;L;;;;;N;;;;; 12036;CUNEIFORM SIGN ARKAB;Lo;0;L;;;;;N;;;;; 12037;CUNEIFORM SIGN ASAL2;Lo;0;L;;;;;N;;;;; -12038;CUNEIFORM SIGN ASH;Lo;0;L;;;;;N;;;;; -12039;CUNEIFORM SIGN ASH ZIDA TENU;Lo;0;L;;;;;N;;;;; +12038;CUNEIFORM SIGN ASH;Lo;0;L;;;;1;N;;;;; +12039;CUNEIFORM SIGN ASH ZIDA TENU;Lo;0;L;;;;1;N;;;;; 1203A;CUNEIFORM SIGN ASH KABA TENU;Lo;0;L;;;;;N;;;;; 1203B;CUNEIFORM SIGN ASH OVER ASH TUG2 OVER TUG2 TUG2 OVER TUG2 PAP;Lo;0;L;;;;;N;;;;; 1203C;CUNEIFORM SIGN ASH OVER ASH OVER ASH;Lo;0;L;;;;;N;;;;; @@ -21861,7 +22317,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 12076;CUNEIFORM SIGN DIM2;Lo;0;L;;;;;N;;;;; 12077;CUNEIFORM SIGN DIN;Lo;0;L;;;;;N;;;;; 12078;CUNEIFORM SIGN DIN KASKAL U GUNU DISH;Lo;0;L;;;;;N;;;;; -12079;CUNEIFORM SIGN DISH;Lo;0;L;;;;;N;;;;; +12079;CUNEIFORM SIGN DISH;Lo;0;L;;;;1;N;;;;; 1207A;CUNEIFORM SIGN DU;Lo;0;L;;;;;N;;;;; 1207B;CUNEIFORM SIGN DU OVER DU;Lo;0;L;;;;;N;;;;; 1207C;CUNEIFORM SIGN DU GUNU;Lo;0;L;;;;;N;;;;; @@ -22290,12 +22746,12 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 12223;CUNEIFORM SIGN MA2;Lo;0;L;;;;;N;;;;; 12224;CUNEIFORM SIGN MAH;Lo;0;L;;;;;N;;;;; 12225;CUNEIFORM SIGN MAR;Lo;0;L;;;;;N;;;;; -12226;CUNEIFORM SIGN MASH;Lo;0;L;;;;;N;;;;; +12226;CUNEIFORM SIGN MASH;Lo;0;L;;;;1/2;N;;;;; 12227;CUNEIFORM SIGN MASH2;Lo;0;L;;;;;N;;;;; 12228;CUNEIFORM SIGN ME;Lo;0;L;;;;;N;;;;; 12229;CUNEIFORM SIGN MES;Lo;0;L;;;;;N;;;;; 1222A;CUNEIFORM SIGN MI;Lo;0;L;;;;;N;;;;; -1222B;CUNEIFORM SIGN MIN;Lo;0;L;;;;;N;;;;; +1222B;CUNEIFORM SIGN MIN;Lo;0;L;;;;2;N;;;;; 1222C;CUNEIFORM SIGN MU;Lo;0;L;;;;;N;;;;; 1222D;CUNEIFORM SIGN MU OVER MU;Lo;0;L;;;;;N;;;;; 1222E;CUNEIFORM SIGN MUG;Lo;0;L;;;;;N;;;;; @@ -22519,9 +22975,9 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 12308;CUNEIFORM SIGN TUM;Lo;0;L;;;;;N;;;;; 12309;CUNEIFORM SIGN TUR;Lo;0;L;;;;;N;;;;; 1230A;CUNEIFORM SIGN TUR OVER TUR ZA OVER ZA;Lo;0;L;;;;;N;;;;; -1230B;CUNEIFORM SIGN U;Lo;0;L;;;;;N;;;;; +1230B;CUNEIFORM SIGN U;Lo;0;L;;;;1;N;;;;; 1230C;CUNEIFORM SIGN U GUD;Lo;0;L;;;;;N;;;;; -1230D;CUNEIFORM SIGN U U U;Lo;0;L;;;;;N;;;;; +1230D;CUNEIFORM SIGN U U U;Lo;0;L;;;;3;N;;;;; 1230E;CUNEIFORM SIGN U OVER U PA OVER PA GAR OVER GAR;Lo;0;L;;;;;N;;;;; 1230F;CUNEIFORM SIGN U OVER U SUR OVER SUR;Lo;0;L;;;;;N;;;;; 12310;CUNEIFORM SIGN U OVER U U REVERSED OVER U REVERSED;Lo;0;L;;;;;N;;;;; @@ -22661,7 +23117,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 12396;CUNEIFORM SIGN SAG TIMES IGI GUNU;Lo;0;L;;;;;N;;;;; 12397;CUNEIFORM SIGN TI2;Lo;0;L;;;;;N;;;;; 12398;CUNEIFORM SIGN UM TIMES ME;Lo;0;L;;;;;N;;;;; -12399;CUNEIFORM SIGN U U;Lo;0;L;;;;;N;;;;; +12399;CUNEIFORM SIGN U U;Lo;0;L;;;;2;N;;;;; 12400;CUNEIFORM NUMERIC SIGN TWO ASH;Nl;0;L;;;;2;N;;;;; 12401;CUNEIFORM NUMERIC SIGN THREE ASH;Nl;0;L;;;;3;N;;;;; 12402;CUNEIFORM NUMERIC SIGN FOUR ASH;Nl;0;L;;;;4;N;;;;; @@ -24183,6 +24639,4001 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 13453;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT TOP AND END;Mn;0;NSM;;;;;N;;;;; 13454;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED AT BOTTOM AND END;Mn;0;NSM;;;;;N;;;;; 13455;EGYPTIAN HIEROGLYPH MODIFIER DAMAGED;Mn;0;NSM;;;;;N;;;;; +13460;EGYPTIAN HIEROGLYPH-13460;Lo;0;L;;;;;N;;;;; +13461;EGYPTIAN HIEROGLYPH-13461;Lo;0;L;;;;;N;;;;; +13462;EGYPTIAN HIEROGLYPH-13462;Lo;0;L;;;;;N;;;;; +13463;EGYPTIAN HIEROGLYPH-13463;Lo;0;L;;;;;N;;;;; +13464;EGYPTIAN HIEROGLYPH-13464;Lo;0;L;;;;;N;;;;; +13465;EGYPTIAN HIEROGLYPH-13465;Lo;0;L;;;;;N;;;;; +13466;EGYPTIAN HIEROGLYPH-13466;Lo;0;L;;;;;N;;;;; +13467;EGYPTIAN HIEROGLYPH-13467;Lo;0;L;;;;;N;;;;; +13468;EGYPTIAN HIEROGLYPH-13468;Lo;0;L;;;;;N;;;;; +13469;EGYPTIAN HIEROGLYPH-13469;Lo;0;L;;;;;N;;;;; +1346A;EGYPTIAN HIEROGLYPH-1346A;Lo;0;L;;;;;N;;;;; +1346B;EGYPTIAN HIEROGLYPH-1346B;Lo;0;L;;;;;N;;;;; +1346C;EGYPTIAN HIEROGLYPH-1346C;Lo;0;L;;;;;N;;;;; +1346D;EGYPTIAN HIEROGLYPH-1346D;Lo;0;L;;;;;N;;;;; +1346E;EGYPTIAN HIEROGLYPH-1346E;Lo;0;L;;;;;N;;;;; +1346F;EGYPTIAN HIEROGLYPH-1346F;Lo;0;L;;;;;N;;;;; +13470;EGYPTIAN HIEROGLYPH-13470;Lo;0;L;;;;;N;;;;; +13471;EGYPTIAN HIEROGLYPH-13471;Lo;0;L;;;;;N;;;;; +13472;EGYPTIAN HIEROGLYPH-13472;Lo;0;L;;;;;N;;;;; +13473;EGYPTIAN HIEROGLYPH-13473;Lo;0;L;;;;;N;;;;; +13474;EGYPTIAN HIEROGLYPH-13474;Lo;0;L;;;;;N;;;;; +13475;EGYPTIAN HIEROGLYPH-13475;Lo;0;L;;;;;N;;;;; +13476;EGYPTIAN HIEROGLYPH-13476;Lo;0;L;;;;;N;;;;; +13477;EGYPTIAN HIEROGLYPH-13477;Lo;0;L;;;;;N;;;;; +13478;EGYPTIAN HIEROGLYPH-13478;Lo;0;L;;;;;N;;;;; +13479;EGYPTIAN HIEROGLYPH-13479;Lo;0;L;;;;;N;;;;; +1347A;EGYPTIAN HIEROGLYPH-1347A;Lo;0;L;;;;;N;;;;; +1347B;EGYPTIAN HIEROGLYPH-1347B;Lo;0;L;;;;;N;;;;; +1347C;EGYPTIAN HIEROGLYPH-1347C;Lo;0;L;;;;;N;;;;; +1347D;EGYPTIAN HIEROGLYPH-1347D;Lo;0;L;;;;;N;;;;; +1347E;EGYPTIAN HIEROGLYPH-1347E;Lo;0;L;;;;;N;;;;; +1347F;EGYPTIAN HIEROGLYPH-1347F;Lo;0;L;;;;;N;;;;; +13480;EGYPTIAN HIEROGLYPH-13480;Lo;0;L;;;;;N;;;;; +13481;EGYPTIAN HIEROGLYPH-13481;Lo;0;L;;;;;N;;;;; +13482;EGYPTIAN HIEROGLYPH-13482;Lo;0;L;;;;;N;;;;; +13483;EGYPTIAN HIEROGLYPH-13483;Lo;0;L;;;;;N;;;;; +13484;EGYPTIAN HIEROGLYPH-13484;Lo;0;L;;;;;N;;;;; +13485;EGYPTIAN HIEROGLYPH-13485;Lo;0;L;;;;;N;;;;; +13486;EGYPTIAN HIEROGLYPH-13486;Lo;0;L;;;;;N;;;;; +13487;EGYPTIAN HIEROGLYPH-13487;Lo;0;L;;;;;N;;;;; +13488;EGYPTIAN HIEROGLYPH-13488;Lo;0;L;;;;;N;;;;; +13489;EGYPTIAN HIEROGLYPH-13489;Lo;0;L;;;;;N;;;;; +1348A;EGYPTIAN HIEROGLYPH-1348A;Lo;0;L;;;;;N;;;;; +1348B;EGYPTIAN HIEROGLYPH-1348B;Lo;0;L;;;;;N;;;;; +1348C;EGYPTIAN HIEROGLYPH-1348C;Lo;0;L;;;;;N;;;;; +1348D;EGYPTIAN HIEROGLYPH-1348D;Lo;0;L;;;;;N;;;;; +1348E;EGYPTIAN HIEROGLYPH-1348E;Lo;0;L;;;;;N;;;;; +1348F;EGYPTIAN HIEROGLYPH-1348F;Lo;0;L;;;;;N;;;;; +13490;EGYPTIAN HIEROGLYPH-13490;Lo;0;L;;;;;N;;;;; +13491;EGYPTIAN HIEROGLYPH-13491;Lo;0;L;;;;;N;;;;; +13492;EGYPTIAN HIEROGLYPH-13492;Lo;0;L;;;;;N;;;;; +13493;EGYPTIAN HIEROGLYPH-13493;Lo;0;L;;;;;N;;;;; +13494;EGYPTIAN HIEROGLYPH-13494;Lo;0;L;;;;;N;;;;; +13495;EGYPTIAN HIEROGLYPH-13495;Lo;0;L;;;;;N;;;;; +13496;EGYPTIAN HIEROGLYPH-13496;Lo;0;L;;;;;N;;;;; +13497;EGYPTIAN HIEROGLYPH-13497;Lo;0;L;;;;;N;;;;; +13498;EGYPTIAN HIEROGLYPH-13498;Lo;0;L;;;;;N;;;;; +13499;EGYPTIAN HIEROGLYPH-13499;Lo;0;L;;;;;N;;;;; +1349A;EGYPTIAN HIEROGLYPH-1349A;Lo;0;L;;;;;N;;;;; +1349B;EGYPTIAN HIEROGLYPH-1349B;Lo;0;L;;;;;N;;;;; +1349C;EGYPTIAN HIEROGLYPH-1349C;Lo;0;L;;;;;N;;;;; +1349D;EGYPTIAN HIEROGLYPH-1349D;Lo;0;L;;;;;N;;;;; +1349E;EGYPTIAN HIEROGLYPH-1349E;Lo;0;L;;;;;N;;;;; +1349F;EGYPTIAN HIEROGLYPH-1349F;Lo;0;L;;;;;N;;;;; +134A0;EGYPTIAN HIEROGLYPH-134A0;Lo;0;L;;;;;N;;;;; +134A1;EGYPTIAN HIEROGLYPH-134A1;Lo;0;L;;;;;N;;;;; +134A2;EGYPTIAN HIEROGLYPH-134A2;Lo;0;L;;;;;N;;;;; +134A3;EGYPTIAN HIEROGLYPH-134A3;Lo;0;L;;;;;N;;;;; +134A4;EGYPTIAN HIEROGLYPH-134A4;Lo;0;L;;;;;N;;;;; +134A5;EGYPTIAN HIEROGLYPH-134A5;Lo;0;L;;;;;N;;;;; +134A6;EGYPTIAN HIEROGLYPH-134A6;Lo;0;L;;;;;N;;;;; +134A7;EGYPTIAN HIEROGLYPH-134A7;Lo;0;L;;;;;N;;;;; +134A8;EGYPTIAN HIEROGLYPH-134A8;Lo;0;L;;;;;N;;;;; +134A9;EGYPTIAN HIEROGLYPH-134A9;Lo;0;L;;;;;N;;;;; +134AA;EGYPTIAN HIEROGLYPH-134AA;Lo;0;L;;;;;N;;;;; +134AB;EGYPTIAN HIEROGLYPH-134AB;Lo;0;L;;;;;N;;;;; +134AC;EGYPTIAN HIEROGLYPH-134AC;Lo;0;L;;;;;N;;;;; +134AD;EGYPTIAN HIEROGLYPH-134AD;Lo;0;L;;;;;N;;;;; +134AE;EGYPTIAN HIEROGLYPH-134AE;Lo;0;L;;;;;N;;;;; +134AF;EGYPTIAN HIEROGLYPH-134AF;Lo;0;L;;;;;N;;;;; +134B0;EGYPTIAN HIEROGLYPH-134B0;Lo;0;L;;;;;N;;;;; +134B1;EGYPTIAN HIEROGLYPH-134B1;Lo;0;L;;;;;N;;;;; +134B2;EGYPTIAN HIEROGLYPH-134B2;Lo;0;L;;;;;N;;;;; +134B3;EGYPTIAN HIEROGLYPH-134B3;Lo;0;L;;;;;N;;;;; +134B4;EGYPTIAN HIEROGLYPH-134B4;Lo;0;L;;;;;N;;;;; +134B5;EGYPTIAN HIEROGLYPH-134B5;Lo;0;L;;;;;N;;;;; +134B6;EGYPTIAN HIEROGLYPH-134B6;Lo;0;L;;;;;N;;;;; +134B7;EGYPTIAN HIEROGLYPH-134B7;Lo;0;L;;;;;N;;;;; +134B8;EGYPTIAN HIEROGLYPH-134B8;Lo;0;L;;;;;N;;;;; +134B9;EGYPTIAN HIEROGLYPH-134B9;Lo;0;L;;;;;N;;;;; +134BA;EGYPTIAN HIEROGLYPH-134BA;Lo;0;L;;;;;N;;;;; +134BB;EGYPTIAN HIEROGLYPH-134BB;Lo;0;L;;;;;N;;;;; +134BC;EGYPTIAN HIEROGLYPH-134BC;Lo;0;L;;;;;N;;;;; +134BD;EGYPTIAN HIEROGLYPH-134BD;Lo;0;L;;;;;N;;;;; +134BE;EGYPTIAN HIEROGLYPH-134BE;Lo;0;L;;;;;N;;;;; +134BF;EGYPTIAN HIEROGLYPH-134BF;Lo;0;L;;;;;N;;;;; +134C0;EGYPTIAN HIEROGLYPH-134C0;Lo;0;L;;;;;N;;;;; +134C1;EGYPTIAN HIEROGLYPH-134C1;Lo;0;L;;;;;N;;;;; +134C2;EGYPTIAN HIEROGLYPH-134C2;Lo;0;L;;;;;N;;;;; +134C3;EGYPTIAN HIEROGLYPH-134C3;Lo;0;L;;;;;N;;;;; +134C4;EGYPTIAN HIEROGLYPH-134C4;Lo;0;L;;;;;N;;;;; +134C5;EGYPTIAN HIEROGLYPH-134C5;Lo;0;L;;;;;N;;;;; +134C6;EGYPTIAN HIEROGLYPH-134C6;Lo;0;L;;;;;N;;;;; +134C7;EGYPTIAN HIEROGLYPH-134C7;Lo;0;L;;;;;N;;;;; +134C8;EGYPTIAN HIEROGLYPH-134C8;Lo;0;L;;;;;N;;;;; +134C9;EGYPTIAN HIEROGLYPH-134C9;Lo;0;L;;;;;N;;;;; +134CA;EGYPTIAN HIEROGLYPH-134CA;Lo;0;L;;;;;N;;;;; +134CB;EGYPTIAN HIEROGLYPH-134CB;Lo;0;L;;;;;N;;;;; +134CC;EGYPTIAN HIEROGLYPH-134CC;Lo;0;L;;;;;N;;;;; +134CD;EGYPTIAN HIEROGLYPH-134CD;Lo;0;L;;;;;N;;;;; +134CE;EGYPTIAN HIEROGLYPH-134CE;Lo;0;L;;;;;N;;;;; +134CF;EGYPTIAN HIEROGLYPH-134CF;Lo;0;L;;;;;N;;;;; +134D0;EGYPTIAN HIEROGLYPH-134D0;Lo;0;L;;;;;N;;;;; +134D1;EGYPTIAN HIEROGLYPH-134D1;Lo;0;L;;;;;N;;;;; +134D2;EGYPTIAN HIEROGLYPH-134D2;Lo;0;L;;;;;N;;;;; +134D3;EGYPTIAN HIEROGLYPH-134D3;Lo;0;L;;;;;N;;;;; +134D4;EGYPTIAN HIEROGLYPH-134D4;Lo;0;L;;;;;N;;;;; +134D5;EGYPTIAN HIEROGLYPH-134D5;Lo;0;L;;;;;N;;;;; +134D6;EGYPTIAN HIEROGLYPH-134D6;Lo;0;L;;;;;N;;;;; +134D7;EGYPTIAN HIEROGLYPH-134D7;Lo;0;L;;;;;N;;;;; +134D8;EGYPTIAN HIEROGLYPH-134D8;Lo;0;L;;;;;N;;;;; +134D9;EGYPTIAN HIEROGLYPH-134D9;Lo;0;L;;;;;N;;;;; +134DA;EGYPTIAN HIEROGLYPH-134DA;Lo;0;L;;;;;N;;;;; +134DB;EGYPTIAN HIEROGLYPH-134DB;Lo;0;L;;;;;N;;;;; +134DC;EGYPTIAN HIEROGLYPH-134DC;Lo;0;L;;;;;N;;;;; +134DD;EGYPTIAN HIEROGLYPH-134DD;Lo;0;L;;;;;N;;;;; +134DE;EGYPTIAN HIEROGLYPH-134DE;Lo;0;L;;;;;N;;;;; +134DF;EGYPTIAN HIEROGLYPH-134DF;Lo;0;L;;;;;N;;;;; +134E0;EGYPTIAN HIEROGLYPH-134E0;Lo;0;L;;;;;N;;;;; +134E1;EGYPTIAN HIEROGLYPH-134E1;Lo;0;L;;;;;N;;;;; +134E2;EGYPTIAN HIEROGLYPH-134E2;Lo;0;L;;;;;N;;;;; +134E3;EGYPTIAN HIEROGLYPH-134E3;Lo;0;L;;;;;N;;;;; +134E4;EGYPTIAN HIEROGLYPH-134E4;Lo;0;L;;;;;N;;;;; +134E5;EGYPTIAN HIEROGLYPH-134E5;Lo;0;L;;;;;N;;;;; +134E6;EGYPTIAN HIEROGLYPH-134E6;Lo;0;L;;;;;N;;;;; +134E7;EGYPTIAN HIEROGLYPH-134E7;Lo;0;L;;;;;N;;;;; +134E8;EGYPTIAN HIEROGLYPH-134E8;Lo;0;L;;;;;N;;;;; +134E9;EGYPTIAN HIEROGLYPH-134E9;Lo;0;L;;;;;N;;;;; +134EA;EGYPTIAN HIEROGLYPH-134EA;Lo;0;L;;;;;N;;;;; +134EB;EGYPTIAN HIEROGLYPH-134EB;Lo;0;L;;;;;N;;;;; +134EC;EGYPTIAN HIEROGLYPH-134EC;Lo;0;L;;;;;N;;;;; +134ED;EGYPTIAN HIEROGLYPH-134ED;Lo;0;L;;;;;N;;;;; +134EE;EGYPTIAN HIEROGLYPH-134EE;Lo;0;L;;;;;N;;;;; +134EF;EGYPTIAN HIEROGLYPH-134EF;Lo;0;L;;;;;N;;;;; +134F0;EGYPTIAN HIEROGLYPH-134F0;Lo;0;L;;;;;N;;;;; +134F1;EGYPTIAN HIEROGLYPH-134F1;Lo;0;L;;;;;N;;;;; +134F2;EGYPTIAN HIEROGLYPH-134F2;Lo;0;L;;;;;N;;;;; +134F3;EGYPTIAN HIEROGLYPH-134F3;Lo;0;L;;;;;N;;;;; +134F4;EGYPTIAN HIEROGLYPH-134F4;Lo;0;L;;;;;N;;;;; +134F5;EGYPTIAN HIEROGLYPH-134F5;Lo;0;L;;;;;N;;;;; +134F6;EGYPTIAN HIEROGLYPH-134F6;Lo;0;L;;;;;N;;;;; +134F7;EGYPTIAN HIEROGLYPH-134F7;Lo;0;L;;;;;N;;;;; +134F8;EGYPTIAN HIEROGLYPH-134F8;Lo;0;L;;;;;N;;;;; +134F9;EGYPTIAN HIEROGLYPH-134F9;Lo;0;L;;;;;N;;;;; +134FA;EGYPTIAN HIEROGLYPH-134FA;Lo;0;L;;;;;N;;;;; +134FB;EGYPTIAN HIEROGLYPH-134FB;Lo;0;L;;;;;N;;;;; +134FC;EGYPTIAN HIEROGLYPH-134FC;Lo;0;L;;;;;N;;;;; +134FD;EGYPTIAN HIEROGLYPH-134FD;Lo;0;L;;;;;N;;;;; +134FE;EGYPTIAN HIEROGLYPH-134FE;Lo;0;L;;;;;N;;;;; +134FF;EGYPTIAN HIEROGLYPH-134FF;Lo;0;L;;;;;N;;;;; +13500;EGYPTIAN HIEROGLYPH-13500;Lo;0;L;;;;;N;;;;; +13501;EGYPTIAN HIEROGLYPH-13501;Lo;0;L;;;;;N;;;;; +13502;EGYPTIAN HIEROGLYPH-13502;Lo;0;L;;;;;N;;;;; +13503;EGYPTIAN HIEROGLYPH-13503;Lo;0;L;;;;;N;;;;; +13504;EGYPTIAN HIEROGLYPH-13504;Lo;0;L;;;;;N;;;;; +13505;EGYPTIAN HIEROGLYPH-13505;Lo;0;L;;;;;N;;;;; +13506;EGYPTIAN HIEROGLYPH-13506;Lo;0;L;;;;;N;;;;; +13507;EGYPTIAN HIEROGLYPH-13507;Lo;0;L;;;;;N;;;;; +13508;EGYPTIAN HIEROGLYPH-13508;Lo;0;L;;;;;N;;;;; +13509;EGYPTIAN HIEROGLYPH-13509;Lo;0;L;;;;;N;;;;; +1350A;EGYPTIAN HIEROGLYPH-1350A;Lo;0;L;;;;;N;;;;; +1350B;EGYPTIAN HIEROGLYPH-1350B;Lo;0;L;;;;;N;;;;; +1350C;EGYPTIAN HIEROGLYPH-1350C;Lo;0;L;;;;;N;;;;; +1350D;EGYPTIAN HIEROGLYPH-1350D;Lo;0;L;;;;;N;;;;; +1350E;EGYPTIAN HIEROGLYPH-1350E;Lo;0;L;;;;;N;;;;; +1350F;EGYPTIAN HIEROGLYPH-1350F;Lo;0;L;;;;;N;;;;; +13510;EGYPTIAN HIEROGLYPH-13510;Lo;0;L;;;;;N;;;;; +13511;EGYPTIAN HIEROGLYPH-13511;Lo;0;L;;;;;N;;;;; +13512;EGYPTIAN HIEROGLYPH-13512;Lo;0;L;;;;;N;;;;; +13513;EGYPTIAN HIEROGLYPH-13513;Lo;0;L;;;;;N;;;;; +13514;EGYPTIAN HIEROGLYPH-13514;Lo;0;L;;;;;N;;;;; +13515;EGYPTIAN HIEROGLYPH-13515;Lo;0;L;;;;;N;;;;; +13516;EGYPTIAN HIEROGLYPH-13516;Lo;0;L;;;;;N;;;;; +13517;EGYPTIAN HIEROGLYPH-13517;Lo;0;L;;;;;N;;;;; +13518;EGYPTIAN HIEROGLYPH-13518;Lo;0;L;;;;;N;;;;; +13519;EGYPTIAN HIEROGLYPH-13519;Lo;0;L;;;;;N;;;;; +1351A;EGYPTIAN HIEROGLYPH-1351A;Lo;0;L;;;;;N;;;;; +1351B;EGYPTIAN HIEROGLYPH-1351B;Lo;0;L;;;;;N;;;;; +1351C;EGYPTIAN HIEROGLYPH-1351C;Lo;0;L;;;;;N;;;;; +1351D;EGYPTIAN HIEROGLYPH-1351D;Lo;0;L;;;;;N;;;;; +1351E;EGYPTIAN HIEROGLYPH-1351E;Lo;0;L;;;;;N;;;;; +1351F;EGYPTIAN HIEROGLYPH-1351F;Lo;0;L;;;;;N;;;;; +13520;EGYPTIAN HIEROGLYPH-13520;Lo;0;L;;;;;N;;;;; +13521;EGYPTIAN HIEROGLYPH-13521;Lo;0;L;;;;;N;;;;; +13522;EGYPTIAN HIEROGLYPH-13522;Lo;0;L;;;;;N;;;;; +13523;EGYPTIAN HIEROGLYPH-13523;Lo;0;L;;;;;N;;;;; +13524;EGYPTIAN HIEROGLYPH-13524;Lo;0;L;;;;;N;;;;; +13525;EGYPTIAN HIEROGLYPH-13525;Lo;0;L;;;;;N;;;;; +13526;EGYPTIAN HIEROGLYPH-13526;Lo;0;L;;;;;N;;;;; +13527;EGYPTIAN HIEROGLYPH-13527;Lo;0;L;;;;;N;;;;; +13528;EGYPTIAN HIEROGLYPH-13528;Lo;0;L;;;;;N;;;;; +13529;EGYPTIAN HIEROGLYPH-13529;Lo;0;L;;;;;N;;;;; +1352A;EGYPTIAN HIEROGLYPH-1352A;Lo;0;L;;;;;N;;;;; +1352B;EGYPTIAN HIEROGLYPH-1352B;Lo;0;L;;;;;N;;;;; +1352C;EGYPTIAN HIEROGLYPH-1352C;Lo;0;L;;;;;N;;;;; +1352D;EGYPTIAN HIEROGLYPH-1352D;Lo;0;L;;;;;N;;;;; +1352E;EGYPTIAN HIEROGLYPH-1352E;Lo;0;L;;;;;N;;;;; +1352F;EGYPTIAN HIEROGLYPH-1352F;Lo;0;L;;;;;N;;;;; +13530;EGYPTIAN HIEROGLYPH-13530;Lo;0;L;;;;;N;;;;; +13531;EGYPTIAN HIEROGLYPH-13531;Lo;0;L;;;;;N;;;;; +13532;EGYPTIAN HIEROGLYPH-13532;Lo;0;L;;;;;N;;;;; +13533;EGYPTIAN HIEROGLYPH-13533;Lo;0;L;;;;;N;;;;; +13534;EGYPTIAN HIEROGLYPH-13534;Lo;0;L;;;;;N;;;;; +13535;EGYPTIAN HIEROGLYPH-13535;Lo;0;L;;;;;N;;;;; +13536;EGYPTIAN HIEROGLYPH-13536;Lo;0;L;;;;;N;;;;; +13537;EGYPTIAN HIEROGLYPH-13537;Lo;0;L;;;;;N;;;;; +13538;EGYPTIAN HIEROGLYPH-13538;Lo;0;L;;;;;N;;;;; +13539;EGYPTIAN HIEROGLYPH-13539;Lo;0;L;;;;;N;;;;; +1353A;EGYPTIAN HIEROGLYPH-1353A;Lo;0;L;;;;;N;;;;; +1353B;EGYPTIAN HIEROGLYPH-1353B;Lo;0;L;;;;;N;;;;; +1353C;EGYPTIAN HIEROGLYPH-1353C;Lo;0;L;;;;;N;;;;; +1353D;EGYPTIAN HIEROGLYPH-1353D;Lo;0;L;;;;;N;;;;; +1353E;EGYPTIAN HIEROGLYPH-1353E;Lo;0;L;;;;;N;;;;; +1353F;EGYPTIAN HIEROGLYPH-1353F;Lo;0;L;;;;;N;;;;; +13540;EGYPTIAN HIEROGLYPH-13540;Lo;0;L;;;;;N;;;;; +13541;EGYPTIAN HIEROGLYPH-13541;Lo;0;L;;;;;N;;;;; +13542;EGYPTIAN HIEROGLYPH-13542;Lo;0;L;;;;;N;;;;; +13543;EGYPTIAN HIEROGLYPH-13543;Lo;0;L;;;;;N;;;;; +13544;EGYPTIAN HIEROGLYPH-13544;Lo;0;L;;;;;N;;;;; +13545;EGYPTIAN HIEROGLYPH-13545;Lo;0;L;;;;;N;;;;; +13546;EGYPTIAN HIEROGLYPH-13546;Lo;0;L;;;;;N;;;;; +13547;EGYPTIAN HIEROGLYPH-13547;Lo;0;L;;;;;N;;;;; +13548;EGYPTIAN HIEROGLYPH-13548;Lo;0;L;;;;;N;;;;; +13549;EGYPTIAN HIEROGLYPH-13549;Lo;0;L;;;;;N;;;;; +1354A;EGYPTIAN HIEROGLYPH-1354A;Lo;0;L;;;;;N;;;;; +1354B;EGYPTIAN HIEROGLYPH-1354B;Lo;0;L;;;;;N;;;;; +1354C;EGYPTIAN HIEROGLYPH-1354C;Lo;0;L;;;;;N;;;;; +1354D;EGYPTIAN HIEROGLYPH-1354D;Lo;0;L;;;;;N;;;;; +1354E;EGYPTIAN HIEROGLYPH-1354E;Lo;0;L;;;;;N;;;;; +1354F;EGYPTIAN HIEROGLYPH-1354F;Lo;0;L;;;;;N;;;;; +13550;EGYPTIAN HIEROGLYPH-13550;Lo;0;L;;;;;N;;;;; +13551;EGYPTIAN HIEROGLYPH-13551;Lo;0;L;;;;;N;;;;; +13552;EGYPTIAN HIEROGLYPH-13552;Lo;0;L;;;;;N;;;;; +13553;EGYPTIAN HIEROGLYPH-13553;Lo;0;L;;;;;N;;;;; +13554;EGYPTIAN HIEROGLYPH-13554;Lo;0;L;;;;;N;;;;; +13555;EGYPTIAN HIEROGLYPH-13555;Lo;0;L;;;;;N;;;;; +13556;EGYPTIAN HIEROGLYPH-13556;Lo;0;L;;;;;N;;;;; +13557;EGYPTIAN HIEROGLYPH-13557;Lo;0;L;;;;;N;;;;; +13558;EGYPTIAN HIEROGLYPH-13558;Lo;0;L;;;;;N;;;;; +13559;EGYPTIAN HIEROGLYPH-13559;Lo;0;L;;;;;N;;;;; +1355A;EGYPTIAN HIEROGLYPH-1355A;Lo;0;L;;;;;N;;;;; +1355B;EGYPTIAN HIEROGLYPH-1355B;Lo;0;L;;;;;N;;;;; +1355C;EGYPTIAN HIEROGLYPH-1355C;Lo;0;L;;;;;N;;;;; +1355D;EGYPTIAN HIEROGLYPH-1355D;Lo;0;L;;;;;N;;;;; +1355E;EGYPTIAN HIEROGLYPH-1355E;Lo;0;L;;;;;N;;;;; +1355F;EGYPTIAN HIEROGLYPH-1355F;Lo;0;L;;;;;N;;;;; +13560;EGYPTIAN HIEROGLYPH-13560;Lo;0;L;;;;;N;;;;; +13561;EGYPTIAN HIEROGLYPH-13561;Lo;0;L;;;;;N;;;;; +13562;EGYPTIAN HIEROGLYPH-13562;Lo;0;L;;;;;N;;;;; +13563;EGYPTIAN HIEROGLYPH-13563;Lo;0;L;;;;;N;;;;; +13564;EGYPTIAN HIEROGLYPH-13564;Lo;0;L;;;;;N;;;;; +13565;EGYPTIAN HIEROGLYPH-13565;Lo;0;L;;;;;N;;;;; +13566;EGYPTIAN HIEROGLYPH-13566;Lo;0;L;;;;;N;;;;; +13567;EGYPTIAN HIEROGLYPH-13567;Lo;0;L;;;;;N;;;;; +13568;EGYPTIAN HIEROGLYPH-13568;Lo;0;L;;;;;N;;;;; +13569;EGYPTIAN HIEROGLYPH-13569;Lo;0;L;;;;;N;;;;; +1356A;EGYPTIAN HIEROGLYPH-1356A;Lo;0;L;;;;;N;;;;; +1356B;EGYPTIAN HIEROGLYPH-1356B;Lo;0;L;;;;;N;;;;; +1356C;EGYPTIAN HIEROGLYPH-1356C;Lo;0;L;;;;;N;;;;; +1356D;EGYPTIAN HIEROGLYPH-1356D;Lo;0;L;;;;;N;;;;; +1356E;EGYPTIAN HIEROGLYPH-1356E;Lo;0;L;;;;;N;;;;; +1356F;EGYPTIAN HIEROGLYPH-1356F;Lo;0;L;;;;;N;;;;; +13570;EGYPTIAN HIEROGLYPH-13570;Lo;0;L;;;;;N;;;;; +13571;EGYPTIAN HIEROGLYPH-13571;Lo;0;L;;;;;N;;;;; +13572;EGYPTIAN HIEROGLYPH-13572;Lo;0;L;;;;;N;;;;; +13573;EGYPTIAN HIEROGLYPH-13573;Lo;0;L;;;;;N;;;;; +13574;EGYPTIAN HIEROGLYPH-13574;Lo;0;L;;;;;N;;;;; +13575;EGYPTIAN HIEROGLYPH-13575;Lo;0;L;;;;;N;;;;; +13576;EGYPTIAN HIEROGLYPH-13576;Lo;0;L;;;;;N;;;;; +13577;EGYPTIAN HIEROGLYPH-13577;Lo;0;L;;;;;N;;;;; +13578;EGYPTIAN HIEROGLYPH-13578;Lo;0;L;;;;;N;;;;; +13579;EGYPTIAN HIEROGLYPH-13579;Lo;0;L;;;;;N;;;;; +1357A;EGYPTIAN HIEROGLYPH-1357A;Lo;0;L;;;;;N;;;;; +1357B;EGYPTIAN HIEROGLYPH-1357B;Lo;0;L;;;;;N;;;;; +1357C;EGYPTIAN HIEROGLYPH-1357C;Lo;0;L;;;;;N;;;;; +1357D;EGYPTIAN HIEROGLYPH-1357D;Lo;0;L;;;;;N;;;;; +1357E;EGYPTIAN HIEROGLYPH-1357E;Lo;0;L;;;;;N;;;;; +1357F;EGYPTIAN HIEROGLYPH-1357F;Lo;0;L;;;;;N;;;;; +13580;EGYPTIAN HIEROGLYPH-13580;Lo;0;L;;;;;N;;;;; +13581;EGYPTIAN HIEROGLYPH-13581;Lo;0;L;;;;;N;;;;; +13582;EGYPTIAN HIEROGLYPH-13582;Lo;0;L;;;;;N;;;;; +13583;EGYPTIAN HIEROGLYPH-13583;Lo;0;L;;;;;N;;;;; +13584;EGYPTIAN HIEROGLYPH-13584;Lo;0;L;;;;;N;;;;; +13585;EGYPTIAN HIEROGLYPH-13585;Lo;0;L;;;;;N;;;;; +13586;EGYPTIAN HIEROGLYPH-13586;Lo;0;L;;;;;N;;;;; +13587;EGYPTIAN HIEROGLYPH-13587;Lo;0;L;;;;;N;;;;; +13588;EGYPTIAN HIEROGLYPH-13588;Lo;0;L;;;;;N;;;;; +13589;EGYPTIAN HIEROGLYPH-13589;Lo;0;L;;;;;N;;;;; +1358A;EGYPTIAN HIEROGLYPH-1358A;Lo;0;L;;;;;N;;;;; +1358B;EGYPTIAN HIEROGLYPH-1358B;Lo;0;L;;;;;N;;;;; +1358C;EGYPTIAN HIEROGLYPH-1358C;Lo;0;L;;;;;N;;;;; +1358D;EGYPTIAN HIEROGLYPH-1358D;Lo;0;L;;;;;N;;;;; +1358E;EGYPTIAN HIEROGLYPH-1358E;Lo;0;L;;;;;N;;;;; +1358F;EGYPTIAN HIEROGLYPH-1358F;Lo;0;L;;;;;N;;;;; +13590;EGYPTIAN HIEROGLYPH-13590;Lo;0;L;;;;;N;;;;; +13591;EGYPTIAN HIEROGLYPH-13591;Lo;0;L;;;;;N;;;;; +13592;EGYPTIAN HIEROGLYPH-13592;Lo;0;L;;;;;N;;;;; +13593;EGYPTIAN HIEROGLYPH-13593;Lo;0;L;;;;;N;;;;; +13594;EGYPTIAN HIEROGLYPH-13594;Lo;0;L;;;;;N;;;;; +13595;EGYPTIAN HIEROGLYPH-13595;Lo;0;L;;;;;N;;;;; +13596;EGYPTIAN HIEROGLYPH-13596;Lo;0;L;;;;;N;;;;; +13597;EGYPTIAN HIEROGLYPH-13597;Lo;0;L;;;;;N;;;;; +13598;EGYPTIAN HIEROGLYPH-13598;Lo;0;L;;;;;N;;;;; +13599;EGYPTIAN HIEROGLYPH-13599;Lo;0;L;;;;;N;;;;; +1359A;EGYPTIAN HIEROGLYPH-1359A;Lo;0;L;;;;;N;;;;; +1359B;EGYPTIAN HIEROGLYPH-1359B;Lo;0;L;;;;;N;;;;; +1359C;EGYPTIAN HIEROGLYPH-1359C;Lo;0;L;;;;;N;;;;; +1359D;EGYPTIAN HIEROGLYPH-1359D;Lo;0;L;;;;;N;;;;; +1359E;EGYPTIAN HIEROGLYPH-1359E;Lo;0;L;;;;;N;;;;; +1359F;EGYPTIAN HIEROGLYPH-1359F;Lo;0;L;;;;;N;;;;; +135A0;EGYPTIAN HIEROGLYPH-135A0;Lo;0;L;;;;;N;;;;; +135A1;EGYPTIAN HIEROGLYPH-135A1;Lo;0;L;;;;;N;;;;; +135A2;EGYPTIAN HIEROGLYPH-135A2;Lo;0;L;;;;;N;;;;; +135A3;EGYPTIAN HIEROGLYPH-135A3;Lo;0;L;;;;;N;;;;; +135A4;EGYPTIAN HIEROGLYPH-135A4;Lo;0;L;;;;;N;;;;; +135A5;EGYPTIAN HIEROGLYPH-135A5;Lo;0;L;;;;;N;;;;; +135A6;EGYPTIAN HIEROGLYPH-135A6;Lo;0;L;;;;;N;;;;; +135A7;EGYPTIAN HIEROGLYPH-135A7;Lo;0;L;;;;;N;;;;; +135A8;EGYPTIAN HIEROGLYPH-135A8;Lo;0;L;;;;;N;;;;; +135A9;EGYPTIAN HIEROGLYPH-135A9;Lo;0;L;;;;;N;;;;; +135AA;EGYPTIAN HIEROGLYPH-135AA;Lo;0;L;;;;;N;;;;; +135AB;EGYPTIAN HIEROGLYPH-135AB;Lo;0;L;;;;;N;;;;; +135AC;EGYPTIAN HIEROGLYPH-135AC;Lo;0;L;;;;;N;;;;; +135AD;EGYPTIAN HIEROGLYPH-135AD;Lo;0;L;;;;;N;;;;; +135AE;EGYPTIAN HIEROGLYPH-135AE;Lo;0;L;;;;;N;;;;; +135AF;EGYPTIAN HIEROGLYPH-135AF;Lo;0;L;;;;;N;;;;; +135B0;EGYPTIAN HIEROGLYPH-135B0;Lo;0;L;;;;;N;;;;; +135B1;EGYPTIAN HIEROGLYPH-135B1;Lo;0;L;;;;;N;;;;; +135B2;EGYPTIAN HIEROGLYPH-135B2;Lo;0;L;;;;;N;;;;; +135B3;EGYPTIAN HIEROGLYPH-135B3;Lo;0;L;;;;;N;;;;; +135B4;EGYPTIAN HIEROGLYPH-135B4;Lo;0;L;;;;;N;;;;; +135B5;EGYPTIAN HIEROGLYPH-135B5;Lo;0;L;;;;;N;;;;; +135B6;EGYPTIAN HIEROGLYPH-135B6;Lo;0;L;;;;;N;;;;; +135B7;EGYPTIAN HIEROGLYPH-135B7;Lo;0;L;;;;;N;;;;; +135B8;EGYPTIAN HIEROGLYPH-135B8;Lo;0;L;;;;;N;;;;; +135B9;EGYPTIAN HIEROGLYPH-135B9;Lo;0;L;;;;;N;;;;; +135BA;EGYPTIAN HIEROGLYPH-135BA;Lo;0;L;;;;;N;;;;; +135BB;EGYPTIAN HIEROGLYPH-135BB;Lo;0;L;;;;;N;;;;; +135BC;EGYPTIAN HIEROGLYPH-135BC;Lo;0;L;;;;;N;;;;; +135BD;EGYPTIAN HIEROGLYPH-135BD;Lo;0;L;;;;;N;;;;; +135BE;EGYPTIAN HIEROGLYPH-135BE;Lo;0;L;;;;;N;;;;; +135BF;EGYPTIAN HIEROGLYPH-135BF;Lo;0;L;;;;;N;;;;; +135C0;EGYPTIAN HIEROGLYPH-135C0;Lo;0;L;;;;;N;;;;; +135C1;EGYPTIAN HIEROGLYPH-135C1;Lo;0;L;;;;;N;;;;; +135C2;EGYPTIAN HIEROGLYPH-135C2;Lo;0;L;;;;;N;;;;; +135C3;EGYPTIAN HIEROGLYPH-135C3;Lo;0;L;;;;;N;;;;; +135C4;EGYPTIAN HIEROGLYPH-135C4;Lo;0;L;;;;;N;;;;; +135C5;EGYPTIAN HIEROGLYPH-135C5;Lo;0;L;;;;;N;;;;; +135C6;EGYPTIAN HIEROGLYPH-135C6;Lo;0;L;;;;;N;;;;; +135C7;EGYPTIAN HIEROGLYPH-135C7;Lo;0;L;;;;;N;;;;; +135C8;EGYPTIAN HIEROGLYPH-135C8;Lo;0;L;;;;;N;;;;; +135C9;EGYPTIAN HIEROGLYPH-135C9;Lo;0;L;;;;;N;;;;; +135CA;EGYPTIAN HIEROGLYPH-135CA;Lo;0;L;;;;;N;;;;; +135CB;EGYPTIAN HIEROGLYPH-135CB;Lo;0;L;;;;;N;;;;; +135CC;EGYPTIAN HIEROGLYPH-135CC;Lo;0;L;;;;;N;;;;; +135CD;EGYPTIAN HIEROGLYPH-135CD;Lo;0;L;;;;;N;;;;; +135CE;EGYPTIAN HIEROGLYPH-135CE;Lo;0;L;;;;;N;;;;; +135CF;EGYPTIAN HIEROGLYPH-135CF;Lo;0;L;;;;;N;;;;; +135D0;EGYPTIAN HIEROGLYPH-135D0;Lo;0;L;;;;;N;;;;; +135D1;EGYPTIAN HIEROGLYPH-135D1;Lo;0;L;;;;;N;;;;; +135D2;EGYPTIAN HIEROGLYPH-135D2;Lo;0;L;;;;;N;;;;; +135D3;EGYPTIAN HIEROGLYPH-135D3;Lo;0;L;;;;;N;;;;; +135D4;EGYPTIAN HIEROGLYPH-135D4;Lo;0;L;;;;;N;;;;; +135D5;EGYPTIAN HIEROGLYPH-135D5;Lo;0;L;;;;;N;;;;; +135D6;EGYPTIAN HIEROGLYPH-135D6;Lo;0;L;;;;;N;;;;; +135D7;EGYPTIAN HIEROGLYPH-135D7;Lo;0;L;;;;;N;;;;; +135D8;EGYPTIAN HIEROGLYPH-135D8;Lo;0;L;;;;;N;;;;; +135D9;EGYPTIAN HIEROGLYPH-135D9;Lo;0;L;;;;;N;;;;; +135DA;EGYPTIAN HIEROGLYPH-135DA;Lo;0;L;;;;;N;;;;; +135DB;EGYPTIAN HIEROGLYPH-135DB;Lo;0;L;;;;;N;;;;; +135DC;EGYPTIAN HIEROGLYPH-135DC;Lo;0;L;;;;;N;;;;; +135DD;EGYPTIAN HIEROGLYPH-135DD;Lo;0;L;;;;;N;;;;; +135DE;EGYPTIAN HIEROGLYPH-135DE;Lo;0;L;;;;;N;;;;; +135DF;EGYPTIAN HIEROGLYPH-135DF;Lo;0;L;;;;;N;;;;; +135E0;EGYPTIAN HIEROGLYPH-135E0;Lo;0;L;;;;;N;;;;; +135E1;EGYPTIAN HIEROGLYPH-135E1;Lo;0;L;;;;;N;;;;; +135E2;EGYPTIAN HIEROGLYPH-135E2;Lo;0;L;;;;;N;;;;; +135E3;EGYPTIAN HIEROGLYPH-135E3;Lo;0;L;;;;;N;;;;; +135E4;EGYPTIAN HIEROGLYPH-135E4;Lo;0;L;;;;;N;;;;; +135E5;EGYPTIAN HIEROGLYPH-135E5;Lo;0;L;;;;;N;;;;; +135E6;EGYPTIAN HIEROGLYPH-135E6;Lo;0;L;;;;;N;;;;; +135E7;EGYPTIAN HIEROGLYPH-135E7;Lo;0;L;;;;;N;;;;; +135E8;EGYPTIAN HIEROGLYPH-135E8;Lo;0;L;;;;;N;;;;; +135E9;EGYPTIAN HIEROGLYPH-135E9;Lo;0;L;;;;;N;;;;; +135EA;EGYPTIAN HIEROGLYPH-135EA;Lo;0;L;;;;;N;;;;; +135EB;EGYPTIAN HIEROGLYPH-135EB;Lo;0;L;;;;;N;;;;; +135EC;EGYPTIAN HIEROGLYPH-135EC;Lo;0;L;;;;;N;;;;; +135ED;EGYPTIAN HIEROGLYPH-135ED;Lo;0;L;;;;;N;;;;; +135EE;EGYPTIAN HIEROGLYPH-135EE;Lo;0;L;;;;;N;;;;; +135EF;EGYPTIAN HIEROGLYPH-135EF;Lo;0;L;;;;;N;;;;; +135F0;EGYPTIAN HIEROGLYPH-135F0;Lo;0;L;;;;;N;;;;; +135F1;EGYPTIAN HIEROGLYPH-135F1;Lo;0;L;;;;;N;;;;; +135F2;EGYPTIAN HIEROGLYPH-135F2;Lo;0;L;;;;;N;;;;; +135F3;EGYPTIAN HIEROGLYPH-135F3;Lo;0;L;;;;;N;;;;; +135F4;EGYPTIAN HIEROGLYPH-135F4;Lo;0;L;;;;;N;;;;; +135F5;EGYPTIAN HIEROGLYPH-135F5;Lo;0;L;;;;;N;;;;; +135F6;EGYPTIAN HIEROGLYPH-135F6;Lo;0;L;;;;;N;;;;; +135F7;EGYPTIAN HIEROGLYPH-135F7;Lo;0;L;;;;;N;;;;; +135F8;EGYPTIAN HIEROGLYPH-135F8;Lo;0;L;;;;;N;;;;; +135F9;EGYPTIAN HIEROGLYPH-135F9;Lo;0;L;;;;;N;;;;; +135FA;EGYPTIAN HIEROGLYPH-135FA;Lo;0;L;;;;;N;;;;; +135FB;EGYPTIAN HIEROGLYPH-135FB;Lo;0;L;;;;;N;;;;; +135FC;EGYPTIAN HIEROGLYPH-135FC;Lo;0;L;;;;;N;;;;; +135FD;EGYPTIAN HIEROGLYPH-135FD;Lo;0;L;;;;;N;;;;; +135FE;EGYPTIAN HIEROGLYPH-135FE;Lo;0;L;;;;;N;;;;; +135FF;EGYPTIAN HIEROGLYPH-135FF;Lo;0;L;;;;;N;;;;; +13600;EGYPTIAN HIEROGLYPH-13600;Lo;0;L;;;;;N;;;;; +13601;EGYPTIAN HIEROGLYPH-13601;Lo;0;L;;;;;N;;;;; +13602;EGYPTIAN HIEROGLYPH-13602;Lo;0;L;;;;;N;;;;; +13603;EGYPTIAN HIEROGLYPH-13603;Lo;0;L;;;;;N;;;;; +13604;EGYPTIAN HIEROGLYPH-13604;Lo;0;L;;;;;N;;;;; +13605;EGYPTIAN HIEROGLYPH-13605;Lo;0;L;;;;;N;;;;; +13606;EGYPTIAN HIEROGLYPH-13606;Lo;0;L;;;;;N;;;;; +13607;EGYPTIAN HIEROGLYPH-13607;Lo;0;L;;;;;N;;;;; +13608;EGYPTIAN HIEROGLYPH-13608;Lo;0;L;;;;;N;;;;; +13609;EGYPTIAN HIEROGLYPH-13609;Lo;0;L;;;;;N;;;;; +1360A;EGYPTIAN HIEROGLYPH-1360A;Lo;0;L;;;;;N;;;;; +1360B;EGYPTIAN HIEROGLYPH-1360B;Lo;0;L;;;;;N;;;;; +1360C;EGYPTIAN HIEROGLYPH-1360C;Lo;0;L;;;;;N;;;;; +1360D;EGYPTIAN HIEROGLYPH-1360D;Lo;0;L;;;;;N;;;;; +1360E;EGYPTIAN HIEROGLYPH-1360E;Lo;0;L;;;;;N;;;;; +1360F;EGYPTIAN HIEROGLYPH-1360F;Lo;0;L;;;;;N;;;;; +13610;EGYPTIAN HIEROGLYPH-13610;Lo;0;L;;;;;N;;;;; +13611;EGYPTIAN HIEROGLYPH-13611;Lo;0;L;;;;;N;;;;; +13612;EGYPTIAN HIEROGLYPH-13612;Lo;0;L;;;;;N;;;;; +13613;EGYPTIAN HIEROGLYPH-13613;Lo;0;L;;;;;N;;;;; +13614;EGYPTIAN HIEROGLYPH-13614;Lo;0;L;;;;;N;;;;; +13615;EGYPTIAN HIEROGLYPH-13615;Lo;0;L;;;;;N;;;;; +13616;EGYPTIAN HIEROGLYPH-13616;Lo;0;L;;;;;N;;;;; +13617;EGYPTIAN HIEROGLYPH-13617;Lo;0;L;;;;;N;;;;; +13618;EGYPTIAN HIEROGLYPH-13618;Lo;0;L;;;;;N;;;;; +13619;EGYPTIAN HIEROGLYPH-13619;Lo;0;L;;;;;N;;;;; +1361A;EGYPTIAN HIEROGLYPH-1361A;Lo;0;L;;;;;N;;;;; +1361B;EGYPTIAN HIEROGLYPH-1361B;Lo;0;L;;;;;N;;;;; +1361C;EGYPTIAN HIEROGLYPH-1361C;Lo;0;L;;;;;N;;;;; +1361D;EGYPTIAN HIEROGLYPH-1361D;Lo;0;L;;;;;N;;;;; +1361E;EGYPTIAN HIEROGLYPH-1361E;Lo;0;L;;;;;N;;;;; +1361F;EGYPTIAN HIEROGLYPH-1361F;Lo;0;L;;;;;N;;;;; +13620;EGYPTIAN HIEROGLYPH-13620;Lo;0;L;;;;;N;;;;; +13621;EGYPTIAN HIEROGLYPH-13621;Lo;0;L;;;;;N;;;;; +13622;EGYPTIAN HIEROGLYPH-13622;Lo;0;L;;;;;N;;;;; +13623;EGYPTIAN HIEROGLYPH-13623;Lo;0;L;;;;;N;;;;; +13624;EGYPTIAN HIEROGLYPH-13624;Lo;0;L;;;;;N;;;;; +13625;EGYPTIAN HIEROGLYPH-13625;Lo;0;L;;;;;N;;;;; +13626;EGYPTIAN HIEROGLYPH-13626;Lo;0;L;;;;;N;;;;; +13627;EGYPTIAN HIEROGLYPH-13627;Lo;0;L;;;;;N;;;;; +13628;EGYPTIAN HIEROGLYPH-13628;Lo;0;L;;;;;N;;;;; +13629;EGYPTIAN HIEROGLYPH-13629;Lo;0;L;;;;;N;;;;; +1362A;EGYPTIAN HIEROGLYPH-1362A;Lo;0;L;;;;;N;;;;; +1362B;EGYPTIAN HIEROGLYPH-1362B;Lo;0;L;;;;;N;;;;; +1362C;EGYPTIAN HIEROGLYPH-1362C;Lo;0;L;;;;;N;;;;; +1362D;EGYPTIAN HIEROGLYPH-1362D;Lo;0;L;;;;;N;;;;; +1362E;EGYPTIAN HIEROGLYPH-1362E;Lo;0;L;;;;;N;;;;; +1362F;EGYPTIAN HIEROGLYPH-1362F;Lo;0;L;;;;;N;;;;; +13630;EGYPTIAN HIEROGLYPH-13630;Lo;0;L;;;;;N;;;;; +13631;EGYPTIAN HIEROGLYPH-13631;Lo;0;L;;;;;N;;;;; +13632;EGYPTIAN HIEROGLYPH-13632;Lo;0;L;;;;;N;;;;; +13633;EGYPTIAN HIEROGLYPH-13633;Lo;0;L;;;;;N;;;;; +13634;EGYPTIAN HIEROGLYPH-13634;Lo;0;L;;;;;N;;;;; +13635;EGYPTIAN HIEROGLYPH-13635;Lo;0;L;;;;;N;;;;; +13636;EGYPTIAN HIEROGLYPH-13636;Lo;0;L;;;;;N;;;;; +13637;EGYPTIAN HIEROGLYPH-13637;Lo;0;L;;;;;N;;;;; +13638;EGYPTIAN HIEROGLYPH-13638;Lo;0;L;;;;;N;;;;; +13639;EGYPTIAN HIEROGLYPH-13639;Lo;0;L;;;;;N;;;;; +1363A;EGYPTIAN HIEROGLYPH-1363A;Lo;0;L;;;;;N;;;;; +1363B;EGYPTIAN HIEROGLYPH-1363B;Lo;0;L;;;;;N;;;;; +1363C;EGYPTIAN HIEROGLYPH-1363C;Lo;0;L;;;;;N;;;;; +1363D;EGYPTIAN HIEROGLYPH-1363D;Lo;0;L;;;;;N;;;;; +1363E;EGYPTIAN HIEROGLYPH-1363E;Lo;0;L;;;;;N;;;;; +1363F;EGYPTIAN HIEROGLYPH-1363F;Lo;0;L;;;;;N;;;;; +13640;EGYPTIAN HIEROGLYPH-13640;Lo;0;L;;;;;N;;;;; +13641;EGYPTIAN HIEROGLYPH-13641;Lo;0;L;;;;;N;;;;; +13642;EGYPTIAN HIEROGLYPH-13642;Lo;0;L;;;;;N;;;;; +13643;EGYPTIAN HIEROGLYPH-13643;Lo;0;L;;;;;N;;;;; +13644;EGYPTIAN HIEROGLYPH-13644;Lo;0;L;;;;;N;;;;; +13645;EGYPTIAN HIEROGLYPH-13645;Lo;0;L;;;;;N;;;;; +13646;EGYPTIAN HIEROGLYPH-13646;Lo;0;L;;;;;N;;;;; +13647;EGYPTIAN HIEROGLYPH-13647;Lo;0;L;;;;;N;;;;; +13648;EGYPTIAN HIEROGLYPH-13648;Lo;0;L;;;;;N;;;;; +13649;EGYPTIAN HIEROGLYPH-13649;Lo;0;L;;;;;N;;;;; +1364A;EGYPTIAN HIEROGLYPH-1364A;Lo;0;L;;;;;N;;;;; +1364B;EGYPTIAN HIEROGLYPH-1364B;Lo;0;L;;;;;N;;;;; +1364C;EGYPTIAN HIEROGLYPH-1364C;Lo;0;L;;;;;N;;;;; +1364D;EGYPTIAN HIEROGLYPH-1364D;Lo;0;L;;;;;N;;;;; +1364E;EGYPTIAN HIEROGLYPH-1364E;Lo;0;L;;;;;N;;;;; +1364F;EGYPTIAN HIEROGLYPH-1364F;Lo;0;L;;;;;N;;;;; +13650;EGYPTIAN HIEROGLYPH-13650;Lo;0;L;;;;;N;;;;; +13651;EGYPTIAN HIEROGLYPH-13651;Lo;0;L;;;;;N;;;;; +13652;EGYPTIAN HIEROGLYPH-13652;Lo;0;L;;;;;N;;;;; +13653;EGYPTIAN HIEROGLYPH-13653;Lo;0;L;;;;;N;;;;; +13654;EGYPTIAN HIEROGLYPH-13654;Lo;0;L;;;;;N;;;;; +13655;EGYPTIAN HIEROGLYPH-13655;Lo;0;L;;;;;N;;;;; +13656;EGYPTIAN HIEROGLYPH-13656;Lo;0;L;;;;;N;;;;; +13657;EGYPTIAN HIEROGLYPH-13657;Lo;0;L;;;;;N;;;;; +13658;EGYPTIAN HIEROGLYPH-13658;Lo;0;L;;;;;N;;;;; +13659;EGYPTIAN HIEROGLYPH-13659;Lo;0;L;;;;;N;;;;; +1365A;EGYPTIAN HIEROGLYPH-1365A;Lo;0;L;;;;;N;;;;; +1365B;EGYPTIAN HIEROGLYPH-1365B;Lo;0;L;;;;;N;;;;; +1365C;EGYPTIAN HIEROGLYPH-1365C;Lo;0;L;;;;;N;;;;; +1365D;EGYPTIAN HIEROGLYPH-1365D;Lo;0;L;;;;;N;;;;; +1365E;EGYPTIAN HIEROGLYPH-1365E;Lo;0;L;;;;;N;;;;; +1365F;EGYPTIAN HIEROGLYPH-1365F;Lo;0;L;;;;;N;;;;; +13660;EGYPTIAN HIEROGLYPH-13660;Lo;0;L;;;;;N;;;;; +13661;EGYPTIAN HIEROGLYPH-13661;Lo;0;L;;;;;N;;;;; +13662;EGYPTIAN HIEROGLYPH-13662;Lo;0;L;;;;;N;;;;; +13663;EGYPTIAN HIEROGLYPH-13663;Lo;0;L;;;;;N;;;;; +13664;EGYPTIAN HIEROGLYPH-13664;Lo;0;L;;;;;N;;;;; +13665;EGYPTIAN HIEROGLYPH-13665;Lo;0;L;;;;;N;;;;; +13666;EGYPTIAN HIEROGLYPH-13666;Lo;0;L;;;;;N;;;;; +13667;EGYPTIAN HIEROGLYPH-13667;Lo;0;L;;;;;N;;;;; +13668;EGYPTIAN HIEROGLYPH-13668;Lo;0;L;;;;;N;;;;; +13669;EGYPTIAN HIEROGLYPH-13669;Lo;0;L;;;;;N;;;;; +1366A;EGYPTIAN HIEROGLYPH-1366A;Lo;0;L;;;;;N;;;;; +1366B;EGYPTIAN HIEROGLYPH-1366B;Lo;0;L;;;;;N;;;;; +1366C;EGYPTIAN HIEROGLYPH-1366C;Lo;0;L;;;;;N;;;;; +1366D;EGYPTIAN HIEROGLYPH-1366D;Lo;0;L;;;;;N;;;;; +1366E;EGYPTIAN HIEROGLYPH-1366E;Lo;0;L;;;;;N;;;;; +1366F;EGYPTIAN HIEROGLYPH-1366F;Lo;0;L;;;;;N;;;;; +13670;EGYPTIAN HIEROGLYPH-13670;Lo;0;L;;;;;N;;;;; +13671;EGYPTIAN HIEROGLYPH-13671;Lo;0;L;;;;;N;;;;; +13672;EGYPTIAN HIEROGLYPH-13672;Lo;0;L;;;;;N;;;;; +13673;EGYPTIAN HIEROGLYPH-13673;Lo;0;L;;;;;N;;;;; +13674;EGYPTIAN HIEROGLYPH-13674;Lo;0;L;;;;;N;;;;; +13675;EGYPTIAN HIEROGLYPH-13675;Lo;0;L;;;;;N;;;;; +13676;EGYPTIAN HIEROGLYPH-13676;Lo;0;L;;;;;N;;;;; +13677;EGYPTIAN HIEROGLYPH-13677;Lo;0;L;;;;;N;;;;; +13678;EGYPTIAN HIEROGLYPH-13678;Lo;0;L;;;;;N;;;;; +13679;EGYPTIAN HIEROGLYPH-13679;Lo;0;L;;;;;N;;;;; +1367A;EGYPTIAN HIEROGLYPH-1367A;Lo;0;L;;;;;N;;;;; +1367B;EGYPTIAN HIEROGLYPH-1367B;Lo;0;L;;;;;N;;;;; +1367C;EGYPTIAN HIEROGLYPH-1367C;Lo;0;L;;;;;N;;;;; +1367D;EGYPTIAN HIEROGLYPH-1367D;Lo;0;L;;;;;N;;;;; +1367E;EGYPTIAN HIEROGLYPH-1367E;Lo;0;L;;;;;N;;;;; +1367F;EGYPTIAN HIEROGLYPH-1367F;Lo;0;L;;;;;N;;;;; +13680;EGYPTIAN HIEROGLYPH-13680;Lo;0;L;;;;;N;;;;; +13681;EGYPTIAN HIEROGLYPH-13681;Lo;0;L;;;;;N;;;;; +13682;EGYPTIAN HIEROGLYPH-13682;Lo;0;L;;;;;N;;;;; +13683;EGYPTIAN HIEROGLYPH-13683;Lo;0;L;;;;;N;;;;; +13684;EGYPTIAN HIEROGLYPH-13684;Lo;0;L;;;;;N;;;;; +13685;EGYPTIAN HIEROGLYPH-13685;Lo;0;L;;;;;N;;;;; +13686;EGYPTIAN HIEROGLYPH-13686;Lo;0;L;;;;;N;;;;; +13687;EGYPTIAN HIEROGLYPH-13687;Lo;0;L;;;;;N;;;;; +13688;EGYPTIAN HIEROGLYPH-13688;Lo;0;L;;;;;N;;;;; +13689;EGYPTIAN HIEROGLYPH-13689;Lo;0;L;;;;;N;;;;; +1368A;EGYPTIAN HIEROGLYPH-1368A;Lo;0;L;;;;;N;;;;; +1368B;EGYPTIAN HIEROGLYPH-1368B;Lo;0;L;;;;;N;;;;; +1368C;EGYPTIAN HIEROGLYPH-1368C;Lo;0;L;;;;;N;;;;; +1368D;EGYPTIAN HIEROGLYPH-1368D;Lo;0;L;;;;;N;;;;; +1368E;EGYPTIAN HIEROGLYPH-1368E;Lo;0;L;;;;;N;;;;; +1368F;EGYPTIAN HIEROGLYPH-1368F;Lo;0;L;;;;;N;;;;; +13690;EGYPTIAN HIEROGLYPH-13690;Lo;0;L;;;;;N;;;;; +13691;EGYPTIAN HIEROGLYPH-13691;Lo;0;L;;;;;N;;;;; +13692;EGYPTIAN HIEROGLYPH-13692;Lo;0;L;;;;;N;;;;; +13693;EGYPTIAN HIEROGLYPH-13693;Lo;0;L;;;;;N;;;;; +13694;EGYPTIAN HIEROGLYPH-13694;Lo;0;L;;;;;N;;;;; +13695;EGYPTIAN HIEROGLYPH-13695;Lo;0;L;;;;;N;;;;; +13696;EGYPTIAN HIEROGLYPH-13696;Lo;0;L;;;;;N;;;;; +13697;EGYPTIAN HIEROGLYPH-13697;Lo;0;L;;;;;N;;;;; +13698;EGYPTIAN HIEROGLYPH-13698;Lo;0;L;;;;;N;;;;; +13699;EGYPTIAN HIEROGLYPH-13699;Lo;0;L;;;;;N;;;;; +1369A;EGYPTIAN HIEROGLYPH-1369A;Lo;0;L;;;;;N;;;;; +1369B;EGYPTIAN HIEROGLYPH-1369B;Lo;0;L;;;;;N;;;;; +1369C;EGYPTIAN HIEROGLYPH-1369C;Lo;0;L;;;;;N;;;;; +1369D;EGYPTIAN HIEROGLYPH-1369D;Lo;0;L;;;;;N;;;;; +1369E;EGYPTIAN HIEROGLYPH-1369E;Lo;0;L;;;;;N;;;;; +1369F;EGYPTIAN HIEROGLYPH-1369F;Lo;0;L;;;;;N;;;;; +136A0;EGYPTIAN HIEROGLYPH-136A0;Lo;0;L;;;;;N;;;;; +136A1;EGYPTIAN HIEROGLYPH-136A1;Lo;0;L;;;;;N;;;;; +136A2;EGYPTIAN HIEROGLYPH-136A2;Lo;0;L;;;;;N;;;;; +136A3;EGYPTIAN HIEROGLYPH-136A3;Lo;0;L;;;;;N;;;;; +136A4;EGYPTIAN HIEROGLYPH-136A4;Lo;0;L;;;;;N;;;;; +136A5;EGYPTIAN HIEROGLYPH-136A5;Lo;0;L;;;;;N;;;;; +136A6;EGYPTIAN HIEROGLYPH-136A6;Lo;0;L;;;;;N;;;;; +136A7;EGYPTIAN HIEROGLYPH-136A7;Lo;0;L;;;;;N;;;;; +136A8;EGYPTIAN HIEROGLYPH-136A8;Lo;0;L;;;;;N;;;;; +136A9;EGYPTIAN HIEROGLYPH-136A9;Lo;0;L;;;;;N;;;;; +136AA;EGYPTIAN HIEROGLYPH-136AA;Lo;0;L;;;;;N;;;;; +136AB;EGYPTIAN HIEROGLYPH-136AB;Lo;0;L;;;;;N;;;;; +136AC;EGYPTIAN HIEROGLYPH-136AC;Lo;0;L;;;;;N;;;;; +136AD;EGYPTIAN HIEROGLYPH-136AD;Lo;0;L;;;;;N;;;;; +136AE;EGYPTIAN HIEROGLYPH-136AE;Lo;0;L;;;;;N;;;;; +136AF;EGYPTIAN HIEROGLYPH-136AF;Lo;0;L;;;;;N;;;;; +136B0;EGYPTIAN HIEROGLYPH-136B0;Lo;0;L;;;;;N;;;;; +136B1;EGYPTIAN HIEROGLYPH-136B1;Lo;0;L;;;;;N;;;;; +136B2;EGYPTIAN HIEROGLYPH-136B2;Lo;0;L;;;;;N;;;;; +136B3;EGYPTIAN HIEROGLYPH-136B3;Lo;0;L;;;;;N;;;;; +136B4;EGYPTIAN HIEROGLYPH-136B4;Lo;0;L;;;;;N;;;;; +136B5;EGYPTIAN HIEROGLYPH-136B5;Lo;0;L;;;;;N;;;;; +136B6;EGYPTIAN HIEROGLYPH-136B6;Lo;0;L;;;;;N;;;;; +136B7;EGYPTIAN HIEROGLYPH-136B7;Lo;0;L;;;;;N;;;;; +136B8;EGYPTIAN HIEROGLYPH-136B8;Lo;0;L;;;;;N;;;;; +136B9;EGYPTIAN HIEROGLYPH-136B9;Lo;0;L;;;;;N;;;;; +136BA;EGYPTIAN HIEROGLYPH-136BA;Lo;0;L;;;;;N;;;;; +136BB;EGYPTIAN HIEROGLYPH-136BB;Lo;0;L;;;;;N;;;;; +136BC;EGYPTIAN HIEROGLYPH-136BC;Lo;0;L;;;;;N;;;;; +136BD;EGYPTIAN HIEROGLYPH-136BD;Lo;0;L;;;;;N;;;;; +136BE;EGYPTIAN HIEROGLYPH-136BE;Lo;0;L;;;;;N;;;;; +136BF;EGYPTIAN HIEROGLYPH-136BF;Lo;0;L;;;;;N;;;;; +136C0;EGYPTIAN HIEROGLYPH-136C0;Lo;0;L;;;;;N;;;;; +136C1;EGYPTIAN HIEROGLYPH-136C1;Lo;0;L;;;;;N;;;;; +136C2;EGYPTIAN HIEROGLYPH-136C2;Lo;0;L;;;;;N;;;;; +136C3;EGYPTIAN HIEROGLYPH-136C3;Lo;0;L;;;;;N;;;;; +136C4;EGYPTIAN HIEROGLYPH-136C4;Lo;0;L;;;;;N;;;;; +136C5;EGYPTIAN HIEROGLYPH-136C5;Lo;0;L;;;;;N;;;;; +136C6;EGYPTIAN HIEROGLYPH-136C6;Lo;0;L;;;;;N;;;;; +136C7;EGYPTIAN HIEROGLYPH-136C7;Lo;0;L;;;;;N;;;;; +136C8;EGYPTIAN HIEROGLYPH-136C8;Lo;0;L;;;;;N;;;;; +136C9;EGYPTIAN HIEROGLYPH-136C9;Lo;0;L;;;;;N;;;;; +136CA;EGYPTIAN HIEROGLYPH-136CA;Lo;0;L;;;;;N;;;;; +136CB;EGYPTIAN HIEROGLYPH-136CB;Lo;0;L;;;;;N;;;;; +136CC;EGYPTIAN HIEROGLYPH-136CC;Lo;0;L;;;;;N;;;;; +136CD;EGYPTIAN HIEROGLYPH-136CD;Lo;0;L;;;;;N;;;;; +136CE;EGYPTIAN HIEROGLYPH-136CE;Lo;0;L;;;;;N;;;;; +136CF;EGYPTIAN HIEROGLYPH-136CF;Lo;0;L;;;;;N;;;;; +136D0;EGYPTIAN HIEROGLYPH-136D0;Lo;0;L;;;;;N;;;;; +136D1;EGYPTIAN HIEROGLYPH-136D1;Lo;0;L;;;;;N;;;;; +136D2;EGYPTIAN HIEROGLYPH-136D2;Lo;0;L;;;;;N;;;;; +136D3;EGYPTIAN HIEROGLYPH-136D3;Lo;0;L;;;;;N;;;;; +136D4;EGYPTIAN HIEROGLYPH-136D4;Lo;0;L;;;;;N;;;;; +136D5;EGYPTIAN HIEROGLYPH-136D5;Lo;0;L;;;;;N;;;;; +136D6;EGYPTIAN HIEROGLYPH-136D6;Lo;0;L;;;;;N;;;;; +136D7;EGYPTIAN HIEROGLYPH-136D7;Lo;0;L;;;;;N;;;;; +136D8;EGYPTIAN HIEROGLYPH-136D8;Lo;0;L;;;;;N;;;;; +136D9;EGYPTIAN HIEROGLYPH-136D9;Lo;0;L;;;;;N;;;;; +136DA;EGYPTIAN HIEROGLYPH-136DA;Lo;0;L;;;;;N;;;;; +136DB;EGYPTIAN HIEROGLYPH-136DB;Lo;0;L;;;;;N;;;;; +136DC;EGYPTIAN HIEROGLYPH-136DC;Lo;0;L;;;;;N;;;;; +136DD;EGYPTIAN HIEROGLYPH-136DD;Lo;0;L;;;;;N;;;;; +136DE;EGYPTIAN HIEROGLYPH-136DE;Lo;0;L;;;;;N;;;;; +136DF;EGYPTIAN HIEROGLYPH-136DF;Lo;0;L;;;;;N;;;;; +136E0;EGYPTIAN HIEROGLYPH-136E0;Lo;0;L;;;;;N;;;;; +136E1;EGYPTIAN HIEROGLYPH-136E1;Lo;0;L;;;;;N;;;;; +136E2;EGYPTIAN HIEROGLYPH-136E2;Lo;0;L;;;;;N;;;;; +136E3;EGYPTIAN HIEROGLYPH-136E3;Lo;0;L;;;;;N;;;;; +136E4;EGYPTIAN HIEROGLYPH-136E4;Lo;0;L;;;;;N;;;;; +136E5;EGYPTIAN HIEROGLYPH-136E5;Lo;0;L;;;;;N;;;;; +136E6;EGYPTIAN HIEROGLYPH-136E6;Lo;0;L;;;;;N;;;;; +136E7;EGYPTIAN HIEROGLYPH-136E7;Lo;0;L;;;;;N;;;;; +136E8;EGYPTIAN HIEROGLYPH-136E8;Lo;0;L;;;;;N;;;;; +136E9;EGYPTIAN HIEROGLYPH-136E9;Lo;0;L;;;;;N;;;;; +136EA;EGYPTIAN HIEROGLYPH-136EA;Lo;0;L;;;;;N;;;;; +136EB;EGYPTIAN HIEROGLYPH-136EB;Lo;0;L;;;;;N;;;;; +136EC;EGYPTIAN HIEROGLYPH-136EC;Lo;0;L;;;;;N;;;;; +136ED;EGYPTIAN HIEROGLYPH-136ED;Lo;0;L;;;;;N;;;;; +136EE;EGYPTIAN HIEROGLYPH-136EE;Lo;0;L;;;;;N;;;;; +136EF;EGYPTIAN HIEROGLYPH-136EF;Lo;0;L;;;;;N;;;;; +136F0;EGYPTIAN HIEROGLYPH-136F0;Lo;0;L;;;;;N;;;;; +136F1;EGYPTIAN HIEROGLYPH-136F1;Lo;0;L;;;;;N;;;;; +136F2;EGYPTIAN HIEROGLYPH-136F2;Lo;0;L;;;;;N;;;;; +136F3;EGYPTIAN HIEROGLYPH-136F3;Lo;0;L;;;;;N;;;;; +136F4;EGYPTIAN HIEROGLYPH-136F4;Lo;0;L;;;;;N;;;;; +136F5;EGYPTIAN HIEROGLYPH-136F5;Lo;0;L;;;;;N;;;;; +136F6;EGYPTIAN HIEROGLYPH-136F6;Lo;0;L;;;;;N;;;;; +136F7;EGYPTIAN HIEROGLYPH-136F7;Lo;0;L;;;;;N;;;;; +136F8;EGYPTIAN HIEROGLYPH-136F8;Lo;0;L;;;;;N;;;;; +136F9;EGYPTIAN HIEROGLYPH-136F9;Lo;0;L;;;;;N;;;;; +136FA;EGYPTIAN HIEROGLYPH-136FA;Lo;0;L;;;;;N;;;;; +136FB;EGYPTIAN HIEROGLYPH-136FB;Lo;0;L;;;;;N;;;;; +136FC;EGYPTIAN HIEROGLYPH-136FC;Lo;0;L;;;;;N;;;;; +136FD;EGYPTIAN HIEROGLYPH-136FD;Lo;0;L;;;;;N;;;;; +136FE;EGYPTIAN HIEROGLYPH-136FE;Lo;0;L;;;;;N;;;;; +136FF;EGYPTIAN HIEROGLYPH-136FF;Lo;0;L;;;;;N;;;;; +13700;EGYPTIAN HIEROGLYPH-13700;Lo;0;L;;;;;N;;;;; +13701;EGYPTIAN HIEROGLYPH-13701;Lo;0;L;;;;;N;;;;; +13702;EGYPTIAN HIEROGLYPH-13702;Lo;0;L;;;;;N;;;;; +13703;EGYPTIAN HIEROGLYPH-13703;Lo;0;L;;;;;N;;;;; +13704;EGYPTIAN HIEROGLYPH-13704;Lo;0;L;;;;;N;;;;; +13705;EGYPTIAN HIEROGLYPH-13705;Lo;0;L;;;;;N;;;;; +13706;EGYPTIAN HIEROGLYPH-13706;Lo;0;L;;;;;N;;;;; +13707;EGYPTIAN HIEROGLYPH-13707;Lo;0;L;;;;;N;;;;; +13708;EGYPTIAN HIEROGLYPH-13708;Lo;0;L;;;;;N;;;;; +13709;EGYPTIAN HIEROGLYPH-13709;Lo;0;L;;;;;N;;;;; +1370A;EGYPTIAN HIEROGLYPH-1370A;Lo;0;L;;;;;N;;;;; +1370B;EGYPTIAN HIEROGLYPH-1370B;Lo;0;L;;;;;N;;;;; +1370C;EGYPTIAN HIEROGLYPH-1370C;Lo;0;L;;;;;N;;;;; +1370D;EGYPTIAN HIEROGLYPH-1370D;Lo;0;L;;;;;N;;;;; +1370E;EGYPTIAN HIEROGLYPH-1370E;Lo;0;L;;;;;N;;;;; +1370F;EGYPTIAN HIEROGLYPH-1370F;Lo;0;L;;;;;N;;;;; +13710;EGYPTIAN HIEROGLYPH-13710;Lo;0;L;;;;;N;;;;; +13711;EGYPTIAN HIEROGLYPH-13711;Lo;0;L;;;;;N;;;;; +13712;EGYPTIAN HIEROGLYPH-13712;Lo;0;L;;;;;N;;;;; +13713;EGYPTIAN HIEROGLYPH-13713;Lo;0;L;;;;;N;;;;; +13714;EGYPTIAN HIEROGLYPH-13714;Lo;0;L;;;;;N;;;;; +13715;EGYPTIAN HIEROGLYPH-13715;Lo;0;L;;;;;N;;;;; +13716;EGYPTIAN HIEROGLYPH-13716;Lo;0;L;;;;;N;;;;; +13717;EGYPTIAN HIEROGLYPH-13717;Lo;0;L;;;;;N;;;;; +13718;EGYPTIAN HIEROGLYPH-13718;Lo;0;L;;;;;N;;;;; +13719;EGYPTIAN HIEROGLYPH-13719;Lo;0;L;;;;;N;;;;; +1371A;EGYPTIAN HIEROGLYPH-1371A;Lo;0;L;;;;;N;;;;; +1371B;EGYPTIAN HIEROGLYPH-1371B;Lo;0;L;;;;;N;;;;; +1371C;EGYPTIAN HIEROGLYPH-1371C;Lo;0;L;;;;;N;;;;; +1371D;EGYPTIAN HIEROGLYPH-1371D;Lo;0;L;;;;;N;;;;; +1371E;EGYPTIAN HIEROGLYPH-1371E;Lo;0;L;;;;;N;;;;; +1371F;EGYPTIAN HIEROGLYPH-1371F;Lo;0;L;;;;;N;;;;; +13720;EGYPTIAN HIEROGLYPH-13720;Lo;0;L;;;;;N;;;;; +13721;EGYPTIAN HIEROGLYPH-13721;Lo;0;L;;;;;N;;;;; +13722;EGYPTIAN HIEROGLYPH-13722;Lo;0;L;;;;;N;;;;; +13723;EGYPTIAN HIEROGLYPH-13723;Lo;0;L;;;;;N;;;;; +13724;EGYPTIAN HIEROGLYPH-13724;Lo;0;L;;;;;N;;;;; +13725;EGYPTIAN HIEROGLYPH-13725;Lo;0;L;;;;;N;;;;; +13726;EGYPTIAN HIEROGLYPH-13726;Lo;0;L;;;;;N;;;;; +13727;EGYPTIAN HIEROGLYPH-13727;Lo;0;L;;;;;N;;;;; +13728;EGYPTIAN HIEROGLYPH-13728;Lo;0;L;;;;;N;;;;; +13729;EGYPTIAN HIEROGLYPH-13729;Lo;0;L;;;;;N;;;;; +1372A;EGYPTIAN HIEROGLYPH-1372A;Lo;0;L;;;;;N;;;;; +1372B;EGYPTIAN HIEROGLYPH-1372B;Lo;0;L;;;;;N;;;;; +1372C;EGYPTIAN HIEROGLYPH-1372C;Lo;0;L;;;;;N;;;;; +1372D;EGYPTIAN HIEROGLYPH-1372D;Lo;0;L;;;;;N;;;;; +1372E;EGYPTIAN HIEROGLYPH-1372E;Lo;0;L;;;;;N;;;;; +1372F;EGYPTIAN HIEROGLYPH-1372F;Lo;0;L;;;;;N;;;;; +13730;EGYPTIAN HIEROGLYPH-13730;Lo;0;L;;;;;N;;;;; +13731;EGYPTIAN HIEROGLYPH-13731;Lo;0;L;;;;;N;;;;; +13732;EGYPTIAN HIEROGLYPH-13732;Lo;0;L;;;;;N;;;;; +13733;EGYPTIAN HIEROGLYPH-13733;Lo;0;L;;;;;N;;;;; +13734;EGYPTIAN HIEROGLYPH-13734;Lo;0;L;;;;;N;;;;; +13735;EGYPTIAN HIEROGLYPH-13735;Lo;0;L;;;;;N;;;;; +13736;EGYPTIAN HIEROGLYPH-13736;Lo;0;L;;;;;N;;;;; +13737;EGYPTIAN HIEROGLYPH-13737;Lo;0;L;;;;;N;;;;; +13738;EGYPTIAN HIEROGLYPH-13738;Lo;0;L;;;;;N;;;;; +13739;EGYPTIAN HIEROGLYPH-13739;Lo;0;L;;;;;N;;;;; +1373A;EGYPTIAN HIEROGLYPH-1373A;Lo;0;L;;;;;N;;;;; +1373B;EGYPTIAN HIEROGLYPH-1373B;Lo;0;L;;;;;N;;;;; +1373C;EGYPTIAN HIEROGLYPH-1373C;Lo;0;L;;;;;N;;;;; +1373D;EGYPTIAN HIEROGLYPH-1373D;Lo;0;L;;;;;N;;;;; +1373E;EGYPTIAN HIEROGLYPH-1373E;Lo;0;L;;;;;N;;;;; +1373F;EGYPTIAN HIEROGLYPH-1373F;Lo;0;L;;;;;N;;;;; +13740;EGYPTIAN HIEROGLYPH-13740;Lo;0;L;;;;;N;;;;; +13741;EGYPTIAN HIEROGLYPH-13741;Lo;0;L;;;;;N;;;;; +13742;EGYPTIAN HIEROGLYPH-13742;Lo;0;L;;;;;N;;;;; +13743;EGYPTIAN HIEROGLYPH-13743;Lo;0;L;;;;;N;;;;; +13744;EGYPTIAN HIEROGLYPH-13744;Lo;0;L;;;;;N;;;;; +13745;EGYPTIAN HIEROGLYPH-13745;Lo;0;L;;;;;N;;;;; +13746;EGYPTIAN HIEROGLYPH-13746;Lo;0;L;;;;;N;;;;; +13747;EGYPTIAN HIEROGLYPH-13747;Lo;0;L;;;;;N;;;;; +13748;EGYPTIAN HIEROGLYPH-13748;Lo;0;L;;;;;N;;;;; +13749;EGYPTIAN HIEROGLYPH-13749;Lo;0;L;;;;;N;;;;; +1374A;EGYPTIAN HIEROGLYPH-1374A;Lo;0;L;;;;;N;;;;; +1374B;EGYPTIAN HIEROGLYPH-1374B;Lo;0;L;;;;;N;;;;; +1374C;EGYPTIAN HIEROGLYPH-1374C;Lo;0;L;;;;;N;;;;; +1374D;EGYPTIAN HIEROGLYPH-1374D;Lo;0;L;;;;;N;;;;; +1374E;EGYPTIAN HIEROGLYPH-1374E;Lo;0;L;;;;;N;;;;; +1374F;EGYPTIAN HIEROGLYPH-1374F;Lo;0;L;;;;;N;;;;; +13750;EGYPTIAN HIEROGLYPH-13750;Lo;0;L;;;;;N;;;;; +13751;EGYPTIAN HIEROGLYPH-13751;Lo;0;L;;;;;N;;;;; +13752;EGYPTIAN HIEROGLYPH-13752;Lo;0;L;;;;;N;;;;; +13753;EGYPTIAN HIEROGLYPH-13753;Lo;0;L;;;;;N;;;;; +13754;EGYPTIAN HIEROGLYPH-13754;Lo;0;L;;;;;N;;;;; +13755;EGYPTIAN HIEROGLYPH-13755;Lo;0;L;;;;;N;;;;; +13756;EGYPTIAN HIEROGLYPH-13756;Lo;0;L;;;;;N;;;;; +13757;EGYPTIAN HIEROGLYPH-13757;Lo;0;L;;;;;N;;;;; +13758;EGYPTIAN HIEROGLYPH-13758;Lo;0;L;;;;;N;;;;; +13759;EGYPTIAN HIEROGLYPH-13759;Lo;0;L;;;;;N;;;;; +1375A;EGYPTIAN HIEROGLYPH-1375A;Lo;0;L;;;;;N;;;;; +1375B;EGYPTIAN HIEROGLYPH-1375B;Lo;0;L;;;;;N;;;;; +1375C;EGYPTIAN HIEROGLYPH-1375C;Lo;0;L;;;;;N;;;;; +1375D;EGYPTIAN HIEROGLYPH-1375D;Lo;0;L;;;;;N;;;;; +1375E;EGYPTIAN HIEROGLYPH-1375E;Lo;0;L;;;;;N;;;;; +1375F;EGYPTIAN HIEROGLYPH-1375F;Lo;0;L;;;;;N;;;;; +13760;EGYPTIAN HIEROGLYPH-13760;Lo;0;L;;;;;N;;;;; +13761;EGYPTIAN HIEROGLYPH-13761;Lo;0;L;;;;;N;;;;; +13762;EGYPTIAN HIEROGLYPH-13762;Lo;0;L;;;;;N;;;;; +13763;EGYPTIAN HIEROGLYPH-13763;Lo;0;L;;;;;N;;;;; +13764;EGYPTIAN HIEROGLYPH-13764;Lo;0;L;;;;;N;;;;; +13765;EGYPTIAN HIEROGLYPH-13765;Lo;0;L;;;;;N;;;;; +13766;EGYPTIAN HIEROGLYPH-13766;Lo;0;L;;;;;N;;;;; +13767;EGYPTIAN HIEROGLYPH-13767;Lo;0;L;;;;;N;;;;; +13768;EGYPTIAN HIEROGLYPH-13768;Lo;0;L;;;;;N;;;;; +13769;EGYPTIAN HIEROGLYPH-13769;Lo;0;L;;;;;N;;;;; +1376A;EGYPTIAN HIEROGLYPH-1376A;Lo;0;L;;;;;N;;;;; +1376B;EGYPTIAN HIEROGLYPH-1376B;Lo;0;L;;;;;N;;;;; +1376C;EGYPTIAN HIEROGLYPH-1376C;Lo;0;L;;;;;N;;;;; +1376D;EGYPTIAN HIEROGLYPH-1376D;Lo;0;L;;;;;N;;;;; +1376E;EGYPTIAN HIEROGLYPH-1376E;Lo;0;L;;;;;N;;;;; +1376F;EGYPTIAN HIEROGLYPH-1376F;Lo;0;L;;;;;N;;;;; +13770;EGYPTIAN HIEROGLYPH-13770;Lo;0;L;;;;;N;;;;; +13771;EGYPTIAN HIEROGLYPH-13771;Lo;0;L;;;;;N;;;;; +13772;EGYPTIAN HIEROGLYPH-13772;Lo;0;L;;;;;N;;;;; +13773;EGYPTIAN HIEROGLYPH-13773;Lo;0;L;;;;;N;;;;; +13774;EGYPTIAN HIEROGLYPH-13774;Lo;0;L;;;;;N;;;;; +13775;EGYPTIAN HIEROGLYPH-13775;Lo;0;L;;;;;N;;;;; +13776;EGYPTIAN HIEROGLYPH-13776;Lo;0;L;;;;;N;;;;; +13777;EGYPTIAN HIEROGLYPH-13777;Lo;0;L;;;;;N;;;;; +13778;EGYPTIAN HIEROGLYPH-13778;Lo;0;L;;;;;N;;;;; +13779;EGYPTIAN HIEROGLYPH-13779;Lo;0;L;;;;;N;;;;; +1377A;EGYPTIAN HIEROGLYPH-1377A;Lo;0;L;;;;;N;;;;; +1377B;EGYPTIAN HIEROGLYPH-1377B;Lo;0;L;;;;;N;;;;; +1377C;EGYPTIAN HIEROGLYPH-1377C;Lo;0;L;;;;;N;;;;; +1377D;EGYPTIAN HIEROGLYPH-1377D;Lo;0;L;;;;;N;;;;; +1377E;EGYPTIAN HIEROGLYPH-1377E;Lo;0;L;;;;;N;;;;; +1377F;EGYPTIAN HIEROGLYPH-1377F;Lo;0;L;;;;;N;;;;; +13780;EGYPTIAN HIEROGLYPH-13780;Lo;0;L;;;;;N;;;;; +13781;EGYPTIAN HIEROGLYPH-13781;Lo;0;L;;;;;N;;;;; +13782;EGYPTIAN HIEROGLYPH-13782;Lo;0;L;;;;;N;;;;; +13783;EGYPTIAN HIEROGLYPH-13783;Lo;0;L;;;;;N;;;;; +13784;EGYPTIAN HIEROGLYPH-13784;Lo;0;L;;;;;N;;;;; +13785;EGYPTIAN HIEROGLYPH-13785;Lo;0;L;;;;;N;;;;; +13786;EGYPTIAN HIEROGLYPH-13786;Lo;0;L;;;;;N;;;;; +13787;EGYPTIAN HIEROGLYPH-13787;Lo;0;L;;;;;N;;;;; +13788;EGYPTIAN HIEROGLYPH-13788;Lo;0;L;;;;;N;;;;; +13789;EGYPTIAN HIEROGLYPH-13789;Lo;0;L;;;;;N;;;;; +1378A;EGYPTIAN HIEROGLYPH-1378A;Lo;0;L;;;;;N;;;;; +1378B;EGYPTIAN HIEROGLYPH-1378B;Lo;0;L;;;;;N;;;;; +1378C;EGYPTIAN HIEROGLYPH-1378C;Lo;0;L;;;;;N;;;;; +1378D;EGYPTIAN HIEROGLYPH-1378D;Lo;0;L;;;;;N;;;;; +1378E;EGYPTIAN HIEROGLYPH-1378E;Lo;0;L;;;;;N;;;;; +1378F;EGYPTIAN HIEROGLYPH-1378F;Lo;0;L;;;;;N;;;;; +13790;EGYPTIAN HIEROGLYPH-13790;Lo;0;L;;;;;N;;;;; +13791;EGYPTIAN HIEROGLYPH-13791;Lo;0;L;;;;;N;;;;; +13792;EGYPTIAN HIEROGLYPH-13792;Lo;0;L;;;;;N;;;;; +13793;EGYPTIAN HIEROGLYPH-13793;Lo;0;L;;;;;N;;;;; +13794;EGYPTIAN HIEROGLYPH-13794;Lo;0;L;;;;;N;;;;; +13795;EGYPTIAN HIEROGLYPH-13795;Lo;0;L;;;;;N;;;;; +13796;EGYPTIAN HIEROGLYPH-13796;Lo;0;L;;;;;N;;;;; +13797;EGYPTIAN HIEROGLYPH-13797;Lo;0;L;;;;;N;;;;; +13798;EGYPTIAN HIEROGLYPH-13798;Lo;0;L;;;;;N;;;;; +13799;EGYPTIAN HIEROGLYPH-13799;Lo;0;L;;;;;N;;;;; +1379A;EGYPTIAN HIEROGLYPH-1379A;Lo;0;L;;;;;N;;;;; +1379B;EGYPTIAN HIEROGLYPH-1379B;Lo;0;L;;;;;N;;;;; +1379C;EGYPTIAN HIEROGLYPH-1379C;Lo;0;L;;;;;N;;;;; +1379D;EGYPTIAN HIEROGLYPH-1379D;Lo;0;L;;;;;N;;;;; +1379E;EGYPTIAN HIEROGLYPH-1379E;Lo;0;L;;;;;N;;;;; +1379F;EGYPTIAN HIEROGLYPH-1379F;Lo;0;L;;;;;N;;;;; +137A0;EGYPTIAN HIEROGLYPH-137A0;Lo;0;L;;;;;N;;;;; +137A1;EGYPTIAN HIEROGLYPH-137A1;Lo;0;L;;;;;N;;;;; +137A2;EGYPTIAN HIEROGLYPH-137A2;Lo;0;L;;;;;N;;;;; +137A3;EGYPTIAN HIEROGLYPH-137A3;Lo;0;L;;;;;N;;;;; +137A4;EGYPTIAN HIEROGLYPH-137A4;Lo;0;L;;;;;N;;;;; +137A5;EGYPTIAN HIEROGLYPH-137A5;Lo;0;L;;;;;N;;;;; +137A6;EGYPTIAN HIEROGLYPH-137A6;Lo;0;L;;;;;N;;;;; +137A7;EGYPTIAN HIEROGLYPH-137A7;Lo;0;L;;;;;N;;;;; +137A8;EGYPTIAN HIEROGLYPH-137A8;Lo;0;L;;;;;N;;;;; +137A9;EGYPTIAN HIEROGLYPH-137A9;Lo;0;L;;;;;N;;;;; +137AA;EGYPTIAN HIEROGLYPH-137AA;Lo;0;L;;;;;N;;;;; +137AB;EGYPTIAN HIEROGLYPH-137AB;Lo;0;L;;;;;N;;;;; +137AC;EGYPTIAN HIEROGLYPH-137AC;Lo;0;L;;;;;N;;;;; +137AD;EGYPTIAN HIEROGLYPH-137AD;Lo;0;L;;;;;N;;;;; +137AE;EGYPTIAN HIEROGLYPH-137AE;Lo;0;L;;;;;N;;;;; +137AF;EGYPTIAN HIEROGLYPH-137AF;Lo;0;L;;;;;N;;;;; +137B0;EGYPTIAN HIEROGLYPH-137B0;Lo;0;L;;;;;N;;;;; +137B1;EGYPTIAN HIEROGLYPH-137B1;Lo;0;L;;;;;N;;;;; +137B2;EGYPTIAN HIEROGLYPH-137B2;Lo;0;L;;;;;N;;;;; +137B3;EGYPTIAN HIEROGLYPH-137B3;Lo;0;L;;;;;N;;;;; +137B4;EGYPTIAN HIEROGLYPH-137B4;Lo;0;L;;;;;N;;;;; +137B5;EGYPTIAN HIEROGLYPH-137B5;Lo;0;L;;;;;N;;;;; +137B6;EGYPTIAN HIEROGLYPH-137B6;Lo;0;L;;;;;N;;;;; +137B7;EGYPTIAN HIEROGLYPH-137B7;Lo;0;L;;;;;N;;;;; +137B8;EGYPTIAN HIEROGLYPH-137B8;Lo;0;L;;;;;N;;;;; +137B9;EGYPTIAN HIEROGLYPH-137B9;Lo;0;L;;;;;N;;;;; +137BA;EGYPTIAN HIEROGLYPH-137BA;Lo;0;L;;;;;N;;;;; +137BB;EGYPTIAN HIEROGLYPH-137BB;Lo;0;L;;;;;N;;;;; +137BC;EGYPTIAN HIEROGLYPH-137BC;Lo;0;L;;;;;N;;;;; +137BD;EGYPTIAN HIEROGLYPH-137BD;Lo;0;L;;;;;N;;;;; +137BE;EGYPTIAN HIEROGLYPH-137BE;Lo;0;L;;;;;N;;;;; +137BF;EGYPTIAN HIEROGLYPH-137BF;Lo;0;L;;;;;N;;;;; +137C0;EGYPTIAN HIEROGLYPH-137C0;Lo;0;L;;;;;N;;;;; +137C1;EGYPTIAN HIEROGLYPH-137C1;Lo;0;L;;;;;N;;;;; +137C2;EGYPTIAN HIEROGLYPH-137C2;Lo;0;L;;;;;N;;;;; +137C3;EGYPTIAN HIEROGLYPH-137C3;Lo;0;L;;;;;N;;;;; +137C4;EGYPTIAN HIEROGLYPH-137C4;Lo;0;L;;;;;N;;;;; +137C5;EGYPTIAN HIEROGLYPH-137C5;Lo;0;L;;;;;N;;;;; +137C6;EGYPTIAN HIEROGLYPH-137C6;Lo;0;L;;;;;N;;;;; +137C7;EGYPTIAN HIEROGLYPH-137C7;Lo;0;L;;;;;N;;;;; +137C8;EGYPTIAN HIEROGLYPH-137C8;Lo;0;L;;;;;N;;;;; +137C9;EGYPTIAN HIEROGLYPH-137C9;Lo;0;L;;;;;N;;;;; +137CA;EGYPTIAN HIEROGLYPH-137CA;Lo;0;L;;;;;N;;;;; +137CB;EGYPTIAN HIEROGLYPH-137CB;Lo;0;L;;;;;N;;;;; +137CC;EGYPTIAN HIEROGLYPH-137CC;Lo;0;L;;;;;N;;;;; +137CD;EGYPTIAN HIEROGLYPH-137CD;Lo;0;L;;;;;N;;;;; +137CE;EGYPTIAN HIEROGLYPH-137CE;Lo;0;L;;;;;N;;;;; +137CF;EGYPTIAN HIEROGLYPH-137CF;Lo;0;L;;;;;N;;;;; +137D0;EGYPTIAN HIEROGLYPH-137D0;Lo;0;L;;;;;N;;;;; +137D1;EGYPTIAN HIEROGLYPH-137D1;Lo;0;L;;;;;N;;;;; +137D2;EGYPTIAN HIEROGLYPH-137D2;Lo;0;L;;;;;N;;;;; +137D3;EGYPTIAN HIEROGLYPH-137D3;Lo;0;L;;;;;N;;;;; +137D4;EGYPTIAN HIEROGLYPH-137D4;Lo;0;L;;;;;N;;;;; +137D5;EGYPTIAN HIEROGLYPH-137D5;Lo;0;L;;;;;N;;;;; +137D6;EGYPTIAN HIEROGLYPH-137D6;Lo;0;L;;;;;N;;;;; +137D7;EGYPTIAN HIEROGLYPH-137D7;Lo;0;L;;;;;N;;;;; +137D8;EGYPTIAN HIEROGLYPH-137D8;Lo;0;L;;;;;N;;;;; +137D9;EGYPTIAN HIEROGLYPH-137D9;Lo;0;L;;;;;N;;;;; +137DA;EGYPTIAN HIEROGLYPH-137DA;Lo;0;L;;;;;N;;;;; +137DB;EGYPTIAN HIEROGLYPH-137DB;Lo;0;L;;;;;N;;;;; +137DC;EGYPTIAN HIEROGLYPH-137DC;Lo;0;L;;;;;N;;;;; +137DD;EGYPTIAN HIEROGLYPH-137DD;Lo;0;L;;;;;N;;;;; +137DE;EGYPTIAN HIEROGLYPH-137DE;Lo;0;L;;;;;N;;;;; +137DF;EGYPTIAN HIEROGLYPH-137DF;Lo;0;L;;;;;N;;;;; +137E0;EGYPTIAN HIEROGLYPH-137E0;Lo;0;L;;;;;N;;;;; +137E1;EGYPTIAN HIEROGLYPH-137E1;Lo;0;L;;;;;N;;;;; +137E2;EGYPTIAN HIEROGLYPH-137E2;Lo;0;L;;;;;N;;;;; +137E3;EGYPTIAN HIEROGLYPH-137E3;Lo;0;L;;;;;N;;;;; +137E4;EGYPTIAN HIEROGLYPH-137E4;Lo;0;L;;;;;N;;;;; +137E5;EGYPTIAN HIEROGLYPH-137E5;Lo;0;L;;;;;N;;;;; +137E6;EGYPTIAN HIEROGLYPH-137E6;Lo;0;L;;;;;N;;;;; +137E7;EGYPTIAN HIEROGLYPH-137E7;Lo;0;L;;;;;N;;;;; +137E8;EGYPTIAN HIEROGLYPH-137E8;Lo;0;L;;;;;N;;;;; +137E9;EGYPTIAN HIEROGLYPH-137E9;Lo;0;L;;;;;N;;;;; +137EA;EGYPTIAN HIEROGLYPH-137EA;Lo;0;L;;;;;N;;;;; +137EB;EGYPTIAN HIEROGLYPH-137EB;Lo;0;L;;;;;N;;;;; +137EC;EGYPTIAN HIEROGLYPH-137EC;Lo;0;L;;;;;N;;;;; +137ED;EGYPTIAN HIEROGLYPH-137ED;Lo;0;L;;;;;N;;;;; +137EE;EGYPTIAN HIEROGLYPH-137EE;Lo;0;L;;;;;N;;;;; +137EF;EGYPTIAN HIEROGLYPH-137EF;Lo;0;L;;;;;N;;;;; +137F0;EGYPTIAN HIEROGLYPH-137F0;Lo;0;L;;;;;N;;;;; +137F1;EGYPTIAN HIEROGLYPH-137F1;Lo;0;L;;;;;N;;;;; +137F2;EGYPTIAN HIEROGLYPH-137F2;Lo;0;L;;;;;N;;;;; +137F3;EGYPTIAN HIEROGLYPH-137F3;Lo;0;L;;;;;N;;;;; +137F4;EGYPTIAN HIEROGLYPH-137F4;Lo;0;L;;;;;N;;;;; +137F5;EGYPTIAN HIEROGLYPH-137F5;Lo;0;L;;;;;N;;;;; +137F6;EGYPTIAN HIEROGLYPH-137F6;Lo;0;L;;;;;N;;;;; +137F7;EGYPTIAN HIEROGLYPH-137F7;Lo;0;L;;;;;N;;;;; +137F8;EGYPTIAN HIEROGLYPH-137F8;Lo;0;L;;;;;N;;;;; +137F9;EGYPTIAN HIEROGLYPH-137F9;Lo;0;L;;;;;N;;;;; +137FA;EGYPTIAN HIEROGLYPH-137FA;Lo;0;L;;;;;N;;;;; +137FB;EGYPTIAN HIEROGLYPH-137FB;Lo;0;L;;;;;N;;;;; +137FC;EGYPTIAN HIEROGLYPH-137FC;Lo;0;L;;;;;N;;;;; +137FD;EGYPTIAN HIEROGLYPH-137FD;Lo;0;L;;;;;N;;;;; +137FE;EGYPTIAN HIEROGLYPH-137FE;Lo;0;L;;;;;N;;;;; +137FF;EGYPTIAN HIEROGLYPH-137FF;Lo;0;L;;;;;N;;;;; +13800;EGYPTIAN HIEROGLYPH-13800;Lo;0;L;;;;;N;;;;; +13801;EGYPTIAN HIEROGLYPH-13801;Lo;0;L;;;;;N;;;;; +13802;EGYPTIAN HIEROGLYPH-13802;Lo;0;L;;;;;N;;;;; +13803;EGYPTIAN HIEROGLYPH-13803;Lo;0;L;;;;;N;;;;; +13804;EGYPTIAN HIEROGLYPH-13804;Lo;0;L;;;;;N;;;;; +13805;EGYPTIAN HIEROGLYPH-13805;Lo;0;L;;;;;N;;;;; +13806;EGYPTIAN HIEROGLYPH-13806;Lo;0;L;;;;;N;;;;; +13807;EGYPTIAN HIEROGLYPH-13807;Lo;0;L;;;;;N;;;;; +13808;EGYPTIAN HIEROGLYPH-13808;Lo;0;L;;;;;N;;;;; +13809;EGYPTIAN HIEROGLYPH-13809;Lo;0;L;;;;;N;;;;; +1380A;EGYPTIAN HIEROGLYPH-1380A;Lo;0;L;;;;;N;;;;; +1380B;EGYPTIAN HIEROGLYPH-1380B;Lo;0;L;;;;;N;;;;; +1380C;EGYPTIAN HIEROGLYPH-1380C;Lo;0;L;;;;;N;;;;; +1380D;EGYPTIAN HIEROGLYPH-1380D;Lo;0;L;;;;;N;;;;; +1380E;EGYPTIAN HIEROGLYPH-1380E;Lo;0;L;;;;;N;;;;; +1380F;EGYPTIAN HIEROGLYPH-1380F;Lo;0;L;;;;;N;;;;; +13810;EGYPTIAN HIEROGLYPH-13810;Lo;0;L;;;;;N;;;;; +13811;EGYPTIAN HIEROGLYPH-13811;Lo;0;L;;;;;N;;;;; +13812;EGYPTIAN HIEROGLYPH-13812;Lo;0;L;;;;;N;;;;; +13813;EGYPTIAN HIEROGLYPH-13813;Lo;0;L;;;;;N;;;;; +13814;EGYPTIAN HIEROGLYPH-13814;Lo;0;L;;;;;N;;;;; +13815;EGYPTIAN HIEROGLYPH-13815;Lo;0;L;;;;;N;;;;; +13816;EGYPTIAN HIEROGLYPH-13816;Lo;0;L;;;;;N;;;;; +13817;EGYPTIAN HIEROGLYPH-13817;Lo;0;L;;;;;N;;;;; +13818;EGYPTIAN HIEROGLYPH-13818;Lo;0;L;;;;;N;;;;; +13819;EGYPTIAN HIEROGLYPH-13819;Lo;0;L;;;;;N;;;;; +1381A;EGYPTIAN HIEROGLYPH-1381A;Lo;0;L;;;;;N;;;;; +1381B;EGYPTIAN HIEROGLYPH-1381B;Lo;0;L;;;;;N;;;;; +1381C;EGYPTIAN HIEROGLYPH-1381C;Lo;0;L;;;;;N;;;;; +1381D;EGYPTIAN HIEROGLYPH-1381D;Lo;0;L;;;;;N;;;;; +1381E;EGYPTIAN HIEROGLYPH-1381E;Lo;0;L;;;;;N;;;;; +1381F;EGYPTIAN HIEROGLYPH-1381F;Lo;0;L;;;;;N;;;;; +13820;EGYPTIAN HIEROGLYPH-13820;Lo;0;L;;;;;N;;;;; +13821;EGYPTIAN HIEROGLYPH-13821;Lo;0;L;;;;;N;;;;; +13822;EGYPTIAN HIEROGLYPH-13822;Lo;0;L;;;;;N;;;;; +13823;EGYPTIAN HIEROGLYPH-13823;Lo;0;L;;;;;N;;;;; +13824;EGYPTIAN HIEROGLYPH-13824;Lo;0;L;;;;;N;;;;; +13825;EGYPTIAN HIEROGLYPH-13825;Lo;0;L;;;;;N;;;;; +13826;EGYPTIAN HIEROGLYPH-13826;Lo;0;L;;;;;N;;;;; +13827;EGYPTIAN HIEROGLYPH-13827;Lo;0;L;;;;;N;;;;; +13828;EGYPTIAN HIEROGLYPH-13828;Lo;0;L;;;;;N;;;;; +13829;EGYPTIAN HIEROGLYPH-13829;Lo;0;L;;;;;N;;;;; +1382A;EGYPTIAN HIEROGLYPH-1382A;Lo;0;L;;;;;N;;;;; +1382B;EGYPTIAN HIEROGLYPH-1382B;Lo;0;L;;;;;N;;;;; +1382C;EGYPTIAN HIEROGLYPH-1382C;Lo;0;L;;;;;N;;;;; +1382D;EGYPTIAN HIEROGLYPH-1382D;Lo;0;L;;;;;N;;;;; +1382E;EGYPTIAN HIEROGLYPH-1382E;Lo;0;L;;;;;N;;;;; +1382F;EGYPTIAN HIEROGLYPH-1382F;Lo;0;L;;;;;N;;;;; +13830;EGYPTIAN HIEROGLYPH-13830;Lo;0;L;;;;;N;;;;; +13831;EGYPTIAN HIEROGLYPH-13831;Lo;0;L;;;;;N;;;;; +13832;EGYPTIAN HIEROGLYPH-13832;Lo;0;L;;;;;N;;;;; +13833;EGYPTIAN HIEROGLYPH-13833;Lo;0;L;;;;;N;;;;; +13834;EGYPTIAN HIEROGLYPH-13834;Lo;0;L;;;;;N;;;;; +13835;EGYPTIAN HIEROGLYPH-13835;Lo;0;L;;;;;N;;;;; +13836;EGYPTIAN HIEROGLYPH-13836;Lo;0;L;;;;;N;;;;; +13837;EGYPTIAN HIEROGLYPH-13837;Lo;0;L;;;;;N;;;;; +13838;EGYPTIAN HIEROGLYPH-13838;Lo;0;L;;;;;N;;;;; +13839;EGYPTIAN HIEROGLYPH-13839;Lo;0;L;;;;;N;;;;; +1383A;EGYPTIAN HIEROGLYPH-1383A;Lo;0;L;;;;;N;;;;; +1383B;EGYPTIAN HIEROGLYPH-1383B;Lo;0;L;;;;;N;;;;; +1383C;EGYPTIAN HIEROGLYPH-1383C;Lo;0;L;;;;;N;;;;; +1383D;EGYPTIAN HIEROGLYPH-1383D;Lo;0;L;;;;;N;;;;; +1383E;EGYPTIAN HIEROGLYPH-1383E;Lo;0;L;;;;;N;;;;; +1383F;EGYPTIAN HIEROGLYPH-1383F;Lo;0;L;;;;;N;;;;; +13840;EGYPTIAN HIEROGLYPH-13840;Lo;0;L;;;;;N;;;;; +13841;EGYPTIAN HIEROGLYPH-13841;Lo;0;L;;;;;N;;;;; +13842;EGYPTIAN HIEROGLYPH-13842;Lo;0;L;;;;;N;;;;; +13843;EGYPTIAN HIEROGLYPH-13843;Lo;0;L;;;;;N;;;;; +13844;EGYPTIAN HIEROGLYPH-13844;Lo;0;L;;;;;N;;;;; +13845;EGYPTIAN HIEROGLYPH-13845;Lo;0;L;;;;;N;;;;; +13846;EGYPTIAN HIEROGLYPH-13846;Lo;0;L;;;;;N;;;;; +13847;EGYPTIAN HIEROGLYPH-13847;Lo;0;L;;;;;N;;;;; +13848;EGYPTIAN HIEROGLYPH-13848;Lo;0;L;;;;;N;;;;; +13849;EGYPTIAN HIEROGLYPH-13849;Lo;0;L;;;;;N;;;;; +1384A;EGYPTIAN HIEROGLYPH-1384A;Lo;0;L;;;;;N;;;;; +1384B;EGYPTIAN HIEROGLYPH-1384B;Lo;0;L;;;;;N;;;;; +1384C;EGYPTIAN HIEROGLYPH-1384C;Lo;0;L;;;;;N;;;;; +1384D;EGYPTIAN HIEROGLYPH-1384D;Lo;0;L;;;;;N;;;;; +1384E;EGYPTIAN HIEROGLYPH-1384E;Lo;0;L;;;;;N;;;;; +1384F;EGYPTIAN HIEROGLYPH-1384F;Lo;0;L;;;;;N;;;;; +13850;EGYPTIAN HIEROGLYPH-13850;Lo;0;L;;;;;N;;;;; +13851;EGYPTIAN HIEROGLYPH-13851;Lo;0;L;;;;;N;;;;; +13852;EGYPTIAN HIEROGLYPH-13852;Lo;0;L;;;;;N;;;;; +13853;EGYPTIAN HIEROGLYPH-13853;Lo;0;L;;;;;N;;;;; +13854;EGYPTIAN HIEROGLYPH-13854;Lo;0;L;;;;;N;;;;; +13855;EGYPTIAN HIEROGLYPH-13855;Lo;0;L;;;;;N;;;;; +13856;EGYPTIAN HIEROGLYPH-13856;Lo;0;L;;;;;N;;;;; +13857;EGYPTIAN HIEROGLYPH-13857;Lo;0;L;;;;;N;;;;; +13858;EGYPTIAN HIEROGLYPH-13858;Lo;0;L;;;;;N;;;;; +13859;EGYPTIAN HIEROGLYPH-13859;Lo;0;L;;;;;N;;;;; +1385A;EGYPTIAN HIEROGLYPH-1385A;Lo;0;L;;;;;N;;;;; +1385B;EGYPTIAN HIEROGLYPH-1385B;Lo;0;L;;;;;N;;;;; +1385C;EGYPTIAN HIEROGLYPH-1385C;Lo;0;L;;;;;N;;;;; +1385D;EGYPTIAN HIEROGLYPH-1385D;Lo;0;L;;;;;N;;;;; +1385E;EGYPTIAN HIEROGLYPH-1385E;Lo;0;L;;;;;N;;;;; +1385F;EGYPTIAN HIEROGLYPH-1385F;Lo;0;L;;;;;N;;;;; +13860;EGYPTIAN HIEROGLYPH-13860;Lo;0;L;;;;;N;;;;; +13861;EGYPTIAN HIEROGLYPH-13861;Lo;0;L;;;;;N;;;;; +13862;EGYPTIAN HIEROGLYPH-13862;Lo;0;L;;;;;N;;;;; +13863;EGYPTIAN HIEROGLYPH-13863;Lo;0;L;;;;;N;;;;; +13864;EGYPTIAN HIEROGLYPH-13864;Lo;0;L;;;;;N;;;;; +13865;EGYPTIAN HIEROGLYPH-13865;Lo;0;L;;;;;N;;;;; +13866;EGYPTIAN HIEROGLYPH-13866;Lo;0;L;;;;;N;;;;; +13867;EGYPTIAN HIEROGLYPH-13867;Lo;0;L;;;;;N;;;;; +13868;EGYPTIAN HIEROGLYPH-13868;Lo;0;L;;;;;N;;;;; +13869;EGYPTIAN HIEROGLYPH-13869;Lo;0;L;;;;;N;;;;; +1386A;EGYPTIAN HIEROGLYPH-1386A;Lo;0;L;;;;;N;;;;; +1386B;EGYPTIAN HIEROGLYPH-1386B;Lo;0;L;;;;;N;;;;; +1386C;EGYPTIAN HIEROGLYPH-1386C;Lo;0;L;;;;;N;;;;; +1386D;EGYPTIAN HIEROGLYPH-1386D;Lo;0;L;;;;;N;;;;; +1386E;EGYPTIAN HIEROGLYPH-1386E;Lo;0;L;;;;;N;;;;; +1386F;EGYPTIAN HIEROGLYPH-1386F;Lo;0;L;;;;;N;;;;; +13870;EGYPTIAN HIEROGLYPH-13870;Lo;0;L;;;;;N;;;;; +13871;EGYPTIAN HIEROGLYPH-13871;Lo;0;L;;;;;N;;;;; +13872;EGYPTIAN HIEROGLYPH-13872;Lo;0;L;;;;;N;;;;; +13873;EGYPTIAN HIEROGLYPH-13873;Lo;0;L;;;;;N;;;;; +13874;EGYPTIAN HIEROGLYPH-13874;Lo;0;L;;;;;N;;;;; +13875;EGYPTIAN HIEROGLYPH-13875;Lo;0;L;;;;;N;;;;; +13876;EGYPTIAN HIEROGLYPH-13876;Lo;0;L;;;;;N;;;;; +13877;EGYPTIAN HIEROGLYPH-13877;Lo;0;L;;;;;N;;;;; +13878;EGYPTIAN HIEROGLYPH-13878;Lo;0;L;;;;;N;;;;; +13879;EGYPTIAN HIEROGLYPH-13879;Lo;0;L;;;;;N;;;;; +1387A;EGYPTIAN HIEROGLYPH-1387A;Lo;0;L;;;;;N;;;;; +1387B;EGYPTIAN HIEROGLYPH-1387B;Lo;0;L;;;;;N;;;;; +1387C;EGYPTIAN HIEROGLYPH-1387C;Lo;0;L;;;;;N;;;;; +1387D;EGYPTIAN HIEROGLYPH-1387D;Lo;0;L;;;;;N;;;;; +1387E;EGYPTIAN HIEROGLYPH-1387E;Lo;0;L;;;;;N;;;;; +1387F;EGYPTIAN HIEROGLYPH-1387F;Lo;0;L;;;;;N;;;;; +13880;EGYPTIAN HIEROGLYPH-13880;Lo;0;L;;;;;N;;;;; +13881;EGYPTIAN HIEROGLYPH-13881;Lo;0;L;;;;;N;;;;; +13882;EGYPTIAN HIEROGLYPH-13882;Lo;0;L;;;;;N;;;;; +13883;EGYPTIAN HIEROGLYPH-13883;Lo;0;L;;;;;N;;;;; +13884;EGYPTIAN HIEROGLYPH-13884;Lo;0;L;;;;;N;;;;; +13885;EGYPTIAN HIEROGLYPH-13885;Lo;0;L;;;;;N;;;;; +13886;EGYPTIAN HIEROGLYPH-13886;Lo;0;L;;;;;N;;;;; +13887;EGYPTIAN HIEROGLYPH-13887;Lo;0;L;;;;;N;;;;; +13888;EGYPTIAN HIEROGLYPH-13888;Lo;0;L;;;;;N;;;;; +13889;EGYPTIAN HIEROGLYPH-13889;Lo;0;L;;;;;N;;;;; +1388A;EGYPTIAN HIEROGLYPH-1388A;Lo;0;L;;;;;N;;;;; +1388B;EGYPTIAN HIEROGLYPH-1388B;Lo;0;L;;;;;N;;;;; +1388C;EGYPTIAN HIEROGLYPH-1388C;Lo;0;L;;;;;N;;;;; +1388D;EGYPTIAN HIEROGLYPH-1388D;Lo;0;L;;;;;N;;;;; +1388E;EGYPTIAN HIEROGLYPH-1388E;Lo;0;L;;;;;N;;;;; +1388F;EGYPTIAN HIEROGLYPH-1388F;Lo;0;L;;;;;N;;;;; +13890;EGYPTIAN HIEROGLYPH-13890;Lo;0;L;;;;;N;;;;; +13891;EGYPTIAN HIEROGLYPH-13891;Lo;0;L;;;;;N;;;;; +13892;EGYPTIAN HIEROGLYPH-13892;Lo;0;L;;;;;N;;;;; +13893;EGYPTIAN HIEROGLYPH-13893;Lo;0;L;;;;;N;;;;; +13894;EGYPTIAN HIEROGLYPH-13894;Lo;0;L;;;;;N;;;;; +13895;EGYPTIAN HIEROGLYPH-13895;Lo;0;L;;;;;N;;;;; +13896;EGYPTIAN HIEROGLYPH-13896;Lo;0;L;;;;;N;;;;; +13897;EGYPTIAN HIEROGLYPH-13897;Lo;0;L;;;;;N;;;;; +13898;EGYPTIAN HIEROGLYPH-13898;Lo;0;L;;;;;N;;;;; +13899;EGYPTIAN HIEROGLYPH-13899;Lo;0;L;;;;;N;;;;; +1389A;EGYPTIAN HIEROGLYPH-1389A;Lo;0;L;;;;;N;;;;; +1389B;EGYPTIAN HIEROGLYPH-1389B;Lo;0;L;;;;;N;;;;; +1389C;EGYPTIAN HIEROGLYPH-1389C;Lo;0;L;;;;;N;;;;; +1389D;EGYPTIAN HIEROGLYPH-1389D;Lo;0;L;;;;;N;;;;; +1389E;EGYPTIAN HIEROGLYPH-1389E;Lo;0;L;;;;;N;;;;; +1389F;EGYPTIAN HIEROGLYPH-1389F;Lo;0;L;;;;;N;;;;; +138A0;EGYPTIAN HIEROGLYPH-138A0;Lo;0;L;;;;;N;;;;; +138A1;EGYPTIAN HIEROGLYPH-138A1;Lo;0;L;;;;;N;;;;; +138A2;EGYPTIAN HIEROGLYPH-138A2;Lo;0;L;;;;;N;;;;; +138A3;EGYPTIAN HIEROGLYPH-138A3;Lo;0;L;;;;;N;;;;; +138A4;EGYPTIAN HIEROGLYPH-138A4;Lo;0;L;;;;;N;;;;; +138A5;EGYPTIAN HIEROGLYPH-138A5;Lo;0;L;;;;;N;;;;; +138A6;EGYPTIAN HIEROGLYPH-138A6;Lo;0;L;;;;;N;;;;; +138A7;EGYPTIAN HIEROGLYPH-138A7;Lo;0;L;;;;;N;;;;; +138A8;EGYPTIAN HIEROGLYPH-138A8;Lo;0;L;;;;;N;;;;; +138A9;EGYPTIAN HIEROGLYPH-138A9;Lo;0;L;;;;;N;;;;; +138AA;EGYPTIAN HIEROGLYPH-138AA;Lo;0;L;;;;;N;;;;; +138AB;EGYPTIAN HIEROGLYPH-138AB;Lo;0;L;;;;;N;;;;; +138AC;EGYPTIAN HIEROGLYPH-138AC;Lo;0;L;;;;;N;;;;; +138AD;EGYPTIAN HIEROGLYPH-138AD;Lo;0;L;;;;;N;;;;; +138AE;EGYPTIAN HIEROGLYPH-138AE;Lo;0;L;;;;;N;;;;; +138AF;EGYPTIAN HIEROGLYPH-138AF;Lo;0;L;;;;;N;;;;; +138B0;EGYPTIAN HIEROGLYPH-138B0;Lo;0;L;;;;;N;;;;; +138B1;EGYPTIAN HIEROGLYPH-138B1;Lo;0;L;;;;;N;;;;; +138B2;EGYPTIAN HIEROGLYPH-138B2;Lo;0;L;;;;;N;;;;; +138B3;EGYPTIAN HIEROGLYPH-138B3;Lo;0;L;;;;;N;;;;; +138B4;EGYPTIAN HIEROGLYPH-138B4;Lo;0;L;;;;;N;;;;; +138B5;EGYPTIAN HIEROGLYPH-138B5;Lo;0;L;;;;;N;;;;; +138B6;EGYPTIAN HIEROGLYPH-138B6;Lo;0;L;;;;;N;;;;; +138B7;EGYPTIAN HIEROGLYPH-138B7;Lo;0;L;;;;;N;;;;; +138B8;EGYPTIAN HIEROGLYPH-138B8;Lo;0;L;;;;;N;;;;; +138B9;EGYPTIAN HIEROGLYPH-138B9;Lo;0;L;;;;;N;;;;; +138BA;EGYPTIAN HIEROGLYPH-138BA;Lo;0;L;;;;;N;;;;; +138BB;EGYPTIAN HIEROGLYPH-138BB;Lo;0;L;;;;;N;;;;; +138BC;EGYPTIAN HIEROGLYPH-138BC;Lo;0;L;;;;;N;;;;; +138BD;EGYPTIAN HIEROGLYPH-138BD;Lo;0;L;;;;;N;;;;; +138BE;EGYPTIAN HIEROGLYPH-138BE;Lo;0;L;;;;;N;;;;; +138BF;EGYPTIAN HIEROGLYPH-138BF;Lo;0;L;;;;;N;;;;; +138C0;EGYPTIAN HIEROGLYPH-138C0;Lo;0;L;;;;;N;;;;; +138C1;EGYPTIAN HIEROGLYPH-138C1;Lo;0;L;;;;;N;;;;; +138C2;EGYPTIAN HIEROGLYPH-138C2;Lo;0;L;;;;;N;;;;; +138C3;EGYPTIAN HIEROGLYPH-138C3;Lo;0;L;;;;;N;;;;; +138C4;EGYPTIAN HIEROGLYPH-138C4;Lo;0;L;;;;;N;;;;; +138C5;EGYPTIAN HIEROGLYPH-138C5;Lo;0;L;;;;;N;;;;; +138C6;EGYPTIAN HIEROGLYPH-138C6;Lo;0;L;;;;;N;;;;; +138C7;EGYPTIAN HIEROGLYPH-138C7;Lo;0;L;;;;;N;;;;; +138C8;EGYPTIAN HIEROGLYPH-138C8;Lo;0;L;;;;;N;;;;; +138C9;EGYPTIAN HIEROGLYPH-138C9;Lo;0;L;;;;;N;;;;; +138CA;EGYPTIAN HIEROGLYPH-138CA;Lo;0;L;;;;;N;;;;; +138CB;EGYPTIAN HIEROGLYPH-138CB;Lo;0;L;;;;;N;;;;; +138CC;EGYPTIAN HIEROGLYPH-138CC;Lo;0;L;;;;;N;;;;; +138CD;EGYPTIAN HIEROGLYPH-138CD;Lo;0;L;;;;;N;;;;; +138CE;EGYPTIAN HIEROGLYPH-138CE;Lo;0;L;;;;;N;;;;; +138CF;EGYPTIAN HIEROGLYPH-138CF;Lo;0;L;;;;;N;;;;; +138D0;EGYPTIAN HIEROGLYPH-138D0;Lo;0;L;;;;;N;;;;; +138D1;EGYPTIAN HIEROGLYPH-138D1;Lo;0;L;;;;;N;;;;; +138D2;EGYPTIAN HIEROGLYPH-138D2;Lo;0;L;;;;;N;;;;; +138D3;EGYPTIAN HIEROGLYPH-138D3;Lo;0;L;;;;;N;;;;; +138D4;EGYPTIAN HIEROGLYPH-138D4;Lo;0;L;;;;;N;;;;; +138D5;EGYPTIAN HIEROGLYPH-138D5;Lo;0;L;;;;;N;;;;; +138D6;EGYPTIAN HIEROGLYPH-138D6;Lo;0;L;;;;;N;;;;; +138D7;EGYPTIAN HIEROGLYPH-138D7;Lo;0;L;;;;;N;;;;; +138D8;EGYPTIAN HIEROGLYPH-138D8;Lo;0;L;;;;;N;;;;; +138D9;EGYPTIAN HIEROGLYPH-138D9;Lo;0;L;;;;;N;;;;; +138DA;EGYPTIAN HIEROGLYPH-138DA;Lo;0;L;;;;;N;;;;; +138DB;EGYPTIAN HIEROGLYPH-138DB;Lo;0;L;;;;;N;;;;; +138DC;EGYPTIAN HIEROGLYPH-138DC;Lo;0;L;;;;;N;;;;; +138DD;EGYPTIAN HIEROGLYPH-138DD;Lo;0;L;;;;;N;;;;; +138DE;EGYPTIAN HIEROGLYPH-138DE;Lo;0;L;;;;;N;;;;; +138DF;EGYPTIAN HIEROGLYPH-138DF;Lo;0;L;;;;;N;;;;; +138E0;EGYPTIAN HIEROGLYPH-138E0;Lo;0;L;;;;;N;;;;; +138E1;EGYPTIAN HIEROGLYPH-138E1;Lo;0;L;;;;;N;;;;; +138E2;EGYPTIAN HIEROGLYPH-138E2;Lo;0;L;;;;;N;;;;; +138E3;EGYPTIAN HIEROGLYPH-138E3;Lo;0;L;;;;;N;;;;; +138E4;EGYPTIAN HIEROGLYPH-138E4;Lo;0;L;;;;;N;;;;; +138E5;EGYPTIAN HIEROGLYPH-138E5;Lo;0;L;;;;;N;;;;; +138E6;EGYPTIAN HIEROGLYPH-138E6;Lo;0;L;;;;;N;;;;; +138E7;EGYPTIAN HIEROGLYPH-138E7;Lo;0;L;;;;;N;;;;; +138E8;EGYPTIAN HIEROGLYPH-138E8;Lo;0;L;;;;;N;;;;; +138E9;EGYPTIAN HIEROGLYPH-138E9;Lo;0;L;;;;;N;;;;; +138EA;EGYPTIAN HIEROGLYPH-138EA;Lo;0;L;;;;;N;;;;; +138EB;EGYPTIAN HIEROGLYPH-138EB;Lo;0;L;;;;;N;;;;; +138EC;EGYPTIAN HIEROGLYPH-138EC;Lo;0;L;;;;;N;;;;; +138ED;EGYPTIAN HIEROGLYPH-138ED;Lo;0;L;;;;;N;;;;; +138EE;EGYPTIAN HIEROGLYPH-138EE;Lo;0;L;;;;;N;;;;; +138EF;EGYPTIAN HIEROGLYPH-138EF;Lo;0;L;;;;;N;;;;; +138F0;EGYPTIAN HIEROGLYPH-138F0;Lo;0;L;;;;;N;;;;; +138F1;EGYPTIAN HIEROGLYPH-138F1;Lo;0;L;;;;;N;;;;; +138F2;EGYPTIAN HIEROGLYPH-138F2;Lo;0;L;;;;;N;;;;; +138F3;EGYPTIAN HIEROGLYPH-138F3;Lo;0;L;;;;;N;;;;; +138F4;EGYPTIAN HIEROGLYPH-138F4;Lo;0;L;;;;;N;;;;; +138F5;EGYPTIAN HIEROGLYPH-138F5;Lo;0;L;;;;;N;;;;; +138F6;EGYPTIAN HIEROGLYPH-138F6;Lo;0;L;;;;;N;;;;; +138F7;EGYPTIAN HIEROGLYPH-138F7;Lo;0;L;;;;;N;;;;; +138F8;EGYPTIAN HIEROGLYPH-138F8;Lo;0;L;;;;;N;;;;; +138F9;EGYPTIAN HIEROGLYPH-138F9;Lo;0;L;;;;;N;;;;; +138FA;EGYPTIAN HIEROGLYPH-138FA;Lo;0;L;;;;;N;;;;; +138FB;EGYPTIAN HIEROGLYPH-138FB;Lo;0;L;;;;;N;;;;; +138FC;EGYPTIAN HIEROGLYPH-138FC;Lo;0;L;;;;;N;;;;; +138FD;EGYPTIAN HIEROGLYPH-138FD;Lo;0;L;;;;;N;;;;; +138FE;EGYPTIAN HIEROGLYPH-138FE;Lo;0;L;;;;;N;;;;; +138FF;EGYPTIAN HIEROGLYPH-138FF;Lo;0;L;;;;;N;;;;; +13900;EGYPTIAN HIEROGLYPH-13900;Lo;0;L;;;;;N;;;;; +13901;EGYPTIAN HIEROGLYPH-13901;Lo;0;L;;;;;N;;;;; +13902;EGYPTIAN HIEROGLYPH-13902;Lo;0;L;;;;;N;;;;; +13903;EGYPTIAN HIEROGLYPH-13903;Lo;0;L;;;;;N;;;;; +13904;EGYPTIAN HIEROGLYPH-13904;Lo;0;L;;;;;N;;;;; +13905;EGYPTIAN HIEROGLYPH-13905;Lo;0;L;;;;;N;;;;; +13906;EGYPTIAN HIEROGLYPH-13906;Lo;0;L;;;;;N;;;;; +13907;EGYPTIAN HIEROGLYPH-13907;Lo;0;L;;;;;N;;;;; +13908;EGYPTIAN HIEROGLYPH-13908;Lo;0;L;;;;;N;;;;; +13909;EGYPTIAN HIEROGLYPH-13909;Lo;0;L;;;;;N;;;;; +1390A;EGYPTIAN HIEROGLYPH-1390A;Lo;0;L;;;;;N;;;;; +1390B;EGYPTIAN HIEROGLYPH-1390B;Lo;0;L;;;;;N;;;;; +1390C;EGYPTIAN HIEROGLYPH-1390C;Lo;0;L;;;;;N;;;;; +1390D;EGYPTIAN HIEROGLYPH-1390D;Lo;0;L;;;;;N;;;;; +1390E;EGYPTIAN HIEROGLYPH-1390E;Lo;0;L;;;;;N;;;;; +1390F;EGYPTIAN HIEROGLYPH-1390F;Lo;0;L;;;;;N;;;;; +13910;EGYPTIAN HIEROGLYPH-13910;Lo;0;L;;;;;N;;;;; +13911;EGYPTIAN HIEROGLYPH-13911;Lo;0;L;;;;;N;;;;; +13912;EGYPTIAN HIEROGLYPH-13912;Lo;0;L;;;;;N;;;;; +13913;EGYPTIAN HIEROGLYPH-13913;Lo;0;L;;;;;N;;;;; +13914;EGYPTIAN HIEROGLYPH-13914;Lo;0;L;;;;;N;;;;; +13915;EGYPTIAN HIEROGLYPH-13915;Lo;0;L;;;;;N;;;;; +13916;EGYPTIAN HIEROGLYPH-13916;Lo;0;L;;;;;N;;;;; +13917;EGYPTIAN HIEROGLYPH-13917;Lo;0;L;;;;;N;;;;; +13918;EGYPTIAN HIEROGLYPH-13918;Lo;0;L;;;;;N;;;;; +13919;EGYPTIAN HIEROGLYPH-13919;Lo;0;L;;;;;N;;;;; +1391A;EGYPTIAN HIEROGLYPH-1391A;Lo;0;L;;;;;N;;;;; +1391B;EGYPTIAN HIEROGLYPH-1391B;Lo;0;L;;;;;N;;;;; +1391C;EGYPTIAN HIEROGLYPH-1391C;Lo;0;L;;;;;N;;;;; +1391D;EGYPTIAN HIEROGLYPH-1391D;Lo;0;L;;;;;N;;;;; +1391E;EGYPTIAN HIEROGLYPH-1391E;Lo;0;L;;;;;N;;;;; +1391F;EGYPTIAN HIEROGLYPH-1391F;Lo;0;L;;;;;N;;;;; +13920;EGYPTIAN HIEROGLYPH-13920;Lo;0;L;;;;;N;;;;; +13921;EGYPTIAN HIEROGLYPH-13921;Lo;0;L;;;;;N;;;;; +13922;EGYPTIAN HIEROGLYPH-13922;Lo;0;L;;;;;N;;;;; +13923;EGYPTIAN HIEROGLYPH-13923;Lo;0;L;;;;;N;;;;; +13924;EGYPTIAN HIEROGLYPH-13924;Lo;0;L;;;;;N;;;;; +13925;EGYPTIAN HIEROGLYPH-13925;Lo;0;L;;;;;N;;;;; +13926;EGYPTIAN HIEROGLYPH-13926;Lo;0;L;;;;;N;;;;; +13927;EGYPTIAN HIEROGLYPH-13927;Lo;0;L;;;;;N;;;;; +13928;EGYPTIAN HIEROGLYPH-13928;Lo;0;L;;;;;N;;;;; +13929;EGYPTIAN HIEROGLYPH-13929;Lo;0;L;;;;;N;;;;; +1392A;EGYPTIAN HIEROGLYPH-1392A;Lo;0;L;;;;;N;;;;; +1392B;EGYPTIAN HIEROGLYPH-1392B;Lo;0;L;;;;;N;;;;; +1392C;EGYPTIAN HIEROGLYPH-1392C;Lo;0;L;;;;;N;;;;; +1392D;EGYPTIAN HIEROGLYPH-1392D;Lo;0;L;;;;;N;;;;; +1392E;EGYPTIAN HIEROGLYPH-1392E;Lo;0;L;;;;;N;;;;; +1392F;EGYPTIAN HIEROGLYPH-1392F;Lo;0;L;;;;;N;;;;; +13930;EGYPTIAN HIEROGLYPH-13930;Lo;0;L;;;;;N;;;;; +13931;EGYPTIAN HIEROGLYPH-13931;Lo;0;L;;;;;N;;;;; +13932;EGYPTIAN HIEROGLYPH-13932;Lo;0;L;;;;;N;;;;; +13933;EGYPTIAN HIEROGLYPH-13933;Lo;0;L;;;;;N;;;;; +13934;EGYPTIAN HIEROGLYPH-13934;Lo;0;L;;;;;N;;;;; +13935;EGYPTIAN HIEROGLYPH-13935;Lo;0;L;;;;;N;;;;; +13936;EGYPTIAN HIEROGLYPH-13936;Lo;0;L;;;;;N;;;;; +13937;EGYPTIAN HIEROGLYPH-13937;Lo;0;L;;;;;N;;;;; +13938;EGYPTIAN HIEROGLYPH-13938;Lo;0;L;;;;;N;;;;; +13939;EGYPTIAN HIEROGLYPH-13939;Lo;0;L;;;;;N;;;;; +1393A;EGYPTIAN HIEROGLYPH-1393A;Lo;0;L;;;;;N;;;;; +1393B;EGYPTIAN HIEROGLYPH-1393B;Lo;0;L;;;;;N;;;;; +1393C;EGYPTIAN HIEROGLYPH-1393C;Lo;0;L;;;;;N;;;;; +1393D;EGYPTIAN HIEROGLYPH-1393D;Lo;0;L;;;;;N;;;;; +1393E;EGYPTIAN HIEROGLYPH-1393E;Lo;0;L;;;;;N;;;;; +1393F;EGYPTIAN HIEROGLYPH-1393F;Lo;0;L;;;;;N;;;;; +13940;EGYPTIAN HIEROGLYPH-13940;Lo;0;L;;;;;N;;;;; +13941;EGYPTIAN HIEROGLYPH-13941;Lo;0;L;;;;;N;;;;; +13942;EGYPTIAN HIEROGLYPH-13942;Lo;0;L;;;;;N;;;;; +13943;EGYPTIAN HIEROGLYPH-13943;Lo;0;L;;;;;N;;;;; +13944;EGYPTIAN HIEROGLYPH-13944;Lo;0;L;;;;;N;;;;; +13945;EGYPTIAN HIEROGLYPH-13945;Lo;0;L;;;;;N;;;;; +13946;EGYPTIAN HIEROGLYPH-13946;Lo;0;L;;;;;N;;;;; +13947;EGYPTIAN HIEROGLYPH-13947;Lo;0;L;;;;;N;;;;; +13948;EGYPTIAN HIEROGLYPH-13948;Lo;0;L;;;;;N;;;;; +13949;EGYPTIAN HIEROGLYPH-13949;Lo;0;L;;;;;N;;;;; +1394A;EGYPTIAN HIEROGLYPH-1394A;Lo;0;L;;;;;N;;;;; +1394B;EGYPTIAN HIEROGLYPH-1394B;Lo;0;L;;;;;N;;;;; +1394C;EGYPTIAN HIEROGLYPH-1394C;Lo;0;L;;;;;N;;;;; +1394D;EGYPTIAN HIEROGLYPH-1394D;Lo;0;L;;;;;N;;;;; +1394E;EGYPTIAN HIEROGLYPH-1394E;Lo;0;L;;;;;N;;;;; +1394F;EGYPTIAN HIEROGLYPH-1394F;Lo;0;L;;;;;N;;;;; +13950;EGYPTIAN HIEROGLYPH-13950;Lo;0;L;;;;;N;;;;; +13951;EGYPTIAN HIEROGLYPH-13951;Lo;0;L;;;;;N;;;;; +13952;EGYPTIAN HIEROGLYPH-13952;Lo;0;L;;;;;N;;;;; +13953;EGYPTIAN HIEROGLYPH-13953;Lo;0;L;;;;;N;;;;; +13954;EGYPTIAN HIEROGLYPH-13954;Lo;0;L;;;;;N;;;;; +13955;EGYPTIAN HIEROGLYPH-13955;Lo;0;L;;;;;N;;;;; +13956;EGYPTIAN HIEROGLYPH-13956;Lo;0;L;;;;;N;;;;; +13957;EGYPTIAN HIEROGLYPH-13957;Lo;0;L;;;;;N;;;;; +13958;EGYPTIAN HIEROGLYPH-13958;Lo;0;L;;;;;N;;;;; +13959;EGYPTIAN HIEROGLYPH-13959;Lo;0;L;;;;;N;;;;; +1395A;EGYPTIAN HIEROGLYPH-1395A;Lo;0;L;;;;;N;;;;; +1395B;EGYPTIAN HIEROGLYPH-1395B;Lo;0;L;;;;;N;;;;; +1395C;EGYPTIAN HIEROGLYPH-1395C;Lo;0;L;;;;;N;;;;; +1395D;EGYPTIAN HIEROGLYPH-1395D;Lo;0;L;;;;;N;;;;; +1395E;EGYPTIAN HIEROGLYPH-1395E;Lo;0;L;;;;;N;;;;; +1395F;EGYPTIAN HIEROGLYPH-1395F;Lo;0;L;;;;;N;;;;; +13960;EGYPTIAN HIEROGLYPH-13960;Lo;0;L;;;;;N;;;;; +13961;EGYPTIAN HIEROGLYPH-13961;Lo;0;L;;;;;N;;;;; +13962;EGYPTIAN HIEROGLYPH-13962;Lo;0;L;;;;;N;;;;; +13963;EGYPTIAN HIEROGLYPH-13963;Lo;0;L;;;;;N;;;;; +13964;EGYPTIAN HIEROGLYPH-13964;Lo;0;L;;;;;N;;;;; +13965;EGYPTIAN HIEROGLYPH-13965;Lo;0;L;;;;;N;;;;; +13966;EGYPTIAN HIEROGLYPH-13966;Lo;0;L;;;;;N;;;;; +13967;EGYPTIAN HIEROGLYPH-13967;Lo;0;L;;;;;N;;;;; +13968;EGYPTIAN HIEROGLYPH-13968;Lo;0;L;;;;;N;;;;; +13969;EGYPTIAN HIEROGLYPH-13969;Lo;0;L;;;;;N;;;;; +1396A;EGYPTIAN HIEROGLYPH-1396A;Lo;0;L;;;;;N;;;;; +1396B;EGYPTIAN HIEROGLYPH-1396B;Lo;0;L;;;;;N;;;;; +1396C;EGYPTIAN HIEROGLYPH-1396C;Lo;0;L;;;;;N;;;;; +1396D;EGYPTIAN HIEROGLYPH-1396D;Lo;0;L;;;;;N;;;;; +1396E;EGYPTIAN HIEROGLYPH-1396E;Lo;0;L;;;;;N;;;;; +1396F;EGYPTIAN HIEROGLYPH-1396F;Lo;0;L;;;;;N;;;;; +13970;EGYPTIAN HIEROGLYPH-13970;Lo;0;L;;;;;N;;;;; +13971;EGYPTIAN HIEROGLYPH-13971;Lo;0;L;;;;;N;;;;; +13972;EGYPTIAN HIEROGLYPH-13972;Lo;0;L;;;;;N;;;;; +13973;EGYPTIAN HIEROGLYPH-13973;Lo;0;L;;;;;N;;;;; +13974;EGYPTIAN HIEROGLYPH-13974;Lo;0;L;;;;;N;;;;; +13975;EGYPTIAN HIEROGLYPH-13975;Lo;0;L;;;;;N;;;;; +13976;EGYPTIAN HIEROGLYPH-13976;Lo;0;L;;;;;N;;;;; +13977;EGYPTIAN HIEROGLYPH-13977;Lo;0;L;;;;;N;;;;; +13978;EGYPTIAN HIEROGLYPH-13978;Lo;0;L;;;;;N;;;;; +13979;EGYPTIAN HIEROGLYPH-13979;Lo;0;L;;;;;N;;;;; +1397A;EGYPTIAN HIEROGLYPH-1397A;Lo;0;L;;;;;N;;;;; +1397B;EGYPTIAN HIEROGLYPH-1397B;Lo;0;L;;;;;N;;;;; +1397C;EGYPTIAN HIEROGLYPH-1397C;Lo;0;L;;;;;N;;;;; +1397D;EGYPTIAN HIEROGLYPH-1397D;Lo;0;L;;;;;N;;;;; +1397E;EGYPTIAN HIEROGLYPH-1397E;Lo;0;L;;;;;N;;;;; +1397F;EGYPTIAN HIEROGLYPH-1397F;Lo;0;L;;;;;N;;;;; +13980;EGYPTIAN HIEROGLYPH-13980;Lo;0;L;;;;;N;;;;; +13981;EGYPTIAN HIEROGLYPH-13981;Lo;0;L;;;;;N;;;;; +13982;EGYPTIAN HIEROGLYPH-13982;Lo;0;L;;;;;N;;;;; +13983;EGYPTIAN HIEROGLYPH-13983;Lo;0;L;;;;;N;;;;; +13984;EGYPTIAN HIEROGLYPH-13984;Lo;0;L;;;;;N;;;;; +13985;EGYPTIAN HIEROGLYPH-13985;Lo;0;L;;;;;N;;;;; +13986;EGYPTIAN HIEROGLYPH-13986;Lo;0;L;;;;;N;;;;; +13987;EGYPTIAN HIEROGLYPH-13987;Lo;0;L;;;;;N;;;;; +13988;EGYPTIAN HIEROGLYPH-13988;Lo;0;L;;;;;N;;;;; +13989;EGYPTIAN HIEROGLYPH-13989;Lo;0;L;;;;;N;;;;; +1398A;EGYPTIAN HIEROGLYPH-1398A;Lo;0;L;;;;;N;;;;; +1398B;EGYPTIAN HIEROGLYPH-1398B;Lo;0;L;;;;;N;;;;; +1398C;EGYPTIAN HIEROGLYPH-1398C;Lo;0;L;;;;;N;;;;; +1398D;EGYPTIAN HIEROGLYPH-1398D;Lo;0;L;;;;;N;;;;; +1398E;EGYPTIAN HIEROGLYPH-1398E;Lo;0;L;;;;;N;;;;; +1398F;EGYPTIAN HIEROGLYPH-1398F;Lo;0;L;;;;;N;;;;; +13990;EGYPTIAN HIEROGLYPH-13990;Lo;0;L;;;;;N;;;;; +13991;EGYPTIAN HIEROGLYPH-13991;Lo;0;L;;;;;N;;;;; +13992;EGYPTIAN HIEROGLYPH-13992;Lo;0;L;;;;;N;;;;; +13993;EGYPTIAN HIEROGLYPH-13993;Lo;0;L;;;;;N;;;;; +13994;EGYPTIAN HIEROGLYPH-13994;Lo;0;L;;;;;N;;;;; +13995;EGYPTIAN HIEROGLYPH-13995;Lo;0;L;;;;;N;;;;; +13996;EGYPTIAN HIEROGLYPH-13996;Lo;0;L;;;;;N;;;;; +13997;EGYPTIAN HIEROGLYPH-13997;Lo;0;L;;;;;N;;;;; +13998;EGYPTIAN HIEROGLYPH-13998;Lo;0;L;;;;;N;;;;; +13999;EGYPTIAN HIEROGLYPH-13999;Lo;0;L;;;;;N;;;;; +1399A;EGYPTIAN HIEROGLYPH-1399A;Lo;0;L;;;;;N;;;;; +1399B;EGYPTIAN HIEROGLYPH-1399B;Lo;0;L;;;;;N;;;;; +1399C;EGYPTIAN HIEROGLYPH-1399C;Lo;0;L;;;;;N;;;;; +1399D;EGYPTIAN HIEROGLYPH-1399D;Lo;0;L;;;;;N;;;;; +1399E;EGYPTIAN HIEROGLYPH-1399E;Lo;0;L;;;;;N;;;;; +1399F;EGYPTIAN HIEROGLYPH-1399F;Lo;0;L;;;;;N;;;;; +139A0;EGYPTIAN HIEROGLYPH-139A0;Lo;0;L;;;;;N;;;;; +139A1;EGYPTIAN HIEROGLYPH-139A1;Lo;0;L;;;;;N;;;;; +139A2;EGYPTIAN HIEROGLYPH-139A2;Lo;0;L;;;;;N;;;;; +139A3;EGYPTIAN HIEROGLYPH-139A3;Lo;0;L;;;;;N;;;;; +139A4;EGYPTIAN HIEROGLYPH-139A4;Lo;0;L;;;;;N;;;;; +139A5;EGYPTIAN HIEROGLYPH-139A5;Lo;0;L;;;;;N;;;;; +139A6;EGYPTIAN HIEROGLYPH-139A6;Lo;0;L;;;;;N;;;;; +139A7;EGYPTIAN HIEROGLYPH-139A7;Lo;0;L;;;;;N;;;;; +139A8;EGYPTIAN HIEROGLYPH-139A8;Lo;0;L;;;;;N;;;;; +139A9;EGYPTIAN HIEROGLYPH-139A9;Lo;0;L;;;;;N;;;;; +139AA;EGYPTIAN HIEROGLYPH-139AA;Lo;0;L;;;;;N;;;;; +139AB;EGYPTIAN HIEROGLYPH-139AB;Lo;0;L;;;;;N;;;;; +139AC;EGYPTIAN HIEROGLYPH-139AC;Lo;0;L;;;;;N;;;;; +139AD;EGYPTIAN HIEROGLYPH-139AD;Lo;0;L;;;;;N;;;;; +139AE;EGYPTIAN HIEROGLYPH-139AE;Lo;0;L;;;;;N;;;;; +139AF;EGYPTIAN HIEROGLYPH-139AF;Lo;0;L;;;;;N;;;;; +139B0;EGYPTIAN HIEROGLYPH-139B0;Lo;0;L;;;;;N;;;;; +139B1;EGYPTIAN HIEROGLYPH-139B1;Lo;0;L;;;;;N;;;;; +139B2;EGYPTIAN HIEROGLYPH-139B2;Lo;0;L;;;;;N;;;;; +139B3;EGYPTIAN HIEROGLYPH-139B3;Lo;0;L;;;;;N;;;;; +139B4;EGYPTIAN HIEROGLYPH-139B4;Lo;0;L;;;;;N;;;;; +139B5;EGYPTIAN HIEROGLYPH-139B5;Lo;0;L;;;;;N;;;;; +139B6;EGYPTIAN HIEROGLYPH-139B6;Lo;0;L;;;;;N;;;;; +139B7;EGYPTIAN HIEROGLYPH-139B7;Lo;0;L;;;;;N;;;;; +139B8;EGYPTIAN HIEROGLYPH-139B8;Lo;0;L;;;;;N;;;;; +139B9;EGYPTIAN HIEROGLYPH-139B9;Lo;0;L;;;;;N;;;;; +139BA;EGYPTIAN HIEROGLYPH-139BA;Lo;0;L;;;;;N;;;;; +139BB;EGYPTIAN HIEROGLYPH-139BB;Lo;0;L;;;;;N;;;;; +139BC;EGYPTIAN HIEROGLYPH-139BC;Lo;0;L;;;;;N;;;;; +139BD;EGYPTIAN HIEROGLYPH-139BD;Lo;0;L;;;;;N;;;;; +139BE;EGYPTIAN HIEROGLYPH-139BE;Lo;0;L;;;;;N;;;;; +139BF;EGYPTIAN HIEROGLYPH-139BF;Lo;0;L;;;;;N;;;;; +139C0;EGYPTIAN HIEROGLYPH-139C0;Lo;0;L;;;;;N;;;;; +139C1;EGYPTIAN HIEROGLYPH-139C1;Lo;0;L;;;;;N;;;;; +139C2;EGYPTIAN HIEROGLYPH-139C2;Lo;0;L;;;;;N;;;;; +139C3;EGYPTIAN HIEROGLYPH-139C3;Lo;0;L;;;;;N;;;;; +139C4;EGYPTIAN HIEROGLYPH-139C4;Lo;0;L;;;;;N;;;;; +139C5;EGYPTIAN HIEROGLYPH-139C5;Lo;0;L;;;;;N;;;;; +139C6;EGYPTIAN HIEROGLYPH-139C6;Lo;0;L;;;;;N;;;;; +139C7;EGYPTIAN HIEROGLYPH-139C7;Lo;0;L;;;;;N;;;;; +139C8;EGYPTIAN HIEROGLYPH-139C8;Lo;0;L;;;;;N;;;;; +139C9;EGYPTIAN HIEROGLYPH-139C9;Lo;0;L;;;;;N;;;;; +139CA;EGYPTIAN HIEROGLYPH-139CA;Lo;0;L;;;;;N;;;;; +139CB;EGYPTIAN HIEROGLYPH-139CB;Lo;0;L;;;;;N;;;;; +139CC;EGYPTIAN HIEROGLYPH-139CC;Lo;0;L;;;;;N;;;;; +139CD;EGYPTIAN HIEROGLYPH-139CD;Lo;0;L;;;;;N;;;;; +139CE;EGYPTIAN HIEROGLYPH-139CE;Lo;0;L;;;;;N;;;;; +139CF;EGYPTIAN HIEROGLYPH-139CF;Lo;0;L;;;;;N;;;;; +139D0;EGYPTIAN HIEROGLYPH-139D0;Lo;0;L;;;;;N;;;;; +139D1;EGYPTIAN HIEROGLYPH-139D1;Lo;0;L;;;;;N;;;;; +139D2;EGYPTIAN HIEROGLYPH-139D2;Lo;0;L;;;;;N;;;;; +139D3;EGYPTIAN HIEROGLYPH-139D3;Lo;0;L;;;;;N;;;;; +139D4;EGYPTIAN HIEROGLYPH-139D4;Lo;0;L;;;;;N;;;;; +139D5;EGYPTIAN HIEROGLYPH-139D5;Lo;0;L;;;;;N;;;;; +139D6;EGYPTIAN HIEROGLYPH-139D6;Lo;0;L;;;;;N;;;;; +139D7;EGYPTIAN HIEROGLYPH-139D7;Lo;0;L;;;;;N;;;;; +139D8;EGYPTIAN HIEROGLYPH-139D8;Lo;0;L;;;;;N;;;;; +139D9;EGYPTIAN HIEROGLYPH-139D9;Lo;0;L;;;;;N;;;;; +139DA;EGYPTIAN HIEROGLYPH-139DA;Lo;0;L;;;;;N;;;;; +139DB;EGYPTIAN HIEROGLYPH-139DB;Lo;0;L;;;;;N;;;;; +139DC;EGYPTIAN HIEROGLYPH-139DC;Lo;0;L;;;;;N;;;;; +139DD;EGYPTIAN HIEROGLYPH-139DD;Lo;0;L;;;;;N;;;;; +139DE;EGYPTIAN HIEROGLYPH-139DE;Lo;0;L;;;;;N;;;;; +139DF;EGYPTIAN HIEROGLYPH-139DF;Lo;0;L;;;;;N;;;;; +139E0;EGYPTIAN HIEROGLYPH-139E0;Lo;0;L;;;;;N;;;;; +139E1;EGYPTIAN HIEROGLYPH-139E1;Lo;0;L;;;;;N;;;;; +139E2;EGYPTIAN HIEROGLYPH-139E2;Lo;0;L;;;;;N;;;;; +139E3;EGYPTIAN HIEROGLYPH-139E3;Lo;0;L;;;;;N;;;;; +139E4;EGYPTIAN HIEROGLYPH-139E4;Lo;0;L;;;;;N;;;;; +139E5;EGYPTIAN HIEROGLYPH-139E5;Lo;0;L;;;;;N;;;;; +139E6;EGYPTIAN HIEROGLYPH-139E6;Lo;0;L;;;;;N;;;;; +139E7;EGYPTIAN HIEROGLYPH-139E7;Lo;0;L;;;;;N;;;;; +139E8;EGYPTIAN HIEROGLYPH-139E8;Lo;0;L;;;;;N;;;;; +139E9;EGYPTIAN HIEROGLYPH-139E9;Lo;0;L;;;;;N;;;;; +139EA;EGYPTIAN HIEROGLYPH-139EA;Lo;0;L;;;;;N;;;;; +139EB;EGYPTIAN HIEROGLYPH-139EB;Lo;0;L;;;;;N;;;;; +139EC;EGYPTIAN HIEROGLYPH-139EC;Lo;0;L;;;;;N;;;;; +139ED;EGYPTIAN HIEROGLYPH-139ED;Lo;0;L;;;;;N;;;;; +139EE;EGYPTIAN HIEROGLYPH-139EE;Lo;0;L;;;;;N;;;;; +139EF;EGYPTIAN HIEROGLYPH-139EF;Lo;0;L;;;;;N;;;;; +139F0;EGYPTIAN HIEROGLYPH-139F0;Lo;0;L;;;;;N;;;;; +139F1;EGYPTIAN HIEROGLYPH-139F1;Lo;0;L;;;;;N;;;;; +139F2;EGYPTIAN HIEROGLYPH-139F2;Lo;0;L;;;;;N;;;;; +139F3;EGYPTIAN HIEROGLYPH-139F3;Lo;0;L;;;;;N;;;;; +139F4;EGYPTIAN HIEROGLYPH-139F4;Lo;0;L;;;;;N;;;;; +139F5;EGYPTIAN HIEROGLYPH-139F5;Lo;0;L;;;;;N;;;;; +139F6;EGYPTIAN HIEROGLYPH-139F6;Lo;0;L;;;;;N;;;;; +139F7;EGYPTIAN HIEROGLYPH-139F7;Lo;0;L;;;;;N;;;;; +139F8;EGYPTIAN HIEROGLYPH-139F8;Lo;0;L;;;;;N;;;;; +139F9;EGYPTIAN HIEROGLYPH-139F9;Lo;0;L;;;;;N;;;;; +139FA;EGYPTIAN HIEROGLYPH-139FA;Lo;0;L;;;;;N;;;;; +139FB;EGYPTIAN HIEROGLYPH-139FB;Lo;0;L;;;;;N;;;;; +139FC;EGYPTIAN HIEROGLYPH-139FC;Lo;0;L;;;;;N;;;;; +139FD;EGYPTIAN HIEROGLYPH-139FD;Lo;0;L;;;;;N;;;;; +139FE;EGYPTIAN HIEROGLYPH-139FE;Lo;0;L;;;;;N;;;;; +139FF;EGYPTIAN HIEROGLYPH-139FF;Lo;0;L;;;;;N;;;;; +13A00;EGYPTIAN HIEROGLYPH-13A00;Lo;0;L;;;;;N;;;;; +13A01;EGYPTIAN HIEROGLYPH-13A01;Lo;0;L;;;;;N;;;;; +13A02;EGYPTIAN HIEROGLYPH-13A02;Lo;0;L;;;;;N;;;;; +13A03;EGYPTIAN HIEROGLYPH-13A03;Lo;0;L;;;;;N;;;;; +13A04;EGYPTIAN HIEROGLYPH-13A04;Lo;0;L;;;;;N;;;;; +13A05;EGYPTIAN HIEROGLYPH-13A05;Lo;0;L;;;;;N;;;;; +13A06;EGYPTIAN HIEROGLYPH-13A06;Lo;0;L;;;;;N;;;;; +13A07;EGYPTIAN HIEROGLYPH-13A07;Lo;0;L;;;;;N;;;;; +13A08;EGYPTIAN HIEROGLYPH-13A08;Lo;0;L;;;;;N;;;;; +13A09;EGYPTIAN HIEROGLYPH-13A09;Lo;0;L;;;;;N;;;;; +13A0A;EGYPTIAN HIEROGLYPH-13A0A;Lo;0;L;;;;;N;;;;; +13A0B;EGYPTIAN HIEROGLYPH-13A0B;Lo;0;L;;;;;N;;;;; +13A0C;EGYPTIAN HIEROGLYPH-13A0C;Lo;0;L;;;;;N;;;;; +13A0D;EGYPTIAN HIEROGLYPH-13A0D;Lo;0;L;;;;;N;;;;; +13A0E;EGYPTIAN HIEROGLYPH-13A0E;Lo;0;L;;;;;N;;;;; +13A0F;EGYPTIAN HIEROGLYPH-13A0F;Lo;0;L;;;;;N;;;;; +13A10;EGYPTIAN HIEROGLYPH-13A10;Lo;0;L;;;;;N;;;;; +13A11;EGYPTIAN HIEROGLYPH-13A11;Lo;0;L;;;;;N;;;;; +13A12;EGYPTIAN HIEROGLYPH-13A12;Lo;0;L;;;;;N;;;;; +13A13;EGYPTIAN HIEROGLYPH-13A13;Lo;0;L;;;;;N;;;;; +13A14;EGYPTIAN HIEROGLYPH-13A14;Lo;0;L;;;;;N;;;;; +13A15;EGYPTIAN HIEROGLYPH-13A15;Lo;0;L;;;;;N;;;;; +13A16;EGYPTIAN HIEROGLYPH-13A16;Lo;0;L;;;;;N;;;;; +13A17;EGYPTIAN HIEROGLYPH-13A17;Lo;0;L;;;;;N;;;;; +13A18;EGYPTIAN HIEROGLYPH-13A18;Lo;0;L;;;;;N;;;;; +13A19;EGYPTIAN HIEROGLYPH-13A19;Lo;0;L;;;;;N;;;;; +13A1A;EGYPTIAN HIEROGLYPH-13A1A;Lo;0;L;;;;;N;;;;; +13A1B;EGYPTIAN HIEROGLYPH-13A1B;Lo;0;L;;;;;N;;;;; +13A1C;EGYPTIAN HIEROGLYPH-13A1C;Lo;0;L;;;;;N;;;;; +13A1D;EGYPTIAN HIEROGLYPH-13A1D;Lo;0;L;;;;;N;;;;; +13A1E;EGYPTIAN HIEROGLYPH-13A1E;Lo;0;L;;;;;N;;;;; +13A1F;EGYPTIAN HIEROGLYPH-13A1F;Lo;0;L;;;;;N;;;;; +13A20;EGYPTIAN HIEROGLYPH-13A20;Lo;0;L;;;;;N;;;;; +13A21;EGYPTIAN HIEROGLYPH-13A21;Lo;0;L;;;;;N;;;;; +13A22;EGYPTIAN HIEROGLYPH-13A22;Lo;0;L;;;;;N;;;;; +13A23;EGYPTIAN HIEROGLYPH-13A23;Lo;0;L;;;;;N;;;;; +13A24;EGYPTIAN HIEROGLYPH-13A24;Lo;0;L;;;;;N;;;;; +13A25;EGYPTIAN HIEROGLYPH-13A25;Lo;0;L;;;;;N;;;;; +13A26;EGYPTIAN HIEROGLYPH-13A26;Lo;0;L;;;;;N;;;;; +13A27;EGYPTIAN HIEROGLYPH-13A27;Lo;0;L;;;;;N;;;;; +13A28;EGYPTIAN HIEROGLYPH-13A28;Lo;0;L;;;;;N;;;;; +13A29;EGYPTIAN HIEROGLYPH-13A29;Lo;0;L;;;;;N;;;;; +13A2A;EGYPTIAN HIEROGLYPH-13A2A;Lo;0;L;;;;;N;;;;; +13A2B;EGYPTIAN HIEROGLYPH-13A2B;Lo;0;L;;;;;N;;;;; +13A2C;EGYPTIAN HIEROGLYPH-13A2C;Lo;0;L;;;;;N;;;;; +13A2D;EGYPTIAN HIEROGLYPH-13A2D;Lo;0;L;;;;;N;;;;; +13A2E;EGYPTIAN HIEROGLYPH-13A2E;Lo;0;L;;;;;N;;;;; +13A2F;EGYPTIAN HIEROGLYPH-13A2F;Lo;0;L;;;;;N;;;;; +13A30;EGYPTIAN HIEROGLYPH-13A30;Lo;0;L;;;;;N;;;;; +13A31;EGYPTIAN HIEROGLYPH-13A31;Lo;0;L;;;;;N;;;;; +13A32;EGYPTIAN HIEROGLYPH-13A32;Lo;0;L;;;;;N;;;;; +13A33;EGYPTIAN HIEROGLYPH-13A33;Lo;0;L;;;;;N;;;;; +13A34;EGYPTIAN HIEROGLYPH-13A34;Lo;0;L;;;;;N;;;;; +13A35;EGYPTIAN HIEROGLYPH-13A35;Lo;0;L;;;;;N;;;;; +13A36;EGYPTIAN HIEROGLYPH-13A36;Lo;0;L;;;;;N;;;;; +13A37;EGYPTIAN HIEROGLYPH-13A37;Lo;0;L;;;;;N;;;;; +13A38;EGYPTIAN HIEROGLYPH-13A38;Lo;0;L;;;;;N;;;;; +13A39;EGYPTIAN HIEROGLYPH-13A39;Lo;0;L;;;;;N;;;;; +13A3A;EGYPTIAN HIEROGLYPH-13A3A;Lo;0;L;;;;;N;;;;; +13A3B;EGYPTIAN HIEROGLYPH-13A3B;Lo;0;L;;;;;N;;;;; +13A3C;EGYPTIAN HIEROGLYPH-13A3C;Lo;0;L;;;;;N;;;;; +13A3D;EGYPTIAN HIEROGLYPH-13A3D;Lo;0;L;;;;;N;;;;; +13A3E;EGYPTIAN HIEROGLYPH-13A3E;Lo;0;L;;;;;N;;;;; +13A3F;EGYPTIAN HIEROGLYPH-13A3F;Lo;0;L;;;;;N;;;;; +13A40;EGYPTIAN HIEROGLYPH-13A40;Lo;0;L;;;;;N;;;;; +13A41;EGYPTIAN HIEROGLYPH-13A41;Lo;0;L;;;;;N;;;;; +13A42;EGYPTIAN HIEROGLYPH-13A42;Lo;0;L;;;;;N;;;;; +13A43;EGYPTIAN HIEROGLYPH-13A43;Lo;0;L;;;;;N;;;;; +13A44;EGYPTIAN HIEROGLYPH-13A44;Lo;0;L;;;;;N;;;;; +13A45;EGYPTIAN HIEROGLYPH-13A45;Lo;0;L;;;;;N;;;;; +13A46;EGYPTIAN HIEROGLYPH-13A46;Lo;0;L;;;;;N;;;;; +13A47;EGYPTIAN HIEROGLYPH-13A47;Lo;0;L;;;;;N;;;;; +13A48;EGYPTIAN HIEROGLYPH-13A48;Lo;0;L;;;;;N;;;;; +13A49;EGYPTIAN HIEROGLYPH-13A49;Lo;0;L;;;;;N;;;;; +13A4A;EGYPTIAN HIEROGLYPH-13A4A;Lo;0;L;;;;;N;;;;; +13A4B;EGYPTIAN HIEROGLYPH-13A4B;Lo;0;L;;;;;N;;;;; +13A4C;EGYPTIAN HIEROGLYPH-13A4C;Lo;0;L;;;;;N;;;;; +13A4D;EGYPTIAN HIEROGLYPH-13A4D;Lo;0;L;;;;;N;;;;; +13A4E;EGYPTIAN HIEROGLYPH-13A4E;Lo;0;L;;;;;N;;;;; +13A4F;EGYPTIAN HIEROGLYPH-13A4F;Lo;0;L;;;;;N;;;;; +13A50;EGYPTIAN HIEROGLYPH-13A50;Lo;0;L;;;;;N;;;;; +13A51;EGYPTIAN HIEROGLYPH-13A51;Lo;0;L;;;;;N;;;;; +13A52;EGYPTIAN HIEROGLYPH-13A52;Lo;0;L;;;;;N;;;;; +13A53;EGYPTIAN HIEROGLYPH-13A53;Lo;0;L;;;;;N;;;;; +13A54;EGYPTIAN HIEROGLYPH-13A54;Lo;0;L;;;;;N;;;;; +13A55;EGYPTIAN HIEROGLYPH-13A55;Lo;0;L;;;;;N;;;;; +13A56;EGYPTIAN HIEROGLYPH-13A56;Lo;0;L;;;;;N;;;;; +13A57;EGYPTIAN HIEROGLYPH-13A57;Lo;0;L;;;;;N;;;;; +13A58;EGYPTIAN HIEROGLYPH-13A58;Lo;0;L;;;;;N;;;;; +13A59;EGYPTIAN HIEROGLYPH-13A59;Lo;0;L;;;;;N;;;;; +13A5A;EGYPTIAN HIEROGLYPH-13A5A;Lo;0;L;;;;;N;;;;; +13A5B;EGYPTIAN HIEROGLYPH-13A5B;Lo;0;L;;;;;N;;;;; +13A5C;EGYPTIAN HIEROGLYPH-13A5C;Lo;0;L;;;;;N;;;;; +13A5D;EGYPTIAN HIEROGLYPH-13A5D;Lo;0;L;;;;;N;;;;; +13A5E;EGYPTIAN HIEROGLYPH-13A5E;Lo;0;L;;;;;N;;;;; +13A5F;EGYPTIAN HIEROGLYPH-13A5F;Lo;0;L;;;;;N;;;;; +13A60;EGYPTIAN HIEROGLYPH-13A60;Lo;0;L;;;;;N;;;;; +13A61;EGYPTIAN HIEROGLYPH-13A61;Lo;0;L;;;;;N;;;;; +13A62;EGYPTIAN HIEROGLYPH-13A62;Lo;0;L;;;;;N;;;;; +13A63;EGYPTIAN HIEROGLYPH-13A63;Lo;0;L;;;;;N;;;;; +13A64;EGYPTIAN HIEROGLYPH-13A64;Lo;0;L;;;;;N;;;;; +13A65;EGYPTIAN HIEROGLYPH-13A65;Lo;0;L;;;;;N;;;;; +13A66;EGYPTIAN HIEROGLYPH-13A66;Lo;0;L;;;;;N;;;;; +13A67;EGYPTIAN HIEROGLYPH-13A67;Lo;0;L;;;;;N;;;;; +13A68;EGYPTIAN HIEROGLYPH-13A68;Lo;0;L;;;;;N;;;;; +13A69;EGYPTIAN HIEROGLYPH-13A69;Lo;0;L;;;;;N;;;;; +13A6A;EGYPTIAN HIEROGLYPH-13A6A;Lo;0;L;;;;;N;;;;; +13A6B;EGYPTIAN HIEROGLYPH-13A6B;Lo;0;L;;;;;N;;;;; +13A6C;EGYPTIAN HIEROGLYPH-13A6C;Lo;0;L;;;;;N;;;;; +13A6D;EGYPTIAN HIEROGLYPH-13A6D;Lo;0;L;;;;;N;;;;; +13A6E;EGYPTIAN HIEROGLYPH-13A6E;Lo;0;L;;;;;N;;;;; +13A6F;EGYPTIAN HIEROGLYPH-13A6F;Lo;0;L;;;;;N;;;;; +13A70;EGYPTIAN HIEROGLYPH-13A70;Lo;0;L;;;;;N;;;;; +13A71;EGYPTIAN HIEROGLYPH-13A71;Lo;0;L;;;;;N;;;;; +13A72;EGYPTIAN HIEROGLYPH-13A72;Lo;0;L;;;;;N;;;;; +13A73;EGYPTIAN HIEROGLYPH-13A73;Lo;0;L;;;;;N;;;;; +13A74;EGYPTIAN HIEROGLYPH-13A74;Lo;0;L;;;;;N;;;;; +13A75;EGYPTIAN HIEROGLYPH-13A75;Lo;0;L;;;;;N;;;;; +13A76;EGYPTIAN HIEROGLYPH-13A76;Lo;0;L;;;;;N;;;;; +13A77;EGYPTIAN HIEROGLYPH-13A77;Lo;0;L;;;;;N;;;;; +13A78;EGYPTIAN HIEROGLYPH-13A78;Lo;0;L;;;;;N;;;;; +13A79;EGYPTIAN HIEROGLYPH-13A79;Lo;0;L;;;;;N;;;;; +13A7A;EGYPTIAN HIEROGLYPH-13A7A;Lo;0;L;;;;;N;;;;; +13A7B;EGYPTIAN HIEROGLYPH-13A7B;Lo;0;L;;;;;N;;;;; +13A7C;EGYPTIAN HIEROGLYPH-13A7C;Lo;0;L;;;;;N;;;;; +13A7D;EGYPTIAN HIEROGLYPH-13A7D;Lo;0;L;;;;;N;;;;; +13A7E;EGYPTIAN HIEROGLYPH-13A7E;Lo;0;L;;;;;N;;;;; +13A7F;EGYPTIAN HIEROGLYPH-13A7F;Lo;0;L;;;;;N;;;;; +13A80;EGYPTIAN HIEROGLYPH-13A80;Lo;0;L;;;;;N;;;;; +13A81;EGYPTIAN HIEROGLYPH-13A81;Lo;0;L;;;;;N;;;;; +13A82;EGYPTIAN HIEROGLYPH-13A82;Lo;0;L;;;;;N;;;;; +13A83;EGYPTIAN HIEROGLYPH-13A83;Lo;0;L;;;;;N;;;;; +13A84;EGYPTIAN HIEROGLYPH-13A84;Lo;0;L;;;;;N;;;;; +13A85;EGYPTIAN HIEROGLYPH-13A85;Lo;0;L;;;;;N;;;;; +13A86;EGYPTIAN HIEROGLYPH-13A86;Lo;0;L;;;;;N;;;;; +13A87;EGYPTIAN HIEROGLYPH-13A87;Lo;0;L;;;;;N;;;;; +13A88;EGYPTIAN HIEROGLYPH-13A88;Lo;0;L;;;;;N;;;;; +13A89;EGYPTIAN HIEROGLYPH-13A89;Lo;0;L;;;;;N;;;;; +13A8A;EGYPTIAN HIEROGLYPH-13A8A;Lo;0;L;;;;;N;;;;; +13A8B;EGYPTIAN HIEROGLYPH-13A8B;Lo;0;L;;;;;N;;;;; +13A8C;EGYPTIAN HIEROGLYPH-13A8C;Lo;0;L;;;;;N;;;;; +13A8D;EGYPTIAN HIEROGLYPH-13A8D;Lo;0;L;;;;;N;;;;; +13A8E;EGYPTIAN HIEROGLYPH-13A8E;Lo;0;L;;;;;N;;;;; +13A8F;EGYPTIAN HIEROGLYPH-13A8F;Lo;0;L;;;;;N;;;;; +13A90;EGYPTIAN HIEROGLYPH-13A90;Lo;0;L;;;;;N;;;;; +13A91;EGYPTIAN HIEROGLYPH-13A91;Lo;0;L;;;;;N;;;;; +13A92;EGYPTIAN HIEROGLYPH-13A92;Lo;0;L;;;;;N;;;;; +13A93;EGYPTIAN HIEROGLYPH-13A93;Lo;0;L;;;;;N;;;;; +13A94;EGYPTIAN HIEROGLYPH-13A94;Lo;0;L;;;;;N;;;;; +13A95;EGYPTIAN HIEROGLYPH-13A95;Lo;0;L;;;;;N;;;;; +13A96;EGYPTIAN HIEROGLYPH-13A96;Lo;0;L;;;;;N;;;;; +13A97;EGYPTIAN HIEROGLYPH-13A97;Lo;0;L;;;;;N;;;;; +13A98;EGYPTIAN HIEROGLYPH-13A98;Lo;0;L;;;;;N;;;;; +13A99;EGYPTIAN HIEROGLYPH-13A99;Lo;0;L;;;;;N;;;;; +13A9A;EGYPTIAN HIEROGLYPH-13A9A;Lo;0;L;;;;;N;;;;; +13A9B;EGYPTIAN HIEROGLYPH-13A9B;Lo;0;L;;;;;N;;;;; +13A9C;EGYPTIAN HIEROGLYPH-13A9C;Lo;0;L;;;;;N;;;;; +13A9D;EGYPTIAN HIEROGLYPH-13A9D;Lo;0;L;;;;;N;;;;; +13A9E;EGYPTIAN HIEROGLYPH-13A9E;Lo;0;L;;;;;N;;;;; +13A9F;EGYPTIAN HIEROGLYPH-13A9F;Lo;0;L;;;;;N;;;;; +13AA0;EGYPTIAN HIEROGLYPH-13AA0;Lo;0;L;;;;;N;;;;; +13AA1;EGYPTIAN HIEROGLYPH-13AA1;Lo;0;L;;;;;N;;;;; +13AA2;EGYPTIAN HIEROGLYPH-13AA2;Lo;0;L;;;;;N;;;;; +13AA3;EGYPTIAN HIEROGLYPH-13AA3;Lo;0;L;;;;;N;;;;; +13AA4;EGYPTIAN HIEROGLYPH-13AA4;Lo;0;L;;;;;N;;;;; +13AA5;EGYPTIAN HIEROGLYPH-13AA5;Lo;0;L;;;;;N;;;;; +13AA6;EGYPTIAN HIEROGLYPH-13AA6;Lo;0;L;;;;;N;;;;; +13AA7;EGYPTIAN HIEROGLYPH-13AA7;Lo;0;L;;;;;N;;;;; +13AA8;EGYPTIAN HIEROGLYPH-13AA8;Lo;0;L;;;;;N;;;;; +13AA9;EGYPTIAN HIEROGLYPH-13AA9;Lo;0;L;;;;;N;;;;; +13AAA;EGYPTIAN HIEROGLYPH-13AAA;Lo;0;L;;;;;N;;;;; +13AAB;EGYPTIAN HIEROGLYPH-13AAB;Lo;0;L;;;;;N;;;;; +13AAC;EGYPTIAN HIEROGLYPH-13AAC;Lo;0;L;;;;;N;;;;; +13AAD;EGYPTIAN HIEROGLYPH-13AAD;Lo;0;L;;;;;N;;;;; +13AAE;EGYPTIAN HIEROGLYPH-13AAE;Lo;0;L;;;;;N;;;;; +13AAF;EGYPTIAN HIEROGLYPH-13AAF;Lo;0;L;;;;;N;;;;; +13AB0;EGYPTIAN HIEROGLYPH-13AB0;Lo;0;L;;;;;N;;;;; +13AB1;EGYPTIAN HIEROGLYPH-13AB1;Lo;0;L;;;;;N;;;;; +13AB2;EGYPTIAN HIEROGLYPH-13AB2;Lo;0;L;;;;;N;;;;; +13AB3;EGYPTIAN HIEROGLYPH-13AB3;Lo;0;L;;;;;N;;;;; +13AB4;EGYPTIAN HIEROGLYPH-13AB4;Lo;0;L;;;;;N;;;;; +13AB5;EGYPTIAN HIEROGLYPH-13AB5;Lo;0;L;;;;;N;;;;; +13AB6;EGYPTIAN HIEROGLYPH-13AB6;Lo;0;L;;;;;N;;;;; +13AB7;EGYPTIAN HIEROGLYPH-13AB7;Lo;0;L;;;;;N;;;;; +13AB8;EGYPTIAN HIEROGLYPH-13AB8;Lo;0;L;;;;;N;;;;; +13AB9;EGYPTIAN HIEROGLYPH-13AB9;Lo;0;L;;;;;N;;;;; +13ABA;EGYPTIAN HIEROGLYPH-13ABA;Lo;0;L;;;;;N;;;;; +13ABB;EGYPTIAN HIEROGLYPH-13ABB;Lo;0;L;;;;;N;;;;; +13ABC;EGYPTIAN HIEROGLYPH-13ABC;Lo;0;L;;;;;N;;;;; +13ABD;EGYPTIAN HIEROGLYPH-13ABD;Lo;0;L;;;;;N;;;;; +13ABE;EGYPTIAN HIEROGLYPH-13ABE;Lo;0;L;;;;;N;;;;; +13ABF;EGYPTIAN HIEROGLYPH-13ABF;Lo;0;L;;;;;N;;;;; +13AC0;EGYPTIAN HIEROGLYPH-13AC0;Lo;0;L;;;;;N;;;;; +13AC1;EGYPTIAN HIEROGLYPH-13AC1;Lo;0;L;;;;;N;;;;; +13AC2;EGYPTIAN HIEROGLYPH-13AC2;Lo;0;L;;;;;N;;;;; +13AC3;EGYPTIAN HIEROGLYPH-13AC3;Lo;0;L;;;;;N;;;;; +13AC4;EGYPTIAN HIEROGLYPH-13AC4;Lo;0;L;;;;;N;;;;; +13AC5;EGYPTIAN HIEROGLYPH-13AC5;Lo;0;L;;;;;N;;;;; +13AC6;EGYPTIAN HIEROGLYPH-13AC6;Lo;0;L;;;;;N;;;;; +13AC7;EGYPTIAN HIEROGLYPH-13AC7;Lo;0;L;;;;;N;;;;; +13AC8;EGYPTIAN HIEROGLYPH-13AC8;Lo;0;L;;;;;N;;;;; +13AC9;EGYPTIAN HIEROGLYPH-13AC9;Lo;0;L;;;;;N;;;;; +13ACA;EGYPTIAN HIEROGLYPH-13ACA;Lo;0;L;;;;;N;;;;; +13ACB;EGYPTIAN HIEROGLYPH-13ACB;Lo;0;L;;;;;N;;;;; +13ACC;EGYPTIAN HIEROGLYPH-13ACC;Lo;0;L;;;;;N;;;;; +13ACD;EGYPTIAN HIEROGLYPH-13ACD;Lo;0;L;;;;;N;;;;; +13ACE;EGYPTIAN HIEROGLYPH-13ACE;Lo;0;L;;;;;N;;;;; +13ACF;EGYPTIAN HIEROGLYPH-13ACF;Lo;0;L;;;;;N;;;;; +13AD0;EGYPTIAN HIEROGLYPH-13AD0;Lo;0;L;;;;;N;;;;; +13AD1;EGYPTIAN HIEROGLYPH-13AD1;Lo;0;L;;;;;N;;;;; +13AD2;EGYPTIAN HIEROGLYPH-13AD2;Lo;0;L;;;;;N;;;;; +13AD3;EGYPTIAN HIEROGLYPH-13AD3;Lo;0;L;;;;;N;;;;; +13AD4;EGYPTIAN HIEROGLYPH-13AD4;Lo;0;L;;;;;N;;;;; +13AD5;EGYPTIAN HIEROGLYPH-13AD5;Lo;0;L;;;;;N;;;;; +13AD6;EGYPTIAN HIEROGLYPH-13AD6;Lo;0;L;;;;;N;;;;; +13AD7;EGYPTIAN HIEROGLYPH-13AD7;Lo;0;L;;;;;N;;;;; +13AD8;EGYPTIAN HIEROGLYPH-13AD8;Lo;0;L;;;;;N;;;;; +13AD9;EGYPTIAN HIEROGLYPH-13AD9;Lo;0;L;;;;;N;;;;; +13ADA;EGYPTIAN HIEROGLYPH-13ADA;Lo;0;L;;;;;N;;;;; +13ADB;EGYPTIAN HIEROGLYPH-13ADB;Lo;0;L;;;;;N;;;;; +13ADC;EGYPTIAN HIEROGLYPH-13ADC;Lo;0;L;;;;;N;;;;; +13ADD;EGYPTIAN HIEROGLYPH-13ADD;Lo;0;L;;;;;N;;;;; +13ADE;EGYPTIAN HIEROGLYPH-13ADE;Lo;0;L;;;;;N;;;;; +13ADF;EGYPTIAN HIEROGLYPH-13ADF;Lo;0;L;;;;;N;;;;; +13AE0;EGYPTIAN HIEROGLYPH-13AE0;Lo;0;L;;;;;N;;;;; +13AE1;EGYPTIAN HIEROGLYPH-13AE1;Lo;0;L;;;;;N;;;;; +13AE2;EGYPTIAN HIEROGLYPH-13AE2;Lo;0;L;;;;;N;;;;; +13AE3;EGYPTIAN HIEROGLYPH-13AE3;Lo;0;L;;;;;N;;;;; +13AE4;EGYPTIAN HIEROGLYPH-13AE4;Lo;0;L;;;;;N;;;;; +13AE5;EGYPTIAN HIEROGLYPH-13AE5;Lo;0;L;;;;;N;;;;; +13AE6;EGYPTIAN HIEROGLYPH-13AE6;Lo;0;L;;;;;N;;;;; +13AE7;EGYPTIAN HIEROGLYPH-13AE7;Lo;0;L;;;;;N;;;;; +13AE8;EGYPTIAN HIEROGLYPH-13AE8;Lo;0;L;;;;;N;;;;; +13AE9;EGYPTIAN HIEROGLYPH-13AE9;Lo;0;L;;;;;N;;;;; +13AEA;EGYPTIAN HIEROGLYPH-13AEA;Lo;0;L;;;;;N;;;;; +13AEB;EGYPTIAN HIEROGLYPH-13AEB;Lo;0;L;;;;;N;;;;; +13AEC;EGYPTIAN HIEROGLYPH-13AEC;Lo;0;L;;;;;N;;;;; +13AED;EGYPTIAN HIEROGLYPH-13AED;Lo;0;L;;;;;N;;;;; +13AEE;EGYPTIAN HIEROGLYPH-13AEE;Lo;0;L;;;;;N;;;;; +13AEF;EGYPTIAN HIEROGLYPH-13AEF;Lo;0;L;;;;;N;;;;; +13AF0;EGYPTIAN HIEROGLYPH-13AF0;Lo;0;L;;;;;N;;;;; +13AF1;EGYPTIAN HIEROGLYPH-13AF1;Lo;0;L;;;;;N;;;;; +13AF2;EGYPTIAN HIEROGLYPH-13AF2;Lo;0;L;;;;;N;;;;; +13AF3;EGYPTIAN HIEROGLYPH-13AF3;Lo;0;L;;;;;N;;;;; +13AF4;EGYPTIAN HIEROGLYPH-13AF4;Lo;0;L;;;;;N;;;;; +13AF5;EGYPTIAN HIEROGLYPH-13AF5;Lo;0;L;;;;;N;;;;; +13AF6;EGYPTIAN HIEROGLYPH-13AF6;Lo;0;L;;;;;N;;;;; +13AF7;EGYPTIAN HIEROGLYPH-13AF7;Lo;0;L;;;;;N;;;;; +13AF8;EGYPTIAN HIEROGLYPH-13AF8;Lo;0;L;;;;;N;;;;; +13AF9;EGYPTIAN HIEROGLYPH-13AF9;Lo;0;L;;;;;N;;;;; +13AFA;EGYPTIAN HIEROGLYPH-13AFA;Lo;0;L;;;;;N;;;;; +13AFB;EGYPTIAN HIEROGLYPH-13AFB;Lo;0;L;;;;;N;;;;; +13AFC;EGYPTIAN HIEROGLYPH-13AFC;Lo;0;L;;;;;N;;;;; +13AFD;EGYPTIAN HIEROGLYPH-13AFD;Lo;0;L;;;;;N;;;;; +13AFE;EGYPTIAN HIEROGLYPH-13AFE;Lo;0;L;;;;;N;;;;; +13AFF;EGYPTIAN HIEROGLYPH-13AFF;Lo;0;L;;;;;N;;;;; +13B00;EGYPTIAN HIEROGLYPH-13B00;Lo;0;L;;;;;N;;;;; +13B01;EGYPTIAN HIEROGLYPH-13B01;Lo;0;L;;;;;N;;;;; +13B02;EGYPTIAN HIEROGLYPH-13B02;Lo;0;L;;;;;N;;;;; +13B03;EGYPTIAN HIEROGLYPH-13B03;Lo;0;L;;;;;N;;;;; +13B04;EGYPTIAN HIEROGLYPH-13B04;Lo;0;L;;;;;N;;;;; +13B05;EGYPTIAN HIEROGLYPH-13B05;Lo;0;L;;;;;N;;;;; +13B06;EGYPTIAN HIEROGLYPH-13B06;Lo;0;L;;;;;N;;;;; +13B07;EGYPTIAN HIEROGLYPH-13B07;Lo;0;L;;;;;N;;;;; +13B08;EGYPTIAN HIEROGLYPH-13B08;Lo;0;L;;;;;N;;;;; +13B09;EGYPTIAN HIEROGLYPH-13B09;Lo;0;L;;;;;N;;;;; +13B0A;EGYPTIAN HIEROGLYPH-13B0A;Lo;0;L;;;;;N;;;;; +13B0B;EGYPTIAN HIEROGLYPH-13B0B;Lo;0;L;;;;;N;;;;; +13B0C;EGYPTIAN HIEROGLYPH-13B0C;Lo;0;L;;;;;N;;;;; +13B0D;EGYPTIAN HIEROGLYPH-13B0D;Lo;0;L;;;;;N;;;;; +13B0E;EGYPTIAN HIEROGLYPH-13B0E;Lo;0;L;;;;;N;;;;; +13B0F;EGYPTIAN HIEROGLYPH-13B0F;Lo;0;L;;;;;N;;;;; +13B10;EGYPTIAN HIEROGLYPH-13B10;Lo;0;L;;;;;N;;;;; +13B11;EGYPTIAN HIEROGLYPH-13B11;Lo;0;L;;;;;N;;;;; +13B12;EGYPTIAN HIEROGLYPH-13B12;Lo;0;L;;;;;N;;;;; +13B13;EGYPTIAN HIEROGLYPH-13B13;Lo;0;L;;;;;N;;;;; +13B14;EGYPTIAN HIEROGLYPH-13B14;Lo;0;L;;;;;N;;;;; +13B15;EGYPTIAN HIEROGLYPH-13B15;Lo;0;L;;;;;N;;;;; +13B16;EGYPTIAN HIEROGLYPH-13B16;Lo;0;L;;;;;N;;;;; +13B17;EGYPTIAN HIEROGLYPH-13B17;Lo;0;L;;;;;N;;;;; +13B18;EGYPTIAN HIEROGLYPH-13B18;Lo;0;L;;;;;N;;;;; +13B19;EGYPTIAN HIEROGLYPH-13B19;Lo;0;L;;;;;N;;;;; +13B1A;EGYPTIAN HIEROGLYPH-13B1A;Lo;0;L;;;;;N;;;;; +13B1B;EGYPTIAN HIEROGLYPH-13B1B;Lo;0;L;;;;;N;;;;; +13B1C;EGYPTIAN HIEROGLYPH-13B1C;Lo;0;L;;;;;N;;;;; +13B1D;EGYPTIAN HIEROGLYPH-13B1D;Lo;0;L;;;;;N;;;;; +13B1E;EGYPTIAN HIEROGLYPH-13B1E;Lo;0;L;;;;;N;;;;; +13B1F;EGYPTIAN HIEROGLYPH-13B1F;Lo;0;L;;;;;N;;;;; +13B20;EGYPTIAN HIEROGLYPH-13B20;Lo;0;L;;;;;N;;;;; +13B21;EGYPTIAN HIEROGLYPH-13B21;Lo;0;L;;;;;N;;;;; +13B22;EGYPTIAN HIEROGLYPH-13B22;Lo;0;L;;;;;N;;;;; +13B23;EGYPTIAN HIEROGLYPH-13B23;Lo;0;L;;;;;N;;;;; +13B24;EGYPTIAN HIEROGLYPH-13B24;Lo;0;L;;;;;N;;;;; +13B25;EGYPTIAN HIEROGLYPH-13B25;Lo;0;L;;;;;N;;;;; +13B26;EGYPTIAN HIEROGLYPH-13B26;Lo;0;L;;;;;N;;;;; +13B27;EGYPTIAN HIEROGLYPH-13B27;Lo;0;L;;;;;N;;;;; +13B28;EGYPTIAN HIEROGLYPH-13B28;Lo;0;L;;;;;N;;;;; +13B29;EGYPTIAN HIEROGLYPH-13B29;Lo;0;L;;;;;N;;;;; +13B2A;EGYPTIAN HIEROGLYPH-13B2A;Lo;0;L;;;;;N;;;;; +13B2B;EGYPTIAN HIEROGLYPH-13B2B;Lo;0;L;;;;;N;;;;; +13B2C;EGYPTIAN HIEROGLYPH-13B2C;Lo;0;L;;;;;N;;;;; +13B2D;EGYPTIAN HIEROGLYPH-13B2D;Lo;0;L;;;;;N;;;;; +13B2E;EGYPTIAN HIEROGLYPH-13B2E;Lo;0;L;;;;;N;;;;; +13B2F;EGYPTIAN HIEROGLYPH-13B2F;Lo;0;L;;;;;N;;;;; +13B30;EGYPTIAN HIEROGLYPH-13B30;Lo;0;L;;;;;N;;;;; +13B31;EGYPTIAN HIEROGLYPH-13B31;Lo;0;L;;;;;N;;;;; +13B32;EGYPTIAN HIEROGLYPH-13B32;Lo;0;L;;;;;N;;;;; +13B33;EGYPTIAN HIEROGLYPH-13B33;Lo;0;L;;;;;N;;;;; +13B34;EGYPTIAN HIEROGLYPH-13B34;Lo;0;L;;;;;N;;;;; +13B35;EGYPTIAN HIEROGLYPH-13B35;Lo;0;L;;;;;N;;;;; +13B36;EGYPTIAN HIEROGLYPH-13B36;Lo;0;L;;;;;N;;;;; +13B37;EGYPTIAN HIEROGLYPH-13B37;Lo;0;L;;;;;N;;;;; +13B38;EGYPTIAN HIEROGLYPH-13B38;Lo;0;L;;;;;N;;;;; +13B39;EGYPTIAN HIEROGLYPH-13B39;Lo;0;L;;;;;N;;;;; +13B3A;EGYPTIAN HIEROGLYPH-13B3A;Lo;0;L;;;;;N;;;;; +13B3B;EGYPTIAN HIEROGLYPH-13B3B;Lo;0;L;;;;;N;;;;; +13B3C;EGYPTIAN HIEROGLYPH-13B3C;Lo;0;L;;;;;N;;;;; +13B3D;EGYPTIAN HIEROGLYPH-13B3D;Lo;0;L;;;;;N;;;;; +13B3E;EGYPTIAN HIEROGLYPH-13B3E;Lo;0;L;;;;;N;;;;; +13B3F;EGYPTIAN HIEROGLYPH-13B3F;Lo;0;L;;;;;N;;;;; +13B40;EGYPTIAN HIEROGLYPH-13B40;Lo;0;L;;;;;N;;;;; +13B41;EGYPTIAN HIEROGLYPH-13B41;Lo;0;L;;;;;N;;;;; +13B42;EGYPTIAN HIEROGLYPH-13B42;Lo;0;L;;;;;N;;;;; +13B43;EGYPTIAN HIEROGLYPH-13B43;Lo;0;L;;;;;N;;;;; +13B44;EGYPTIAN HIEROGLYPH-13B44;Lo;0;L;;;;;N;;;;; +13B45;EGYPTIAN HIEROGLYPH-13B45;Lo;0;L;;;;;N;;;;; +13B46;EGYPTIAN HIEROGLYPH-13B46;Lo;0;L;;;;;N;;;;; +13B47;EGYPTIAN HIEROGLYPH-13B47;Lo;0;L;;;;;N;;;;; +13B48;EGYPTIAN HIEROGLYPH-13B48;Lo;0;L;;;;;N;;;;; +13B49;EGYPTIAN HIEROGLYPH-13B49;Lo;0;L;;;;;N;;;;; +13B4A;EGYPTIAN HIEROGLYPH-13B4A;Lo;0;L;;;;;N;;;;; +13B4B;EGYPTIAN HIEROGLYPH-13B4B;Lo;0;L;;;;;N;;;;; +13B4C;EGYPTIAN HIEROGLYPH-13B4C;Lo;0;L;;;;;N;;;;; +13B4D;EGYPTIAN HIEROGLYPH-13B4D;Lo;0;L;;;;;N;;;;; +13B4E;EGYPTIAN HIEROGLYPH-13B4E;Lo;0;L;;;;;N;;;;; +13B4F;EGYPTIAN HIEROGLYPH-13B4F;Lo;0;L;;;;;N;;;;; +13B50;EGYPTIAN HIEROGLYPH-13B50;Lo;0;L;;;;;N;;;;; +13B51;EGYPTIAN HIEROGLYPH-13B51;Lo;0;L;;;;;N;;;;; +13B52;EGYPTIAN HIEROGLYPH-13B52;Lo;0;L;;;;;N;;;;; +13B53;EGYPTIAN HIEROGLYPH-13B53;Lo;0;L;;;;;N;;;;; +13B54;EGYPTIAN HIEROGLYPH-13B54;Lo;0;L;;;;;N;;;;; +13B55;EGYPTIAN HIEROGLYPH-13B55;Lo;0;L;;;;;N;;;;; +13B56;EGYPTIAN HIEROGLYPH-13B56;Lo;0;L;;;;;N;;;;; +13B57;EGYPTIAN HIEROGLYPH-13B57;Lo;0;L;;;;;N;;;;; +13B58;EGYPTIAN HIEROGLYPH-13B58;Lo;0;L;;;;;N;;;;; +13B59;EGYPTIAN HIEROGLYPH-13B59;Lo;0;L;;;;;N;;;;; +13B5A;EGYPTIAN HIEROGLYPH-13B5A;Lo;0;L;;;;;N;;;;; +13B5B;EGYPTIAN HIEROGLYPH-13B5B;Lo;0;L;;;;;N;;;;; +13B5C;EGYPTIAN HIEROGLYPH-13B5C;Lo;0;L;;;;;N;;;;; +13B5D;EGYPTIAN HIEROGLYPH-13B5D;Lo;0;L;;;;;N;;;;; +13B5E;EGYPTIAN HIEROGLYPH-13B5E;Lo;0;L;;;;;N;;;;; +13B5F;EGYPTIAN HIEROGLYPH-13B5F;Lo;0;L;;;;;N;;;;; +13B60;EGYPTIAN HIEROGLYPH-13B60;Lo;0;L;;;;;N;;;;; +13B61;EGYPTIAN HIEROGLYPH-13B61;Lo;0;L;;;;;N;;;;; +13B62;EGYPTIAN HIEROGLYPH-13B62;Lo;0;L;;;;;N;;;;; +13B63;EGYPTIAN HIEROGLYPH-13B63;Lo;0;L;;;;;N;;;;; +13B64;EGYPTIAN HIEROGLYPH-13B64;Lo;0;L;;;;;N;;;;; +13B65;EGYPTIAN HIEROGLYPH-13B65;Lo;0;L;;;;;N;;;;; +13B66;EGYPTIAN HIEROGLYPH-13B66;Lo;0;L;;;;;N;;;;; +13B67;EGYPTIAN HIEROGLYPH-13B67;Lo;0;L;;;;;N;;;;; +13B68;EGYPTIAN HIEROGLYPH-13B68;Lo;0;L;;;;;N;;;;; +13B69;EGYPTIAN HIEROGLYPH-13B69;Lo;0;L;;;;;N;;;;; +13B6A;EGYPTIAN HIEROGLYPH-13B6A;Lo;0;L;;;;;N;;;;; +13B6B;EGYPTIAN HIEROGLYPH-13B6B;Lo;0;L;;;;;N;;;;; +13B6C;EGYPTIAN HIEROGLYPH-13B6C;Lo;0;L;;;;;N;;;;; +13B6D;EGYPTIAN HIEROGLYPH-13B6D;Lo;0;L;;;;;N;;;;; +13B6E;EGYPTIAN HIEROGLYPH-13B6E;Lo;0;L;;;;;N;;;;; +13B6F;EGYPTIAN HIEROGLYPH-13B6F;Lo;0;L;;;;;N;;;;; +13B70;EGYPTIAN HIEROGLYPH-13B70;Lo;0;L;;;;;N;;;;; +13B71;EGYPTIAN HIEROGLYPH-13B71;Lo;0;L;;;;;N;;;;; +13B72;EGYPTIAN HIEROGLYPH-13B72;Lo;0;L;;;;;N;;;;; +13B73;EGYPTIAN HIEROGLYPH-13B73;Lo;0;L;;;;;N;;;;; +13B74;EGYPTIAN HIEROGLYPH-13B74;Lo;0;L;;;;;N;;;;; +13B75;EGYPTIAN HIEROGLYPH-13B75;Lo;0;L;;;;;N;;;;; +13B76;EGYPTIAN HIEROGLYPH-13B76;Lo;0;L;;;;;N;;;;; +13B77;EGYPTIAN HIEROGLYPH-13B77;Lo;0;L;;;;;N;;;;; +13B78;EGYPTIAN HIEROGLYPH-13B78;Lo;0;L;;;;;N;;;;; +13B79;EGYPTIAN HIEROGLYPH-13B79;Lo;0;L;;;;;N;;;;; +13B7A;EGYPTIAN HIEROGLYPH-13B7A;Lo;0;L;;;;;N;;;;; +13B7B;EGYPTIAN HIEROGLYPH-13B7B;Lo;0;L;;;;;N;;;;; +13B7C;EGYPTIAN HIEROGLYPH-13B7C;Lo;0;L;;;;;N;;;;; +13B7D;EGYPTIAN HIEROGLYPH-13B7D;Lo;0;L;;;;;N;;;;; +13B7E;EGYPTIAN HIEROGLYPH-13B7E;Lo;0;L;;;;;N;;;;; +13B7F;EGYPTIAN HIEROGLYPH-13B7F;Lo;0;L;;;;;N;;;;; +13B80;EGYPTIAN HIEROGLYPH-13B80;Lo;0;L;;;;;N;;;;; +13B81;EGYPTIAN HIEROGLYPH-13B81;Lo;0;L;;;;;N;;;;; +13B82;EGYPTIAN HIEROGLYPH-13B82;Lo;0;L;;;;;N;;;;; +13B83;EGYPTIAN HIEROGLYPH-13B83;Lo;0;L;;;;;N;;;;; +13B84;EGYPTIAN HIEROGLYPH-13B84;Lo;0;L;;;;;N;;;;; +13B85;EGYPTIAN HIEROGLYPH-13B85;Lo;0;L;;;;;N;;;;; +13B86;EGYPTIAN HIEROGLYPH-13B86;Lo;0;L;;;;;N;;;;; +13B87;EGYPTIAN HIEROGLYPH-13B87;Lo;0;L;;;;;N;;;;; +13B88;EGYPTIAN HIEROGLYPH-13B88;Lo;0;L;;;;;N;;;;; +13B89;EGYPTIAN HIEROGLYPH-13B89;Lo;0;L;;;;;N;;;;; +13B8A;EGYPTIAN HIEROGLYPH-13B8A;Lo;0;L;;;;;N;;;;; +13B8B;EGYPTIAN HIEROGLYPH-13B8B;Lo;0;L;;;;;N;;;;; +13B8C;EGYPTIAN HIEROGLYPH-13B8C;Lo;0;L;;;;;N;;;;; +13B8D;EGYPTIAN HIEROGLYPH-13B8D;Lo;0;L;;;;;N;;;;; +13B8E;EGYPTIAN HIEROGLYPH-13B8E;Lo;0;L;;;;;N;;;;; +13B8F;EGYPTIAN HIEROGLYPH-13B8F;Lo;0;L;;;;;N;;;;; +13B90;EGYPTIAN HIEROGLYPH-13B90;Lo;0;L;;;;;N;;;;; +13B91;EGYPTIAN HIEROGLYPH-13B91;Lo;0;L;;;;;N;;;;; +13B92;EGYPTIAN HIEROGLYPH-13B92;Lo;0;L;;;;;N;;;;; +13B93;EGYPTIAN HIEROGLYPH-13B93;Lo;0;L;;;;;N;;;;; +13B94;EGYPTIAN HIEROGLYPH-13B94;Lo;0;L;;;;;N;;;;; +13B95;EGYPTIAN HIEROGLYPH-13B95;Lo;0;L;;;;;N;;;;; +13B96;EGYPTIAN HIEROGLYPH-13B96;Lo;0;L;;;;;N;;;;; +13B97;EGYPTIAN HIEROGLYPH-13B97;Lo;0;L;;;;;N;;;;; +13B98;EGYPTIAN HIEROGLYPH-13B98;Lo;0;L;;;;;N;;;;; +13B99;EGYPTIAN HIEROGLYPH-13B99;Lo;0;L;;;;;N;;;;; +13B9A;EGYPTIAN HIEROGLYPH-13B9A;Lo;0;L;;;;;N;;;;; +13B9B;EGYPTIAN HIEROGLYPH-13B9B;Lo;0;L;;;;;N;;;;; +13B9C;EGYPTIAN HIEROGLYPH-13B9C;Lo;0;L;;;;;N;;;;; +13B9D;EGYPTIAN HIEROGLYPH-13B9D;Lo;0;L;;;;;N;;;;; +13B9E;EGYPTIAN HIEROGLYPH-13B9E;Lo;0;L;;;;;N;;;;; +13B9F;EGYPTIAN HIEROGLYPH-13B9F;Lo;0;L;;;;;N;;;;; +13BA0;EGYPTIAN HIEROGLYPH-13BA0;Lo;0;L;;;;;N;;;;; +13BA1;EGYPTIAN HIEROGLYPH-13BA1;Lo;0;L;;;;;N;;;;; +13BA2;EGYPTIAN HIEROGLYPH-13BA2;Lo;0;L;;;;;N;;;;; +13BA3;EGYPTIAN HIEROGLYPH-13BA3;Lo;0;L;;;;;N;;;;; +13BA4;EGYPTIAN HIEROGLYPH-13BA4;Lo;0;L;;;;;N;;;;; +13BA5;EGYPTIAN HIEROGLYPH-13BA5;Lo;0;L;;;;;N;;;;; +13BA6;EGYPTIAN HIEROGLYPH-13BA6;Lo;0;L;;;;;N;;;;; +13BA7;EGYPTIAN HIEROGLYPH-13BA7;Lo;0;L;;;;;N;;;;; +13BA8;EGYPTIAN HIEROGLYPH-13BA8;Lo;0;L;;;;;N;;;;; +13BA9;EGYPTIAN HIEROGLYPH-13BA9;Lo;0;L;;;;;N;;;;; +13BAA;EGYPTIAN HIEROGLYPH-13BAA;Lo;0;L;;;;;N;;;;; +13BAB;EGYPTIAN HIEROGLYPH-13BAB;Lo;0;L;;;;;N;;;;; +13BAC;EGYPTIAN HIEROGLYPH-13BAC;Lo;0;L;;;;;N;;;;; +13BAD;EGYPTIAN HIEROGLYPH-13BAD;Lo;0;L;;;;;N;;;;; +13BAE;EGYPTIAN HIEROGLYPH-13BAE;Lo;0;L;;;;;N;;;;; +13BAF;EGYPTIAN HIEROGLYPH-13BAF;Lo;0;L;;;;;N;;;;; +13BB0;EGYPTIAN HIEROGLYPH-13BB0;Lo;0;L;;;;;N;;;;; +13BB1;EGYPTIAN HIEROGLYPH-13BB1;Lo;0;L;;;;;N;;;;; +13BB2;EGYPTIAN HIEROGLYPH-13BB2;Lo;0;L;;;;;N;;;;; +13BB3;EGYPTIAN HIEROGLYPH-13BB3;Lo;0;L;;;;;N;;;;; +13BB4;EGYPTIAN HIEROGLYPH-13BB4;Lo;0;L;;;;;N;;;;; +13BB5;EGYPTIAN HIEROGLYPH-13BB5;Lo;0;L;;;;;N;;;;; +13BB6;EGYPTIAN HIEROGLYPH-13BB6;Lo;0;L;;;;;N;;;;; +13BB7;EGYPTIAN HIEROGLYPH-13BB7;Lo;0;L;;;;;N;;;;; +13BB8;EGYPTIAN HIEROGLYPH-13BB8;Lo;0;L;;;;;N;;;;; +13BB9;EGYPTIAN HIEROGLYPH-13BB9;Lo;0;L;;;;;N;;;;; +13BBA;EGYPTIAN HIEROGLYPH-13BBA;Lo;0;L;;;;;N;;;;; +13BBB;EGYPTIAN HIEROGLYPH-13BBB;Lo;0;L;;;;;N;;;;; +13BBC;EGYPTIAN HIEROGLYPH-13BBC;Lo;0;L;;;;;N;;;;; +13BBD;EGYPTIAN HIEROGLYPH-13BBD;Lo;0;L;;;;;N;;;;; +13BBE;EGYPTIAN HIEROGLYPH-13BBE;Lo;0;L;;;;;N;;;;; +13BBF;EGYPTIAN HIEROGLYPH-13BBF;Lo;0;L;;;;;N;;;;; +13BC0;EGYPTIAN HIEROGLYPH-13BC0;Lo;0;L;;;;;N;;;;; +13BC1;EGYPTIAN HIEROGLYPH-13BC1;Lo;0;L;;;;;N;;;;; +13BC2;EGYPTIAN HIEROGLYPH-13BC2;Lo;0;L;;;;;N;;;;; +13BC3;EGYPTIAN HIEROGLYPH-13BC3;Lo;0;L;;;;;N;;;;; +13BC4;EGYPTIAN HIEROGLYPH-13BC4;Lo;0;L;;;;;N;;;;; +13BC5;EGYPTIAN HIEROGLYPH-13BC5;Lo;0;L;;;;;N;;;;; +13BC6;EGYPTIAN HIEROGLYPH-13BC6;Lo;0;L;;;;;N;;;;; +13BC7;EGYPTIAN HIEROGLYPH-13BC7;Lo;0;L;;;;;N;;;;; +13BC8;EGYPTIAN HIEROGLYPH-13BC8;Lo;0;L;;;;;N;;;;; +13BC9;EGYPTIAN HIEROGLYPH-13BC9;Lo;0;L;;;;;N;;;;; +13BCA;EGYPTIAN HIEROGLYPH-13BCA;Lo;0;L;;;;;N;;;;; +13BCB;EGYPTIAN HIEROGLYPH-13BCB;Lo;0;L;;;;;N;;;;; +13BCC;EGYPTIAN HIEROGLYPH-13BCC;Lo;0;L;;;;;N;;;;; +13BCD;EGYPTIAN HIEROGLYPH-13BCD;Lo;0;L;;;;;N;;;;; +13BCE;EGYPTIAN HIEROGLYPH-13BCE;Lo;0;L;;;;;N;;;;; +13BCF;EGYPTIAN HIEROGLYPH-13BCF;Lo;0;L;;;;;N;;;;; +13BD0;EGYPTIAN HIEROGLYPH-13BD0;Lo;0;L;;;;;N;;;;; +13BD1;EGYPTIAN HIEROGLYPH-13BD1;Lo;0;L;;;;;N;;;;; +13BD2;EGYPTIAN HIEROGLYPH-13BD2;Lo;0;L;;;;;N;;;;; +13BD3;EGYPTIAN HIEROGLYPH-13BD3;Lo;0;L;;;;;N;;;;; +13BD4;EGYPTIAN HIEROGLYPH-13BD4;Lo;0;L;;;;;N;;;;; +13BD5;EGYPTIAN HIEROGLYPH-13BD5;Lo;0;L;;;;;N;;;;; +13BD6;EGYPTIAN HIEROGLYPH-13BD6;Lo;0;L;;;;;N;;;;; +13BD7;EGYPTIAN HIEROGLYPH-13BD7;Lo;0;L;;;;;N;;;;; +13BD8;EGYPTIAN HIEROGLYPH-13BD8;Lo;0;L;;;;;N;;;;; +13BD9;EGYPTIAN HIEROGLYPH-13BD9;Lo;0;L;;;;;N;;;;; +13BDA;EGYPTIAN HIEROGLYPH-13BDA;Lo;0;L;;;;;N;;;;; +13BDB;EGYPTIAN HIEROGLYPH-13BDB;Lo;0;L;;;;;N;;;;; +13BDC;EGYPTIAN HIEROGLYPH-13BDC;Lo;0;L;;;;;N;;;;; +13BDD;EGYPTIAN HIEROGLYPH-13BDD;Lo;0;L;;;;;N;;;;; +13BDE;EGYPTIAN HIEROGLYPH-13BDE;Lo;0;L;;;;;N;;;;; +13BDF;EGYPTIAN HIEROGLYPH-13BDF;Lo;0;L;;;;;N;;;;; +13BE0;EGYPTIAN HIEROGLYPH-13BE0;Lo;0;L;;;;;N;;;;; +13BE1;EGYPTIAN HIEROGLYPH-13BE1;Lo;0;L;;;;;N;;;;; +13BE2;EGYPTIAN HIEROGLYPH-13BE2;Lo;0;L;;;;;N;;;;; +13BE3;EGYPTIAN HIEROGLYPH-13BE3;Lo;0;L;;;;;N;;;;; +13BE4;EGYPTIAN HIEROGLYPH-13BE4;Lo;0;L;;;;;N;;;;; +13BE5;EGYPTIAN HIEROGLYPH-13BE5;Lo;0;L;;;;;N;;;;; +13BE6;EGYPTIAN HIEROGLYPH-13BE6;Lo;0;L;;;;;N;;;;; +13BE7;EGYPTIAN HIEROGLYPH-13BE7;Lo;0;L;;;;;N;;;;; +13BE8;EGYPTIAN HIEROGLYPH-13BE8;Lo;0;L;;;;;N;;;;; +13BE9;EGYPTIAN HIEROGLYPH-13BE9;Lo;0;L;;;;;N;;;;; +13BEA;EGYPTIAN HIEROGLYPH-13BEA;Lo;0;L;;;;;N;;;;; +13BEB;EGYPTIAN HIEROGLYPH-13BEB;Lo;0;L;;;;;N;;;;; +13BEC;EGYPTIAN HIEROGLYPH-13BEC;Lo;0;L;;;;;N;;;;; +13BED;EGYPTIAN HIEROGLYPH-13BED;Lo;0;L;;;;;N;;;;; +13BEE;EGYPTIAN HIEROGLYPH-13BEE;Lo;0;L;;;;;N;;;;; +13BEF;EGYPTIAN HIEROGLYPH-13BEF;Lo;0;L;;;;;N;;;;; +13BF0;EGYPTIAN HIEROGLYPH-13BF0;Lo;0;L;;;;;N;;;;; +13BF1;EGYPTIAN HIEROGLYPH-13BF1;Lo;0;L;;;;;N;;;;; +13BF2;EGYPTIAN HIEROGLYPH-13BF2;Lo;0;L;;;;;N;;;;; +13BF3;EGYPTIAN HIEROGLYPH-13BF3;Lo;0;L;;;;;N;;;;; +13BF4;EGYPTIAN HIEROGLYPH-13BF4;Lo;0;L;;;;;N;;;;; +13BF5;EGYPTIAN HIEROGLYPH-13BF5;Lo;0;L;;;;;N;;;;; +13BF6;EGYPTIAN HIEROGLYPH-13BF6;Lo;0;L;;;;;N;;;;; +13BF7;EGYPTIAN HIEROGLYPH-13BF7;Lo;0;L;;;;;N;;;;; +13BF8;EGYPTIAN HIEROGLYPH-13BF8;Lo;0;L;;;;;N;;;;; +13BF9;EGYPTIAN HIEROGLYPH-13BF9;Lo;0;L;;;;;N;;;;; +13BFA;EGYPTIAN HIEROGLYPH-13BFA;Lo;0;L;;;;;N;;;;; +13BFB;EGYPTIAN HIEROGLYPH-13BFB;Lo;0;L;;;;;N;;;;; +13BFC;EGYPTIAN HIEROGLYPH-13BFC;Lo;0;L;;;;;N;;;;; +13BFD;EGYPTIAN HIEROGLYPH-13BFD;Lo;0;L;;;;;N;;;;; +13BFE;EGYPTIAN HIEROGLYPH-13BFE;Lo;0;L;;;;;N;;;;; +13BFF;EGYPTIAN HIEROGLYPH-13BFF;Lo;0;L;;;;;N;;;;; +13C00;EGYPTIAN HIEROGLYPH-13C00;Lo;0;L;;;;;N;;;;; +13C01;EGYPTIAN HIEROGLYPH-13C01;Lo;0;L;;;;;N;;;;; +13C02;EGYPTIAN HIEROGLYPH-13C02;Lo;0;L;;;;;N;;;;; +13C03;EGYPTIAN HIEROGLYPH-13C03;Lo;0;L;;;;;N;;;;; +13C04;EGYPTIAN HIEROGLYPH-13C04;Lo;0;L;;;;;N;;;;; +13C05;EGYPTIAN HIEROGLYPH-13C05;Lo;0;L;;;;;N;;;;; +13C06;EGYPTIAN HIEROGLYPH-13C06;Lo;0;L;;;;;N;;;;; +13C07;EGYPTIAN HIEROGLYPH-13C07;Lo;0;L;;;;;N;;;;; +13C08;EGYPTIAN HIEROGLYPH-13C08;Lo;0;L;;;;;N;;;;; +13C09;EGYPTIAN HIEROGLYPH-13C09;Lo;0;L;;;;;N;;;;; +13C0A;EGYPTIAN HIEROGLYPH-13C0A;Lo;0;L;;;;;N;;;;; +13C0B;EGYPTIAN HIEROGLYPH-13C0B;Lo;0;L;;;;;N;;;;; +13C0C;EGYPTIAN HIEROGLYPH-13C0C;Lo;0;L;;;;;N;;;;; +13C0D;EGYPTIAN HIEROGLYPH-13C0D;Lo;0;L;;;;;N;;;;; +13C0E;EGYPTIAN HIEROGLYPH-13C0E;Lo;0;L;;;;;N;;;;; +13C0F;EGYPTIAN HIEROGLYPH-13C0F;Lo;0;L;;;;;N;;;;; +13C10;EGYPTIAN HIEROGLYPH-13C10;Lo;0;L;;;;;N;;;;; +13C11;EGYPTIAN HIEROGLYPH-13C11;Lo;0;L;;;;;N;;;;; +13C12;EGYPTIAN HIEROGLYPH-13C12;Lo;0;L;;;;;N;;;;; +13C13;EGYPTIAN HIEROGLYPH-13C13;Lo;0;L;;;;;N;;;;; +13C14;EGYPTIAN HIEROGLYPH-13C14;Lo;0;L;;;;;N;;;;; +13C15;EGYPTIAN HIEROGLYPH-13C15;Lo;0;L;;;;;N;;;;; +13C16;EGYPTIAN HIEROGLYPH-13C16;Lo;0;L;;;;;N;;;;; +13C17;EGYPTIAN HIEROGLYPH-13C17;Lo;0;L;;;;;N;;;;; +13C18;EGYPTIAN HIEROGLYPH-13C18;Lo;0;L;;;;;N;;;;; +13C19;EGYPTIAN HIEROGLYPH-13C19;Lo;0;L;;;;;N;;;;; +13C1A;EGYPTIAN HIEROGLYPH-13C1A;Lo;0;L;;;;;N;;;;; +13C1B;EGYPTIAN HIEROGLYPH-13C1B;Lo;0;L;;;;;N;;;;; +13C1C;EGYPTIAN HIEROGLYPH-13C1C;Lo;0;L;;;;;N;;;;; +13C1D;EGYPTIAN HIEROGLYPH-13C1D;Lo;0;L;;;;;N;;;;; +13C1E;EGYPTIAN HIEROGLYPH-13C1E;Lo;0;L;;;;;N;;;;; +13C1F;EGYPTIAN HIEROGLYPH-13C1F;Lo;0;L;;;;;N;;;;; +13C20;EGYPTIAN HIEROGLYPH-13C20;Lo;0;L;;;;;N;;;;; +13C21;EGYPTIAN HIEROGLYPH-13C21;Lo;0;L;;;;;N;;;;; +13C22;EGYPTIAN HIEROGLYPH-13C22;Lo;0;L;;;;;N;;;;; +13C23;EGYPTIAN HIEROGLYPH-13C23;Lo;0;L;;;;;N;;;;; +13C24;EGYPTIAN HIEROGLYPH-13C24;Lo;0;L;;;;;N;;;;; +13C25;EGYPTIAN HIEROGLYPH-13C25;Lo;0;L;;;;;N;;;;; +13C26;EGYPTIAN HIEROGLYPH-13C26;Lo;0;L;;;;;N;;;;; +13C27;EGYPTIAN HIEROGLYPH-13C27;Lo;0;L;;;;;N;;;;; +13C28;EGYPTIAN HIEROGLYPH-13C28;Lo;0;L;;;;;N;;;;; +13C29;EGYPTIAN HIEROGLYPH-13C29;Lo;0;L;;;;;N;;;;; +13C2A;EGYPTIAN HIEROGLYPH-13C2A;Lo;0;L;;;;;N;;;;; +13C2B;EGYPTIAN HIEROGLYPH-13C2B;Lo;0;L;;;;;N;;;;; +13C2C;EGYPTIAN HIEROGLYPH-13C2C;Lo;0;L;;;;;N;;;;; +13C2D;EGYPTIAN HIEROGLYPH-13C2D;Lo;0;L;;;;;N;;;;; +13C2E;EGYPTIAN HIEROGLYPH-13C2E;Lo;0;L;;;;;N;;;;; +13C2F;EGYPTIAN HIEROGLYPH-13C2F;Lo;0;L;;;;;N;;;;; +13C30;EGYPTIAN HIEROGLYPH-13C30;Lo;0;L;;;;;N;;;;; +13C31;EGYPTIAN HIEROGLYPH-13C31;Lo;0;L;;;;;N;;;;; +13C32;EGYPTIAN HIEROGLYPH-13C32;Lo;0;L;;;;;N;;;;; +13C33;EGYPTIAN HIEROGLYPH-13C33;Lo;0;L;;;;;N;;;;; +13C34;EGYPTIAN HIEROGLYPH-13C34;Lo;0;L;;;;;N;;;;; +13C35;EGYPTIAN HIEROGLYPH-13C35;Lo;0;L;;;;;N;;;;; +13C36;EGYPTIAN HIEROGLYPH-13C36;Lo;0;L;;;;;N;;;;; +13C37;EGYPTIAN HIEROGLYPH-13C37;Lo;0;L;;;;;N;;;;; +13C38;EGYPTIAN HIEROGLYPH-13C38;Lo;0;L;;;;;N;;;;; +13C39;EGYPTIAN HIEROGLYPH-13C39;Lo;0;L;;;;;N;;;;; +13C3A;EGYPTIAN HIEROGLYPH-13C3A;Lo;0;L;;;;;N;;;;; +13C3B;EGYPTIAN HIEROGLYPH-13C3B;Lo;0;L;;;;;N;;;;; +13C3C;EGYPTIAN HIEROGLYPH-13C3C;Lo;0;L;;;;;N;;;;; +13C3D;EGYPTIAN HIEROGLYPH-13C3D;Lo;0;L;;;;;N;;;;; +13C3E;EGYPTIAN HIEROGLYPH-13C3E;Lo;0;L;;;;;N;;;;; +13C3F;EGYPTIAN HIEROGLYPH-13C3F;Lo;0;L;;;;;N;;;;; +13C40;EGYPTIAN HIEROGLYPH-13C40;Lo;0;L;;;;;N;;;;; +13C41;EGYPTIAN HIEROGLYPH-13C41;Lo;0;L;;;;;N;;;;; +13C42;EGYPTIAN HIEROGLYPH-13C42;Lo;0;L;;;;;N;;;;; +13C43;EGYPTIAN HIEROGLYPH-13C43;Lo;0;L;;;;;N;;;;; +13C44;EGYPTIAN HIEROGLYPH-13C44;Lo;0;L;;;;;N;;;;; +13C45;EGYPTIAN HIEROGLYPH-13C45;Lo;0;L;;;;;N;;;;; +13C46;EGYPTIAN HIEROGLYPH-13C46;Lo;0;L;;;;;N;;;;; +13C47;EGYPTIAN HIEROGLYPH-13C47;Lo;0;L;;;;;N;;;;; +13C48;EGYPTIAN HIEROGLYPH-13C48;Lo;0;L;;;;;N;;;;; +13C49;EGYPTIAN HIEROGLYPH-13C49;Lo;0;L;;;;;N;;;;; +13C4A;EGYPTIAN HIEROGLYPH-13C4A;Lo;0;L;;;;;N;;;;; +13C4B;EGYPTIAN HIEROGLYPH-13C4B;Lo;0;L;;;;;N;;;;; +13C4C;EGYPTIAN HIEROGLYPH-13C4C;Lo;0;L;;;;;N;;;;; +13C4D;EGYPTIAN HIEROGLYPH-13C4D;Lo;0;L;;;;;N;;;;; +13C4E;EGYPTIAN HIEROGLYPH-13C4E;Lo;0;L;;;;;N;;;;; +13C4F;EGYPTIAN HIEROGLYPH-13C4F;Lo;0;L;;;;;N;;;;; +13C50;EGYPTIAN HIEROGLYPH-13C50;Lo;0;L;;;;;N;;;;; +13C51;EGYPTIAN HIEROGLYPH-13C51;Lo;0;L;;;;;N;;;;; +13C52;EGYPTIAN HIEROGLYPH-13C52;Lo;0;L;;;;;N;;;;; +13C53;EGYPTIAN HIEROGLYPH-13C53;Lo;0;L;;;;;N;;;;; +13C54;EGYPTIAN HIEROGLYPH-13C54;Lo;0;L;;;;;N;;;;; +13C55;EGYPTIAN HIEROGLYPH-13C55;Lo;0;L;;;;;N;;;;; +13C56;EGYPTIAN HIEROGLYPH-13C56;Lo;0;L;;;;;N;;;;; +13C57;EGYPTIAN HIEROGLYPH-13C57;Lo;0;L;;;;;N;;;;; +13C58;EGYPTIAN HIEROGLYPH-13C58;Lo;0;L;;;;;N;;;;; +13C59;EGYPTIAN HIEROGLYPH-13C59;Lo;0;L;;;;;N;;;;; +13C5A;EGYPTIAN HIEROGLYPH-13C5A;Lo;0;L;;;;;N;;;;; +13C5B;EGYPTIAN HIEROGLYPH-13C5B;Lo;0;L;;;;;N;;;;; +13C5C;EGYPTIAN HIEROGLYPH-13C5C;Lo;0;L;;;;;N;;;;; +13C5D;EGYPTIAN HIEROGLYPH-13C5D;Lo;0;L;;;;;N;;;;; +13C5E;EGYPTIAN HIEROGLYPH-13C5E;Lo;0;L;;;;;N;;;;; +13C5F;EGYPTIAN HIEROGLYPH-13C5F;Lo;0;L;;;;;N;;;;; +13C60;EGYPTIAN HIEROGLYPH-13C60;Lo;0;L;;;;;N;;;;; +13C61;EGYPTIAN HIEROGLYPH-13C61;Lo;0;L;;;;;N;;;;; +13C62;EGYPTIAN HIEROGLYPH-13C62;Lo;0;L;;;;;N;;;;; +13C63;EGYPTIAN HIEROGLYPH-13C63;Lo;0;L;;;;;N;;;;; +13C64;EGYPTIAN HIEROGLYPH-13C64;Lo;0;L;;;;;N;;;;; +13C65;EGYPTIAN HIEROGLYPH-13C65;Lo;0;L;;;;;N;;;;; +13C66;EGYPTIAN HIEROGLYPH-13C66;Lo;0;L;;;;;N;;;;; +13C67;EGYPTIAN HIEROGLYPH-13C67;Lo;0;L;;;;;N;;;;; +13C68;EGYPTIAN HIEROGLYPH-13C68;Lo;0;L;;;;;N;;;;; +13C69;EGYPTIAN HIEROGLYPH-13C69;Lo;0;L;;;;;N;;;;; +13C6A;EGYPTIAN HIEROGLYPH-13C6A;Lo;0;L;;;;;N;;;;; +13C6B;EGYPTIAN HIEROGLYPH-13C6B;Lo;0;L;;;;;N;;;;; +13C6C;EGYPTIAN HIEROGLYPH-13C6C;Lo;0;L;;;;;N;;;;; +13C6D;EGYPTIAN HIEROGLYPH-13C6D;Lo;0;L;;;;;N;;;;; +13C6E;EGYPTIAN HIEROGLYPH-13C6E;Lo;0;L;;;;;N;;;;; +13C6F;EGYPTIAN HIEROGLYPH-13C6F;Lo;0;L;;;;;N;;;;; +13C70;EGYPTIAN HIEROGLYPH-13C70;Lo;0;L;;;;;N;;;;; +13C71;EGYPTIAN HIEROGLYPH-13C71;Lo;0;L;;;;;N;;;;; +13C72;EGYPTIAN HIEROGLYPH-13C72;Lo;0;L;;;;;N;;;;; +13C73;EGYPTIAN HIEROGLYPH-13C73;Lo;0;L;;;;;N;;;;; +13C74;EGYPTIAN HIEROGLYPH-13C74;Lo;0;L;;;;;N;;;;; +13C75;EGYPTIAN HIEROGLYPH-13C75;Lo;0;L;;;;;N;;;;; +13C76;EGYPTIAN HIEROGLYPH-13C76;Lo;0;L;;;;;N;;;;; +13C77;EGYPTIAN HIEROGLYPH-13C77;Lo;0;L;;;;;N;;;;; +13C78;EGYPTIAN HIEROGLYPH-13C78;Lo;0;L;;;;;N;;;;; +13C79;EGYPTIAN HIEROGLYPH-13C79;Lo;0;L;;;;;N;;;;; +13C7A;EGYPTIAN HIEROGLYPH-13C7A;Lo;0;L;;;;;N;;;;; +13C7B;EGYPTIAN HIEROGLYPH-13C7B;Lo;0;L;;;;;N;;;;; +13C7C;EGYPTIAN HIEROGLYPH-13C7C;Lo;0;L;;;;;N;;;;; +13C7D;EGYPTIAN HIEROGLYPH-13C7D;Lo;0;L;;;;;N;;;;; +13C7E;EGYPTIAN HIEROGLYPH-13C7E;Lo;0;L;;;;;N;;;;; +13C7F;EGYPTIAN HIEROGLYPH-13C7F;Lo;0;L;;;;;N;;;;; +13C80;EGYPTIAN HIEROGLYPH-13C80;Lo;0;L;;;;;N;;;;; +13C81;EGYPTIAN HIEROGLYPH-13C81;Lo;0;L;;;;;N;;;;; +13C82;EGYPTIAN HIEROGLYPH-13C82;Lo;0;L;;;;;N;;;;; +13C83;EGYPTIAN HIEROGLYPH-13C83;Lo;0;L;;;;;N;;;;; +13C84;EGYPTIAN HIEROGLYPH-13C84;Lo;0;L;;;;;N;;;;; +13C85;EGYPTIAN HIEROGLYPH-13C85;Lo;0;L;;;;;N;;;;; +13C86;EGYPTIAN HIEROGLYPH-13C86;Lo;0;L;;;;;N;;;;; +13C87;EGYPTIAN HIEROGLYPH-13C87;Lo;0;L;;;;;N;;;;; +13C88;EGYPTIAN HIEROGLYPH-13C88;Lo;0;L;;;;;N;;;;; +13C89;EGYPTIAN HIEROGLYPH-13C89;Lo;0;L;;;;;N;;;;; +13C8A;EGYPTIAN HIEROGLYPH-13C8A;Lo;0;L;;;;;N;;;;; +13C8B;EGYPTIAN HIEROGLYPH-13C8B;Lo;0;L;;;;;N;;;;; +13C8C;EGYPTIAN HIEROGLYPH-13C8C;Lo;0;L;;;;;N;;;;; +13C8D;EGYPTIAN HIEROGLYPH-13C8D;Lo;0;L;;;;;N;;;;; +13C8E;EGYPTIAN HIEROGLYPH-13C8E;Lo;0;L;;;;;N;;;;; +13C8F;EGYPTIAN HIEROGLYPH-13C8F;Lo;0;L;;;;;N;;;;; +13C90;EGYPTIAN HIEROGLYPH-13C90;Lo;0;L;;;;;N;;;;; +13C91;EGYPTIAN HIEROGLYPH-13C91;Lo;0;L;;;;;N;;;;; +13C92;EGYPTIAN HIEROGLYPH-13C92;Lo;0;L;;;;;N;;;;; +13C93;EGYPTIAN HIEROGLYPH-13C93;Lo;0;L;;;;;N;;;;; +13C94;EGYPTIAN HIEROGLYPH-13C94;Lo;0;L;;;;;N;;;;; +13C95;EGYPTIAN HIEROGLYPH-13C95;Lo;0;L;;;;;N;;;;; +13C96;EGYPTIAN HIEROGLYPH-13C96;Lo;0;L;;;;;N;;;;; +13C97;EGYPTIAN HIEROGLYPH-13C97;Lo;0;L;;;;;N;;;;; +13C98;EGYPTIAN HIEROGLYPH-13C98;Lo;0;L;;;;;N;;;;; +13C99;EGYPTIAN HIEROGLYPH-13C99;Lo;0;L;;;;;N;;;;; +13C9A;EGYPTIAN HIEROGLYPH-13C9A;Lo;0;L;;;;;N;;;;; +13C9B;EGYPTIAN HIEROGLYPH-13C9B;Lo;0;L;;;;;N;;;;; +13C9C;EGYPTIAN HIEROGLYPH-13C9C;Lo;0;L;;;;;N;;;;; +13C9D;EGYPTIAN HIEROGLYPH-13C9D;Lo;0;L;;;;;N;;;;; +13C9E;EGYPTIAN HIEROGLYPH-13C9E;Lo;0;L;;;;;N;;;;; +13C9F;EGYPTIAN HIEROGLYPH-13C9F;Lo;0;L;;;;;N;;;;; +13CA0;EGYPTIAN HIEROGLYPH-13CA0;Lo;0;L;;;;;N;;;;; +13CA1;EGYPTIAN HIEROGLYPH-13CA1;Lo;0;L;;;;;N;;;;; +13CA2;EGYPTIAN HIEROGLYPH-13CA2;Lo;0;L;;;;;N;;;;; +13CA3;EGYPTIAN HIEROGLYPH-13CA3;Lo;0;L;;;;;N;;;;; +13CA4;EGYPTIAN HIEROGLYPH-13CA4;Lo;0;L;;;;;N;;;;; +13CA5;EGYPTIAN HIEROGLYPH-13CA5;Lo;0;L;;;;;N;;;;; +13CA6;EGYPTIAN HIEROGLYPH-13CA6;Lo;0;L;;;;;N;;;;; +13CA7;EGYPTIAN HIEROGLYPH-13CA7;Lo;0;L;;;;;N;;;;; +13CA8;EGYPTIAN HIEROGLYPH-13CA8;Lo;0;L;;;;;N;;;;; +13CA9;EGYPTIAN HIEROGLYPH-13CA9;Lo;0;L;;;;;N;;;;; +13CAA;EGYPTIAN HIEROGLYPH-13CAA;Lo;0;L;;;;;N;;;;; +13CAB;EGYPTIAN HIEROGLYPH-13CAB;Lo;0;L;;;;;N;;;;; +13CAC;EGYPTIAN HIEROGLYPH-13CAC;Lo;0;L;;;;;N;;;;; +13CAD;EGYPTIAN HIEROGLYPH-13CAD;Lo;0;L;;;;;N;;;;; +13CAE;EGYPTIAN HIEROGLYPH-13CAE;Lo;0;L;;;;;N;;;;; +13CAF;EGYPTIAN HIEROGLYPH-13CAF;Lo;0;L;;;;;N;;;;; +13CB0;EGYPTIAN HIEROGLYPH-13CB0;Lo;0;L;;;;;N;;;;; +13CB1;EGYPTIAN HIEROGLYPH-13CB1;Lo;0;L;;;;;N;;;;; +13CB2;EGYPTIAN HIEROGLYPH-13CB2;Lo;0;L;;;;;N;;;;; +13CB3;EGYPTIAN HIEROGLYPH-13CB3;Lo;0;L;;;;;N;;;;; +13CB4;EGYPTIAN HIEROGLYPH-13CB4;Lo;0;L;;;;;N;;;;; +13CB5;EGYPTIAN HIEROGLYPH-13CB5;Lo;0;L;;;;;N;;;;; +13CB6;EGYPTIAN HIEROGLYPH-13CB6;Lo;0;L;;;;;N;;;;; +13CB7;EGYPTIAN HIEROGLYPH-13CB7;Lo;0;L;;;;;N;;;;; +13CB8;EGYPTIAN HIEROGLYPH-13CB8;Lo;0;L;;;;;N;;;;; +13CB9;EGYPTIAN HIEROGLYPH-13CB9;Lo;0;L;;;;;N;;;;; +13CBA;EGYPTIAN HIEROGLYPH-13CBA;Lo;0;L;;;;;N;;;;; +13CBB;EGYPTIAN HIEROGLYPH-13CBB;Lo;0;L;;;;;N;;;;; +13CBC;EGYPTIAN HIEROGLYPH-13CBC;Lo;0;L;;;;;N;;;;; +13CBD;EGYPTIAN HIEROGLYPH-13CBD;Lo;0;L;;;;;N;;;;; +13CBE;EGYPTIAN HIEROGLYPH-13CBE;Lo;0;L;;;;;N;;;;; +13CBF;EGYPTIAN HIEROGLYPH-13CBF;Lo;0;L;;;;;N;;;;; +13CC0;EGYPTIAN HIEROGLYPH-13CC0;Lo;0;L;;;;;N;;;;; +13CC1;EGYPTIAN HIEROGLYPH-13CC1;Lo;0;L;;;;;N;;;;; +13CC2;EGYPTIAN HIEROGLYPH-13CC2;Lo;0;L;;;;;N;;;;; +13CC3;EGYPTIAN HIEROGLYPH-13CC3;Lo;0;L;;;;;N;;;;; +13CC4;EGYPTIAN HIEROGLYPH-13CC4;Lo;0;L;;;;;N;;;;; +13CC5;EGYPTIAN HIEROGLYPH-13CC5;Lo;0;L;;;;;N;;;;; +13CC6;EGYPTIAN HIEROGLYPH-13CC6;Lo;0;L;;;;;N;;;;; +13CC7;EGYPTIAN HIEROGLYPH-13CC7;Lo;0;L;;;;;N;;;;; +13CC8;EGYPTIAN HIEROGLYPH-13CC8;Lo;0;L;;;;;N;;;;; +13CC9;EGYPTIAN HIEROGLYPH-13CC9;Lo;0;L;;;;;N;;;;; +13CCA;EGYPTIAN HIEROGLYPH-13CCA;Lo;0;L;;;;;N;;;;; +13CCB;EGYPTIAN HIEROGLYPH-13CCB;Lo;0;L;;;;;N;;;;; +13CCC;EGYPTIAN HIEROGLYPH-13CCC;Lo;0;L;;;;;N;;;;; +13CCD;EGYPTIAN HIEROGLYPH-13CCD;Lo;0;L;;;;;N;;;;; +13CCE;EGYPTIAN HIEROGLYPH-13CCE;Lo;0;L;;;;;N;;;;; +13CCF;EGYPTIAN HIEROGLYPH-13CCF;Lo;0;L;;;;;N;;;;; +13CD0;EGYPTIAN HIEROGLYPH-13CD0;Lo;0;L;;;;;N;;;;; +13CD1;EGYPTIAN HIEROGLYPH-13CD1;Lo;0;L;;;;;N;;;;; +13CD2;EGYPTIAN HIEROGLYPH-13CD2;Lo;0;L;;;;;N;;;;; +13CD3;EGYPTIAN HIEROGLYPH-13CD3;Lo;0;L;;;;;N;;;;; +13CD4;EGYPTIAN HIEROGLYPH-13CD4;Lo;0;L;;;;;N;;;;; +13CD5;EGYPTIAN HIEROGLYPH-13CD5;Lo;0;L;;;;;N;;;;; +13CD6;EGYPTIAN HIEROGLYPH-13CD6;Lo;0;L;;;;;N;;;;; +13CD7;EGYPTIAN HIEROGLYPH-13CD7;Lo;0;L;;;;;N;;;;; +13CD8;EGYPTIAN HIEROGLYPH-13CD8;Lo;0;L;;;;;N;;;;; +13CD9;EGYPTIAN HIEROGLYPH-13CD9;Lo;0;L;;;;;N;;;;; +13CDA;EGYPTIAN HIEROGLYPH-13CDA;Lo;0;L;;;;;N;;;;; +13CDB;EGYPTIAN HIEROGLYPH-13CDB;Lo;0;L;;;;;N;;;;; +13CDC;EGYPTIAN HIEROGLYPH-13CDC;Lo;0;L;;;;;N;;;;; +13CDD;EGYPTIAN HIEROGLYPH-13CDD;Lo;0;L;;;;;N;;;;; +13CDE;EGYPTIAN HIEROGLYPH-13CDE;Lo;0;L;;;;;N;;;;; +13CDF;EGYPTIAN HIEROGLYPH-13CDF;Lo;0;L;;;;;N;;;;; +13CE0;EGYPTIAN HIEROGLYPH-13CE0;Lo;0;L;;;;;N;;;;; +13CE1;EGYPTIAN HIEROGLYPH-13CE1;Lo;0;L;;;;;N;;;;; +13CE2;EGYPTIAN HIEROGLYPH-13CE2;Lo;0;L;;;;;N;;;;; +13CE3;EGYPTIAN HIEROGLYPH-13CE3;Lo;0;L;;;;;N;;;;; +13CE4;EGYPTIAN HIEROGLYPH-13CE4;Lo;0;L;;;;;N;;;;; +13CE5;EGYPTIAN HIEROGLYPH-13CE5;Lo;0;L;;;;;N;;;;; +13CE6;EGYPTIAN HIEROGLYPH-13CE6;Lo;0;L;;;;;N;;;;; +13CE7;EGYPTIAN HIEROGLYPH-13CE7;Lo;0;L;;;;;N;;;;; +13CE8;EGYPTIAN HIEROGLYPH-13CE8;Lo;0;L;;;;;N;;;;; +13CE9;EGYPTIAN HIEROGLYPH-13CE9;Lo;0;L;;;;;N;;;;; +13CEA;EGYPTIAN HIEROGLYPH-13CEA;Lo;0;L;;;;;N;;;;; +13CEB;EGYPTIAN HIEROGLYPH-13CEB;Lo;0;L;;;;;N;;;;; +13CEC;EGYPTIAN HIEROGLYPH-13CEC;Lo;0;L;;;;;N;;;;; +13CED;EGYPTIAN HIEROGLYPH-13CED;Lo;0;L;;;;;N;;;;; +13CEE;EGYPTIAN HIEROGLYPH-13CEE;Lo;0;L;;;;;N;;;;; +13CEF;EGYPTIAN HIEROGLYPH-13CEF;Lo;0;L;;;;;N;;;;; +13CF0;EGYPTIAN HIEROGLYPH-13CF0;Lo;0;L;;;;;N;;;;; +13CF1;EGYPTIAN HIEROGLYPH-13CF1;Lo;0;L;;;;;N;;;;; +13CF2;EGYPTIAN HIEROGLYPH-13CF2;Lo;0;L;;;;;N;;;;; +13CF3;EGYPTIAN HIEROGLYPH-13CF3;Lo;0;L;;;;;N;;;;; +13CF4;EGYPTIAN HIEROGLYPH-13CF4;Lo;0;L;;;;;N;;;;; +13CF5;EGYPTIAN HIEROGLYPH-13CF5;Lo;0;L;;;;;N;;;;; +13CF6;EGYPTIAN HIEROGLYPH-13CF6;Lo;0;L;;;;;N;;;;; +13CF7;EGYPTIAN HIEROGLYPH-13CF7;Lo;0;L;;;;;N;;;;; +13CF8;EGYPTIAN HIEROGLYPH-13CF8;Lo;0;L;;;;;N;;;;; +13CF9;EGYPTIAN HIEROGLYPH-13CF9;Lo;0;L;;;;;N;;;;; +13CFA;EGYPTIAN HIEROGLYPH-13CFA;Lo;0;L;;;;;N;;;;; +13CFB;EGYPTIAN HIEROGLYPH-13CFB;Lo;0;L;;;;;N;;;;; +13CFC;EGYPTIAN HIEROGLYPH-13CFC;Lo;0;L;;;;;N;;;;; +13CFD;EGYPTIAN HIEROGLYPH-13CFD;Lo;0;L;;;;;N;;;;; +13CFE;EGYPTIAN HIEROGLYPH-13CFE;Lo;0;L;;;;;N;;;;; +13CFF;EGYPTIAN HIEROGLYPH-13CFF;Lo;0;L;;;;;N;;;;; +13D00;EGYPTIAN HIEROGLYPH-13D00;Lo;0;L;;;;;N;;;;; +13D01;EGYPTIAN HIEROGLYPH-13D01;Lo;0;L;;;;;N;;;;; +13D02;EGYPTIAN HIEROGLYPH-13D02;Lo;0;L;;;;;N;;;;; +13D03;EGYPTIAN HIEROGLYPH-13D03;Lo;0;L;;;;;N;;;;; +13D04;EGYPTIAN HIEROGLYPH-13D04;Lo;0;L;;;;;N;;;;; +13D05;EGYPTIAN HIEROGLYPH-13D05;Lo;0;L;;;;;N;;;;; +13D06;EGYPTIAN HIEROGLYPH-13D06;Lo;0;L;;;;;N;;;;; +13D07;EGYPTIAN HIEROGLYPH-13D07;Lo;0;L;;;;;N;;;;; +13D08;EGYPTIAN HIEROGLYPH-13D08;Lo;0;L;;;;;N;;;;; +13D09;EGYPTIAN HIEROGLYPH-13D09;Lo;0;L;;;;;N;;;;; +13D0A;EGYPTIAN HIEROGLYPH-13D0A;Lo;0;L;;;;;N;;;;; +13D0B;EGYPTIAN HIEROGLYPH-13D0B;Lo;0;L;;;;;N;;;;; +13D0C;EGYPTIAN HIEROGLYPH-13D0C;Lo;0;L;;;;;N;;;;; +13D0D;EGYPTIAN HIEROGLYPH-13D0D;Lo;0;L;;;;;N;;;;; +13D0E;EGYPTIAN HIEROGLYPH-13D0E;Lo;0;L;;;;;N;;;;; +13D0F;EGYPTIAN HIEROGLYPH-13D0F;Lo;0;L;;;;;N;;;;; +13D10;EGYPTIAN HIEROGLYPH-13D10;Lo;0;L;;;;;N;;;;; +13D11;EGYPTIAN HIEROGLYPH-13D11;Lo;0;L;;;;;N;;;;; +13D12;EGYPTIAN HIEROGLYPH-13D12;Lo;0;L;;;;;N;;;;; +13D13;EGYPTIAN HIEROGLYPH-13D13;Lo;0;L;;;;;N;;;;; +13D14;EGYPTIAN HIEROGLYPH-13D14;Lo;0;L;;;;;N;;;;; +13D15;EGYPTIAN HIEROGLYPH-13D15;Lo;0;L;;;;;N;;;;; +13D16;EGYPTIAN HIEROGLYPH-13D16;Lo;0;L;;;;;N;;;;; +13D17;EGYPTIAN HIEROGLYPH-13D17;Lo;0;L;;;;;N;;;;; +13D18;EGYPTIAN HIEROGLYPH-13D18;Lo;0;L;;;;;N;;;;; +13D19;EGYPTIAN HIEROGLYPH-13D19;Lo;0;L;;;;;N;;;;; +13D1A;EGYPTIAN HIEROGLYPH-13D1A;Lo;0;L;;;;;N;;;;; +13D1B;EGYPTIAN HIEROGLYPH-13D1B;Lo;0;L;;;;;N;;;;; +13D1C;EGYPTIAN HIEROGLYPH-13D1C;Lo;0;L;;;;;N;;;;; +13D1D;EGYPTIAN HIEROGLYPH-13D1D;Lo;0;L;;;;;N;;;;; +13D1E;EGYPTIAN HIEROGLYPH-13D1E;Lo;0;L;;;;;N;;;;; +13D1F;EGYPTIAN HIEROGLYPH-13D1F;Lo;0;L;;;;;N;;;;; +13D20;EGYPTIAN HIEROGLYPH-13D20;Lo;0;L;;;;;N;;;;; +13D21;EGYPTIAN HIEROGLYPH-13D21;Lo;0;L;;;;;N;;;;; +13D22;EGYPTIAN HIEROGLYPH-13D22;Lo;0;L;;;;;N;;;;; +13D23;EGYPTIAN HIEROGLYPH-13D23;Lo;0;L;;;;;N;;;;; +13D24;EGYPTIAN HIEROGLYPH-13D24;Lo;0;L;;;;;N;;;;; +13D25;EGYPTIAN HIEROGLYPH-13D25;Lo;0;L;;;;;N;;;;; +13D26;EGYPTIAN HIEROGLYPH-13D26;Lo;0;L;;;;;N;;;;; +13D27;EGYPTIAN HIEROGLYPH-13D27;Lo;0;L;;;;;N;;;;; +13D28;EGYPTIAN HIEROGLYPH-13D28;Lo;0;L;;;;;N;;;;; +13D29;EGYPTIAN HIEROGLYPH-13D29;Lo;0;L;;;;;N;;;;; +13D2A;EGYPTIAN HIEROGLYPH-13D2A;Lo;0;L;;;;;N;;;;; +13D2B;EGYPTIAN HIEROGLYPH-13D2B;Lo;0;L;;;;;N;;;;; +13D2C;EGYPTIAN HIEROGLYPH-13D2C;Lo;0;L;;;;;N;;;;; +13D2D;EGYPTIAN HIEROGLYPH-13D2D;Lo;0;L;;;;;N;;;;; +13D2E;EGYPTIAN HIEROGLYPH-13D2E;Lo;0;L;;;;;N;;;;; +13D2F;EGYPTIAN HIEROGLYPH-13D2F;Lo;0;L;;;;;N;;;;; +13D30;EGYPTIAN HIEROGLYPH-13D30;Lo;0;L;;;;;N;;;;; +13D31;EGYPTIAN HIEROGLYPH-13D31;Lo;0;L;;;;;N;;;;; +13D32;EGYPTIAN HIEROGLYPH-13D32;Lo;0;L;;;;;N;;;;; +13D33;EGYPTIAN HIEROGLYPH-13D33;Lo;0;L;;;;;N;;;;; +13D34;EGYPTIAN HIEROGLYPH-13D34;Lo;0;L;;;;;N;;;;; +13D35;EGYPTIAN HIEROGLYPH-13D35;Lo;0;L;;;;;N;;;;; +13D36;EGYPTIAN HIEROGLYPH-13D36;Lo;0;L;;;;;N;;;;; +13D37;EGYPTIAN HIEROGLYPH-13D37;Lo;0;L;;;;;N;;;;; +13D38;EGYPTIAN HIEROGLYPH-13D38;Lo;0;L;;;;;N;;;;; +13D39;EGYPTIAN HIEROGLYPH-13D39;Lo;0;L;;;;;N;;;;; +13D3A;EGYPTIAN HIEROGLYPH-13D3A;Lo;0;L;;;;;N;;;;; +13D3B;EGYPTIAN HIEROGLYPH-13D3B;Lo;0;L;;;;;N;;;;; +13D3C;EGYPTIAN HIEROGLYPH-13D3C;Lo;0;L;;;;;N;;;;; +13D3D;EGYPTIAN HIEROGLYPH-13D3D;Lo;0;L;;;;;N;;;;; +13D3E;EGYPTIAN HIEROGLYPH-13D3E;Lo;0;L;;;;;N;;;;; +13D3F;EGYPTIAN HIEROGLYPH-13D3F;Lo;0;L;;;;;N;;;;; +13D40;EGYPTIAN HIEROGLYPH-13D40;Lo;0;L;;;;;N;;;;; +13D41;EGYPTIAN HIEROGLYPH-13D41;Lo;0;L;;;;;N;;;;; +13D42;EGYPTIAN HIEROGLYPH-13D42;Lo;0;L;;;;;N;;;;; +13D43;EGYPTIAN HIEROGLYPH-13D43;Lo;0;L;;;;;N;;;;; +13D44;EGYPTIAN HIEROGLYPH-13D44;Lo;0;L;;;;;N;;;;; +13D45;EGYPTIAN HIEROGLYPH-13D45;Lo;0;L;;;;;N;;;;; +13D46;EGYPTIAN HIEROGLYPH-13D46;Lo;0;L;;;;;N;;;;; +13D47;EGYPTIAN HIEROGLYPH-13D47;Lo;0;L;;;;;N;;;;; +13D48;EGYPTIAN HIEROGLYPH-13D48;Lo;0;L;;;;;N;;;;; +13D49;EGYPTIAN HIEROGLYPH-13D49;Lo;0;L;;;;;N;;;;; +13D4A;EGYPTIAN HIEROGLYPH-13D4A;Lo;0;L;;;;;N;;;;; +13D4B;EGYPTIAN HIEROGLYPH-13D4B;Lo;0;L;;;;;N;;;;; +13D4C;EGYPTIAN HIEROGLYPH-13D4C;Lo;0;L;;;;;N;;;;; +13D4D;EGYPTIAN HIEROGLYPH-13D4D;Lo;0;L;;;;;N;;;;; +13D4E;EGYPTIAN HIEROGLYPH-13D4E;Lo;0;L;;;;;N;;;;; +13D4F;EGYPTIAN HIEROGLYPH-13D4F;Lo;0;L;;;;;N;;;;; +13D50;EGYPTIAN HIEROGLYPH-13D50;Lo;0;L;;;;;N;;;;; +13D51;EGYPTIAN HIEROGLYPH-13D51;Lo;0;L;;;;;N;;;;; +13D52;EGYPTIAN HIEROGLYPH-13D52;Lo;0;L;;;;;N;;;;; +13D53;EGYPTIAN HIEROGLYPH-13D53;Lo;0;L;;;;;N;;;;; +13D54;EGYPTIAN HIEROGLYPH-13D54;Lo;0;L;;;;;N;;;;; +13D55;EGYPTIAN HIEROGLYPH-13D55;Lo;0;L;;;;;N;;;;; +13D56;EGYPTIAN HIEROGLYPH-13D56;Lo;0;L;;;;;N;;;;; +13D57;EGYPTIAN HIEROGLYPH-13D57;Lo;0;L;;;;;N;;;;; +13D58;EGYPTIAN HIEROGLYPH-13D58;Lo;0;L;;;;;N;;;;; +13D59;EGYPTIAN HIEROGLYPH-13D59;Lo;0;L;;;;;N;;;;; +13D5A;EGYPTIAN HIEROGLYPH-13D5A;Lo;0;L;;;;;N;;;;; +13D5B;EGYPTIAN HIEROGLYPH-13D5B;Lo;0;L;;;;;N;;;;; +13D5C;EGYPTIAN HIEROGLYPH-13D5C;Lo;0;L;;;;;N;;;;; +13D5D;EGYPTIAN HIEROGLYPH-13D5D;Lo;0;L;;;;;N;;;;; +13D5E;EGYPTIAN HIEROGLYPH-13D5E;Lo;0;L;;;;;N;;;;; +13D5F;EGYPTIAN HIEROGLYPH-13D5F;Lo;0;L;;;;;N;;;;; +13D60;EGYPTIAN HIEROGLYPH-13D60;Lo;0;L;;;;;N;;;;; +13D61;EGYPTIAN HIEROGLYPH-13D61;Lo;0;L;;;;;N;;;;; +13D62;EGYPTIAN HIEROGLYPH-13D62;Lo;0;L;;;;;N;;;;; +13D63;EGYPTIAN HIEROGLYPH-13D63;Lo;0;L;;;;;N;;;;; +13D64;EGYPTIAN HIEROGLYPH-13D64;Lo;0;L;;;;;N;;;;; +13D65;EGYPTIAN HIEROGLYPH-13D65;Lo;0;L;;;;;N;;;;; +13D66;EGYPTIAN HIEROGLYPH-13D66;Lo;0;L;;;;;N;;;;; +13D67;EGYPTIAN HIEROGLYPH-13D67;Lo;0;L;;;;;N;;;;; +13D68;EGYPTIAN HIEROGLYPH-13D68;Lo;0;L;;;;;N;;;;; +13D69;EGYPTIAN HIEROGLYPH-13D69;Lo;0;L;;;;;N;;;;; +13D6A;EGYPTIAN HIEROGLYPH-13D6A;Lo;0;L;;;;;N;;;;; +13D6B;EGYPTIAN HIEROGLYPH-13D6B;Lo;0;L;;;;;N;;;;; +13D6C;EGYPTIAN HIEROGLYPH-13D6C;Lo;0;L;;;;;N;;;;; +13D6D;EGYPTIAN HIEROGLYPH-13D6D;Lo;0;L;;;;;N;;;;; +13D6E;EGYPTIAN HIEROGLYPH-13D6E;Lo;0;L;;;;;N;;;;; +13D6F;EGYPTIAN HIEROGLYPH-13D6F;Lo;0;L;;;;;N;;;;; +13D70;EGYPTIAN HIEROGLYPH-13D70;Lo;0;L;;;;;N;;;;; +13D71;EGYPTIAN HIEROGLYPH-13D71;Lo;0;L;;;;;N;;;;; +13D72;EGYPTIAN HIEROGLYPH-13D72;Lo;0;L;;;;;N;;;;; +13D73;EGYPTIAN HIEROGLYPH-13D73;Lo;0;L;;;;;N;;;;; +13D74;EGYPTIAN HIEROGLYPH-13D74;Lo;0;L;;;;;N;;;;; +13D75;EGYPTIAN HIEROGLYPH-13D75;Lo;0;L;;;;;N;;;;; +13D76;EGYPTIAN HIEROGLYPH-13D76;Lo;0;L;;;;;N;;;;; +13D77;EGYPTIAN HIEROGLYPH-13D77;Lo;0;L;;;;;N;;;;; +13D78;EGYPTIAN HIEROGLYPH-13D78;Lo;0;L;;;;;N;;;;; +13D79;EGYPTIAN HIEROGLYPH-13D79;Lo;0;L;;;;;N;;;;; +13D7A;EGYPTIAN HIEROGLYPH-13D7A;Lo;0;L;;;;;N;;;;; +13D7B;EGYPTIAN HIEROGLYPH-13D7B;Lo;0;L;;;;;N;;;;; +13D7C;EGYPTIAN HIEROGLYPH-13D7C;Lo;0;L;;;;;N;;;;; +13D7D;EGYPTIAN HIEROGLYPH-13D7D;Lo;0;L;;;;;N;;;;; +13D7E;EGYPTIAN HIEROGLYPH-13D7E;Lo;0;L;;;;;N;;;;; +13D7F;EGYPTIAN HIEROGLYPH-13D7F;Lo;0;L;;;;;N;;;;; +13D80;EGYPTIAN HIEROGLYPH-13D80;Lo;0;L;;;;;N;;;;; +13D81;EGYPTIAN HIEROGLYPH-13D81;Lo;0;L;;;;;N;;;;; +13D82;EGYPTIAN HIEROGLYPH-13D82;Lo;0;L;;;;;N;;;;; +13D83;EGYPTIAN HIEROGLYPH-13D83;Lo;0;L;;;;;N;;;;; +13D84;EGYPTIAN HIEROGLYPH-13D84;Lo;0;L;;;;;N;;;;; +13D85;EGYPTIAN HIEROGLYPH-13D85;Lo;0;L;;;;;N;;;;; +13D86;EGYPTIAN HIEROGLYPH-13D86;Lo;0;L;;;;;N;;;;; +13D87;EGYPTIAN HIEROGLYPH-13D87;Lo;0;L;;;;;N;;;;; +13D88;EGYPTIAN HIEROGLYPH-13D88;Lo;0;L;;;;;N;;;;; +13D89;EGYPTIAN HIEROGLYPH-13D89;Lo;0;L;;;;;N;;;;; +13D8A;EGYPTIAN HIEROGLYPH-13D8A;Lo;0;L;;;;;N;;;;; +13D8B;EGYPTIAN HIEROGLYPH-13D8B;Lo;0;L;;;;;N;;;;; +13D8C;EGYPTIAN HIEROGLYPH-13D8C;Lo;0;L;;;;;N;;;;; +13D8D;EGYPTIAN HIEROGLYPH-13D8D;Lo;0;L;;;;;N;;;;; +13D8E;EGYPTIAN HIEROGLYPH-13D8E;Lo;0;L;;;;;N;;;;; +13D8F;EGYPTIAN HIEROGLYPH-13D8F;Lo;0;L;;;;;N;;;;; +13D90;EGYPTIAN HIEROGLYPH-13D90;Lo;0;L;;;;;N;;;;; +13D91;EGYPTIAN HIEROGLYPH-13D91;Lo;0;L;;;;;N;;;;; +13D92;EGYPTIAN HIEROGLYPH-13D92;Lo;0;L;;;;;N;;;;; +13D93;EGYPTIAN HIEROGLYPH-13D93;Lo;0;L;;;;;N;;;;; +13D94;EGYPTIAN HIEROGLYPH-13D94;Lo;0;L;;;;;N;;;;; +13D95;EGYPTIAN HIEROGLYPH-13D95;Lo;0;L;;;;;N;;;;; +13D96;EGYPTIAN HIEROGLYPH-13D96;Lo;0;L;;;;;N;;;;; +13D97;EGYPTIAN HIEROGLYPH-13D97;Lo;0;L;;;;;N;;;;; +13D98;EGYPTIAN HIEROGLYPH-13D98;Lo;0;L;;;;;N;;;;; +13D99;EGYPTIAN HIEROGLYPH-13D99;Lo;0;L;;;;;N;;;;; +13D9A;EGYPTIAN HIEROGLYPH-13D9A;Lo;0;L;;;;;N;;;;; +13D9B;EGYPTIAN HIEROGLYPH-13D9B;Lo;0;L;;;;;N;;;;; +13D9C;EGYPTIAN HIEROGLYPH-13D9C;Lo;0;L;;;;;N;;;;; +13D9D;EGYPTIAN HIEROGLYPH-13D9D;Lo;0;L;;;;;N;;;;; +13D9E;EGYPTIAN HIEROGLYPH-13D9E;Lo;0;L;;;;;N;;;;; +13D9F;EGYPTIAN HIEROGLYPH-13D9F;Lo;0;L;;;;;N;;;;; +13DA0;EGYPTIAN HIEROGLYPH-13DA0;Lo;0;L;;;;;N;;;;; +13DA1;EGYPTIAN HIEROGLYPH-13DA1;Lo;0;L;;;;;N;;;;; +13DA2;EGYPTIAN HIEROGLYPH-13DA2;Lo;0;L;;;;;N;;;;; +13DA3;EGYPTIAN HIEROGLYPH-13DA3;Lo;0;L;;;;;N;;;;; +13DA4;EGYPTIAN HIEROGLYPH-13DA4;Lo;0;L;;;;;N;;;;; +13DA5;EGYPTIAN HIEROGLYPH-13DA5;Lo;0;L;;;;;N;;;;; +13DA6;EGYPTIAN HIEROGLYPH-13DA6;Lo;0;L;;;;;N;;;;; +13DA7;EGYPTIAN HIEROGLYPH-13DA7;Lo;0;L;;;;;N;;;;; +13DA8;EGYPTIAN HIEROGLYPH-13DA8;Lo;0;L;;;;;N;;;;; +13DA9;EGYPTIAN HIEROGLYPH-13DA9;Lo;0;L;;;;;N;;;;; +13DAA;EGYPTIAN HIEROGLYPH-13DAA;Lo;0;L;;;;;N;;;;; +13DAB;EGYPTIAN HIEROGLYPH-13DAB;Lo;0;L;;;;;N;;;;; +13DAC;EGYPTIAN HIEROGLYPH-13DAC;Lo;0;L;;;;;N;;;;; +13DAD;EGYPTIAN HIEROGLYPH-13DAD;Lo;0;L;;;;;N;;;;; +13DAE;EGYPTIAN HIEROGLYPH-13DAE;Lo;0;L;;;;;N;;;;; +13DAF;EGYPTIAN HIEROGLYPH-13DAF;Lo;0;L;;;;;N;;;;; +13DB0;EGYPTIAN HIEROGLYPH-13DB0;Lo;0;L;;;;;N;;;;; +13DB1;EGYPTIAN HIEROGLYPH-13DB1;Lo;0;L;;;;;N;;;;; +13DB2;EGYPTIAN HIEROGLYPH-13DB2;Lo;0;L;;;;;N;;;;; +13DB3;EGYPTIAN HIEROGLYPH-13DB3;Lo;0;L;;;;;N;;;;; +13DB4;EGYPTIAN HIEROGLYPH-13DB4;Lo;0;L;;;;;N;;;;; +13DB5;EGYPTIAN HIEROGLYPH-13DB5;Lo;0;L;;;;;N;;;;; +13DB6;EGYPTIAN HIEROGLYPH-13DB6;Lo;0;L;;;;;N;;;;; +13DB7;EGYPTIAN HIEROGLYPH-13DB7;Lo;0;L;;;;;N;;;;; +13DB8;EGYPTIAN HIEROGLYPH-13DB8;Lo;0;L;;;;;N;;;;; +13DB9;EGYPTIAN HIEROGLYPH-13DB9;Lo;0;L;;;;;N;;;;; +13DBA;EGYPTIAN HIEROGLYPH-13DBA;Lo;0;L;;;;;N;;;;; +13DBB;EGYPTIAN HIEROGLYPH-13DBB;Lo;0;L;;;;;N;;;;; +13DBC;EGYPTIAN HIEROGLYPH-13DBC;Lo;0;L;;;;;N;;;;; +13DBD;EGYPTIAN HIEROGLYPH-13DBD;Lo;0;L;;;;;N;;;;; +13DBE;EGYPTIAN HIEROGLYPH-13DBE;Lo;0;L;;;;;N;;;;; +13DBF;EGYPTIAN HIEROGLYPH-13DBF;Lo;0;L;;;;;N;;;;; +13DC0;EGYPTIAN HIEROGLYPH-13DC0;Lo;0;L;;;;;N;;;;; +13DC1;EGYPTIAN HIEROGLYPH-13DC1;Lo;0;L;;;;;N;;;;; +13DC2;EGYPTIAN HIEROGLYPH-13DC2;Lo;0;L;;;;;N;;;;; +13DC3;EGYPTIAN HIEROGLYPH-13DC3;Lo;0;L;;;;;N;;;;; +13DC4;EGYPTIAN HIEROGLYPH-13DC4;Lo;0;L;;;;;N;;;;; +13DC5;EGYPTIAN HIEROGLYPH-13DC5;Lo;0;L;;;;;N;;;;; +13DC6;EGYPTIAN HIEROGLYPH-13DC6;Lo;0;L;;;;;N;;;;; +13DC7;EGYPTIAN HIEROGLYPH-13DC7;Lo;0;L;;;;;N;;;;; +13DC8;EGYPTIAN HIEROGLYPH-13DC8;Lo;0;L;;;;;N;;;;; +13DC9;EGYPTIAN HIEROGLYPH-13DC9;Lo;0;L;;;;;N;;;;; +13DCA;EGYPTIAN HIEROGLYPH-13DCA;Lo;0;L;;;;;N;;;;; +13DCB;EGYPTIAN HIEROGLYPH-13DCB;Lo;0;L;;;;;N;;;;; +13DCC;EGYPTIAN HIEROGLYPH-13DCC;Lo;0;L;;;;;N;;;;; +13DCD;EGYPTIAN HIEROGLYPH-13DCD;Lo;0;L;;;;;N;;;;; +13DCE;EGYPTIAN HIEROGLYPH-13DCE;Lo;0;L;;;;;N;;;;; +13DCF;EGYPTIAN HIEROGLYPH-13DCF;Lo;0;L;;;;;N;;;;; +13DD0;EGYPTIAN HIEROGLYPH-13DD0;Lo;0;L;;;;;N;;;;; +13DD1;EGYPTIAN HIEROGLYPH-13DD1;Lo;0;L;;;;;N;;;;; +13DD2;EGYPTIAN HIEROGLYPH-13DD2;Lo;0;L;;;;;N;;;;; +13DD3;EGYPTIAN HIEROGLYPH-13DD3;Lo;0;L;;;;;N;;;;; +13DD4;EGYPTIAN HIEROGLYPH-13DD4;Lo;0;L;;;;;N;;;;; +13DD5;EGYPTIAN HIEROGLYPH-13DD5;Lo;0;L;;;;;N;;;;; +13DD6;EGYPTIAN HIEROGLYPH-13DD6;Lo;0;L;;;;;N;;;;; +13DD7;EGYPTIAN HIEROGLYPH-13DD7;Lo;0;L;;;;;N;;;;; +13DD8;EGYPTIAN HIEROGLYPH-13DD8;Lo;0;L;;;;;N;;;;; +13DD9;EGYPTIAN HIEROGLYPH-13DD9;Lo;0;L;;;;;N;;;;; +13DDA;EGYPTIAN HIEROGLYPH-13DDA;Lo;0;L;;;;;N;;;;; +13DDB;EGYPTIAN HIEROGLYPH-13DDB;Lo;0;L;;;;;N;;;;; +13DDC;EGYPTIAN HIEROGLYPH-13DDC;Lo;0;L;;;;;N;;;;; +13DDD;EGYPTIAN HIEROGLYPH-13DDD;Lo;0;L;;;;;N;;;;; +13DDE;EGYPTIAN HIEROGLYPH-13DDE;Lo;0;L;;;;;N;;;;; +13DDF;EGYPTIAN HIEROGLYPH-13DDF;Lo;0;L;;;;;N;;;;; +13DE0;EGYPTIAN HIEROGLYPH-13DE0;Lo;0;L;;;;;N;;;;; +13DE1;EGYPTIAN HIEROGLYPH-13DE1;Lo;0;L;;;;;N;;;;; +13DE2;EGYPTIAN HIEROGLYPH-13DE2;Lo;0;L;;;;;N;;;;; +13DE3;EGYPTIAN HIEROGLYPH-13DE3;Lo;0;L;;;;;N;;;;; +13DE4;EGYPTIAN HIEROGLYPH-13DE4;Lo;0;L;;;;;N;;;;; +13DE5;EGYPTIAN HIEROGLYPH-13DE5;Lo;0;L;;;;;N;;;;; +13DE6;EGYPTIAN HIEROGLYPH-13DE6;Lo;0;L;;;;;N;;;;; +13DE7;EGYPTIAN HIEROGLYPH-13DE7;Lo;0;L;;;;;N;;;;; +13DE8;EGYPTIAN HIEROGLYPH-13DE8;Lo;0;L;;;;;N;;;;; +13DE9;EGYPTIAN HIEROGLYPH-13DE9;Lo;0;L;;;;;N;;;;; +13DEA;EGYPTIAN HIEROGLYPH-13DEA;Lo;0;L;;;;;N;;;;; +13DEB;EGYPTIAN HIEROGLYPH-13DEB;Lo;0;L;;;;;N;;;;; +13DEC;EGYPTIAN HIEROGLYPH-13DEC;Lo;0;L;;;;;N;;;;; +13DED;EGYPTIAN HIEROGLYPH-13DED;Lo;0;L;;;;;N;;;;; +13DEE;EGYPTIAN HIEROGLYPH-13DEE;Lo;0;L;;;;;N;;;;; +13DEF;EGYPTIAN HIEROGLYPH-13DEF;Lo;0;L;;;;;N;;;;; +13DF0;EGYPTIAN HIEROGLYPH-13DF0;Lo;0;L;;;;;N;;;;; +13DF1;EGYPTIAN HIEROGLYPH-13DF1;Lo;0;L;;;;;N;;;;; +13DF2;EGYPTIAN HIEROGLYPH-13DF2;Lo;0;L;;;;;N;;;;; +13DF3;EGYPTIAN HIEROGLYPH-13DF3;Lo;0;L;;;;;N;;;;; +13DF4;EGYPTIAN HIEROGLYPH-13DF4;Lo;0;L;;;;;N;;;;; +13DF5;EGYPTIAN HIEROGLYPH-13DF5;Lo;0;L;;;;;N;;;;; +13DF6;EGYPTIAN HIEROGLYPH-13DF6;Lo;0;L;;;;;N;;;;; +13DF7;EGYPTIAN HIEROGLYPH-13DF7;Lo;0;L;;;;;N;;;;; +13DF8;EGYPTIAN HIEROGLYPH-13DF8;Lo;0;L;;;;;N;;;;; +13DF9;EGYPTIAN HIEROGLYPH-13DF9;Lo;0;L;;;;;N;;;;; +13DFA;EGYPTIAN HIEROGLYPH-13DFA;Lo;0;L;;;;;N;;;;; +13DFB;EGYPTIAN HIEROGLYPH-13DFB;Lo;0;L;;;;;N;;;;; +13DFC;EGYPTIAN HIEROGLYPH-13DFC;Lo;0;L;;;;;N;;;;; +13DFD;EGYPTIAN HIEROGLYPH-13DFD;Lo;0;L;;;;;N;;;;; +13DFE;EGYPTIAN HIEROGLYPH-13DFE;Lo;0;L;;;;;N;;;;; +13DFF;EGYPTIAN HIEROGLYPH-13DFF;Lo;0;L;;;;;N;;;;; +13E00;EGYPTIAN HIEROGLYPH-13E00;Lo;0;L;;;;;N;;;;; +13E01;EGYPTIAN HIEROGLYPH-13E01;Lo;0;L;;;;;N;;;;; +13E02;EGYPTIAN HIEROGLYPH-13E02;Lo;0;L;;;;;N;;;;; +13E03;EGYPTIAN HIEROGLYPH-13E03;Lo;0;L;;;;;N;;;;; +13E04;EGYPTIAN HIEROGLYPH-13E04;Lo;0;L;;;;;N;;;;; +13E05;EGYPTIAN HIEROGLYPH-13E05;Lo;0;L;;;;;N;;;;; +13E06;EGYPTIAN HIEROGLYPH-13E06;Lo;0;L;;;;;N;;;;; +13E07;EGYPTIAN HIEROGLYPH-13E07;Lo;0;L;;;;;N;;;;; +13E08;EGYPTIAN HIEROGLYPH-13E08;Lo;0;L;;;;;N;;;;; +13E09;EGYPTIAN HIEROGLYPH-13E09;Lo;0;L;;;;;N;;;;; +13E0A;EGYPTIAN HIEROGLYPH-13E0A;Lo;0;L;;;;;N;;;;; +13E0B;EGYPTIAN HIEROGLYPH-13E0B;Lo;0;L;;;;;N;;;;; +13E0C;EGYPTIAN HIEROGLYPH-13E0C;Lo;0;L;;;;;N;;;;; +13E0D;EGYPTIAN HIEROGLYPH-13E0D;Lo;0;L;;;;;N;;;;; +13E0E;EGYPTIAN HIEROGLYPH-13E0E;Lo;0;L;;;;;N;;;;; +13E0F;EGYPTIAN HIEROGLYPH-13E0F;Lo;0;L;;;;;N;;;;; +13E10;EGYPTIAN HIEROGLYPH-13E10;Lo;0;L;;;;;N;;;;; +13E11;EGYPTIAN HIEROGLYPH-13E11;Lo;0;L;;;;;N;;;;; +13E12;EGYPTIAN HIEROGLYPH-13E12;Lo;0;L;;;;;N;;;;; +13E13;EGYPTIAN HIEROGLYPH-13E13;Lo;0;L;;;;;N;;;;; +13E14;EGYPTIAN HIEROGLYPH-13E14;Lo;0;L;;;;;N;;;;; +13E15;EGYPTIAN HIEROGLYPH-13E15;Lo;0;L;;;;;N;;;;; +13E16;EGYPTIAN HIEROGLYPH-13E16;Lo;0;L;;;;;N;;;;; +13E17;EGYPTIAN HIEROGLYPH-13E17;Lo;0;L;;;;;N;;;;; +13E18;EGYPTIAN HIEROGLYPH-13E18;Lo;0;L;;;;;N;;;;; +13E19;EGYPTIAN HIEROGLYPH-13E19;Lo;0;L;;;;;N;;;;; +13E1A;EGYPTIAN HIEROGLYPH-13E1A;Lo;0;L;;;;;N;;;;; +13E1B;EGYPTIAN HIEROGLYPH-13E1B;Lo;0;L;;;;;N;;;;; +13E1C;EGYPTIAN HIEROGLYPH-13E1C;Lo;0;L;;;;;N;;;;; +13E1D;EGYPTIAN HIEROGLYPH-13E1D;Lo;0;L;;;;;N;;;;; +13E1E;EGYPTIAN HIEROGLYPH-13E1E;Lo;0;L;;;;;N;;;;; +13E1F;EGYPTIAN HIEROGLYPH-13E1F;Lo;0;L;;;;;N;;;;; +13E20;EGYPTIAN HIEROGLYPH-13E20;Lo;0;L;;;;;N;;;;; +13E21;EGYPTIAN HIEROGLYPH-13E21;Lo;0;L;;;;;N;;;;; +13E22;EGYPTIAN HIEROGLYPH-13E22;Lo;0;L;;;;;N;;;;; +13E23;EGYPTIAN HIEROGLYPH-13E23;Lo;0;L;;;;;N;;;;; +13E24;EGYPTIAN HIEROGLYPH-13E24;Lo;0;L;;;;;N;;;;; +13E25;EGYPTIAN HIEROGLYPH-13E25;Lo;0;L;;;;;N;;;;; +13E26;EGYPTIAN HIEROGLYPH-13E26;Lo;0;L;;;;;N;;;;; +13E27;EGYPTIAN HIEROGLYPH-13E27;Lo;0;L;;;;;N;;;;; +13E28;EGYPTIAN HIEROGLYPH-13E28;Lo;0;L;;;;;N;;;;; +13E29;EGYPTIAN HIEROGLYPH-13E29;Lo;0;L;;;;;N;;;;; +13E2A;EGYPTIAN HIEROGLYPH-13E2A;Lo;0;L;;;;;N;;;;; +13E2B;EGYPTIAN HIEROGLYPH-13E2B;Lo;0;L;;;;;N;;;;; +13E2C;EGYPTIAN HIEROGLYPH-13E2C;Lo;0;L;;;;;N;;;;; +13E2D;EGYPTIAN HIEROGLYPH-13E2D;Lo;0;L;;;;;N;;;;; +13E2E;EGYPTIAN HIEROGLYPH-13E2E;Lo;0;L;;;;;N;;;;; +13E2F;EGYPTIAN HIEROGLYPH-13E2F;Lo;0;L;;;;;N;;;;; +13E30;EGYPTIAN HIEROGLYPH-13E30;Lo;0;L;;;;;N;;;;; +13E31;EGYPTIAN HIEROGLYPH-13E31;Lo;0;L;;;;;N;;;;; +13E32;EGYPTIAN HIEROGLYPH-13E32;Lo;0;L;;;;;N;;;;; +13E33;EGYPTIAN HIEROGLYPH-13E33;Lo;0;L;;;;;N;;;;; +13E34;EGYPTIAN HIEROGLYPH-13E34;Lo;0;L;;;;;N;;;;; +13E35;EGYPTIAN HIEROGLYPH-13E35;Lo;0;L;;;;;N;;;;; +13E36;EGYPTIAN HIEROGLYPH-13E36;Lo;0;L;;;;;N;;;;; +13E37;EGYPTIAN HIEROGLYPH-13E37;Lo;0;L;;;;;N;;;;; +13E38;EGYPTIAN HIEROGLYPH-13E38;Lo;0;L;;;;;N;;;;; +13E39;EGYPTIAN HIEROGLYPH-13E39;Lo;0;L;;;;;N;;;;; +13E3A;EGYPTIAN HIEROGLYPH-13E3A;Lo;0;L;;;;;N;;;;; +13E3B;EGYPTIAN HIEROGLYPH-13E3B;Lo;0;L;;;;;N;;;;; +13E3C;EGYPTIAN HIEROGLYPH-13E3C;Lo;0;L;;;;;N;;;;; +13E3D;EGYPTIAN HIEROGLYPH-13E3D;Lo;0;L;;;;;N;;;;; +13E3E;EGYPTIAN HIEROGLYPH-13E3E;Lo;0;L;;;;;N;;;;; +13E3F;EGYPTIAN HIEROGLYPH-13E3F;Lo;0;L;;;;;N;;;;; +13E40;EGYPTIAN HIEROGLYPH-13E40;Lo;0;L;;;;;N;;;;; +13E41;EGYPTIAN HIEROGLYPH-13E41;Lo;0;L;;;;;N;;;;; +13E42;EGYPTIAN HIEROGLYPH-13E42;Lo;0;L;;;;;N;;;;; +13E43;EGYPTIAN HIEROGLYPH-13E43;Lo;0;L;;;;;N;;;;; +13E44;EGYPTIAN HIEROGLYPH-13E44;Lo;0;L;;;;;N;;;;; +13E45;EGYPTIAN HIEROGLYPH-13E45;Lo;0;L;;;;;N;;;;; +13E46;EGYPTIAN HIEROGLYPH-13E46;Lo;0;L;;;;;N;;;;; +13E47;EGYPTIAN HIEROGLYPH-13E47;Lo;0;L;;;;;N;;;;; +13E48;EGYPTIAN HIEROGLYPH-13E48;Lo;0;L;;;;;N;;;;; +13E49;EGYPTIAN HIEROGLYPH-13E49;Lo;0;L;;;;;N;;;;; +13E4A;EGYPTIAN HIEROGLYPH-13E4A;Lo;0;L;;;;;N;;;;; +13E4B;EGYPTIAN HIEROGLYPH-13E4B;Lo;0;L;;;;;N;;;;; +13E4C;EGYPTIAN HIEROGLYPH-13E4C;Lo;0;L;;;;;N;;;;; +13E4D;EGYPTIAN HIEROGLYPH-13E4D;Lo;0;L;;;;;N;;;;; +13E4E;EGYPTIAN HIEROGLYPH-13E4E;Lo;0;L;;;;;N;;;;; +13E4F;EGYPTIAN HIEROGLYPH-13E4F;Lo;0;L;;;;;N;;;;; +13E50;EGYPTIAN HIEROGLYPH-13E50;Lo;0;L;;;;;N;;;;; +13E51;EGYPTIAN HIEROGLYPH-13E51;Lo;0;L;;;;;N;;;;; +13E52;EGYPTIAN HIEROGLYPH-13E52;Lo;0;L;;;;;N;;;;; +13E53;EGYPTIAN HIEROGLYPH-13E53;Lo;0;L;;;;;N;;;;; +13E54;EGYPTIAN HIEROGLYPH-13E54;Lo;0;L;;;;;N;;;;; +13E55;EGYPTIAN HIEROGLYPH-13E55;Lo;0;L;;;;;N;;;;; +13E56;EGYPTIAN HIEROGLYPH-13E56;Lo;0;L;;;;;N;;;;; +13E57;EGYPTIAN HIEROGLYPH-13E57;Lo;0;L;;;;;N;;;;; +13E58;EGYPTIAN HIEROGLYPH-13E58;Lo;0;L;;;;;N;;;;; +13E59;EGYPTIAN HIEROGLYPH-13E59;Lo;0;L;;;;;N;;;;; +13E5A;EGYPTIAN HIEROGLYPH-13E5A;Lo;0;L;;;;;N;;;;; +13E5B;EGYPTIAN HIEROGLYPH-13E5B;Lo;0;L;;;;;N;;;;; +13E5C;EGYPTIAN HIEROGLYPH-13E5C;Lo;0;L;;;;;N;;;;; +13E5D;EGYPTIAN HIEROGLYPH-13E5D;Lo;0;L;;;;;N;;;;; +13E5E;EGYPTIAN HIEROGLYPH-13E5E;Lo;0;L;;;;;N;;;;; +13E5F;EGYPTIAN HIEROGLYPH-13E5F;Lo;0;L;;;;;N;;;;; +13E60;EGYPTIAN HIEROGLYPH-13E60;Lo;0;L;;;;;N;;;;; +13E61;EGYPTIAN HIEROGLYPH-13E61;Lo;0;L;;;;;N;;;;; +13E62;EGYPTIAN HIEROGLYPH-13E62;Lo;0;L;;;;;N;;;;; +13E63;EGYPTIAN HIEROGLYPH-13E63;Lo;0;L;;;;;N;;;;; +13E64;EGYPTIAN HIEROGLYPH-13E64;Lo;0;L;;;;;N;;;;; +13E65;EGYPTIAN HIEROGLYPH-13E65;Lo;0;L;;;;;N;;;;; +13E66;EGYPTIAN HIEROGLYPH-13E66;Lo;0;L;;;;;N;;;;; +13E67;EGYPTIAN HIEROGLYPH-13E67;Lo;0;L;;;;;N;;;;; +13E68;EGYPTIAN HIEROGLYPH-13E68;Lo;0;L;;;;;N;;;;; +13E69;EGYPTIAN HIEROGLYPH-13E69;Lo;0;L;;;;;N;;;;; +13E6A;EGYPTIAN HIEROGLYPH-13E6A;Lo;0;L;;;;;N;;;;; +13E6B;EGYPTIAN HIEROGLYPH-13E6B;Lo;0;L;;;;;N;;;;; +13E6C;EGYPTIAN HIEROGLYPH-13E6C;Lo;0;L;;;;;N;;;;; +13E6D;EGYPTIAN HIEROGLYPH-13E6D;Lo;0;L;;;;;N;;;;; +13E6E;EGYPTIAN HIEROGLYPH-13E6E;Lo;0;L;;;;;N;;;;; +13E6F;EGYPTIAN HIEROGLYPH-13E6F;Lo;0;L;;;;;N;;;;; +13E70;EGYPTIAN HIEROGLYPH-13E70;Lo;0;L;;;;;N;;;;; +13E71;EGYPTIAN HIEROGLYPH-13E71;Lo;0;L;;;;;N;;;;; +13E72;EGYPTIAN HIEROGLYPH-13E72;Lo;0;L;;;;;N;;;;; +13E73;EGYPTIAN HIEROGLYPH-13E73;Lo;0;L;;;;;N;;;;; +13E74;EGYPTIAN HIEROGLYPH-13E74;Lo;0;L;;;;;N;;;;; +13E75;EGYPTIAN HIEROGLYPH-13E75;Lo;0;L;;;;;N;;;;; +13E76;EGYPTIAN HIEROGLYPH-13E76;Lo;0;L;;;;;N;;;;; +13E77;EGYPTIAN HIEROGLYPH-13E77;Lo;0;L;;;;;N;;;;; +13E78;EGYPTIAN HIEROGLYPH-13E78;Lo;0;L;;;;;N;;;;; +13E79;EGYPTIAN HIEROGLYPH-13E79;Lo;0;L;;;;;N;;;;; +13E7A;EGYPTIAN HIEROGLYPH-13E7A;Lo;0;L;;;;;N;;;;; +13E7B;EGYPTIAN HIEROGLYPH-13E7B;Lo;0;L;;;;;N;;;;; +13E7C;EGYPTIAN HIEROGLYPH-13E7C;Lo;0;L;;;;;N;;;;; +13E7D;EGYPTIAN HIEROGLYPH-13E7D;Lo;0;L;;;;;N;;;;; +13E7E;EGYPTIAN HIEROGLYPH-13E7E;Lo;0;L;;;;;N;;;;; +13E7F;EGYPTIAN HIEROGLYPH-13E7F;Lo;0;L;;;;;N;;;;; +13E80;EGYPTIAN HIEROGLYPH-13E80;Lo;0;L;;;;;N;;;;; +13E81;EGYPTIAN HIEROGLYPH-13E81;Lo;0;L;;;;;N;;;;; +13E82;EGYPTIAN HIEROGLYPH-13E82;Lo;0;L;;;;;N;;;;; +13E83;EGYPTIAN HIEROGLYPH-13E83;Lo;0;L;;;;;N;;;;; +13E84;EGYPTIAN HIEROGLYPH-13E84;Lo;0;L;;;;;N;;;;; +13E85;EGYPTIAN HIEROGLYPH-13E85;Lo;0;L;;;;;N;;;;; +13E86;EGYPTIAN HIEROGLYPH-13E86;Lo;0;L;;;;;N;;;;; +13E87;EGYPTIAN HIEROGLYPH-13E87;Lo;0;L;;;;;N;;;;; +13E88;EGYPTIAN HIEROGLYPH-13E88;Lo;0;L;;;;;N;;;;; +13E89;EGYPTIAN HIEROGLYPH-13E89;Lo;0;L;;;;;N;;;;; +13E8A;EGYPTIAN HIEROGLYPH-13E8A;Lo;0;L;;;;;N;;;;; +13E8B;EGYPTIAN HIEROGLYPH-13E8B;Lo;0;L;;;;;N;;;;; +13E8C;EGYPTIAN HIEROGLYPH-13E8C;Lo;0;L;;;;;N;;;;; +13E8D;EGYPTIAN HIEROGLYPH-13E8D;Lo;0;L;;;;;N;;;;; +13E8E;EGYPTIAN HIEROGLYPH-13E8E;Lo;0;L;;;;;N;;;;; +13E8F;EGYPTIAN HIEROGLYPH-13E8F;Lo;0;L;;;;;N;;;;; +13E90;EGYPTIAN HIEROGLYPH-13E90;Lo;0;L;;;;;N;;;;; +13E91;EGYPTIAN HIEROGLYPH-13E91;Lo;0;L;;;;;N;;;;; +13E92;EGYPTIAN HIEROGLYPH-13E92;Lo;0;L;;;;;N;;;;; +13E93;EGYPTIAN HIEROGLYPH-13E93;Lo;0;L;;;;;N;;;;; +13E94;EGYPTIAN HIEROGLYPH-13E94;Lo;0;L;;;;;N;;;;; +13E95;EGYPTIAN HIEROGLYPH-13E95;Lo;0;L;;;;;N;;;;; +13E96;EGYPTIAN HIEROGLYPH-13E96;Lo;0;L;;;;;N;;;;; +13E97;EGYPTIAN HIEROGLYPH-13E97;Lo;0;L;;;;;N;;;;; +13E98;EGYPTIAN HIEROGLYPH-13E98;Lo;0;L;;;;;N;;;;; +13E99;EGYPTIAN HIEROGLYPH-13E99;Lo;0;L;;;;;N;;;;; +13E9A;EGYPTIAN HIEROGLYPH-13E9A;Lo;0;L;;;;;N;;;;; +13E9B;EGYPTIAN HIEROGLYPH-13E9B;Lo;0;L;;;;;N;;;;; +13E9C;EGYPTIAN HIEROGLYPH-13E9C;Lo;0;L;;;;;N;;;;; +13E9D;EGYPTIAN HIEROGLYPH-13E9D;Lo;0;L;;;;;N;;;;; +13E9E;EGYPTIAN HIEROGLYPH-13E9E;Lo;0;L;;;;;N;;;;; +13E9F;EGYPTIAN HIEROGLYPH-13E9F;Lo;0;L;;;;;N;;;;; +13EA0;EGYPTIAN HIEROGLYPH-13EA0;Lo;0;L;;;;;N;;;;; +13EA1;EGYPTIAN HIEROGLYPH-13EA1;Lo;0;L;;;;;N;;;;; +13EA2;EGYPTIAN HIEROGLYPH-13EA2;Lo;0;L;;;;;N;;;;; +13EA3;EGYPTIAN HIEROGLYPH-13EA3;Lo;0;L;;;;;N;;;;; +13EA4;EGYPTIAN HIEROGLYPH-13EA4;Lo;0;L;;;;;N;;;;; +13EA5;EGYPTIAN HIEROGLYPH-13EA5;Lo;0;L;;;;;N;;;;; +13EA6;EGYPTIAN HIEROGLYPH-13EA6;Lo;0;L;;;;;N;;;;; +13EA7;EGYPTIAN HIEROGLYPH-13EA7;Lo;0;L;;;;;N;;;;; +13EA8;EGYPTIAN HIEROGLYPH-13EA8;Lo;0;L;;;;;N;;;;; +13EA9;EGYPTIAN HIEROGLYPH-13EA9;Lo;0;L;;;;;N;;;;; +13EAA;EGYPTIAN HIEROGLYPH-13EAA;Lo;0;L;;;;;N;;;;; +13EAB;EGYPTIAN HIEROGLYPH-13EAB;Lo;0;L;;;;;N;;;;; +13EAC;EGYPTIAN HIEROGLYPH-13EAC;Lo;0;L;;;;;N;;;;; +13EAD;EGYPTIAN HIEROGLYPH-13EAD;Lo;0;L;;;;;N;;;;; +13EAE;EGYPTIAN HIEROGLYPH-13EAE;Lo;0;L;;;;;N;;;;; +13EAF;EGYPTIAN HIEROGLYPH-13EAF;Lo;0;L;;;;;N;;;;; +13EB0;EGYPTIAN HIEROGLYPH-13EB0;Lo;0;L;;;;;N;;;;; +13EB1;EGYPTIAN HIEROGLYPH-13EB1;Lo;0;L;;;;;N;;;;; +13EB2;EGYPTIAN HIEROGLYPH-13EB2;Lo;0;L;;;;;N;;;;; +13EB3;EGYPTIAN HIEROGLYPH-13EB3;Lo;0;L;;;;;N;;;;; +13EB4;EGYPTIAN HIEROGLYPH-13EB4;Lo;0;L;;;;;N;;;;; +13EB5;EGYPTIAN HIEROGLYPH-13EB5;Lo;0;L;;;;;N;;;;; +13EB6;EGYPTIAN HIEROGLYPH-13EB6;Lo;0;L;;;;;N;;;;; +13EB7;EGYPTIAN HIEROGLYPH-13EB7;Lo;0;L;;;;;N;;;;; +13EB8;EGYPTIAN HIEROGLYPH-13EB8;Lo;0;L;;;;;N;;;;; +13EB9;EGYPTIAN HIEROGLYPH-13EB9;Lo;0;L;;;;;N;;;;; +13EBA;EGYPTIAN HIEROGLYPH-13EBA;Lo;0;L;;;;;N;;;;; +13EBB;EGYPTIAN HIEROGLYPH-13EBB;Lo;0;L;;;;;N;;;;; +13EBC;EGYPTIAN HIEROGLYPH-13EBC;Lo;0;L;;;;;N;;;;; +13EBD;EGYPTIAN HIEROGLYPH-13EBD;Lo;0;L;;;;;N;;;;; +13EBE;EGYPTIAN HIEROGLYPH-13EBE;Lo;0;L;;;;;N;;;;; +13EBF;EGYPTIAN HIEROGLYPH-13EBF;Lo;0;L;;;;;N;;;;; +13EC0;EGYPTIAN HIEROGLYPH-13EC0;Lo;0;L;;;;;N;;;;; +13EC1;EGYPTIAN HIEROGLYPH-13EC1;Lo;0;L;;;;;N;;;;; +13EC2;EGYPTIAN HIEROGLYPH-13EC2;Lo;0;L;;;;;N;;;;; +13EC3;EGYPTIAN HIEROGLYPH-13EC3;Lo;0;L;;;;;N;;;;; +13EC4;EGYPTIAN HIEROGLYPH-13EC4;Lo;0;L;;;;;N;;;;; +13EC5;EGYPTIAN HIEROGLYPH-13EC5;Lo;0;L;;;;;N;;;;; +13EC6;EGYPTIAN HIEROGLYPH-13EC6;Lo;0;L;;;;;N;;;;; +13EC7;EGYPTIAN HIEROGLYPH-13EC7;Lo;0;L;;;;;N;;;;; +13EC8;EGYPTIAN HIEROGLYPH-13EC8;Lo;0;L;;;;;N;;;;; +13EC9;EGYPTIAN HIEROGLYPH-13EC9;Lo;0;L;;;;;N;;;;; +13ECA;EGYPTIAN HIEROGLYPH-13ECA;Lo;0;L;;;;;N;;;;; +13ECB;EGYPTIAN HIEROGLYPH-13ECB;Lo;0;L;;;;;N;;;;; +13ECC;EGYPTIAN HIEROGLYPH-13ECC;Lo;0;L;;;;;N;;;;; +13ECD;EGYPTIAN HIEROGLYPH-13ECD;Lo;0;L;;;;;N;;;;; +13ECE;EGYPTIAN HIEROGLYPH-13ECE;Lo;0;L;;;;;N;;;;; +13ECF;EGYPTIAN HIEROGLYPH-13ECF;Lo;0;L;;;;;N;;;;; +13ED0;EGYPTIAN HIEROGLYPH-13ED0;Lo;0;L;;;;;N;;;;; +13ED1;EGYPTIAN HIEROGLYPH-13ED1;Lo;0;L;;;;;N;;;;; +13ED2;EGYPTIAN HIEROGLYPH-13ED2;Lo;0;L;;;;;N;;;;; +13ED3;EGYPTIAN HIEROGLYPH-13ED3;Lo;0;L;;;;;N;;;;; +13ED4;EGYPTIAN HIEROGLYPH-13ED4;Lo;0;L;;;;;N;;;;; +13ED5;EGYPTIAN HIEROGLYPH-13ED5;Lo;0;L;;;;;N;;;;; +13ED6;EGYPTIAN HIEROGLYPH-13ED6;Lo;0;L;;;;;N;;;;; +13ED7;EGYPTIAN HIEROGLYPH-13ED7;Lo;0;L;;;;;N;;;;; +13ED8;EGYPTIAN HIEROGLYPH-13ED8;Lo;0;L;;;;;N;;;;; +13ED9;EGYPTIAN HIEROGLYPH-13ED9;Lo;0;L;;;;;N;;;;; +13EDA;EGYPTIAN HIEROGLYPH-13EDA;Lo;0;L;;;;;N;;;;; +13EDB;EGYPTIAN HIEROGLYPH-13EDB;Lo;0;L;;;;;N;;;;; +13EDC;EGYPTIAN HIEROGLYPH-13EDC;Lo;0;L;;;;;N;;;;; +13EDD;EGYPTIAN HIEROGLYPH-13EDD;Lo;0;L;;;;;N;;;;; +13EDE;EGYPTIAN HIEROGLYPH-13EDE;Lo;0;L;;;;;N;;;;; +13EDF;EGYPTIAN HIEROGLYPH-13EDF;Lo;0;L;;;;;N;;;;; +13EE0;EGYPTIAN HIEROGLYPH-13EE0;Lo;0;L;;;;;N;;;;; +13EE1;EGYPTIAN HIEROGLYPH-13EE1;Lo;0;L;;;;;N;;;;; +13EE2;EGYPTIAN HIEROGLYPH-13EE2;Lo;0;L;;;;;N;;;;; +13EE3;EGYPTIAN HIEROGLYPH-13EE3;Lo;0;L;;;;;N;;;;; +13EE4;EGYPTIAN HIEROGLYPH-13EE4;Lo;0;L;;;;;N;;;;; +13EE5;EGYPTIAN HIEROGLYPH-13EE5;Lo;0;L;;;;;N;;;;; +13EE6;EGYPTIAN HIEROGLYPH-13EE6;Lo;0;L;;;;;N;;;;; +13EE7;EGYPTIAN HIEROGLYPH-13EE7;Lo;0;L;;;;;N;;;;; +13EE8;EGYPTIAN HIEROGLYPH-13EE8;Lo;0;L;;;;;N;;;;; +13EE9;EGYPTIAN HIEROGLYPH-13EE9;Lo;0;L;;;;;N;;;;; +13EEA;EGYPTIAN HIEROGLYPH-13EEA;Lo;0;L;;;;;N;;;;; +13EEB;EGYPTIAN HIEROGLYPH-13EEB;Lo;0;L;;;;;N;;;;; +13EEC;EGYPTIAN HIEROGLYPH-13EEC;Lo;0;L;;;;;N;;;;; +13EED;EGYPTIAN HIEROGLYPH-13EED;Lo;0;L;;;;;N;;;;; +13EEE;EGYPTIAN HIEROGLYPH-13EEE;Lo;0;L;;;;;N;;;;; +13EEF;EGYPTIAN HIEROGLYPH-13EEF;Lo;0;L;;;;;N;;;;; +13EF0;EGYPTIAN HIEROGLYPH-13EF0;Lo;0;L;;;;;N;;;;; +13EF1;EGYPTIAN HIEROGLYPH-13EF1;Lo;0;L;;;;;N;;;;; +13EF2;EGYPTIAN HIEROGLYPH-13EF2;Lo;0;L;;;;;N;;;;; +13EF3;EGYPTIAN HIEROGLYPH-13EF3;Lo;0;L;;;;;N;;;;; +13EF4;EGYPTIAN HIEROGLYPH-13EF4;Lo;0;L;;;;;N;;;;; +13EF5;EGYPTIAN HIEROGLYPH-13EF5;Lo;0;L;;;;;N;;;;; +13EF6;EGYPTIAN HIEROGLYPH-13EF6;Lo;0;L;;;;;N;;;;; +13EF7;EGYPTIAN HIEROGLYPH-13EF7;Lo;0;L;;;;;N;;;;; +13EF8;EGYPTIAN HIEROGLYPH-13EF8;Lo;0;L;;;;;N;;;;; +13EF9;EGYPTIAN HIEROGLYPH-13EF9;Lo;0;L;;;;;N;;;;; +13EFA;EGYPTIAN HIEROGLYPH-13EFA;Lo;0;L;;;;;N;;;;; +13EFB;EGYPTIAN HIEROGLYPH-13EFB;Lo;0;L;;;;;N;;;;; +13EFC;EGYPTIAN HIEROGLYPH-13EFC;Lo;0;L;;;;;N;;;;; +13EFD;EGYPTIAN HIEROGLYPH-13EFD;Lo;0;L;;;;;N;;;;; +13EFE;EGYPTIAN HIEROGLYPH-13EFE;Lo;0;L;;;;;N;;;;; +13EFF;EGYPTIAN HIEROGLYPH-13EFF;Lo;0;L;;;;;N;;;;; +13F00;EGYPTIAN HIEROGLYPH-13F00;Lo;0;L;;;;;N;;;;; +13F01;EGYPTIAN HIEROGLYPH-13F01;Lo;0;L;;;;;N;;;;; +13F02;EGYPTIAN HIEROGLYPH-13F02;Lo;0;L;;;;;N;;;;; +13F03;EGYPTIAN HIEROGLYPH-13F03;Lo;0;L;;;;;N;;;;; +13F04;EGYPTIAN HIEROGLYPH-13F04;Lo;0;L;;;;;N;;;;; +13F05;EGYPTIAN HIEROGLYPH-13F05;Lo;0;L;;;;;N;;;;; +13F06;EGYPTIAN HIEROGLYPH-13F06;Lo;0;L;;;;;N;;;;; +13F07;EGYPTIAN HIEROGLYPH-13F07;Lo;0;L;;;;;N;;;;; +13F08;EGYPTIAN HIEROGLYPH-13F08;Lo;0;L;;;;;N;;;;; +13F09;EGYPTIAN HIEROGLYPH-13F09;Lo;0;L;;;;;N;;;;; +13F0A;EGYPTIAN HIEROGLYPH-13F0A;Lo;0;L;;;;;N;;;;; +13F0B;EGYPTIAN HIEROGLYPH-13F0B;Lo;0;L;;;;;N;;;;; +13F0C;EGYPTIAN HIEROGLYPH-13F0C;Lo;0;L;;;;;N;;;;; +13F0D;EGYPTIAN HIEROGLYPH-13F0D;Lo;0;L;;;;;N;;;;; +13F0E;EGYPTIAN HIEROGLYPH-13F0E;Lo;0;L;;;;;N;;;;; +13F0F;EGYPTIAN HIEROGLYPH-13F0F;Lo;0;L;;;;;N;;;;; +13F10;EGYPTIAN HIEROGLYPH-13F10;Lo;0;L;;;;;N;;;;; +13F11;EGYPTIAN HIEROGLYPH-13F11;Lo;0;L;;;;;N;;;;; +13F12;EGYPTIAN HIEROGLYPH-13F12;Lo;0;L;;;;;N;;;;; +13F13;EGYPTIAN HIEROGLYPH-13F13;Lo;0;L;;;;;N;;;;; +13F14;EGYPTIAN HIEROGLYPH-13F14;Lo;0;L;;;;;N;;;;; +13F15;EGYPTIAN HIEROGLYPH-13F15;Lo;0;L;;;;;N;;;;; +13F16;EGYPTIAN HIEROGLYPH-13F16;Lo;0;L;;;;;N;;;;; +13F17;EGYPTIAN HIEROGLYPH-13F17;Lo;0;L;;;;;N;;;;; +13F18;EGYPTIAN HIEROGLYPH-13F18;Lo;0;L;;;;;N;;;;; +13F19;EGYPTIAN HIEROGLYPH-13F19;Lo;0;L;;;;;N;;;;; +13F1A;EGYPTIAN HIEROGLYPH-13F1A;Lo;0;L;;;;;N;;;;; +13F1B;EGYPTIAN HIEROGLYPH-13F1B;Lo;0;L;;;;;N;;;;; +13F1C;EGYPTIAN HIEROGLYPH-13F1C;Lo;0;L;;;;;N;;;;; +13F1D;EGYPTIAN HIEROGLYPH-13F1D;Lo;0;L;;;;;N;;;;; +13F1E;EGYPTIAN HIEROGLYPH-13F1E;Lo;0;L;;;;;N;;;;; +13F1F;EGYPTIAN HIEROGLYPH-13F1F;Lo;0;L;;;;;N;;;;; +13F20;EGYPTIAN HIEROGLYPH-13F20;Lo;0;L;;;;;N;;;;; +13F21;EGYPTIAN HIEROGLYPH-13F21;Lo;0;L;;;;;N;;;;; +13F22;EGYPTIAN HIEROGLYPH-13F22;Lo;0;L;;;;;N;;;;; +13F23;EGYPTIAN HIEROGLYPH-13F23;Lo;0;L;;;;;N;;;;; +13F24;EGYPTIAN HIEROGLYPH-13F24;Lo;0;L;;;;;N;;;;; +13F25;EGYPTIAN HIEROGLYPH-13F25;Lo;0;L;;;;;N;;;;; +13F26;EGYPTIAN HIEROGLYPH-13F26;Lo;0;L;;;;;N;;;;; +13F27;EGYPTIAN HIEROGLYPH-13F27;Lo;0;L;;;;;N;;;;; +13F28;EGYPTIAN HIEROGLYPH-13F28;Lo;0;L;;;;;N;;;;; +13F29;EGYPTIAN HIEROGLYPH-13F29;Lo;0;L;;;;;N;;;;; +13F2A;EGYPTIAN HIEROGLYPH-13F2A;Lo;0;L;;;;;N;;;;; +13F2B;EGYPTIAN HIEROGLYPH-13F2B;Lo;0;L;;;;;N;;;;; +13F2C;EGYPTIAN HIEROGLYPH-13F2C;Lo;0;L;;;;;N;;;;; +13F2D;EGYPTIAN HIEROGLYPH-13F2D;Lo;0;L;;;;;N;;;;; +13F2E;EGYPTIAN HIEROGLYPH-13F2E;Lo;0;L;;;;;N;;;;; +13F2F;EGYPTIAN HIEROGLYPH-13F2F;Lo;0;L;;;;;N;;;;; +13F30;EGYPTIAN HIEROGLYPH-13F30;Lo;0;L;;;;;N;;;;; +13F31;EGYPTIAN HIEROGLYPH-13F31;Lo;0;L;;;;;N;;;;; +13F32;EGYPTIAN HIEROGLYPH-13F32;Lo;0;L;;;;;N;;;;; +13F33;EGYPTIAN HIEROGLYPH-13F33;Lo;0;L;;;;;N;;;;; +13F34;EGYPTIAN HIEROGLYPH-13F34;Lo;0;L;;;;;N;;;;; +13F35;EGYPTIAN HIEROGLYPH-13F35;Lo;0;L;;;;;N;;;;; +13F36;EGYPTIAN HIEROGLYPH-13F36;Lo;0;L;;;;;N;;;;; +13F37;EGYPTIAN HIEROGLYPH-13F37;Lo;0;L;;;;;N;;;;; +13F38;EGYPTIAN HIEROGLYPH-13F38;Lo;0;L;;;;;N;;;;; +13F39;EGYPTIAN HIEROGLYPH-13F39;Lo;0;L;;;;;N;;;;; +13F3A;EGYPTIAN HIEROGLYPH-13F3A;Lo;0;L;;;;;N;;;;; +13F3B;EGYPTIAN HIEROGLYPH-13F3B;Lo;0;L;;;;;N;;;;; +13F3C;EGYPTIAN HIEROGLYPH-13F3C;Lo;0;L;;;;;N;;;;; +13F3D;EGYPTIAN HIEROGLYPH-13F3D;Lo;0;L;;;;;N;;;;; +13F3E;EGYPTIAN HIEROGLYPH-13F3E;Lo;0;L;;;;;N;;;;; +13F3F;EGYPTIAN HIEROGLYPH-13F3F;Lo;0;L;;;;;N;;;;; +13F40;EGYPTIAN HIEROGLYPH-13F40;Lo;0;L;;;;;N;;;;; +13F41;EGYPTIAN HIEROGLYPH-13F41;Lo;0;L;;;;;N;;;;; +13F42;EGYPTIAN HIEROGLYPH-13F42;Lo;0;L;;;;;N;;;;; +13F43;EGYPTIAN HIEROGLYPH-13F43;Lo;0;L;;;;;N;;;;; +13F44;EGYPTIAN HIEROGLYPH-13F44;Lo;0;L;;;;;N;;;;; +13F45;EGYPTIAN HIEROGLYPH-13F45;Lo;0;L;;;;;N;;;;; +13F46;EGYPTIAN HIEROGLYPH-13F46;Lo;0;L;;;;;N;;;;; +13F47;EGYPTIAN HIEROGLYPH-13F47;Lo;0;L;;;;;N;;;;; +13F48;EGYPTIAN HIEROGLYPH-13F48;Lo;0;L;;;;;N;;;;; +13F49;EGYPTIAN HIEROGLYPH-13F49;Lo;0;L;;;;;N;;;;; +13F4A;EGYPTIAN HIEROGLYPH-13F4A;Lo;0;L;;;;;N;;;;; +13F4B;EGYPTIAN HIEROGLYPH-13F4B;Lo;0;L;;;;;N;;;;; +13F4C;EGYPTIAN HIEROGLYPH-13F4C;Lo;0;L;;;;;N;;;;; +13F4D;EGYPTIAN HIEROGLYPH-13F4D;Lo;0;L;;;;;N;;;;; +13F4E;EGYPTIAN HIEROGLYPH-13F4E;Lo;0;L;;;;;N;;;;; +13F4F;EGYPTIAN HIEROGLYPH-13F4F;Lo;0;L;;;;;N;;;;; +13F50;EGYPTIAN HIEROGLYPH-13F50;Lo;0;L;;;;;N;;;;; +13F51;EGYPTIAN HIEROGLYPH-13F51;Lo;0;L;;;;;N;;;;; +13F52;EGYPTIAN HIEROGLYPH-13F52;Lo;0;L;;;;;N;;;;; +13F53;EGYPTIAN HIEROGLYPH-13F53;Lo;0;L;;;;;N;;;;; +13F54;EGYPTIAN HIEROGLYPH-13F54;Lo;0;L;;;;;N;;;;; +13F55;EGYPTIAN HIEROGLYPH-13F55;Lo;0;L;;;;;N;;;;; +13F56;EGYPTIAN HIEROGLYPH-13F56;Lo;0;L;;;;;N;;;;; +13F57;EGYPTIAN HIEROGLYPH-13F57;Lo;0;L;;;;;N;;;;; +13F58;EGYPTIAN HIEROGLYPH-13F58;Lo;0;L;;;;;N;;;;; +13F59;EGYPTIAN HIEROGLYPH-13F59;Lo;0;L;;;;;N;;;;; +13F5A;EGYPTIAN HIEROGLYPH-13F5A;Lo;0;L;;;;;N;;;;; +13F5B;EGYPTIAN HIEROGLYPH-13F5B;Lo;0;L;;;;;N;;;;; +13F5C;EGYPTIAN HIEROGLYPH-13F5C;Lo;0;L;;;;;N;;;;; +13F5D;EGYPTIAN HIEROGLYPH-13F5D;Lo;0;L;;;;;N;;;;; +13F5E;EGYPTIAN HIEROGLYPH-13F5E;Lo;0;L;;;;;N;;;;; +13F5F;EGYPTIAN HIEROGLYPH-13F5F;Lo;0;L;;;;;N;;;;; +13F60;EGYPTIAN HIEROGLYPH-13F60;Lo;0;L;;;;;N;;;;; +13F61;EGYPTIAN HIEROGLYPH-13F61;Lo;0;L;;;;;N;;;;; +13F62;EGYPTIAN HIEROGLYPH-13F62;Lo;0;L;;;;;N;;;;; +13F63;EGYPTIAN HIEROGLYPH-13F63;Lo;0;L;;;;;N;;;;; +13F64;EGYPTIAN HIEROGLYPH-13F64;Lo;0;L;;;;;N;;;;; +13F65;EGYPTIAN HIEROGLYPH-13F65;Lo;0;L;;;;;N;;;;; +13F66;EGYPTIAN HIEROGLYPH-13F66;Lo;0;L;;;;;N;;;;; +13F67;EGYPTIAN HIEROGLYPH-13F67;Lo;0;L;;;;;N;;;;; +13F68;EGYPTIAN HIEROGLYPH-13F68;Lo;0;L;;;;;N;;;;; +13F69;EGYPTIAN HIEROGLYPH-13F69;Lo;0;L;;;;;N;;;;; +13F6A;EGYPTIAN HIEROGLYPH-13F6A;Lo;0;L;;;;;N;;;;; +13F6B;EGYPTIAN HIEROGLYPH-13F6B;Lo;0;L;;;;;N;;;;; +13F6C;EGYPTIAN HIEROGLYPH-13F6C;Lo;0;L;;;;;N;;;;; +13F6D;EGYPTIAN HIEROGLYPH-13F6D;Lo;0;L;;;;;N;;;;; +13F6E;EGYPTIAN HIEROGLYPH-13F6E;Lo;0;L;;;;;N;;;;; +13F6F;EGYPTIAN HIEROGLYPH-13F6F;Lo;0;L;;;;;N;;;;; +13F70;EGYPTIAN HIEROGLYPH-13F70;Lo;0;L;;;;;N;;;;; +13F71;EGYPTIAN HIEROGLYPH-13F71;Lo;0;L;;;;;N;;;;; +13F72;EGYPTIAN HIEROGLYPH-13F72;Lo;0;L;;;;;N;;;;; +13F73;EGYPTIAN HIEROGLYPH-13F73;Lo;0;L;;;;;N;;;;; +13F74;EGYPTIAN HIEROGLYPH-13F74;Lo;0;L;;;;;N;;;;; +13F75;EGYPTIAN HIEROGLYPH-13F75;Lo;0;L;;;;;N;;;;; +13F76;EGYPTIAN HIEROGLYPH-13F76;Lo;0;L;;;;;N;;;;; +13F77;EGYPTIAN HIEROGLYPH-13F77;Lo;0;L;;;;;N;;;;; +13F78;EGYPTIAN HIEROGLYPH-13F78;Lo;0;L;;;;;N;;;;; +13F79;EGYPTIAN HIEROGLYPH-13F79;Lo;0;L;;;;;N;;;;; +13F7A;EGYPTIAN HIEROGLYPH-13F7A;Lo;0;L;;;;;N;;;;; +13F7B;EGYPTIAN HIEROGLYPH-13F7B;Lo;0;L;;;;;N;;;;; +13F7C;EGYPTIAN HIEROGLYPH-13F7C;Lo;0;L;;;;;N;;;;; +13F7D;EGYPTIAN HIEROGLYPH-13F7D;Lo;0;L;;;;;N;;;;; +13F7E;EGYPTIAN HIEROGLYPH-13F7E;Lo;0;L;;;;;N;;;;; +13F7F;EGYPTIAN HIEROGLYPH-13F7F;Lo;0;L;;;;;N;;;;; +13F80;EGYPTIAN HIEROGLYPH-13F80;Lo;0;L;;;;;N;;;;; +13F81;EGYPTIAN HIEROGLYPH-13F81;Lo;0;L;;;;;N;;;;; +13F82;EGYPTIAN HIEROGLYPH-13F82;Lo;0;L;;;;;N;;;;; +13F83;EGYPTIAN HIEROGLYPH-13F83;Lo;0;L;;;;;N;;;;; +13F84;EGYPTIAN HIEROGLYPH-13F84;Lo;0;L;;;;;N;;;;; +13F85;EGYPTIAN HIEROGLYPH-13F85;Lo;0;L;;;;;N;;;;; +13F86;EGYPTIAN HIEROGLYPH-13F86;Lo;0;L;;;;;N;;;;; +13F87;EGYPTIAN HIEROGLYPH-13F87;Lo;0;L;;;;;N;;;;; +13F88;EGYPTIAN HIEROGLYPH-13F88;Lo;0;L;;;;;N;;;;; +13F89;EGYPTIAN HIEROGLYPH-13F89;Lo;0;L;;;;;N;;;;; +13F8A;EGYPTIAN HIEROGLYPH-13F8A;Lo;0;L;;;;;N;;;;; +13F8B;EGYPTIAN HIEROGLYPH-13F8B;Lo;0;L;;;;;N;;;;; +13F8C;EGYPTIAN HIEROGLYPH-13F8C;Lo;0;L;;;;;N;;;;; +13F8D;EGYPTIAN HIEROGLYPH-13F8D;Lo;0;L;;;;;N;;;;; +13F8E;EGYPTIAN HIEROGLYPH-13F8E;Lo;0;L;;;;;N;;;;; +13F8F;EGYPTIAN HIEROGLYPH-13F8F;Lo;0;L;;;;;N;;;;; +13F90;EGYPTIAN HIEROGLYPH-13F90;Lo;0;L;;;;;N;;;;; +13F91;EGYPTIAN HIEROGLYPH-13F91;Lo;0;L;;;;;N;;;;; +13F92;EGYPTIAN HIEROGLYPH-13F92;Lo;0;L;;;;;N;;;;; +13F93;EGYPTIAN HIEROGLYPH-13F93;Lo;0;L;;;;;N;;;;; +13F94;EGYPTIAN HIEROGLYPH-13F94;Lo;0;L;;;;;N;;;;; +13F95;EGYPTIAN HIEROGLYPH-13F95;Lo;0;L;;;;;N;;;;; +13F96;EGYPTIAN HIEROGLYPH-13F96;Lo;0;L;;;;;N;;;;; +13F97;EGYPTIAN HIEROGLYPH-13F97;Lo;0;L;;;;;N;;;;; +13F98;EGYPTIAN HIEROGLYPH-13F98;Lo;0;L;;;;;N;;;;; +13F99;EGYPTIAN HIEROGLYPH-13F99;Lo;0;L;;;;;N;;;;; +13F9A;EGYPTIAN HIEROGLYPH-13F9A;Lo;0;L;;;;;N;;;;; +13F9B;EGYPTIAN HIEROGLYPH-13F9B;Lo;0;L;;;;;N;;;;; +13F9C;EGYPTIAN HIEROGLYPH-13F9C;Lo;0;L;;;;;N;;;;; +13F9D;EGYPTIAN HIEROGLYPH-13F9D;Lo;0;L;;;;;N;;;;; +13F9E;EGYPTIAN HIEROGLYPH-13F9E;Lo;0;L;;;;;N;;;;; +13F9F;EGYPTIAN HIEROGLYPH-13F9F;Lo;0;L;;;;;N;;;;; +13FA0;EGYPTIAN HIEROGLYPH-13FA0;Lo;0;L;;;;;N;;;;; +13FA1;EGYPTIAN HIEROGLYPH-13FA1;Lo;0;L;;;;;N;;;;; +13FA2;EGYPTIAN HIEROGLYPH-13FA2;Lo;0;L;;;;;N;;;;; +13FA3;EGYPTIAN HIEROGLYPH-13FA3;Lo;0;L;;;;;N;;;;; +13FA4;EGYPTIAN HIEROGLYPH-13FA4;Lo;0;L;;;;;N;;;;; +13FA5;EGYPTIAN HIEROGLYPH-13FA5;Lo;0;L;;;;;N;;;;; +13FA6;EGYPTIAN HIEROGLYPH-13FA6;Lo;0;L;;;;;N;;;;; +13FA7;EGYPTIAN HIEROGLYPH-13FA7;Lo;0;L;;;;;N;;;;; +13FA8;EGYPTIAN HIEROGLYPH-13FA8;Lo;0;L;;;;;N;;;;; +13FA9;EGYPTIAN HIEROGLYPH-13FA9;Lo;0;L;;;;;N;;;;; +13FAA;EGYPTIAN HIEROGLYPH-13FAA;Lo;0;L;;;;;N;;;;; +13FAB;EGYPTIAN HIEROGLYPH-13FAB;Lo;0;L;;;;;N;;;;; +13FAC;EGYPTIAN HIEROGLYPH-13FAC;Lo;0;L;;;;;N;;;;; +13FAD;EGYPTIAN HIEROGLYPH-13FAD;Lo;0;L;;;;;N;;;;; +13FAE;EGYPTIAN HIEROGLYPH-13FAE;Lo;0;L;;;;;N;;;;; +13FAF;EGYPTIAN HIEROGLYPH-13FAF;Lo;0;L;;;;;N;;;;; +13FB0;EGYPTIAN HIEROGLYPH-13FB0;Lo;0;L;;;;;N;;;;; +13FB1;EGYPTIAN HIEROGLYPH-13FB1;Lo;0;L;;;;;N;;;;; +13FB2;EGYPTIAN HIEROGLYPH-13FB2;Lo;0;L;;;;;N;;;;; +13FB3;EGYPTIAN HIEROGLYPH-13FB3;Lo;0;L;;;;;N;;;;; +13FB4;EGYPTIAN HIEROGLYPH-13FB4;Lo;0;L;;;;;N;;;;; +13FB5;EGYPTIAN HIEROGLYPH-13FB5;Lo;0;L;;;;;N;;;;; +13FB6;EGYPTIAN HIEROGLYPH-13FB6;Lo;0;L;;;;;N;;;;; +13FB7;EGYPTIAN HIEROGLYPH-13FB7;Lo;0;L;;;;;N;;;;; +13FB8;EGYPTIAN HIEROGLYPH-13FB8;Lo;0;L;;;;;N;;;;; +13FB9;EGYPTIAN HIEROGLYPH-13FB9;Lo;0;L;;;;;N;;;;; +13FBA;EGYPTIAN HIEROGLYPH-13FBA;Lo;0;L;;;;;N;;;;; +13FBB;EGYPTIAN HIEROGLYPH-13FBB;Lo;0;L;;;;;N;;;;; +13FBC;EGYPTIAN HIEROGLYPH-13FBC;Lo;0;L;;;;;N;;;;; +13FBD;EGYPTIAN HIEROGLYPH-13FBD;Lo;0;L;;;;;N;;;;; +13FBE;EGYPTIAN HIEROGLYPH-13FBE;Lo;0;L;;;;;N;;;;; +13FBF;EGYPTIAN HIEROGLYPH-13FBF;Lo;0;L;;;;;N;;;;; +13FC0;EGYPTIAN HIEROGLYPH-13FC0;Lo;0;L;;;;;N;;;;; +13FC1;EGYPTIAN HIEROGLYPH-13FC1;Lo;0;L;;;;;N;;;;; +13FC2;EGYPTIAN HIEROGLYPH-13FC2;Lo;0;L;;;;;N;;;;; +13FC3;EGYPTIAN HIEROGLYPH-13FC3;Lo;0;L;;;;;N;;;;; +13FC4;EGYPTIAN HIEROGLYPH-13FC4;Lo;0;L;;;;;N;;;;; +13FC5;EGYPTIAN HIEROGLYPH-13FC5;Lo;0;L;;;;;N;;;;; +13FC6;EGYPTIAN HIEROGLYPH-13FC6;Lo;0;L;;;;;N;;;;; +13FC7;EGYPTIAN HIEROGLYPH-13FC7;Lo;0;L;;;;;N;;;;; +13FC8;EGYPTIAN HIEROGLYPH-13FC8;Lo;0;L;;;;;N;;;;; +13FC9;EGYPTIAN HIEROGLYPH-13FC9;Lo;0;L;;;;;N;;;;; +13FCA;EGYPTIAN HIEROGLYPH-13FCA;Lo;0;L;;;;;N;;;;; +13FCB;EGYPTIAN HIEROGLYPH-13FCB;Lo;0;L;;;;;N;;;;; +13FCC;EGYPTIAN HIEROGLYPH-13FCC;Lo;0;L;;;;;N;;;;; +13FCD;EGYPTIAN HIEROGLYPH-13FCD;Lo;0;L;;;;;N;;;;; +13FCE;EGYPTIAN HIEROGLYPH-13FCE;Lo;0;L;;;;;N;;;;; +13FCF;EGYPTIAN HIEROGLYPH-13FCF;Lo;0;L;;;;;N;;;;; +13FD0;EGYPTIAN HIEROGLYPH-13FD0;Lo;0;L;;;;;N;;;;; +13FD1;EGYPTIAN HIEROGLYPH-13FD1;Lo;0;L;;;;;N;;;;; +13FD2;EGYPTIAN HIEROGLYPH-13FD2;Lo;0;L;;;;;N;;;;; +13FD3;EGYPTIAN HIEROGLYPH-13FD3;Lo;0;L;;;;;N;;;;; +13FD4;EGYPTIAN HIEROGLYPH-13FD4;Lo;0;L;;;;;N;;;;; +13FD5;EGYPTIAN HIEROGLYPH-13FD5;Lo;0;L;;;;;N;;;;; +13FD6;EGYPTIAN HIEROGLYPH-13FD6;Lo;0;L;;;;;N;;;;; +13FD7;EGYPTIAN HIEROGLYPH-13FD7;Lo;0;L;;;;;N;;;;; +13FD8;EGYPTIAN HIEROGLYPH-13FD8;Lo;0;L;;;;;N;;;;; +13FD9;EGYPTIAN HIEROGLYPH-13FD9;Lo;0;L;;;;;N;;;;; +13FDA;EGYPTIAN HIEROGLYPH-13FDA;Lo;0;L;;;;;N;;;;; +13FDB;EGYPTIAN HIEROGLYPH-13FDB;Lo;0;L;;;;;N;;;;; +13FDC;EGYPTIAN HIEROGLYPH-13FDC;Lo;0;L;;;;;N;;;;; +13FDD;EGYPTIAN HIEROGLYPH-13FDD;Lo;0;L;;;;;N;;;;; +13FDE;EGYPTIAN HIEROGLYPH-13FDE;Lo;0;L;;;;;N;;;;; +13FDF;EGYPTIAN HIEROGLYPH-13FDF;Lo;0;L;;;;;N;;;;; +13FE0;EGYPTIAN HIEROGLYPH-13FE0;Lo;0;L;;;;;N;;;;; +13FE1;EGYPTIAN HIEROGLYPH-13FE1;Lo;0;L;;;;;N;;;;; +13FE2;EGYPTIAN HIEROGLYPH-13FE2;Lo;0;L;;;;;N;;;;; +13FE3;EGYPTIAN HIEROGLYPH-13FE3;Lo;0;L;;;;;N;;;;; +13FE4;EGYPTIAN HIEROGLYPH-13FE4;Lo;0;L;;;;;N;;;;; +13FE5;EGYPTIAN HIEROGLYPH-13FE5;Lo;0;L;;;;;N;;;;; +13FE6;EGYPTIAN HIEROGLYPH-13FE6;Lo;0;L;;;;;N;;;;; +13FE7;EGYPTIAN HIEROGLYPH-13FE7;Lo;0;L;;;;;N;;;;; +13FE8;EGYPTIAN HIEROGLYPH-13FE8;Lo;0;L;;;;;N;;;;; +13FE9;EGYPTIAN HIEROGLYPH-13FE9;Lo;0;L;;;;;N;;;;; +13FEA;EGYPTIAN HIEROGLYPH-13FEA;Lo;0;L;;;;;N;;;;; +13FEB;EGYPTIAN HIEROGLYPH-13FEB;Lo;0;L;;;;;N;;;;; +13FEC;EGYPTIAN HIEROGLYPH-13FEC;Lo;0;L;;;;;N;;;;; +13FED;EGYPTIAN HIEROGLYPH-13FED;Lo;0;L;;;;;N;;;;; +13FEE;EGYPTIAN HIEROGLYPH-13FEE;Lo;0;L;;;;;N;;;;; +13FEF;EGYPTIAN HIEROGLYPH-13FEF;Lo;0;L;;;;;N;;;;; +13FF0;EGYPTIAN HIEROGLYPH-13FF0;Lo;0;L;;;;;N;;;;; +13FF1;EGYPTIAN HIEROGLYPH-13FF1;Lo;0;L;;;;;N;;;;; +13FF2;EGYPTIAN HIEROGLYPH-13FF2;Lo;0;L;;;;;N;;;;; +13FF3;EGYPTIAN HIEROGLYPH-13FF3;Lo;0;L;;;;;N;;;;; +13FF4;EGYPTIAN HIEROGLYPH-13FF4;Lo;0;L;;;;;N;;;;; +13FF5;EGYPTIAN HIEROGLYPH-13FF5;Lo;0;L;;;;;N;;;;; +13FF6;EGYPTIAN HIEROGLYPH-13FF6;Lo;0;L;;;;;N;;;;; +13FF7;EGYPTIAN HIEROGLYPH-13FF7;Lo;0;L;;;;;N;;;;; +13FF8;EGYPTIAN HIEROGLYPH-13FF8;Lo;0;L;;;;;N;;;;; +13FF9;EGYPTIAN HIEROGLYPH-13FF9;Lo;0;L;;;;;N;;;;; +13FFA;EGYPTIAN HIEROGLYPH-13FFA;Lo;0;L;;;;;N;;;;; +13FFB;EGYPTIAN HIEROGLYPH-13FFB;Lo;0;L;;;;;N;;;;; +13FFC;EGYPTIAN HIEROGLYPH-13FFC;Lo;0;L;;;;;N;;;;; +13FFD;EGYPTIAN HIEROGLYPH-13FFD;Lo;0;L;;;;;N;;;;; +13FFE;EGYPTIAN HIEROGLYPH-13FFE;Lo;0;L;;;;;N;;;;; +13FFF;EGYPTIAN HIEROGLYPH-13FFF;Lo;0;L;;;;;N;;;;; +14000;EGYPTIAN HIEROGLYPH-14000;Lo;0;L;;;;;N;;;;; +14001;EGYPTIAN HIEROGLYPH-14001;Lo;0;L;;;;;N;;;;; +14002;EGYPTIAN HIEROGLYPH-14002;Lo;0;L;;;;;N;;;;; +14003;EGYPTIAN HIEROGLYPH-14003;Lo;0;L;;;;;N;;;;; +14004;EGYPTIAN HIEROGLYPH-14004;Lo;0;L;;;;;N;;;;; +14005;EGYPTIAN HIEROGLYPH-14005;Lo;0;L;;;;;N;;;;; +14006;EGYPTIAN HIEROGLYPH-14006;Lo;0;L;;;;;N;;;;; +14007;EGYPTIAN HIEROGLYPH-14007;Lo;0;L;;;;;N;;;;; +14008;EGYPTIAN HIEROGLYPH-14008;Lo;0;L;;;;;N;;;;; +14009;EGYPTIAN HIEROGLYPH-14009;Lo;0;L;;;;;N;;;;; +1400A;EGYPTIAN HIEROGLYPH-1400A;Lo;0;L;;;;;N;;;;; +1400B;EGYPTIAN HIEROGLYPH-1400B;Lo;0;L;;;;;N;;;;; +1400C;EGYPTIAN HIEROGLYPH-1400C;Lo;0;L;;;;;N;;;;; +1400D;EGYPTIAN HIEROGLYPH-1400D;Lo;0;L;;;;;N;;;;; +1400E;EGYPTIAN HIEROGLYPH-1400E;Lo;0;L;;;;;N;;;;; +1400F;EGYPTIAN HIEROGLYPH-1400F;Lo;0;L;;;;;N;;;;; +14010;EGYPTIAN HIEROGLYPH-14010;Lo;0;L;;;;;N;;;;; +14011;EGYPTIAN HIEROGLYPH-14011;Lo;0;L;;;;;N;;;;; +14012;EGYPTIAN HIEROGLYPH-14012;Lo;0;L;;;;;N;;;;; +14013;EGYPTIAN HIEROGLYPH-14013;Lo;0;L;;;;;N;;;;; +14014;EGYPTIAN HIEROGLYPH-14014;Lo;0;L;;;;;N;;;;; +14015;EGYPTIAN HIEROGLYPH-14015;Lo;0;L;;;;;N;;;;; +14016;EGYPTIAN HIEROGLYPH-14016;Lo;0;L;;;;;N;;;;; +14017;EGYPTIAN HIEROGLYPH-14017;Lo;0;L;;;;;N;;;;; +14018;EGYPTIAN HIEROGLYPH-14018;Lo;0;L;;;;;N;;;;; +14019;EGYPTIAN HIEROGLYPH-14019;Lo;0;L;;;;;N;;;;; +1401A;EGYPTIAN HIEROGLYPH-1401A;Lo;0;L;;;;;N;;;;; +1401B;EGYPTIAN HIEROGLYPH-1401B;Lo;0;L;;;;;N;;;;; +1401C;EGYPTIAN HIEROGLYPH-1401C;Lo;0;L;;;;;N;;;;; +1401D;EGYPTIAN HIEROGLYPH-1401D;Lo;0;L;;;;;N;;;;; +1401E;EGYPTIAN HIEROGLYPH-1401E;Lo;0;L;;;;;N;;;;; +1401F;EGYPTIAN HIEROGLYPH-1401F;Lo;0;L;;;;;N;;;;; +14020;EGYPTIAN HIEROGLYPH-14020;Lo;0;L;;;;;N;;;;; +14021;EGYPTIAN HIEROGLYPH-14021;Lo;0;L;;;;;N;;;;; +14022;EGYPTIAN HIEROGLYPH-14022;Lo;0;L;;;;;N;;;;; +14023;EGYPTIAN HIEROGLYPH-14023;Lo;0;L;;;;;N;;;;; +14024;EGYPTIAN HIEROGLYPH-14024;Lo;0;L;;;;;N;;;;; +14025;EGYPTIAN HIEROGLYPH-14025;Lo;0;L;;;;;N;;;;; +14026;EGYPTIAN HIEROGLYPH-14026;Lo;0;L;;;;;N;;;;; +14027;EGYPTIAN HIEROGLYPH-14027;Lo;0;L;;;;;N;;;;; +14028;EGYPTIAN HIEROGLYPH-14028;Lo;0;L;;;;;N;;;;; +14029;EGYPTIAN HIEROGLYPH-14029;Lo;0;L;;;;;N;;;;; +1402A;EGYPTIAN HIEROGLYPH-1402A;Lo;0;L;;;;;N;;;;; +1402B;EGYPTIAN HIEROGLYPH-1402B;Lo;0;L;;;;;N;;;;; +1402C;EGYPTIAN HIEROGLYPH-1402C;Lo;0;L;;;;;N;;;;; +1402D;EGYPTIAN HIEROGLYPH-1402D;Lo;0;L;;;;;N;;;;; +1402E;EGYPTIAN HIEROGLYPH-1402E;Lo;0;L;;;;;N;;;;; +1402F;EGYPTIAN HIEROGLYPH-1402F;Lo;0;L;;;;;N;;;;; +14030;EGYPTIAN HIEROGLYPH-14030;Lo;0;L;;;;;N;;;;; +14031;EGYPTIAN HIEROGLYPH-14031;Lo;0;L;;;;;N;;;;; +14032;EGYPTIAN HIEROGLYPH-14032;Lo;0;L;;;;;N;;;;; +14033;EGYPTIAN HIEROGLYPH-14033;Lo;0;L;;;;;N;;;;; +14034;EGYPTIAN HIEROGLYPH-14034;Lo;0;L;;;;;N;;;;; +14035;EGYPTIAN HIEROGLYPH-14035;Lo;0;L;;;;;N;;;;; +14036;EGYPTIAN HIEROGLYPH-14036;Lo;0;L;;;;;N;;;;; +14037;EGYPTIAN HIEROGLYPH-14037;Lo;0;L;;;;;N;;;;; +14038;EGYPTIAN HIEROGLYPH-14038;Lo;0;L;;;;;N;;;;; +14039;EGYPTIAN HIEROGLYPH-14039;Lo;0;L;;;;;N;;;;; +1403A;EGYPTIAN HIEROGLYPH-1403A;Lo;0;L;;;;;N;;;;; +1403B;EGYPTIAN HIEROGLYPH-1403B;Lo;0;L;;;;;N;;;;; +1403C;EGYPTIAN HIEROGLYPH-1403C;Lo;0;L;;;;;N;;;;; +1403D;EGYPTIAN HIEROGLYPH-1403D;Lo;0;L;;;;;N;;;;; +1403E;EGYPTIAN HIEROGLYPH-1403E;Lo;0;L;;;;;N;;;;; +1403F;EGYPTIAN HIEROGLYPH-1403F;Lo;0;L;;;;;N;;;;; +14040;EGYPTIAN HIEROGLYPH-14040;Lo;0;L;;;;;N;;;;; +14041;EGYPTIAN HIEROGLYPH-14041;Lo;0;L;;;;;N;;;;; +14042;EGYPTIAN HIEROGLYPH-14042;Lo;0;L;;;;;N;;;;; +14043;EGYPTIAN HIEROGLYPH-14043;Lo;0;L;;;;;N;;;;; +14044;EGYPTIAN HIEROGLYPH-14044;Lo;0;L;;;;;N;;;;; +14045;EGYPTIAN HIEROGLYPH-14045;Lo;0;L;;;;;N;;;;; +14046;EGYPTIAN HIEROGLYPH-14046;Lo;0;L;;;;;N;;;;; +14047;EGYPTIAN HIEROGLYPH-14047;Lo;0;L;;;;;N;;;;; +14048;EGYPTIAN HIEROGLYPH-14048;Lo;0;L;;;;;N;;;;; +14049;EGYPTIAN HIEROGLYPH-14049;Lo;0;L;;;;;N;;;;; +1404A;EGYPTIAN HIEROGLYPH-1404A;Lo;0;L;;;;;N;;;;; +1404B;EGYPTIAN HIEROGLYPH-1404B;Lo;0;L;;;;;N;;;;; +1404C;EGYPTIAN HIEROGLYPH-1404C;Lo;0;L;;;;;N;;;;; +1404D;EGYPTIAN HIEROGLYPH-1404D;Lo;0;L;;;;;N;;;;; +1404E;EGYPTIAN HIEROGLYPH-1404E;Lo;0;L;;;;;N;;;;; +1404F;EGYPTIAN HIEROGLYPH-1404F;Lo;0;L;;;;;N;;;;; +14050;EGYPTIAN HIEROGLYPH-14050;Lo;0;L;;;;;N;;;;; +14051;EGYPTIAN HIEROGLYPH-14051;Lo;0;L;;;;;N;;;;; +14052;EGYPTIAN HIEROGLYPH-14052;Lo;0;L;;;;;N;;;;; +14053;EGYPTIAN HIEROGLYPH-14053;Lo;0;L;;;;;N;;;;; +14054;EGYPTIAN HIEROGLYPH-14054;Lo;0;L;;;;;N;;;;; +14055;EGYPTIAN HIEROGLYPH-14055;Lo;0;L;;;;;N;;;;; +14056;EGYPTIAN HIEROGLYPH-14056;Lo;0;L;;;;;N;;;;; +14057;EGYPTIAN HIEROGLYPH-14057;Lo;0;L;;;;;N;;;;; +14058;EGYPTIAN HIEROGLYPH-14058;Lo;0;L;;;;;N;;;;; +14059;EGYPTIAN HIEROGLYPH-14059;Lo;0;L;;;;;N;;;;; +1405A;EGYPTIAN HIEROGLYPH-1405A;Lo;0;L;;;;;N;;;;; +1405B;EGYPTIAN HIEROGLYPH-1405B;Lo;0;L;;;;;N;;;;; +1405C;EGYPTIAN HIEROGLYPH-1405C;Lo;0;L;;;;;N;;;;; +1405D;EGYPTIAN HIEROGLYPH-1405D;Lo;0;L;;;;;N;;;;; +1405E;EGYPTIAN HIEROGLYPH-1405E;Lo;0;L;;;;;N;;;;; +1405F;EGYPTIAN HIEROGLYPH-1405F;Lo;0;L;;;;;N;;;;; +14060;EGYPTIAN HIEROGLYPH-14060;Lo;0;L;;;;;N;;;;; +14061;EGYPTIAN HIEROGLYPH-14061;Lo;0;L;;;;;N;;;;; +14062;EGYPTIAN HIEROGLYPH-14062;Lo;0;L;;;;;N;;;;; +14063;EGYPTIAN HIEROGLYPH-14063;Lo;0;L;;;;;N;;;;; +14064;EGYPTIAN HIEROGLYPH-14064;Lo;0;L;;;;;N;;;;; +14065;EGYPTIAN HIEROGLYPH-14065;Lo;0;L;;;;;N;;;;; +14066;EGYPTIAN HIEROGLYPH-14066;Lo;0;L;;;;;N;;;;; +14067;EGYPTIAN HIEROGLYPH-14067;Lo;0;L;;;;;N;;;;; +14068;EGYPTIAN HIEROGLYPH-14068;Lo;0;L;;;;;N;;;;; +14069;EGYPTIAN HIEROGLYPH-14069;Lo;0;L;;;;;N;;;;; +1406A;EGYPTIAN HIEROGLYPH-1406A;Lo;0;L;;;;;N;;;;; +1406B;EGYPTIAN HIEROGLYPH-1406B;Lo;0;L;;;;;N;;;;; +1406C;EGYPTIAN HIEROGLYPH-1406C;Lo;0;L;;;;;N;;;;; +1406D;EGYPTIAN HIEROGLYPH-1406D;Lo;0;L;;;;;N;;;;; +1406E;EGYPTIAN HIEROGLYPH-1406E;Lo;0;L;;;;;N;;;;; +1406F;EGYPTIAN HIEROGLYPH-1406F;Lo;0;L;;;;;N;;;;; +14070;EGYPTIAN HIEROGLYPH-14070;Lo;0;L;;;;;N;;;;; +14071;EGYPTIAN HIEROGLYPH-14071;Lo;0;L;;;;;N;;;;; +14072;EGYPTIAN HIEROGLYPH-14072;Lo;0;L;;;;;N;;;;; +14073;EGYPTIAN HIEROGLYPH-14073;Lo;0;L;;;;;N;;;;; +14074;EGYPTIAN HIEROGLYPH-14074;Lo;0;L;;;;;N;;;;; +14075;EGYPTIAN HIEROGLYPH-14075;Lo;0;L;;;;;N;;;;; +14076;EGYPTIAN HIEROGLYPH-14076;Lo;0;L;;;;;N;;;;; +14077;EGYPTIAN HIEROGLYPH-14077;Lo;0;L;;;;;N;;;;; +14078;EGYPTIAN HIEROGLYPH-14078;Lo;0;L;;;;;N;;;;; +14079;EGYPTIAN HIEROGLYPH-14079;Lo;0;L;;;;;N;;;;; +1407A;EGYPTIAN HIEROGLYPH-1407A;Lo;0;L;;;;;N;;;;; +1407B;EGYPTIAN HIEROGLYPH-1407B;Lo;0;L;;;;;N;;;;; +1407C;EGYPTIAN HIEROGLYPH-1407C;Lo;0;L;;;;;N;;;;; +1407D;EGYPTIAN HIEROGLYPH-1407D;Lo;0;L;;;;;N;;;;; +1407E;EGYPTIAN HIEROGLYPH-1407E;Lo;0;L;;;;;N;;;;; +1407F;EGYPTIAN HIEROGLYPH-1407F;Lo;0;L;;;;;N;;;;; +14080;EGYPTIAN HIEROGLYPH-14080;Lo;0;L;;;;;N;;;;; +14081;EGYPTIAN HIEROGLYPH-14081;Lo;0;L;;;;;N;;;;; +14082;EGYPTIAN HIEROGLYPH-14082;Lo;0;L;;;;;N;;;;; +14083;EGYPTIAN HIEROGLYPH-14083;Lo;0;L;;;;;N;;;;; +14084;EGYPTIAN HIEROGLYPH-14084;Lo;0;L;;;;;N;;;;; +14085;EGYPTIAN HIEROGLYPH-14085;Lo;0;L;;;;;N;;;;; +14086;EGYPTIAN HIEROGLYPH-14086;Lo;0;L;;;;;N;;;;; +14087;EGYPTIAN HIEROGLYPH-14087;Lo;0;L;;;;;N;;;;; +14088;EGYPTIAN HIEROGLYPH-14088;Lo;0;L;;;;;N;;;;; +14089;EGYPTIAN HIEROGLYPH-14089;Lo;0;L;;;;;N;;;;; +1408A;EGYPTIAN HIEROGLYPH-1408A;Lo;0;L;;;;;N;;;;; +1408B;EGYPTIAN HIEROGLYPH-1408B;Lo;0;L;;;;;N;;;;; +1408C;EGYPTIAN HIEROGLYPH-1408C;Lo;0;L;;;;;N;;;;; +1408D;EGYPTIAN HIEROGLYPH-1408D;Lo;0;L;;;;;N;;;;; +1408E;EGYPTIAN HIEROGLYPH-1408E;Lo;0;L;;;;;N;;;;; +1408F;EGYPTIAN HIEROGLYPH-1408F;Lo;0;L;;;;;N;;;;; +14090;EGYPTIAN HIEROGLYPH-14090;Lo;0;L;;;;;N;;;;; +14091;EGYPTIAN HIEROGLYPH-14091;Lo;0;L;;;;;N;;;;; +14092;EGYPTIAN HIEROGLYPH-14092;Lo;0;L;;;;;N;;;;; +14093;EGYPTIAN HIEROGLYPH-14093;Lo;0;L;;;;;N;;;;; +14094;EGYPTIAN HIEROGLYPH-14094;Lo;0;L;;;;;N;;;;; +14095;EGYPTIAN HIEROGLYPH-14095;Lo;0;L;;;;;N;;;;; +14096;EGYPTIAN HIEROGLYPH-14096;Lo;0;L;;;;;N;;;;; +14097;EGYPTIAN HIEROGLYPH-14097;Lo;0;L;;;;;N;;;;; +14098;EGYPTIAN HIEROGLYPH-14098;Lo;0;L;;;;;N;;;;; +14099;EGYPTIAN HIEROGLYPH-14099;Lo;0;L;;;;;N;;;;; +1409A;EGYPTIAN HIEROGLYPH-1409A;Lo;0;L;;;;;N;;;;; +1409B;EGYPTIAN HIEROGLYPH-1409B;Lo;0;L;;;;;N;;;;; +1409C;EGYPTIAN HIEROGLYPH-1409C;Lo;0;L;;;;;N;;;;; +1409D;EGYPTIAN HIEROGLYPH-1409D;Lo;0;L;;;;;N;;;;; +1409E;EGYPTIAN HIEROGLYPH-1409E;Lo;0;L;;;;;N;;;;; +1409F;EGYPTIAN HIEROGLYPH-1409F;Lo;0;L;;;;;N;;;;; +140A0;EGYPTIAN HIEROGLYPH-140A0;Lo;0;L;;;;;N;;;;; +140A1;EGYPTIAN HIEROGLYPH-140A1;Lo;0;L;;;;;N;;;;; +140A2;EGYPTIAN HIEROGLYPH-140A2;Lo;0;L;;;;;N;;;;; +140A3;EGYPTIAN HIEROGLYPH-140A3;Lo;0;L;;;;;N;;;;; +140A4;EGYPTIAN HIEROGLYPH-140A4;Lo;0;L;;;;;N;;;;; +140A5;EGYPTIAN HIEROGLYPH-140A5;Lo;0;L;;;;;N;;;;; +140A6;EGYPTIAN HIEROGLYPH-140A6;Lo;0;L;;;;;N;;;;; +140A7;EGYPTIAN HIEROGLYPH-140A7;Lo;0;L;;;;;N;;;;; +140A8;EGYPTIAN HIEROGLYPH-140A8;Lo;0;L;;;;;N;;;;; +140A9;EGYPTIAN HIEROGLYPH-140A9;Lo;0;L;;;;;N;;;;; +140AA;EGYPTIAN HIEROGLYPH-140AA;Lo;0;L;;;;;N;;;;; +140AB;EGYPTIAN HIEROGLYPH-140AB;Lo;0;L;;;;;N;;;;; +140AC;EGYPTIAN HIEROGLYPH-140AC;Lo;0;L;;;;;N;;;;; +140AD;EGYPTIAN HIEROGLYPH-140AD;Lo;0;L;;;;;N;;;;; +140AE;EGYPTIAN HIEROGLYPH-140AE;Lo;0;L;;;;;N;;;;; +140AF;EGYPTIAN HIEROGLYPH-140AF;Lo;0;L;;;;;N;;;;; +140B0;EGYPTIAN HIEROGLYPH-140B0;Lo;0;L;;;;;N;;;;; +140B1;EGYPTIAN HIEROGLYPH-140B1;Lo;0;L;;;;;N;;;;; +140B2;EGYPTIAN HIEROGLYPH-140B2;Lo;0;L;;;;;N;;;;; +140B3;EGYPTIAN HIEROGLYPH-140B3;Lo;0;L;;;;;N;;;;; +140B4;EGYPTIAN HIEROGLYPH-140B4;Lo;0;L;;;;;N;;;;; +140B5;EGYPTIAN HIEROGLYPH-140B5;Lo;0;L;;;;;N;;;;; +140B6;EGYPTIAN HIEROGLYPH-140B6;Lo;0;L;;;;;N;;;;; +140B7;EGYPTIAN HIEROGLYPH-140B7;Lo;0;L;;;;;N;;;;; +140B8;EGYPTIAN HIEROGLYPH-140B8;Lo;0;L;;;;;N;;;;; +140B9;EGYPTIAN HIEROGLYPH-140B9;Lo;0;L;;;;;N;;;;; +140BA;EGYPTIAN HIEROGLYPH-140BA;Lo;0;L;;;;;N;;;;; +140BB;EGYPTIAN HIEROGLYPH-140BB;Lo;0;L;;;;;N;;;;; +140BC;EGYPTIAN HIEROGLYPH-140BC;Lo;0;L;;;;;N;;;;; +140BD;EGYPTIAN HIEROGLYPH-140BD;Lo;0;L;;;;;N;;;;; +140BE;EGYPTIAN HIEROGLYPH-140BE;Lo;0;L;;;;;N;;;;; +140BF;EGYPTIAN HIEROGLYPH-140BF;Lo;0;L;;;;;N;;;;; +140C0;EGYPTIAN HIEROGLYPH-140C0;Lo;0;L;;;;;N;;;;; +140C1;EGYPTIAN HIEROGLYPH-140C1;Lo;0;L;;;;;N;;;;; +140C2;EGYPTIAN HIEROGLYPH-140C2;Lo;0;L;;;;;N;;;;; +140C3;EGYPTIAN HIEROGLYPH-140C3;Lo;0;L;;;;;N;;;;; +140C4;EGYPTIAN HIEROGLYPH-140C4;Lo;0;L;;;;;N;;;;; +140C5;EGYPTIAN HIEROGLYPH-140C5;Lo;0;L;;;;;N;;;;; +140C6;EGYPTIAN HIEROGLYPH-140C6;Lo;0;L;;;;;N;;;;; +140C7;EGYPTIAN HIEROGLYPH-140C7;Lo;0;L;;;;;N;;;;; +140C8;EGYPTIAN HIEROGLYPH-140C8;Lo;0;L;;;;;N;;;;; +140C9;EGYPTIAN HIEROGLYPH-140C9;Lo;0;L;;;;;N;;;;; +140CA;EGYPTIAN HIEROGLYPH-140CA;Lo;0;L;;;;;N;;;;; +140CB;EGYPTIAN HIEROGLYPH-140CB;Lo;0;L;;;;;N;;;;; +140CC;EGYPTIAN HIEROGLYPH-140CC;Lo;0;L;;;;;N;;;;; +140CD;EGYPTIAN HIEROGLYPH-140CD;Lo;0;L;;;;;N;;;;; +140CE;EGYPTIAN HIEROGLYPH-140CE;Lo;0;L;;;;;N;;;;; +140CF;EGYPTIAN HIEROGLYPH-140CF;Lo;0;L;;;;;N;;;;; +140D0;EGYPTIAN HIEROGLYPH-140D0;Lo;0;L;;;;;N;;;;; +140D1;EGYPTIAN HIEROGLYPH-140D1;Lo;0;L;;;;;N;;;;; +140D2;EGYPTIAN HIEROGLYPH-140D2;Lo;0;L;;;;;N;;;;; +140D3;EGYPTIAN HIEROGLYPH-140D3;Lo;0;L;;;;;N;;;;; +140D4;EGYPTIAN HIEROGLYPH-140D4;Lo;0;L;;;;;N;;;;; +140D5;EGYPTIAN HIEROGLYPH-140D5;Lo;0;L;;;;;N;;;;; +140D6;EGYPTIAN HIEROGLYPH-140D6;Lo;0;L;;;;;N;;;;; +140D7;EGYPTIAN HIEROGLYPH-140D7;Lo;0;L;;;;;N;;;;; +140D8;EGYPTIAN HIEROGLYPH-140D8;Lo;0;L;;;;;N;;;;; +140D9;EGYPTIAN HIEROGLYPH-140D9;Lo;0;L;;;;;N;;;;; +140DA;EGYPTIAN HIEROGLYPH-140DA;Lo;0;L;;;;;N;;;;; +140DB;EGYPTIAN HIEROGLYPH-140DB;Lo;0;L;;;;;N;;;;; +140DC;EGYPTIAN HIEROGLYPH-140DC;Lo;0;L;;;;;N;;;;; +140DD;EGYPTIAN HIEROGLYPH-140DD;Lo;0;L;;;;;N;;;;; +140DE;EGYPTIAN HIEROGLYPH-140DE;Lo;0;L;;;;;N;;;;; +140DF;EGYPTIAN HIEROGLYPH-140DF;Lo;0;L;;;;;N;;;;; +140E0;EGYPTIAN HIEROGLYPH-140E0;Lo;0;L;;;;;N;;;;; +140E1;EGYPTIAN HIEROGLYPH-140E1;Lo;0;L;;;;;N;;;;; +140E2;EGYPTIAN HIEROGLYPH-140E2;Lo;0;L;;;;;N;;;;; +140E3;EGYPTIAN HIEROGLYPH-140E3;Lo;0;L;;;;;N;;;;; +140E4;EGYPTIAN HIEROGLYPH-140E4;Lo;0;L;;;;;N;;;;; +140E5;EGYPTIAN HIEROGLYPH-140E5;Lo;0;L;;;;;N;;;;; +140E6;EGYPTIAN HIEROGLYPH-140E6;Lo;0;L;;;;;N;;;;; +140E7;EGYPTIAN HIEROGLYPH-140E7;Lo;0;L;;;;;N;;;;; +140E8;EGYPTIAN HIEROGLYPH-140E8;Lo;0;L;;;;;N;;;;; +140E9;EGYPTIAN HIEROGLYPH-140E9;Lo;0;L;;;;;N;;;;; +140EA;EGYPTIAN HIEROGLYPH-140EA;Lo;0;L;;;;;N;;;;; +140EB;EGYPTIAN HIEROGLYPH-140EB;Lo;0;L;;;;;N;;;;; +140EC;EGYPTIAN HIEROGLYPH-140EC;Lo;0;L;;;;;N;;;;; +140ED;EGYPTIAN HIEROGLYPH-140ED;Lo;0;L;;;;;N;;;;; +140EE;EGYPTIAN HIEROGLYPH-140EE;Lo;0;L;;;;;N;;;;; +140EF;EGYPTIAN HIEROGLYPH-140EF;Lo;0;L;;;;;N;;;;; +140F0;EGYPTIAN HIEROGLYPH-140F0;Lo;0;L;;;;;N;;;;; +140F1;EGYPTIAN HIEROGLYPH-140F1;Lo;0;L;;;;;N;;;;; +140F2;EGYPTIAN HIEROGLYPH-140F2;Lo;0;L;;;;;N;;;;; +140F3;EGYPTIAN HIEROGLYPH-140F3;Lo;0;L;;;;;N;;;;; +140F4;EGYPTIAN HIEROGLYPH-140F4;Lo;0;L;;;;;N;;;;; +140F5;EGYPTIAN HIEROGLYPH-140F5;Lo;0;L;;;;;N;;;;; +140F6;EGYPTIAN HIEROGLYPH-140F6;Lo;0;L;;;;;N;;;;; +140F7;EGYPTIAN HIEROGLYPH-140F7;Lo;0;L;;;;;N;;;;; +140F8;EGYPTIAN HIEROGLYPH-140F8;Lo;0;L;;;;;N;;;;; +140F9;EGYPTIAN HIEROGLYPH-140F9;Lo;0;L;;;;;N;;;;; +140FA;EGYPTIAN HIEROGLYPH-140FA;Lo;0;L;;;;;N;;;;; +140FB;EGYPTIAN HIEROGLYPH-140FB;Lo;0;L;;;;;N;;;;; +140FC;EGYPTIAN HIEROGLYPH-140FC;Lo;0;L;;;;;N;;;;; +140FD;EGYPTIAN HIEROGLYPH-140FD;Lo;0;L;;;;;N;;;;; +140FE;EGYPTIAN HIEROGLYPH-140FE;Lo;0;L;;;;;N;;;;; +140FF;EGYPTIAN HIEROGLYPH-140FF;Lo;0;L;;;;;N;;;;; +14100;EGYPTIAN HIEROGLYPH-14100;Lo;0;L;;;;;N;;;;; +14101;EGYPTIAN HIEROGLYPH-14101;Lo;0;L;;;;;N;;;;; +14102;EGYPTIAN HIEROGLYPH-14102;Lo;0;L;;;;;N;;;;; +14103;EGYPTIAN HIEROGLYPH-14103;Lo;0;L;;;;;N;;;;; +14104;EGYPTIAN HIEROGLYPH-14104;Lo;0;L;;;;;N;;;;; +14105;EGYPTIAN HIEROGLYPH-14105;Lo;0;L;;;;;N;;;;; +14106;EGYPTIAN HIEROGLYPH-14106;Lo;0;L;;;;;N;;;;; +14107;EGYPTIAN HIEROGLYPH-14107;Lo;0;L;;;;;N;;;;; +14108;EGYPTIAN HIEROGLYPH-14108;Lo;0;L;;;;;N;;;;; +14109;EGYPTIAN HIEROGLYPH-14109;Lo;0;L;;;;;N;;;;; +1410A;EGYPTIAN HIEROGLYPH-1410A;Lo;0;L;;;;;N;;;;; +1410B;EGYPTIAN HIEROGLYPH-1410B;Lo;0;L;;;;;N;;;;; +1410C;EGYPTIAN HIEROGLYPH-1410C;Lo;0;L;;;;;N;;;;; +1410D;EGYPTIAN HIEROGLYPH-1410D;Lo;0;L;;;;;N;;;;; +1410E;EGYPTIAN HIEROGLYPH-1410E;Lo;0;L;;;;;N;;;;; +1410F;EGYPTIAN HIEROGLYPH-1410F;Lo;0;L;;;;;N;;;;; +14110;EGYPTIAN HIEROGLYPH-14110;Lo;0;L;;;;;N;;;;; +14111;EGYPTIAN HIEROGLYPH-14111;Lo;0;L;;;;;N;;;;; +14112;EGYPTIAN HIEROGLYPH-14112;Lo;0;L;;;;;N;;;;; +14113;EGYPTIAN HIEROGLYPH-14113;Lo;0;L;;;;;N;;;;; +14114;EGYPTIAN HIEROGLYPH-14114;Lo;0;L;;;;;N;;;;; +14115;EGYPTIAN HIEROGLYPH-14115;Lo;0;L;;;;;N;;;;; +14116;EGYPTIAN HIEROGLYPH-14116;Lo;0;L;;;;;N;;;;; +14117;EGYPTIAN HIEROGLYPH-14117;Lo;0;L;;;;;N;;;;; +14118;EGYPTIAN HIEROGLYPH-14118;Lo;0;L;;;;;N;;;;; +14119;EGYPTIAN HIEROGLYPH-14119;Lo;0;L;;;;;N;;;;; +1411A;EGYPTIAN HIEROGLYPH-1411A;Lo;0;L;;;;;N;;;;; +1411B;EGYPTIAN HIEROGLYPH-1411B;Lo;0;L;;;;;N;;;;; +1411C;EGYPTIAN HIEROGLYPH-1411C;Lo;0;L;;;;;N;;;;; +1411D;EGYPTIAN HIEROGLYPH-1411D;Lo;0;L;;;;;N;;;;; +1411E;EGYPTIAN HIEROGLYPH-1411E;Lo;0;L;;;;;N;;;;; +1411F;EGYPTIAN HIEROGLYPH-1411F;Lo;0;L;;;;;N;;;;; +14120;EGYPTIAN HIEROGLYPH-14120;Lo;0;L;;;;;N;;;;; +14121;EGYPTIAN HIEROGLYPH-14121;Lo;0;L;;;;;N;;;;; +14122;EGYPTIAN HIEROGLYPH-14122;Lo;0;L;;;;;N;;;;; +14123;EGYPTIAN HIEROGLYPH-14123;Lo;0;L;;;;;N;;;;; +14124;EGYPTIAN HIEROGLYPH-14124;Lo;0;L;;;;;N;;;;; +14125;EGYPTIAN HIEROGLYPH-14125;Lo;0;L;;;;;N;;;;; +14126;EGYPTIAN HIEROGLYPH-14126;Lo;0;L;;;;;N;;;;; +14127;EGYPTIAN HIEROGLYPH-14127;Lo;0;L;;;;;N;;;;; +14128;EGYPTIAN HIEROGLYPH-14128;Lo;0;L;;;;;N;;;;; +14129;EGYPTIAN HIEROGLYPH-14129;Lo;0;L;;;;;N;;;;; +1412A;EGYPTIAN HIEROGLYPH-1412A;Lo;0;L;;;;;N;;;;; +1412B;EGYPTIAN HIEROGLYPH-1412B;Lo;0;L;;;;;N;;;;; +1412C;EGYPTIAN HIEROGLYPH-1412C;Lo;0;L;;;;;N;;;;; +1412D;EGYPTIAN HIEROGLYPH-1412D;Lo;0;L;;;;;N;;;;; +1412E;EGYPTIAN HIEROGLYPH-1412E;Lo;0;L;;;;;N;;;;; +1412F;EGYPTIAN HIEROGLYPH-1412F;Lo;0;L;;;;;N;;;;; +14130;EGYPTIAN HIEROGLYPH-14130;Lo;0;L;;;;;N;;;;; +14131;EGYPTIAN HIEROGLYPH-14131;Lo;0;L;;;;;N;;;;; +14132;EGYPTIAN HIEROGLYPH-14132;Lo;0;L;;;;;N;;;;; +14133;EGYPTIAN HIEROGLYPH-14133;Lo;0;L;;;;;N;;;;; +14134;EGYPTIAN HIEROGLYPH-14134;Lo;0;L;;;;;N;;;;; +14135;EGYPTIAN HIEROGLYPH-14135;Lo;0;L;;;;;N;;;;; +14136;EGYPTIAN HIEROGLYPH-14136;Lo;0;L;;;;;N;;;;; +14137;EGYPTIAN HIEROGLYPH-14137;Lo;0;L;;;;;N;;;;; +14138;EGYPTIAN HIEROGLYPH-14138;Lo;0;L;;;;;N;;;;; +14139;EGYPTIAN HIEROGLYPH-14139;Lo;0;L;;;;;N;;;;; +1413A;EGYPTIAN HIEROGLYPH-1413A;Lo;0;L;;;;;N;;;;; +1413B;EGYPTIAN HIEROGLYPH-1413B;Lo;0;L;;;;;N;;;;; +1413C;EGYPTIAN HIEROGLYPH-1413C;Lo;0;L;;;;;N;;;;; +1413D;EGYPTIAN HIEROGLYPH-1413D;Lo;0;L;;;;;N;;;;; +1413E;EGYPTIAN HIEROGLYPH-1413E;Lo;0;L;;;;;N;;;;; +1413F;EGYPTIAN HIEROGLYPH-1413F;Lo;0;L;;;;;N;;;;; +14140;EGYPTIAN HIEROGLYPH-14140;Lo;0;L;;;;;N;;;;; +14141;EGYPTIAN HIEROGLYPH-14141;Lo;0;L;;;;;N;;;;; +14142;EGYPTIAN HIEROGLYPH-14142;Lo;0;L;;;;;N;;;;; +14143;EGYPTIAN HIEROGLYPH-14143;Lo;0;L;;;;;N;;;;; +14144;EGYPTIAN HIEROGLYPH-14144;Lo;0;L;;;;;N;;;;; +14145;EGYPTIAN HIEROGLYPH-14145;Lo;0;L;;;;;N;;;;; +14146;EGYPTIAN HIEROGLYPH-14146;Lo;0;L;;;;;N;;;;; +14147;EGYPTIAN HIEROGLYPH-14147;Lo;0;L;;;;;N;;;;; +14148;EGYPTIAN HIEROGLYPH-14148;Lo;0;L;;;;;N;;;;; +14149;EGYPTIAN HIEROGLYPH-14149;Lo;0;L;;;;;N;;;;; +1414A;EGYPTIAN HIEROGLYPH-1414A;Lo;0;L;;;;;N;;;;; +1414B;EGYPTIAN HIEROGLYPH-1414B;Lo;0;L;;;;;N;;;;; +1414C;EGYPTIAN HIEROGLYPH-1414C;Lo;0;L;;;;;N;;;;; +1414D;EGYPTIAN HIEROGLYPH-1414D;Lo;0;L;;;;;N;;;;; +1414E;EGYPTIAN HIEROGLYPH-1414E;Lo;0;L;;;;;N;;;;; +1414F;EGYPTIAN HIEROGLYPH-1414F;Lo;0;L;;;;;N;;;;; +14150;EGYPTIAN HIEROGLYPH-14150;Lo;0;L;;;;;N;;;;; +14151;EGYPTIAN HIEROGLYPH-14151;Lo;0;L;;;;;N;;;;; +14152;EGYPTIAN HIEROGLYPH-14152;Lo;0;L;;;;;N;;;;; +14153;EGYPTIAN HIEROGLYPH-14153;Lo;0;L;;;;;N;;;;; +14154;EGYPTIAN HIEROGLYPH-14154;Lo;0;L;;;;;N;;;;; +14155;EGYPTIAN HIEROGLYPH-14155;Lo;0;L;;;;;N;;;;; +14156;EGYPTIAN HIEROGLYPH-14156;Lo;0;L;;;;;N;;;;; +14157;EGYPTIAN HIEROGLYPH-14157;Lo;0;L;;;;;N;;;;; +14158;EGYPTIAN HIEROGLYPH-14158;Lo;0;L;;;;;N;;;;; +14159;EGYPTIAN HIEROGLYPH-14159;Lo;0;L;;;;;N;;;;; +1415A;EGYPTIAN HIEROGLYPH-1415A;Lo;0;L;;;;;N;;;;; +1415B;EGYPTIAN HIEROGLYPH-1415B;Lo;0;L;;;;;N;;;;; +1415C;EGYPTIAN HIEROGLYPH-1415C;Lo;0;L;;;;;N;;;;; +1415D;EGYPTIAN HIEROGLYPH-1415D;Lo;0;L;;;;;N;;;;; +1415E;EGYPTIAN HIEROGLYPH-1415E;Lo;0;L;;;;;N;;;;; +1415F;EGYPTIAN HIEROGLYPH-1415F;Lo;0;L;;;;;N;;;;; +14160;EGYPTIAN HIEROGLYPH-14160;Lo;0;L;;;;;N;;;;; +14161;EGYPTIAN HIEROGLYPH-14161;Lo;0;L;;;;;N;;;;; +14162;EGYPTIAN HIEROGLYPH-14162;Lo;0;L;;;;;N;;;;; +14163;EGYPTIAN HIEROGLYPH-14163;Lo;0;L;;;;;N;;;;; +14164;EGYPTIAN HIEROGLYPH-14164;Lo;0;L;;;;;N;;;;; +14165;EGYPTIAN HIEROGLYPH-14165;Lo;0;L;;;;;N;;;;; +14166;EGYPTIAN HIEROGLYPH-14166;Lo;0;L;;;;;N;;;;; +14167;EGYPTIAN HIEROGLYPH-14167;Lo;0;L;;;;;N;;;;; +14168;EGYPTIAN HIEROGLYPH-14168;Lo;0;L;;;;;N;;;;; +14169;EGYPTIAN HIEROGLYPH-14169;Lo;0;L;;;;;N;;;;; +1416A;EGYPTIAN HIEROGLYPH-1416A;Lo;0;L;;;;;N;;;;; +1416B;EGYPTIAN HIEROGLYPH-1416B;Lo;0;L;;;;;N;;;;; +1416C;EGYPTIAN HIEROGLYPH-1416C;Lo;0;L;;;;;N;;;;; +1416D;EGYPTIAN HIEROGLYPH-1416D;Lo;0;L;;;;;N;;;;; +1416E;EGYPTIAN HIEROGLYPH-1416E;Lo;0;L;;;;;N;;;;; +1416F;EGYPTIAN HIEROGLYPH-1416F;Lo;0;L;;;;;N;;;;; +14170;EGYPTIAN HIEROGLYPH-14170;Lo;0;L;;;;;N;;;;; +14171;EGYPTIAN HIEROGLYPH-14171;Lo;0;L;;;;;N;;;;; +14172;EGYPTIAN HIEROGLYPH-14172;Lo;0;L;;;;;N;;;;; +14173;EGYPTIAN HIEROGLYPH-14173;Lo;0;L;;;;;N;;;;; +14174;EGYPTIAN HIEROGLYPH-14174;Lo;0;L;;;;;N;;;;; +14175;EGYPTIAN HIEROGLYPH-14175;Lo;0;L;;;;;N;;;;; +14176;EGYPTIAN HIEROGLYPH-14176;Lo;0;L;;;;;N;;;;; +14177;EGYPTIAN HIEROGLYPH-14177;Lo;0;L;;;;;N;;;;; +14178;EGYPTIAN HIEROGLYPH-14178;Lo;0;L;;;;;N;;;;; +14179;EGYPTIAN HIEROGLYPH-14179;Lo;0;L;;;;;N;;;;; +1417A;EGYPTIAN HIEROGLYPH-1417A;Lo;0;L;;;;;N;;;;; +1417B;EGYPTIAN HIEROGLYPH-1417B;Lo;0;L;;;;;N;;;;; +1417C;EGYPTIAN HIEROGLYPH-1417C;Lo;0;L;;;;;N;;;;; +1417D;EGYPTIAN HIEROGLYPH-1417D;Lo;0;L;;;;;N;;;;; +1417E;EGYPTIAN HIEROGLYPH-1417E;Lo;0;L;;;;;N;;;;; +1417F;EGYPTIAN HIEROGLYPH-1417F;Lo;0;L;;;;;N;;;;; +14180;EGYPTIAN HIEROGLYPH-14180;Lo;0;L;;;;;N;;;;; +14181;EGYPTIAN HIEROGLYPH-14181;Lo;0;L;;;;;N;;;;; +14182;EGYPTIAN HIEROGLYPH-14182;Lo;0;L;;;;;N;;;;; +14183;EGYPTIAN HIEROGLYPH-14183;Lo;0;L;;;;;N;;;;; +14184;EGYPTIAN HIEROGLYPH-14184;Lo;0;L;;;;;N;;;;; +14185;EGYPTIAN HIEROGLYPH-14185;Lo;0;L;;;;;N;;;;; +14186;EGYPTIAN HIEROGLYPH-14186;Lo;0;L;;;;;N;;;;; +14187;EGYPTIAN HIEROGLYPH-14187;Lo;0;L;;;;;N;;;;; +14188;EGYPTIAN HIEROGLYPH-14188;Lo;0;L;;;;;N;;;;; +14189;EGYPTIAN HIEROGLYPH-14189;Lo;0;L;;;;;N;;;;; +1418A;EGYPTIAN HIEROGLYPH-1418A;Lo;0;L;;;;;N;;;;; +1418B;EGYPTIAN HIEROGLYPH-1418B;Lo;0;L;;;;;N;;;;; +1418C;EGYPTIAN HIEROGLYPH-1418C;Lo;0;L;;;;;N;;;;; +1418D;EGYPTIAN HIEROGLYPH-1418D;Lo;0;L;;;;;N;;;;; +1418E;EGYPTIAN HIEROGLYPH-1418E;Lo;0;L;;;;;N;;;;; +1418F;EGYPTIAN HIEROGLYPH-1418F;Lo;0;L;;;;;N;;;;; +14190;EGYPTIAN HIEROGLYPH-14190;Lo;0;L;;;;;N;;;;; +14191;EGYPTIAN HIEROGLYPH-14191;Lo;0;L;;;;;N;;;;; +14192;EGYPTIAN HIEROGLYPH-14192;Lo;0;L;;;;;N;;;;; +14193;EGYPTIAN HIEROGLYPH-14193;Lo;0;L;;;;;N;;;;; +14194;EGYPTIAN HIEROGLYPH-14194;Lo;0;L;;;;;N;;;;; +14195;EGYPTIAN HIEROGLYPH-14195;Lo;0;L;;;;;N;;;;; +14196;EGYPTIAN HIEROGLYPH-14196;Lo;0;L;;;;;N;;;;; +14197;EGYPTIAN HIEROGLYPH-14197;Lo;0;L;;;;;N;;;;; +14198;EGYPTIAN HIEROGLYPH-14198;Lo;0;L;;;;;N;;;;; +14199;EGYPTIAN HIEROGLYPH-14199;Lo;0;L;;;;;N;;;;; +1419A;EGYPTIAN HIEROGLYPH-1419A;Lo;0;L;;;;;N;;;;; +1419B;EGYPTIAN HIEROGLYPH-1419B;Lo;0;L;;;;;N;;;;; +1419C;EGYPTIAN HIEROGLYPH-1419C;Lo;0;L;;;;;N;;;;; +1419D;EGYPTIAN HIEROGLYPH-1419D;Lo;0;L;;;;;N;;;;; +1419E;EGYPTIAN HIEROGLYPH-1419E;Lo;0;L;;;;;N;;;;; +1419F;EGYPTIAN HIEROGLYPH-1419F;Lo;0;L;;;;;N;;;;; +141A0;EGYPTIAN HIEROGLYPH-141A0;Lo;0;L;;;;;N;;;;; +141A1;EGYPTIAN HIEROGLYPH-141A1;Lo;0;L;;;;;N;;;;; +141A2;EGYPTIAN HIEROGLYPH-141A2;Lo;0;L;;;;;N;;;;; +141A3;EGYPTIAN HIEROGLYPH-141A3;Lo;0;L;;;;;N;;;;; +141A4;EGYPTIAN HIEROGLYPH-141A4;Lo;0;L;;;;;N;;;;; +141A5;EGYPTIAN HIEROGLYPH-141A5;Lo;0;L;;;;;N;;;;; +141A6;EGYPTIAN HIEROGLYPH-141A6;Lo;0;L;;;;;N;;;;; +141A7;EGYPTIAN HIEROGLYPH-141A7;Lo;0;L;;;;;N;;;;; +141A8;EGYPTIAN HIEROGLYPH-141A8;Lo;0;L;;;;;N;;;;; +141A9;EGYPTIAN HIEROGLYPH-141A9;Lo;0;L;;;;;N;;;;; +141AA;EGYPTIAN HIEROGLYPH-141AA;Lo;0;L;;;;;N;;;;; +141AB;EGYPTIAN HIEROGLYPH-141AB;Lo;0;L;;;;;N;;;;; +141AC;EGYPTIAN HIEROGLYPH-141AC;Lo;0;L;;;;;N;;;;; +141AD;EGYPTIAN HIEROGLYPH-141AD;Lo;0;L;;;;;N;;;;; +141AE;EGYPTIAN HIEROGLYPH-141AE;Lo;0;L;;;;;N;;;;; +141AF;EGYPTIAN HIEROGLYPH-141AF;Lo;0;L;;;;;N;;;;; +141B0;EGYPTIAN HIEROGLYPH-141B0;Lo;0;L;;;;;N;;;;; +141B1;EGYPTIAN HIEROGLYPH-141B1;Lo;0;L;;;;;N;;;;; +141B2;EGYPTIAN HIEROGLYPH-141B2;Lo;0;L;;;;;N;;;;; +141B3;EGYPTIAN HIEROGLYPH-141B3;Lo;0;L;;;;;N;;;;; +141B4;EGYPTIAN HIEROGLYPH-141B4;Lo;0;L;;;;;N;;;;; +141B5;EGYPTIAN HIEROGLYPH-141B5;Lo;0;L;;;;;N;;;;; +141B6;EGYPTIAN HIEROGLYPH-141B6;Lo;0;L;;;;;N;;;;; +141B7;EGYPTIAN HIEROGLYPH-141B7;Lo;0;L;;;;;N;;;;; +141B8;EGYPTIAN HIEROGLYPH-141B8;Lo;0;L;;;;;N;;;;; +141B9;EGYPTIAN HIEROGLYPH-141B9;Lo;0;L;;;;;N;;;;; +141BA;EGYPTIAN HIEROGLYPH-141BA;Lo;0;L;;;;;N;;;;; +141BB;EGYPTIAN HIEROGLYPH-141BB;Lo;0;L;;;;;N;;;;; +141BC;EGYPTIAN HIEROGLYPH-141BC;Lo;0;L;;;;;N;;;;; +141BD;EGYPTIAN HIEROGLYPH-141BD;Lo;0;L;;;;;N;;;;; +141BE;EGYPTIAN HIEROGLYPH-141BE;Lo;0;L;;;;;N;;;;; +141BF;EGYPTIAN HIEROGLYPH-141BF;Lo;0;L;;;;;N;;;;; +141C0;EGYPTIAN HIEROGLYPH-141C0;Lo;0;L;;;;;N;;;;; +141C1;EGYPTIAN HIEROGLYPH-141C1;Lo;0;L;;;;;N;;;;; +141C2;EGYPTIAN HIEROGLYPH-141C2;Lo;0;L;;;;;N;;;;; +141C3;EGYPTIAN HIEROGLYPH-141C3;Lo;0;L;;;;;N;;;;; +141C4;EGYPTIAN HIEROGLYPH-141C4;Lo;0;L;;;;;N;;;;; +141C5;EGYPTIAN HIEROGLYPH-141C5;Lo;0;L;;;;;N;;;;; +141C6;EGYPTIAN HIEROGLYPH-141C6;Lo;0;L;;;;;N;;;;; +141C7;EGYPTIAN HIEROGLYPH-141C7;Lo;0;L;;;;;N;;;;; +141C8;EGYPTIAN HIEROGLYPH-141C8;Lo;0;L;;;;;N;;;;; +141C9;EGYPTIAN HIEROGLYPH-141C9;Lo;0;L;;;;;N;;;;; +141CA;EGYPTIAN HIEROGLYPH-141CA;Lo;0;L;;;;;N;;;;; +141CB;EGYPTIAN HIEROGLYPH-141CB;Lo;0;L;;;;;N;;;;; +141CC;EGYPTIAN HIEROGLYPH-141CC;Lo;0;L;;;;;N;;;;; +141CD;EGYPTIAN HIEROGLYPH-141CD;Lo;0;L;;;;;N;;;;; +141CE;EGYPTIAN HIEROGLYPH-141CE;Lo;0;L;;;;;N;;;;; +141CF;EGYPTIAN HIEROGLYPH-141CF;Lo;0;L;;;;;N;;;;; +141D0;EGYPTIAN HIEROGLYPH-141D0;Lo;0;L;;;;;N;;;;; +141D1;EGYPTIAN HIEROGLYPH-141D1;Lo;0;L;;;;;N;;;;; +141D2;EGYPTIAN HIEROGLYPH-141D2;Lo;0;L;;;;;N;;;;; +141D3;EGYPTIAN HIEROGLYPH-141D3;Lo;0;L;;;;;N;;;;; +141D4;EGYPTIAN HIEROGLYPH-141D4;Lo;0;L;;;;;N;;;;; +141D5;EGYPTIAN HIEROGLYPH-141D5;Lo;0;L;;;;;N;;;;; +141D6;EGYPTIAN HIEROGLYPH-141D6;Lo;0;L;;;;;N;;;;; +141D7;EGYPTIAN HIEROGLYPH-141D7;Lo;0;L;;;;;N;;;;; +141D8;EGYPTIAN HIEROGLYPH-141D8;Lo;0;L;;;;;N;;;;; +141D9;EGYPTIAN HIEROGLYPH-141D9;Lo;0;L;;;;;N;;;;; +141DA;EGYPTIAN HIEROGLYPH-141DA;Lo;0;L;;;;;N;;;;; +141DB;EGYPTIAN HIEROGLYPH-141DB;Lo;0;L;;;;;N;;;;; +141DC;EGYPTIAN HIEROGLYPH-141DC;Lo;0;L;;;;;N;;;;; +141DD;EGYPTIAN HIEROGLYPH-141DD;Lo;0;L;;;;;N;;;;; +141DE;EGYPTIAN HIEROGLYPH-141DE;Lo;0;L;;;;;N;;;;; +141DF;EGYPTIAN HIEROGLYPH-141DF;Lo;0;L;;;;;N;;;;; +141E0;EGYPTIAN HIEROGLYPH-141E0;Lo;0;L;;;;;N;;;;; +141E1;EGYPTIAN HIEROGLYPH-141E1;Lo;0;L;;;;;N;;;;; +141E2;EGYPTIAN HIEROGLYPH-141E2;Lo;0;L;;;;;N;;;;; +141E3;EGYPTIAN HIEROGLYPH-141E3;Lo;0;L;;;;;N;;;;; +141E4;EGYPTIAN HIEROGLYPH-141E4;Lo;0;L;;;;;N;;;;; +141E5;EGYPTIAN HIEROGLYPH-141E5;Lo;0;L;;;;;N;;;;; +141E6;EGYPTIAN HIEROGLYPH-141E6;Lo;0;L;;;;;N;;;;; +141E7;EGYPTIAN HIEROGLYPH-141E7;Lo;0;L;;;;;N;;;;; +141E8;EGYPTIAN HIEROGLYPH-141E8;Lo;0;L;;;;;N;;;;; +141E9;EGYPTIAN HIEROGLYPH-141E9;Lo;0;L;;;;;N;;;;; +141EA;EGYPTIAN HIEROGLYPH-141EA;Lo;0;L;;;;;N;;;;; +141EB;EGYPTIAN HIEROGLYPH-141EB;Lo;0;L;;;;;N;;;;; +141EC;EGYPTIAN HIEROGLYPH-141EC;Lo;0;L;;;;;N;;;;; +141ED;EGYPTIAN HIEROGLYPH-141ED;Lo;0;L;;;;;N;;;;; +141EE;EGYPTIAN HIEROGLYPH-141EE;Lo;0;L;;;;;N;;;;; +141EF;EGYPTIAN HIEROGLYPH-141EF;Lo;0;L;;;;;N;;;;; +141F0;EGYPTIAN HIEROGLYPH-141F0;Lo;0;L;;;;;N;;;;; +141F1;EGYPTIAN HIEROGLYPH-141F1;Lo;0;L;;;;;N;;;;; +141F2;EGYPTIAN HIEROGLYPH-141F2;Lo;0;L;;;;;N;;;;; +141F3;EGYPTIAN HIEROGLYPH-141F3;Lo;0;L;;;;;N;;;;; +141F4;EGYPTIAN HIEROGLYPH-141F4;Lo;0;L;;;;;N;;;;; +141F5;EGYPTIAN HIEROGLYPH-141F5;Lo;0;L;;;;;N;;;;; +141F6;EGYPTIAN HIEROGLYPH-141F6;Lo;0;L;;;;;N;;;;; +141F7;EGYPTIAN HIEROGLYPH-141F7;Lo;0;L;;;;;N;;;;; +141F8;EGYPTIAN HIEROGLYPH-141F8;Lo;0;L;;;;;N;;;;; +141F9;EGYPTIAN HIEROGLYPH-141F9;Lo;0;L;;;;;N;;;;; +141FA;EGYPTIAN HIEROGLYPH-141FA;Lo;0;L;;;;;N;;;;; +141FB;EGYPTIAN HIEROGLYPH-141FB;Lo;0;L;;;;;N;;;;; +141FC;EGYPTIAN HIEROGLYPH-141FC;Lo;0;L;;;;;N;;;;; +141FD;EGYPTIAN HIEROGLYPH-141FD;Lo;0;L;;;;;N;;;;; +141FE;EGYPTIAN HIEROGLYPH-141FE;Lo;0;L;;;;;N;;;;; +141FF;EGYPTIAN HIEROGLYPH-141FF;Lo;0;L;;;;;N;;;;; +14200;EGYPTIAN HIEROGLYPH-14200;Lo;0;L;;;;;N;;;;; +14201;EGYPTIAN HIEROGLYPH-14201;Lo;0;L;;;;;N;;;;; +14202;EGYPTIAN HIEROGLYPH-14202;Lo;0;L;;;;;N;;;;; +14203;EGYPTIAN HIEROGLYPH-14203;Lo;0;L;;;;;N;;;;; +14204;EGYPTIAN HIEROGLYPH-14204;Lo;0;L;;;;;N;;;;; +14205;EGYPTIAN HIEROGLYPH-14205;Lo;0;L;;;;;N;;;;; +14206;EGYPTIAN HIEROGLYPH-14206;Lo;0;L;;;;;N;;;;; +14207;EGYPTIAN HIEROGLYPH-14207;Lo;0;L;;;;;N;;;;; +14208;EGYPTIAN HIEROGLYPH-14208;Lo;0;L;;;;;N;;;;; +14209;EGYPTIAN HIEROGLYPH-14209;Lo;0;L;;;;;N;;;;; +1420A;EGYPTIAN HIEROGLYPH-1420A;Lo;0;L;;;;;N;;;;; +1420B;EGYPTIAN HIEROGLYPH-1420B;Lo;0;L;;;;;N;;;;; +1420C;EGYPTIAN HIEROGLYPH-1420C;Lo;0;L;;;;;N;;;;; +1420D;EGYPTIAN HIEROGLYPH-1420D;Lo;0;L;;;;;N;;;;; +1420E;EGYPTIAN HIEROGLYPH-1420E;Lo;0;L;;;;;N;;;;; +1420F;EGYPTIAN HIEROGLYPH-1420F;Lo;0;L;;;;;N;;;;; +14210;EGYPTIAN HIEROGLYPH-14210;Lo;0;L;;;;;N;;;;; +14211;EGYPTIAN HIEROGLYPH-14211;Lo;0;L;;;;;N;;;;; +14212;EGYPTIAN HIEROGLYPH-14212;Lo;0;L;;;;;N;;;;; +14213;EGYPTIAN HIEROGLYPH-14213;Lo;0;L;;;;;N;;;;; +14214;EGYPTIAN HIEROGLYPH-14214;Lo;0;L;;;;;N;;;;; +14215;EGYPTIAN HIEROGLYPH-14215;Lo;0;L;;;;;N;;;;; +14216;EGYPTIAN HIEROGLYPH-14216;Lo;0;L;;;;;N;;;;; +14217;EGYPTIAN HIEROGLYPH-14217;Lo;0;L;;;;;N;;;;; +14218;EGYPTIAN HIEROGLYPH-14218;Lo;0;L;;;;;N;;;;; +14219;EGYPTIAN HIEROGLYPH-14219;Lo;0;L;;;;;N;;;;; +1421A;EGYPTIAN HIEROGLYPH-1421A;Lo;0;L;;;;;N;;;;; +1421B;EGYPTIAN HIEROGLYPH-1421B;Lo;0;L;;;;;N;;;;; +1421C;EGYPTIAN HIEROGLYPH-1421C;Lo;0;L;;;;;N;;;;; +1421D;EGYPTIAN HIEROGLYPH-1421D;Lo;0;L;;;;;N;;;;; +1421E;EGYPTIAN HIEROGLYPH-1421E;Lo;0;L;;;;;N;;;;; +1421F;EGYPTIAN HIEROGLYPH-1421F;Lo;0;L;;;;;N;;;;; +14220;EGYPTIAN HIEROGLYPH-14220;Lo;0;L;;;;;N;;;;; +14221;EGYPTIAN HIEROGLYPH-14221;Lo;0;L;;;;;N;;;;; +14222;EGYPTIAN HIEROGLYPH-14222;Lo;0;L;;;;;N;;;;; +14223;EGYPTIAN HIEROGLYPH-14223;Lo;0;L;;;;;N;;;;; +14224;EGYPTIAN HIEROGLYPH-14224;Lo;0;L;;;;;N;;;;; +14225;EGYPTIAN HIEROGLYPH-14225;Lo;0;L;;;;;N;;;;; +14226;EGYPTIAN HIEROGLYPH-14226;Lo;0;L;;;;;N;;;;; +14227;EGYPTIAN HIEROGLYPH-14227;Lo;0;L;;;;;N;;;;; +14228;EGYPTIAN HIEROGLYPH-14228;Lo;0;L;;;;;N;;;;; +14229;EGYPTIAN HIEROGLYPH-14229;Lo;0;L;;;;;N;;;;; +1422A;EGYPTIAN HIEROGLYPH-1422A;Lo;0;L;;;;;N;;;;; +1422B;EGYPTIAN HIEROGLYPH-1422B;Lo;0;L;;;;;N;;;;; +1422C;EGYPTIAN HIEROGLYPH-1422C;Lo;0;L;;;;;N;;;;; +1422D;EGYPTIAN HIEROGLYPH-1422D;Lo;0;L;;;;;N;;;;; +1422E;EGYPTIAN HIEROGLYPH-1422E;Lo;0;L;;;;;N;;;;; +1422F;EGYPTIAN HIEROGLYPH-1422F;Lo;0;L;;;;;N;;;;; +14230;EGYPTIAN HIEROGLYPH-14230;Lo;0;L;;;;;N;;;;; +14231;EGYPTIAN HIEROGLYPH-14231;Lo;0;L;;;;;N;;;;; +14232;EGYPTIAN HIEROGLYPH-14232;Lo;0;L;;;;;N;;;;; +14233;EGYPTIAN HIEROGLYPH-14233;Lo;0;L;;;;;N;;;;; +14234;EGYPTIAN HIEROGLYPH-14234;Lo;0;L;;;;;N;;;;; +14235;EGYPTIAN HIEROGLYPH-14235;Lo;0;L;;;;;N;;;;; +14236;EGYPTIAN HIEROGLYPH-14236;Lo;0;L;;;;;N;;;;; +14237;EGYPTIAN HIEROGLYPH-14237;Lo;0;L;;;;;N;;;;; +14238;EGYPTIAN HIEROGLYPH-14238;Lo;0;L;;;;;N;;;;; +14239;EGYPTIAN HIEROGLYPH-14239;Lo;0;L;;;;;N;;;;; +1423A;EGYPTIAN HIEROGLYPH-1423A;Lo;0;L;;;;;N;;;;; +1423B;EGYPTIAN HIEROGLYPH-1423B;Lo;0;L;;;;;N;;;;; +1423C;EGYPTIAN HIEROGLYPH-1423C;Lo;0;L;;;;;N;;;;; +1423D;EGYPTIAN HIEROGLYPH-1423D;Lo;0;L;;;;;N;;;;; +1423E;EGYPTIAN HIEROGLYPH-1423E;Lo;0;L;;;;;N;;;;; +1423F;EGYPTIAN HIEROGLYPH-1423F;Lo;0;L;;;;;N;;;;; +14240;EGYPTIAN HIEROGLYPH-14240;Lo;0;L;;;;;N;;;;; +14241;EGYPTIAN HIEROGLYPH-14241;Lo;0;L;;;;;N;;;;; +14242;EGYPTIAN HIEROGLYPH-14242;Lo;0;L;;;;;N;;;;; +14243;EGYPTIAN HIEROGLYPH-14243;Lo;0;L;;;;;N;;;;; +14244;EGYPTIAN HIEROGLYPH-14244;Lo;0;L;;;;;N;;;;; +14245;EGYPTIAN HIEROGLYPH-14245;Lo;0;L;;;;;N;;;;; +14246;EGYPTIAN HIEROGLYPH-14246;Lo;0;L;;;;;N;;;;; +14247;EGYPTIAN HIEROGLYPH-14247;Lo;0;L;;;;;N;;;;; +14248;EGYPTIAN HIEROGLYPH-14248;Lo;0;L;;;;;N;;;;; +14249;EGYPTIAN HIEROGLYPH-14249;Lo;0;L;;;;;N;;;;; +1424A;EGYPTIAN HIEROGLYPH-1424A;Lo;0;L;;;;;N;;;;; +1424B;EGYPTIAN HIEROGLYPH-1424B;Lo;0;L;;;;;N;;;;; +1424C;EGYPTIAN HIEROGLYPH-1424C;Lo;0;L;;;;;N;;;;; +1424D;EGYPTIAN HIEROGLYPH-1424D;Lo;0;L;;;;;N;;;;; +1424E;EGYPTIAN HIEROGLYPH-1424E;Lo;0;L;;;;;N;;;;; +1424F;EGYPTIAN HIEROGLYPH-1424F;Lo;0;L;;;;;N;;;;; +14250;EGYPTIAN HIEROGLYPH-14250;Lo;0;L;;;;;N;;;;; +14251;EGYPTIAN HIEROGLYPH-14251;Lo;0;L;;;;;N;;;;; +14252;EGYPTIAN HIEROGLYPH-14252;Lo;0;L;;;;;N;;;;; +14253;EGYPTIAN HIEROGLYPH-14253;Lo;0;L;;;;;N;;;;; +14254;EGYPTIAN HIEROGLYPH-14254;Lo;0;L;;;;;N;;;;; +14255;EGYPTIAN HIEROGLYPH-14255;Lo;0;L;;;;;N;;;;; +14256;EGYPTIAN HIEROGLYPH-14256;Lo;0;L;;;;;N;;;;; +14257;EGYPTIAN HIEROGLYPH-14257;Lo;0;L;;;;;N;;;;; +14258;EGYPTIAN HIEROGLYPH-14258;Lo;0;L;;;;;N;;;;; +14259;EGYPTIAN HIEROGLYPH-14259;Lo;0;L;;;;;N;;;;; +1425A;EGYPTIAN HIEROGLYPH-1425A;Lo;0;L;;;;;N;;;;; +1425B;EGYPTIAN HIEROGLYPH-1425B;Lo;0;L;;;;;N;;;;; +1425C;EGYPTIAN HIEROGLYPH-1425C;Lo;0;L;;;;;N;;;;; +1425D;EGYPTIAN HIEROGLYPH-1425D;Lo;0;L;;;;;N;;;;; +1425E;EGYPTIAN HIEROGLYPH-1425E;Lo;0;L;;;;;N;;;;; +1425F;EGYPTIAN HIEROGLYPH-1425F;Lo;0;L;;;;;N;;;;; +14260;EGYPTIAN HIEROGLYPH-14260;Lo;0;L;;;;;N;;;;; +14261;EGYPTIAN HIEROGLYPH-14261;Lo;0;L;;;;;N;;;;; +14262;EGYPTIAN HIEROGLYPH-14262;Lo;0;L;;;;;N;;;;; +14263;EGYPTIAN HIEROGLYPH-14263;Lo;0;L;;;;;N;;;;; +14264;EGYPTIAN HIEROGLYPH-14264;Lo;0;L;;;;;N;;;;; +14265;EGYPTIAN HIEROGLYPH-14265;Lo;0;L;;;;;N;;;;; +14266;EGYPTIAN HIEROGLYPH-14266;Lo;0;L;;;;;N;;;;; +14267;EGYPTIAN HIEROGLYPH-14267;Lo;0;L;;;;;N;;;;; +14268;EGYPTIAN HIEROGLYPH-14268;Lo;0;L;;;;;N;;;;; +14269;EGYPTIAN HIEROGLYPH-14269;Lo;0;L;;;;;N;;;;; +1426A;EGYPTIAN HIEROGLYPH-1426A;Lo;0;L;;;;;N;;;;; +1426B;EGYPTIAN HIEROGLYPH-1426B;Lo;0;L;;;;;N;;;;; +1426C;EGYPTIAN HIEROGLYPH-1426C;Lo;0;L;;;;;N;;;;; +1426D;EGYPTIAN HIEROGLYPH-1426D;Lo;0;L;;;;;N;;;;; +1426E;EGYPTIAN HIEROGLYPH-1426E;Lo;0;L;;;;;N;;;;; +1426F;EGYPTIAN HIEROGLYPH-1426F;Lo;0;L;;;;;N;;;;; +14270;EGYPTIAN HIEROGLYPH-14270;Lo;0;L;;;;;N;;;;; +14271;EGYPTIAN HIEROGLYPH-14271;Lo;0;L;;;;;N;;;;; +14272;EGYPTIAN HIEROGLYPH-14272;Lo;0;L;;;;;N;;;;; +14273;EGYPTIAN HIEROGLYPH-14273;Lo;0;L;;;;;N;;;;; +14274;EGYPTIAN HIEROGLYPH-14274;Lo;0;L;;;;;N;;;;; +14275;EGYPTIAN HIEROGLYPH-14275;Lo;0;L;;;;;N;;;;; +14276;EGYPTIAN HIEROGLYPH-14276;Lo;0;L;;;;;N;;;;; +14277;EGYPTIAN HIEROGLYPH-14277;Lo;0;L;;;;;N;;;;; +14278;EGYPTIAN HIEROGLYPH-14278;Lo;0;L;;;;;N;;;;; +14279;EGYPTIAN HIEROGLYPH-14279;Lo;0;L;;;;;N;;;;; +1427A;EGYPTIAN HIEROGLYPH-1427A;Lo;0;L;;;;;N;;;;; +1427B;EGYPTIAN HIEROGLYPH-1427B;Lo;0;L;;;;;N;;;;; +1427C;EGYPTIAN HIEROGLYPH-1427C;Lo;0;L;;;;;N;;;;; +1427D;EGYPTIAN HIEROGLYPH-1427D;Lo;0;L;;;;;N;;;;; +1427E;EGYPTIAN HIEROGLYPH-1427E;Lo;0;L;;;;;N;;;;; +1427F;EGYPTIAN HIEROGLYPH-1427F;Lo;0;L;;;;;N;;;;; +14280;EGYPTIAN HIEROGLYPH-14280;Lo;0;L;;;;;N;;;;; +14281;EGYPTIAN HIEROGLYPH-14281;Lo;0;L;;;;;N;;;;; +14282;EGYPTIAN HIEROGLYPH-14282;Lo;0;L;;;;;N;;;;; +14283;EGYPTIAN HIEROGLYPH-14283;Lo;0;L;;;;;N;;;;; +14284;EGYPTIAN HIEROGLYPH-14284;Lo;0;L;;;;;N;;;;; +14285;EGYPTIAN HIEROGLYPH-14285;Lo;0;L;;;;;N;;;;; +14286;EGYPTIAN HIEROGLYPH-14286;Lo;0;L;;;;;N;;;;; +14287;EGYPTIAN HIEROGLYPH-14287;Lo;0;L;;;;;N;;;;; +14288;EGYPTIAN HIEROGLYPH-14288;Lo;0;L;;;;;N;;;;; +14289;EGYPTIAN HIEROGLYPH-14289;Lo;0;L;;;;;N;;;;; +1428A;EGYPTIAN HIEROGLYPH-1428A;Lo;0;L;;;;;N;;;;; +1428B;EGYPTIAN HIEROGLYPH-1428B;Lo;0;L;;;;;N;;;;; +1428C;EGYPTIAN HIEROGLYPH-1428C;Lo;0;L;;;;;N;;;;; +1428D;EGYPTIAN HIEROGLYPH-1428D;Lo;0;L;;;;;N;;;;; +1428E;EGYPTIAN HIEROGLYPH-1428E;Lo;0;L;;;;;N;;;;; +1428F;EGYPTIAN HIEROGLYPH-1428F;Lo;0;L;;;;;N;;;;; +14290;EGYPTIAN HIEROGLYPH-14290;Lo;0;L;;;;;N;;;;; +14291;EGYPTIAN HIEROGLYPH-14291;Lo;0;L;;;;;N;;;;; +14292;EGYPTIAN HIEROGLYPH-14292;Lo;0;L;;;;;N;;;;; +14293;EGYPTIAN HIEROGLYPH-14293;Lo;0;L;;;;;N;;;;; +14294;EGYPTIAN HIEROGLYPH-14294;Lo;0;L;;;;;N;;;;; +14295;EGYPTIAN HIEROGLYPH-14295;Lo;0;L;;;;;N;;;;; +14296;EGYPTIAN HIEROGLYPH-14296;Lo;0;L;;;;;N;;;;; +14297;EGYPTIAN HIEROGLYPH-14297;Lo;0;L;;;;;N;;;;; +14298;EGYPTIAN HIEROGLYPH-14298;Lo;0;L;;;;;N;;;;; +14299;EGYPTIAN HIEROGLYPH-14299;Lo;0;L;;;;;N;;;;; +1429A;EGYPTIAN HIEROGLYPH-1429A;Lo;0;L;;;;;N;;;;; +1429B;EGYPTIAN HIEROGLYPH-1429B;Lo;0;L;;;;;N;;;;; +1429C;EGYPTIAN HIEROGLYPH-1429C;Lo;0;L;;;;;N;;;;; +1429D;EGYPTIAN HIEROGLYPH-1429D;Lo;0;L;;;;;N;;;;; +1429E;EGYPTIAN HIEROGLYPH-1429E;Lo;0;L;;;;;N;;;;; +1429F;EGYPTIAN HIEROGLYPH-1429F;Lo;0;L;;;;;N;;;;; +142A0;EGYPTIAN HIEROGLYPH-142A0;Lo;0;L;;;;;N;;;;; +142A1;EGYPTIAN HIEROGLYPH-142A1;Lo;0;L;;;;;N;;;;; +142A2;EGYPTIAN HIEROGLYPH-142A2;Lo;0;L;;;;;N;;;;; +142A3;EGYPTIAN HIEROGLYPH-142A3;Lo;0;L;;;;;N;;;;; +142A4;EGYPTIAN HIEROGLYPH-142A4;Lo;0;L;;;;;N;;;;; +142A5;EGYPTIAN HIEROGLYPH-142A5;Lo;0;L;;;;;N;;;;; +142A6;EGYPTIAN HIEROGLYPH-142A6;Lo;0;L;;;;;N;;;;; +142A7;EGYPTIAN HIEROGLYPH-142A7;Lo;0;L;;;;;N;;;;; +142A8;EGYPTIAN HIEROGLYPH-142A8;Lo;0;L;;;;;N;;;;; +142A9;EGYPTIAN HIEROGLYPH-142A9;Lo;0;L;;;;;N;;;;; +142AA;EGYPTIAN HIEROGLYPH-142AA;Lo;0;L;;;;;N;;;;; +142AB;EGYPTIAN HIEROGLYPH-142AB;Lo;0;L;;;;;N;;;;; +142AC;EGYPTIAN HIEROGLYPH-142AC;Lo;0;L;;;;;N;;;;; +142AD;EGYPTIAN HIEROGLYPH-142AD;Lo;0;L;;;;;N;;;;; +142AE;EGYPTIAN HIEROGLYPH-142AE;Lo;0;L;;;;;N;;;;; +142AF;EGYPTIAN HIEROGLYPH-142AF;Lo;0;L;;;;;N;;;;; +142B0;EGYPTIAN HIEROGLYPH-142B0;Lo;0;L;;;;;N;;;;; +142B1;EGYPTIAN HIEROGLYPH-142B1;Lo;0;L;;;;;N;;;;; +142B2;EGYPTIAN HIEROGLYPH-142B2;Lo;0;L;;;;;N;;;;; +142B3;EGYPTIAN HIEROGLYPH-142B3;Lo;0;L;;;;;N;;;;; +142B4;EGYPTIAN HIEROGLYPH-142B4;Lo;0;L;;;;;N;;;;; +142B5;EGYPTIAN HIEROGLYPH-142B5;Lo;0;L;;;;;N;;;;; +142B6;EGYPTIAN HIEROGLYPH-142B6;Lo;0;L;;;;;N;;;;; +142B7;EGYPTIAN HIEROGLYPH-142B7;Lo;0;L;;;;;N;;;;; +142B8;EGYPTIAN HIEROGLYPH-142B8;Lo;0;L;;;;;N;;;;; +142B9;EGYPTIAN HIEROGLYPH-142B9;Lo;0;L;;;;;N;;;;; +142BA;EGYPTIAN HIEROGLYPH-142BA;Lo;0;L;;;;;N;;;;; +142BB;EGYPTIAN HIEROGLYPH-142BB;Lo;0;L;;;;;N;;;;; +142BC;EGYPTIAN HIEROGLYPH-142BC;Lo;0;L;;;;;N;;;;; +142BD;EGYPTIAN HIEROGLYPH-142BD;Lo;0;L;;;;;N;;;;; +142BE;EGYPTIAN HIEROGLYPH-142BE;Lo;0;L;;;;;N;;;;; +142BF;EGYPTIAN HIEROGLYPH-142BF;Lo;0;L;;;;;N;;;;; +142C0;EGYPTIAN HIEROGLYPH-142C0;Lo;0;L;;;;;N;;;;; +142C1;EGYPTIAN HIEROGLYPH-142C1;Lo;0;L;;;;;N;;;;; +142C2;EGYPTIAN HIEROGLYPH-142C2;Lo;0;L;;;;;N;;;;; +142C3;EGYPTIAN HIEROGLYPH-142C3;Lo;0;L;;;;;N;;;;; +142C4;EGYPTIAN HIEROGLYPH-142C4;Lo;0;L;;;;;N;;;;; +142C5;EGYPTIAN HIEROGLYPH-142C5;Lo;0;L;;;;;N;;;;; +142C6;EGYPTIAN HIEROGLYPH-142C6;Lo;0;L;;;;;N;;;;; +142C7;EGYPTIAN HIEROGLYPH-142C7;Lo;0;L;;;;;N;;;;; +142C8;EGYPTIAN HIEROGLYPH-142C8;Lo;0;L;;;;;N;;;;; +142C9;EGYPTIAN HIEROGLYPH-142C9;Lo;0;L;;;;;N;;;;; +142CA;EGYPTIAN HIEROGLYPH-142CA;Lo;0;L;;;;;N;;;;; +142CB;EGYPTIAN HIEROGLYPH-142CB;Lo;0;L;;;;;N;;;;; +142CC;EGYPTIAN HIEROGLYPH-142CC;Lo;0;L;;;;;N;;;;; +142CD;EGYPTIAN HIEROGLYPH-142CD;Lo;0;L;;;;;N;;;;; +142CE;EGYPTIAN HIEROGLYPH-142CE;Lo;0;L;;;;;N;;;;; +142CF;EGYPTIAN HIEROGLYPH-142CF;Lo;0;L;;;;;N;;;;; +142D0;EGYPTIAN HIEROGLYPH-142D0;Lo;0;L;;;;;N;;;;; +142D1;EGYPTIAN HIEROGLYPH-142D1;Lo;0;L;;;;;N;;;;; +142D2;EGYPTIAN HIEROGLYPH-142D2;Lo;0;L;;;;;N;;;;; +142D3;EGYPTIAN HIEROGLYPH-142D3;Lo;0;L;;;;;N;;;;; +142D4;EGYPTIAN HIEROGLYPH-142D4;Lo;0;L;;;;;N;;;;; +142D5;EGYPTIAN HIEROGLYPH-142D5;Lo;0;L;;;;;N;;;;; +142D6;EGYPTIAN HIEROGLYPH-142D6;Lo;0;L;;;;;N;;;;; +142D7;EGYPTIAN HIEROGLYPH-142D7;Lo;0;L;;;;;N;;;;; +142D8;EGYPTIAN HIEROGLYPH-142D8;Lo;0;L;;;;;N;;;;; +142D9;EGYPTIAN HIEROGLYPH-142D9;Lo;0;L;;;;;N;;;;; +142DA;EGYPTIAN HIEROGLYPH-142DA;Lo;0;L;;;;;N;;;;; +142DB;EGYPTIAN HIEROGLYPH-142DB;Lo;0;L;;;;;N;;;;; +142DC;EGYPTIAN HIEROGLYPH-142DC;Lo;0;L;;;;;N;;;;; +142DD;EGYPTIAN HIEROGLYPH-142DD;Lo;0;L;;;;;N;;;;; +142DE;EGYPTIAN HIEROGLYPH-142DE;Lo;0;L;;;;;N;;;;; +142DF;EGYPTIAN HIEROGLYPH-142DF;Lo;0;L;;;;;N;;;;; +142E0;EGYPTIAN HIEROGLYPH-142E0;Lo;0;L;;;;;N;;;;; +142E1;EGYPTIAN HIEROGLYPH-142E1;Lo;0;L;;;;;N;;;;; +142E2;EGYPTIAN HIEROGLYPH-142E2;Lo;0;L;;;;;N;;;;; +142E3;EGYPTIAN HIEROGLYPH-142E3;Lo;0;L;;;;;N;;;;; +142E4;EGYPTIAN HIEROGLYPH-142E4;Lo;0;L;;;;;N;;;;; +142E5;EGYPTIAN HIEROGLYPH-142E5;Lo;0;L;;;;;N;;;;; +142E6;EGYPTIAN HIEROGLYPH-142E6;Lo;0;L;;;;;N;;;;; +142E7;EGYPTIAN HIEROGLYPH-142E7;Lo;0;L;;;;;N;;;;; +142E8;EGYPTIAN HIEROGLYPH-142E8;Lo;0;L;;;;;N;;;;; +142E9;EGYPTIAN HIEROGLYPH-142E9;Lo;0;L;;;;;N;;;;; +142EA;EGYPTIAN HIEROGLYPH-142EA;Lo;0;L;;;;;N;;;;; +142EB;EGYPTIAN HIEROGLYPH-142EB;Lo;0;L;;;;;N;;;;; +142EC;EGYPTIAN HIEROGLYPH-142EC;Lo;0;L;;;;;N;;;;; +142ED;EGYPTIAN HIEROGLYPH-142ED;Lo;0;L;;;;;N;;;;; +142EE;EGYPTIAN HIEROGLYPH-142EE;Lo;0;L;;;;;N;;;;; +142EF;EGYPTIAN HIEROGLYPH-142EF;Lo;0;L;;;;;N;;;;; +142F0;EGYPTIAN HIEROGLYPH-142F0;Lo;0;L;;;;;N;;;;; +142F1;EGYPTIAN HIEROGLYPH-142F1;Lo;0;L;;;;;N;;;;; +142F2;EGYPTIAN HIEROGLYPH-142F2;Lo;0;L;;;;;N;;;;; +142F3;EGYPTIAN HIEROGLYPH-142F3;Lo;0;L;;;;;N;;;;; +142F4;EGYPTIAN HIEROGLYPH-142F4;Lo;0;L;;;;;N;;;;; +142F5;EGYPTIAN HIEROGLYPH-142F5;Lo;0;L;;;;;N;;;;; +142F6;EGYPTIAN HIEROGLYPH-142F6;Lo;0;L;;;;;N;;;;; +142F7;EGYPTIAN HIEROGLYPH-142F7;Lo;0;L;;;;;N;;;;; +142F8;EGYPTIAN HIEROGLYPH-142F8;Lo;0;L;;;;;N;;;;; +142F9;EGYPTIAN HIEROGLYPH-142F9;Lo;0;L;;;;;N;;;;; +142FA;EGYPTIAN HIEROGLYPH-142FA;Lo;0;L;;;;;N;;;;; +142FB;EGYPTIAN HIEROGLYPH-142FB;Lo;0;L;;;;;N;;;;; +142FC;EGYPTIAN HIEROGLYPH-142FC;Lo;0;L;;;;;N;;;;; +142FD;EGYPTIAN HIEROGLYPH-142FD;Lo;0;L;;;;;N;;;;; +142FE;EGYPTIAN HIEROGLYPH-142FE;Lo;0;L;;;;;N;;;;; +142FF;EGYPTIAN HIEROGLYPH-142FF;Lo;0;L;;;;;N;;;;; +14300;EGYPTIAN HIEROGLYPH-14300;Lo;0;L;;;;;N;;;;; +14301;EGYPTIAN HIEROGLYPH-14301;Lo;0;L;;;;;N;;;;; +14302;EGYPTIAN HIEROGLYPH-14302;Lo;0;L;;;;;N;;;;; +14303;EGYPTIAN HIEROGLYPH-14303;Lo;0;L;;;;;N;;;;; +14304;EGYPTIAN HIEROGLYPH-14304;Lo;0;L;;;;;N;;;;; +14305;EGYPTIAN HIEROGLYPH-14305;Lo;0;L;;;;;N;;;;; +14306;EGYPTIAN HIEROGLYPH-14306;Lo;0;L;;;;;N;;;;; +14307;EGYPTIAN HIEROGLYPH-14307;Lo;0;L;;;;;N;;;;; +14308;EGYPTIAN HIEROGLYPH-14308;Lo;0;L;;;;;N;;;;; +14309;EGYPTIAN HIEROGLYPH-14309;Lo;0;L;;;;;N;;;;; +1430A;EGYPTIAN HIEROGLYPH-1430A;Lo;0;L;;;;;N;;;;; +1430B;EGYPTIAN HIEROGLYPH-1430B;Lo;0;L;;;;;N;;;;; +1430C;EGYPTIAN HIEROGLYPH-1430C;Lo;0;L;;;;;N;;;;; +1430D;EGYPTIAN HIEROGLYPH-1430D;Lo;0;L;;;;;N;;;;; +1430E;EGYPTIAN HIEROGLYPH-1430E;Lo;0;L;;;;;N;;;;; +1430F;EGYPTIAN HIEROGLYPH-1430F;Lo;0;L;;;;;N;;;;; +14310;EGYPTIAN HIEROGLYPH-14310;Lo;0;L;;;;;N;;;;; +14311;EGYPTIAN HIEROGLYPH-14311;Lo;0;L;;;;;N;;;;; +14312;EGYPTIAN HIEROGLYPH-14312;Lo;0;L;;;;;N;;;;; +14313;EGYPTIAN HIEROGLYPH-14313;Lo;0;L;;;;;N;;;;; +14314;EGYPTIAN HIEROGLYPH-14314;Lo;0;L;;;;;N;;;;; +14315;EGYPTIAN HIEROGLYPH-14315;Lo;0;L;;;;;N;;;;; +14316;EGYPTIAN HIEROGLYPH-14316;Lo;0;L;;;;;N;;;;; +14317;EGYPTIAN HIEROGLYPH-14317;Lo;0;L;;;;;N;;;;; +14318;EGYPTIAN HIEROGLYPH-14318;Lo;0;L;;;;;N;;;;; +14319;EGYPTIAN HIEROGLYPH-14319;Lo;0;L;;;;;N;;;;; +1431A;EGYPTIAN HIEROGLYPH-1431A;Lo;0;L;;;;;N;;;;; +1431B;EGYPTIAN HIEROGLYPH-1431B;Lo;0;L;;;;;N;;;;; +1431C;EGYPTIAN HIEROGLYPH-1431C;Lo;0;L;;;;;N;;;;; +1431D;EGYPTIAN HIEROGLYPH-1431D;Lo;0;L;;;;;N;;;;; +1431E;EGYPTIAN HIEROGLYPH-1431E;Lo;0;L;;;;;N;;;;; +1431F;EGYPTIAN HIEROGLYPH-1431F;Lo;0;L;;;;;N;;;;; +14320;EGYPTIAN HIEROGLYPH-14320;Lo;0;L;;;;;N;;;;; +14321;EGYPTIAN HIEROGLYPH-14321;Lo;0;L;;;;;N;;;;; +14322;EGYPTIAN HIEROGLYPH-14322;Lo;0;L;;;;;N;;;;; +14323;EGYPTIAN HIEROGLYPH-14323;Lo;0;L;;;;;N;;;;; +14324;EGYPTIAN HIEROGLYPH-14324;Lo;0;L;;;;;N;;;;; +14325;EGYPTIAN HIEROGLYPH-14325;Lo;0;L;;;;;N;;;;; +14326;EGYPTIAN HIEROGLYPH-14326;Lo;0;L;;;;;N;;;;; +14327;EGYPTIAN HIEROGLYPH-14327;Lo;0;L;;;;;N;;;;; +14328;EGYPTIAN HIEROGLYPH-14328;Lo;0;L;;;;;N;;;;; +14329;EGYPTIAN HIEROGLYPH-14329;Lo;0;L;;;;;N;;;;; +1432A;EGYPTIAN HIEROGLYPH-1432A;Lo;0;L;;;;;N;;;;; +1432B;EGYPTIAN HIEROGLYPH-1432B;Lo;0;L;;;;;N;;;;; +1432C;EGYPTIAN HIEROGLYPH-1432C;Lo;0;L;;;;;N;;;;; +1432D;EGYPTIAN HIEROGLYPH-1432D;Lo;0;L;;;;;N;;;;; +1432E;EGYPTIAN HIEROGLYPH-1432E;Lo;0;L;;;;;N;;;;; +1432F;EGYPTIAN HIEROGLYPH-1432F;Lo;0;L;;;;;N;;;;; +14330;EGYPTIAN HIEROGLYPH-14330;Lo;0;L;;;;;N;;;;; +14331;EGYPTIAN HIEROGLYPH-14331;Lo;0;L;;;;;N;;;;; +14332;EGYPTIAN HIEROGLYPH-14332;Lo;0;L;;;;;N;;;;; +14333;EGYPTIAN HIEROGLYPH-14333;Lo;0;L;;;;;N;;;;; +14334;EGYPTIAN HIEROGLYPH-14334;Lo;0;L;;;;;N;;;;; +14335;EGYPTIAN HIEROGLYPH-14335;Lo;0;L;;;;;N;;;;; +14336;EGYPTIAN HIEROGLYPH-14336;Lo;0;L;;;;;N;;;;; +14337;EGYPTIAN HIEROGLYPH-14337;Lo;0;L;;;;;N;;;;; +14338;EGYPTIAN HIEROGLYPH-14338;Lo;0;L;;;;;N;;;;; +14339;EGYPTIAN HIEROGLYPH-14339;Lo;0;L;;;;;N;;;;; +1433A;EGYPTIAN HIEROGLYPH-1433A;Lo;0;L;;;;;N;;;;; +1433B;EGYPTIAN HIEROGLYPH-1433B;Lo;0;L;;;;;N;;;;; +1433C;EGYPTIAN HIEROGLYPH-1433C;Lo;0;L;;;;;N;;;;; +1433D;EGYPTIAN HIEROGLYPH-1433D;Lo;0;L;;;;;N;;;;; +1433E;EGYPTIAN HIEROGLYPH-1433E;Lo;0;L;;;;;N;;;;; +1433F;EGYPTIAN HIEROGLYPH-1433F;Lo;0;L;;;;;N;;;;; +14340;EGYPTIAN HIEROGLYPH-14340;Lo;0;L;;;;;N;;;;; +14341;EGYPTIAN HIEROGLYPH-14341;Lo;0;L;;;;;N;;;;; +14342;EGYPTIAN HIEROGLYPH-14342;Lo;0;L;;;;;N;;;;; +14343;EGYPTIAN HIEROGLYPH-14343;Lo;0;L;;;;;N;;;;; +14344;EGYPTIAN HIEROGLYPH-14344;Lo;0;L;;;;;N;;;;; +14345;EGYPTIAN HIEROGLYPH-14345;Lo;0;L;;;;;N;;;;; +14346;EGYPTIAN HIEROGLYPH-14346;Lo;0;L;;;;;N;;;;; +14347;EGYPTIAN HIEROGLYPH-14347;Lo;0;L;;;;;N;;;;; +14348;EGYPTIAN HIEROGLYPH-14348;Lo;0;L;;;;;N;;;;; +14349;EGYPTIAN HIEROGLYPH-14349;Lo;0;L;;;;;N;;;;; +1434A;EGYPTIAN HIEROGLYPH-1434A;Lo;0;L;;;;;N;;;;; +1434B;EGYPTIAN HIEROGLYPH-1434B;Lo;0;L;;;;;N;;;;; +1434C;EGYPTIAN HIEROGLYPH-1434C;Lo;0;L;;;;;N;;;;; +1434D;EGYPTIAN HIEROGLYPH-1434D;Lo;0;L;;;;;N;;;;; +1434E;EGYPTIAN HIEROGLYPH-1434E;Lo;0;L;;;;;N;;;;; +1434F;EGYPTIAN HIEROGLYPH-1434F;Lo;0;L;;;;;N;;;;; +14350;EGYPTIAN HIEROGLYPH-14350;Lo;0;L;;;;;N;;;;; +14351;EGYPTIAN HIEROGLYPH-14351;Lo;0;L;;;;;N;;;;; +14352;EGYPTIAN HIEROGLYPH-14352;Lo;0;L;;;;;N;;;;; +14353;EGYPTIAN HIEROGLYPH-14353;Lo;0;L;;;;;N;;;;; +14354;EGYPTIAN HIEROGLYPH-14354;Lo;0;L;;;;;N;;;;; +14355;EGYPTIAN HIEROGLYPH-14355;Lo;0;L;;;;;N;;;;; +14356;EGYPTIAN HIEROGLYPH-14356;Lo;0;L;;;;;N;;;;; +14357;EGYPTIAN HIEROGLYPH-14357;Lo;0;L;;;;;N;;;;; +14358;EGYPTIAN HIEROGLYPH-14358;Lo;0;L;;;;;N;;;;; +14359;EGYPTIAN HIEROGLYPH-14359;Lo;0;L;;;;;N;;;;; +1435A;EGYPTIAN HIEROGLYPH-1435A;Lo;0;L;;;;;N;;;;; +1435B;EGYPTIAN HIEROGLYPH-1435B;Lo;0;L;;;;;N;;;;; +1435C;EGYPTIAN HIEROGLYPH-1435C;Lo;0;L;;;;;N;;;;; +1435D;EGYPTIAN HIEROGLYPH-1435D;Lo;0;L;;;;;N;;;;; +1435E;EGYPTIAN HIEROGLYPH-1435E;Lo;0;L;;;;;N;;;;; +1435F;EGYPTIAN HIEROGLYPH-1435F;Lo;0;L;;;;;N;;;;; +14360;EGYPTIAN HIEROGLYPH-14360;Lo;0;L;;;;;N;;;;; +14361;EGYPTIAN HIEROGLYPH-14361;Lo;0;L;;;;;N;;;;; +14362;EGYPTIAN HIEROGLYPH-14362;Lo;0;L;;;;;N;;;;; +14363;EGYPTIAN HIEROGLYPH-14363;Lo;0;L;;;;;N;;;;; +14364;EGYPTIAN HIEROGLYPH-14364;Lo;0;L;;;;;N;;;;; +14365;EGYPTIAN HIEROGLYPH-14365;Lo;0;L;;;;;N;;;;; +14366;EGYPTIAN HIEROGLYPH-14366;Lo;0;L;;;;;N;;;;; +14367;EGYPTIAN HIEROGLYPH-14367;Lo;0;L;;;;;N;;;;; +14368;EGYPTIAN HIEROGLYPH-14368;Lo;0;L;;;;;N;;;;; +14369;EGYPTIAN HIEROGLYPH-14369;Lo;0;L;;;;;N;;;;; +1436A;EGYPTIAN HIEROGLYPH-1436A;Lo;0;L;;;;;N;;;;; +1436B;EGYPTIAN HIEROGLYPH-1436B;Lo;0;L;;;;;N;;;;; +1436C;EGYPTIAN HIEROGLYPH-1436C;Lo;0;L;;;;;N;;;;; +1436D;EGYPTIAN HIEROGLYPH-1436D;Lo;0;L;;;;;N;;;;; +1436E;EGYPTIAN HIEROGLYPH-1436E;Lo;0;L;;;;;N;;;;; +1436F;EGYPTIAN HIEROGLYPH-1436F;Lo;0;L;;;;;N;;;;; +14370;EGYPTIAN HIEROGLYPH-14370;Lo;0;L;;;;;N;;;;; +14371;EGYPTIAN HIEROGLYPH-14371;Lo;0;L;;;;;N;;;;; +14372;EGYPTIAN HIEROGLYPH-14372;Lo;0;L;;;;;N;;;;; +14373;EGYPTIAN HIEROGLYPH-14373;Lo;0;L;;;;;N;;;;; +14374;EGYPTIAN HIEROGLYPH-14374;Lo;0;L;;;;;N;;;;; +14375;EGYPTIAN HIEROGLYPH-14375;Lo;0;L;;;;;N;;;;; +14376;EGYPTIAN HIEROGLYPH-14376;Lo;0;L;;;;;N;;;;; +14377;EGYPTIAN HIEROGLYPH-14377;Lo;0;L;;;;;N;;;;; +14378;EGYPTIAN HIEROGLYPH-14378;Lo;0;L;;;;;N;;;;; +14379;EGYPTIAN HIEROGLYPH-14379;Lo;0;L;;;;;N;;;;; +1437A;EGYPTIAN HIEROGLYPH-1437A;Lo;0;L;;;;;N;;;;; +1437B;EGYPTIAN HIEROGLYPH-1437B;Lo;0;L;;;;;N;;;;; +1437C;EGYPTIAN HIEROGLYPH-1437C;Lo;0;L;;;;;N;;;;; +1437D;EGYPTIAN HIEROGLYPH-1437D;Lo;0;L;;;;;N;;;;; +1437E;EGYPTIAN HIEROGLYPH-1437E;Lo;0;L;;;;;N;;;;; +1437F;EGYPTIAN HIEROGLYPH-1437F;Lo;0;L;;;;;N;;;;; +14380;EGYPTIAN HIEROGLYPH-14380;Lo;0;L;;;;;N;;;;; +14381;EGYPTIAN HIEROGLYPH-14381;Lo;0;L;;;;;N;;;;; +14382;EGYPTIAN HIEROGLYPH-14382;Lo;0;L;;;;;N;;;;; +14383;EGYPTIAN HIEROGLYPH-14383;Lo;0;L;;;;;N;;;;; +14384;EGYPTIAN HIEROGLYPH-14384;Lo;0;L;;;;;N;;;;; +14385;EGYPTIAN HIEROGLYPH-14385;Lo;0;L;;;;;N;;;;; +14386;EGYPTIAN HIEROGLYPH-14386;Lo;0;L;;;;;N;;;;; +14387;EGYPTIAN HIEROGLYPH-14387;Lo;0;L;;;;;N;;;;; +14388;EGYPTIAN HIEROGLYPH-14388;Lo;0;L;;;;;N;;;;; +14389;EGYPTIAN HIEROGLYPH-14389;Lo;0;L;;;;;N;;;;; +1438A;EGYPTIAN HIEROGLYPH-1438A;Lo;0;L;;;;;N;;;;; +1438B;EGYPTIAN HIEROGLYPH-1438B;Lo;0;L;;;;;N;;;;; +1438C;EGYPTIAN HIEROGLYPH-1438C;Lo;0;L;;;;;N;;;;; +1438D;EGYPTIAN HIEROGLYPH-1438D;Lo;0;L;;;;;N;;;;; +1438E;EGYPTIAN HIEROGLYPH-1438E;Lo;0;L;;;;;N;;;;; +1438F;EGYPTIAN HIEROGLYPH-1438F;Lo;0;L;;;;;N;;;;; +14390;EGYPTIAN HIEROGLYPH-14390;Lo;0;L;;;;;N;;;;; +14391;EGYPTIAN HIEROGLYPH-14391;Lo;0;L;;;;;N;;;;; +14392;EGYPTIAN HIEROGLYPH-14392;Lo;0;L;;;;;N;;;;; +14393;EGYPTIAN HIEROGLYPH-14393;Lo;0;L;;;;;N;;;;; +14394;EGYPTIAN HIEROGLYPH-14394;Lo;0;L;;;;;N;;;;; +14395;EGYPTIAN HIEROGLYPH-14395;Lo;0;L;;;;;N;;;;; +14396;EGYPTIAN HIEROGLYPH-14396;Lo;0;L;;;;;N;;;;; +14397;EGYPTIAN HIEROGLYPH-14397;Lo;0;L;;;;;N;;;;; +14398;EGYPTIAN HIEROGLYPH-14398;Lo;0;L;;;;;N;;;;; +14399;EGYPTIAN HIEROGLYPH-14399;Lo;0;L;;;;;N;;;;; +1439A;EGYPTIAN HIEROGLYPH-1439A;Lo;0;L;;;;;N;;;;; +1439B;EGYPTIAN HIEROGLYPH-1439B;Lo;0;L;;;;;N;;;;; +1439C;EGYPTIAN HIEROGLYPH-1439C;Lo;0;L;;;;;N;;;;; +1439D;EGYPTIAN HIEROGLYPH-1439D;Lo;0;L;;;;;N;;;;; +1439E;EGYPTIAN HIEROGLYPH-1439E;Lo;0;L;;;;;N;;;;; +1439F;EGYPTIAN HIEROGLYPH-1439F;Lo;0;L;;;;;N;;;;; +143A0;EGYPTIAN HIEROGLYPH-143A0;Lo;0;L;;;;;N;;;;; +143A1;EGYPTIAN HIEROGLYPH-143A1;Lo;0;L;;;;;N;;;;; +143A2;EGYPTIAN HIEROGLYPH-143A2;Lo;0;L;;;;;N;;;;; +143A3;EGYPTIAN HIEROGLYPH-143A3;Lo;0;L;;;;;N;;;;; +143A4;EGYPTIAN HIEROGLYPH-143A4;Lo;0;L;;;;;N;;;;; +143A5;EGYPTIAN HIEROGLYPH-143A5;Lo;0;L;;;;;N;;;;; +143A6;EGYPTIAN HIEROGLYPH-143A6;Lo;0;L;;;;;N;;;;; +143A7;EGYPTIAN HIEROGLYPH-143A7;Lo;0;L;;;;;N;;;;; +143A8;EGYPTIAN HIEROGLYPH-143A8;Lo;0;L;;;;;N;;;;; +143A9;EGYPTIAN HIEROGLYPH-143A9;Lo;0;L;;;;;N;;;;; +143AA;EGYPTIAN HIEROGLYPH-143AA;Lo;0;L;;;;;N;;;;; +143AB;EGYPTIAN HIEROGLYPH-143AB;Lo;0;L;;;;;N;;;;; +143AC;EGYPTIAN HIEROGLYPH-143AC;Lo;0;L;;;;;N;;;;; +143AD;EGYPTIAN HIEROGLYPH-143AD;Lo;0;L;;;;;N;;;;; +143AE;EGYPTIAN HIEROGLYPH-143AE;Lo;0;L;;;;;N;;;;; +143AF;EGYPTIAN HIEROGLYPH-143AF;Lo;0;L;;;;;N;;;;; +143B0;EGYPTIAN HIEROGLYPH-143B0;Lo;0;L;;;;;N;;;;; +143B1;EGYPTIAN HIEROGLYPH-143B1;Lo;0;L;;;;;N;;;;; +143B2;EGYPTIAN HIEROGLYPH-143B2;Lo;0;L;;;;;N;;;;; +143B3;EGYPTIAN HIEROGLYPH-143B3;Lo;0;L;;;;;N;;;;; +143B4;EGYPTIAN HIEROGLYPH-143B4;Lo;0;L;;;;;N;;;;; +143B5;EGYPTIAN HIEROGLYPH-143B5;Lo;0;L;;;;;N;;;;; +143B6;EGYPTIAN HIEROGLYPH-143B6;Lo;0;L;;;;;N;;;;; +143B7;EGYPTIAN HIEROGLYPH-143B7;Lo;0;L;;;;;N;;;;; +143B8;EGYPTIAN HIEROGLYPH-143B8;Lo;0;L;;;;;N;;;;; +143B9;EGYPTIAN HIEROGLYPH-143B9;Lo;0;L;;;;;N;;;;; +143BA;EGYPTIAN HIEROGLYPH-143BA;Lo;0;L;;;;;N;;;;; +143BB;EGYPTIAN HIEROGLYPH-143BB;Lo;0;L;;;;;N;;;;; +143BC;EGYPTIAN HIEROGLYPH-143BC;Lo;0;L;;;;;N;;;;; +143BD;EGYPTIAN HIEROGLYPH-143BD;Lo;0;L;;;;;N;;;;; +143BE;EGYPTIAN HIEROGLYPH-143BE;Lo;0;L;;;;;N;;;;; +143BF;EGYPTIAN HIEROGLYPH-143BF;Lo;0;L;;;;;N;;;;; +143C0;EGYPTIAN HIEROGLYPH-143C0;Lo;0;L;;;;;N;;;;; +143C1;EGYPTIAN HIEROGLYPH-143C1;Lo;0;L;;;;;N;;;;; +143C2;EGYPTIAN HIEROGLYPH-143C2;Lo;0;L;;;;;N;;;;; +143C3;EGYPTIAN HIEROGLYPH-143C3;Lo;0;L;;;;;N;;;;; +143C4;EGYPTIAN HIEROGLYPH-143C4;Lo;0;L;;;;;N;;;;; +143C5;EGYPTIAN HIEROGLYPH-143C5;Lo;0;L;;;;;N;;;;; +143C6;EGYPTIAN HIEROGLYPH-143C6;Lo;0;L;;;;;N;;;;; +143C7;EGYPTIAN HIEROGLYPH-143C7;Lo;0;L;;;;;N;;;;; +143C8;EGYPTIAN HIEROGLYPH-143C8;Lo;0;L;;;;;N;;;;; +143C9;EGYPTIAN HIEROGLYPH-143C9;Lo;0;L;;;;;N;;;;; +143CA;EGYPTIAN HIEROGLYPH-143CA;Lo;0;L;;;;;N;;;;; +143CB;EGYPTIAN HIEROGLYPH-143CB;Lo;0;L;;;;;N;;;;; +143CC;EGYPTIAN HIEROGLYPH-143CC;Lo;0;L;;;;;N;;;;; +143CD;EGYPTIAN HIEROGLYPH-143CD;Lo;0;L;;;;;N;;;;; +143CE;EGYPTIAN HIEROGLYPH-143CE;Lo;0;L;;;;;N;;;;; +143CF;EGYPTIAN HIEROGLYPH-143CF;Lo;0;L;;;;;N;;;;; +143D0;EGYPTIAN HIEROGLYPH-143D0;Lo;0;L;;;;;N;;;;; +143D1;EGYPTIAN HIEROGLYPH-143D1;Lo;0;L;;;;;N;;;;; +143D2;EGYPTIAN HIEROGLYPH-143D2;Lo;0;L;;;;;N;;;;; +143D3;EGYPTIAN HIEROGLYPH-143D3;Lo;0;L;;;;;N;;;;; +143D4;EGYPTIAN HIEROGLYPH-143D4;Lo;0;L;;;;;N;;;;; +143D5;EGYPTIAN HIEROGLYPH-143D5;Lo;0;L;;;;;N;;;;; +143D6;EGYPTIAN HIEROGLYPH-143D6;Lo;0;L;;;;;N;;;;; +143D7;EGYPTIAN HIEROGLYPH-143D7;Lo;0;L;;;;;N;;;;; +143D8;EGYPTIAN HIEROGLYPH-143D8;Lo;0;L;;;;;N;;;;; +143D9;EGYPTIAN HIEROGLYPH-143D9;Lo;0;L;;;;;N;;;;; +143DA;EGYPTIAN HIEROGLYPH-143DA;Lo;0;L;;;;;N;;;;; +143DB;EGYPTIAN HIEROGLYPH-143DB;Lo;0;L;;;;;N;;;;; +143DC;EGYPTIAN HIEROGLYPH-143DC;Lo;0;L;;;;;N;;;;; +143DD;EGYPTIAN HIEROGLYPH-143DD;Lo;0;L;;;;;N;;;;; +143DE;EGYPTIAN HIEROGLYPH-143DE;Lo;0;L;;;;;N;;;;; +143DF;EGYPTIAN HIEROGLYPH-143DF;Lo;0;L;;;;;N;;;;; +143E0;EGYPTIAN HIEROGLYPH-143E0;Lo;0;L;;;;;N;;;;; +143E1;EGYPTIAN HIEROGLYPH-143E1;Lo;0;L;;;;;N;;;;; +143E2;EGYPTIAN HIEROGLYPH-143E2;Lo;0;L;;;;;N;;;;; +143E3;EGYPTIAN HIEROGLYPH-143E3;Lo;0;L;;;;;N;;;;; +143E4;EGYPTIAN HIEROGLYPH-143E4;Lo;0;L;;;;;N;;;;; +143E5;EGYPTIAN HIEROGLYPH-143E5;Lo;0;L;;;;;N;;;;; +143E6;EGYPTIAN HIEROGLYPH-143E6;Lo;0;L;;;;;N;;;;; +143E7;EGYPTIAN HIEROGLYPH-143E7;Lo;0;L;;;;;N;;;;; +143E8;EGYPTIAN HIEROGLYPH-143E8;Lo;0;L;;;;;N;;;;; +143E9;EGYPTIAN HIEROGLYPH-143E9;Lo;0;L;;;;;N;;;;; +143EA;EGYPTIAN HIEROGLYPH-143EA;Lo;0;L;;;;;N;;;;; +143EB;EGYPTIAN HIEROGLYPH-143EB;Lo;0;L;;;;;N;;;;; +143EC;EGYPTIAN HIEROGLYPH-143EC;Lo;0;L;;;;;N;;;;; +143ED;EGYPTIAN HIEROGLYPH-143ED;Lo;0;L;;;;;N;;;;; +143EE;EGYPTIAN HIEROGLYPH-143EE;Lo;0;L;;;;;N;;;;; +143EF;EGYPTIAN HIEROGLYPH-143EF;Lo;0;L;;;;;N;;;;; +143F0;EGYPTIAN HIEROGLYPH-143F0;Lo;0;L;;;;;N;;;;; +143F1;EGYPTIAN HIEROGLYPH-143F1;Lo;0;L;;;;;N;;;;; +143F2;EGYPTIAN HIEROGLYPH-143F2;Lo;0;L;;;;;N;;;;; +143F3;EGYPTIAN HIEROGLYPH-143F3;Lo;0;L;;;;;N;;;;; +143F4;EGYPTIAN HIEROGLYPH-143F4;Lo;0;L;;;;;N;;;;; +143F5;EGYPTIAN HIEROGLYPH-143F5;Lo;0;L;;;;;N;;;;; +143F6;EGYPTIAN HIEROGLYPH-143F6;Lo;0;L;;;;;N;;;;; +143F7;EGYPTIAN HIEROGLYPH-143F7;Lo;0;L;;;;;N;;;;; +143F8;EGYPTIAN HIEROGLYPH-143F8;Lo;0;L;;;;;N;;;;; +143F9;EGYPTIAN HIEROGLYPH-143F9;Lo;0;L;;;;;N;;;;; +143FA;EGYPTIAN HIEROGLYPH-143FA;Lo;0;L;;;;;N;;;;; 14400;ANATOLIAN HIEROGLYPH A001;Lo;0;L;;;;;N;;;;; 14401;ANATOLIAN HIEROGLYPH A002;Lo;0;L;;;;;N;;;;; 14402;ANATOLIAN HIEROGLYPH A003;Lo;0;L;;;;;N;;;;; @@ -24766,6 +29217,64 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 14644;ANATOLIAN HIEROGLYPH A528;Lo;0;L;;;;;N;;;;; 14645;ANATOLIAN HIEROGLYPH A529;Lo;0;L;;;;;N;;;;; 14646;ANATOLIAN HIEROGLYPH A530;Lo;0;L;;;;;N;;;;; +16100;GURUNG KHEMA LETTER A;Lo;0;L;;;;;N;;;;; +16101;GURUNG KHEMA LETTER KA;Lo;0;L;;;;;N;;;;; +16102;GURUNG KHEMA LETTER KHA;Lo;0;L;;;;;N;;;;; +16103;GURUNG KHEMA LETTER GA;Lo;0;L;;;;;N;;;;; +16104;GURUNG KHEMA LETTER GHA;Lo;0;L;;;;;N;;;;; +16105;GURUNG KHEMA LETTER NGA;Lo;0;L;;;;;N;;;;; +16106;GURUNG KHEMA LETTER CA;Lo;0;L;;;;;N;;;;; +16107;GURUNG KHEMA LETTER CHA;Lo;0;L;;;;;N;;;;; +16108;GURUNG KHEMA LETTER JA;Lo;0;L;;;;;N;;;;; +16109;GURUNG KHEMA LETTER JHA;Lo;0;L;;;;;N;;;;; +1610A;GURUNG KHEMA LETTER HA;Lo;0;L;;;;;N;;;;; +1610B;GURUNG KHEMA LETTER TTA;Lo;0;L;;;;;N;;;;; +1610C;GURUNG KHEMA LETTER TTHA;Lo;0;L;;;;;N;;;;; +1610D;GURUNG KHEMA LETTER DDA;Lo;0;L;;;;;N;;;;; +1610E;GURUNG KHEMA LETTER DDHA;Lo;0;L;;;;;N;;;;; +1610F;GURUNG KHEMA LETTER VA;Lo;0;L;;;;;N;;;;; +16110;GURUNG KHEMA LETTER TA;Lo;0;L;;;;;N;;;;; +16111;GURUNG KHEMA LETTER THA;Lo;0;L;;;;;N;;;;; +16112;GURUNG KHEMA LETTER DA;Lo;0;L;;;;;N;;;;; +16113;GURUNG KHEMA LETTER DHA;Lo;0;L;;;;;N;;;;; +16114;GURUNG KHEMA LETTER NA;Lo;0;L;;;;;N;;;;; +16115;GURUNG KHEMA LETTER PA;Lo;0;L;;;;;N;;;;; +16116;GURUNG KHEMA LETTER PHA;Lo;0;L;;;;;N;;;;; +16117;GURUNG KHEMA LETTER BA;Lo;0;L;;;;;N;;;;; +16118;GURUNG KHEMA LETTER BHA;Lo;0;L;;;;;N;;;;; +16119;GURUNG KHEMA LETTER MA;Lo;0;L;;;;;N;;;;; +1611A;GURUNG KHEMA LETTER YA;Lo;0;L;;;;;N;;;;; +1611B;GURUNG KHEMA LETTER RA;Lo;0;L;;;;;N;;;;; +1611C;GURUNG KHEMA LETTER LA;Lo;0;L;;;;;N;;;;; +1611D;GURUNG KHEMA LETTER SA;Lo;0;L;;;;;N;;;;; +1611E;GURUNG KHEMA VOWEL SIGN AA;Mn;0;NSM;;;;;N;;;;; +1611F;GURUNG KHEMA VOWEL SIGN I;Mn;0;NSM;;;;;N;;;;; +16120;GURUNG KHEMA VOWEL SIGN II;Mn;0;NSM;;;;;N;;;;; +16121;GURUNG KHEMA VOWEL SIGN U;Mn;0;NSM;1611E 1611E;;;;N;;;;; +16122;GURUNG KHEMA VOWEL SIGN UU;Mn;0;NSM;1611E 16129;;;;N;;;;; +16123;GURUNG KHEMA VOWEL SIGN E;Mn;0;NSM;1611E 1611F;;;;N;;;;; +16124;GURUNG KHEMA VOWEL SIGN EE;Mn;0;NSM;16129 1611F;;;;N;;;;; +16125;GURUNG KHEMA VOWEL SIGN AI;Mn;0;NSM;1611E 16120;;;;N;;;;; +16126;GURUNG KHEMA VOWEL SIGN O;Mn;0;NSM;16121 1611F;;;;N;;;;; +16127;GURUNG KHEMA VOWEL SIGN OO;Mn;0;NSM;16122 1611F;;;;N;;;;; +16128;GURUNG KHEMA VOWEL SIGN AU;Mn;0;NSM;16121 16120;;;;N;;;;; +16129;GURUNG KHEMA VOWEL LENGTH MARK;Mn;0;NSM;;;;;N;;;;; +1612A;GURUNG KHEMA CONSONANT SIGN MEDIAL YA;Mc;0;L;;;;;N;;;;; +1612B;GURUNG KHEMA CONSONANT SIGN MEDIAL VA;Mc;0;L;;;;;N;;;;; +1612C;GURUNG KHEMA CONSONANT SIGN MEDIAL HA;Mc;0;L;;;;;N;;;;; +1612D;GURUNG KHEMA SIGN ANUSVARA;Mn;0;NSM;;;;;N;;;;; +1612E;GURUNG KHEMA CONSONANT SIGN MEDIAL RA;Mn;0;NSM;;;;;N;;;;; +1612F;GURUNG KHEMA SIGN THOLHOMA;Mn;9;NSM;;;;;N;;;;; +16130;GURUNG KHEMA DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +16131;GURUNG KHEMA DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +16132;GURUNG KHEMA DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +16133;GURUNG KHEMA DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +16134;GURUNG KHEMA DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +16135;GURUNG KHEMA DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +16136;GURUNG KHEMA DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +16137;GURUNG KHEMA DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +16138;GURUNG KHEMA DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +16139;GURUNG KHEMA DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; 16800;BAMUM LETTER PHASE-A NGKUE MFON;Lo;0;L;;;;;N;;;;; 16801;BAMUM LETTER PHASE-A GBIEE FON;Lo;0;L;;;;;N;;;;; 16802;BAMUM LETTER PHASE-A PON MFON PIPAEMGBIEE;Lo;0;L;;;;;N;;;;; @@ -25630,6 +30139,64 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 16B8D;PAHAWH HMONG CLAN SIGN TSWB;Lo;0;L;;;;;N;;;;; 16B8E;PAHAWH HMONG CLAN SIGN KWM;Lo;0;L;;;;;N;;;;; 16B8F;PAHAWH HMONG CLAN SIGN VWJ;Lo;0;L;;;;;N;;;;; +16D40;KIRAT RAI SIGN ANUSVARA;Lm;0;L;;;;;N;;;;; +16D41;KIRAT RAI SIGN TONPI;Lm;0;L;;;;;N;;;;; +16D42;KIRAT RAI SIGN VISARGA;Lm;0;L;;;;;N;;;;; +16D43;KIRAT RAI LETTER A;Lo;0;L;;;;;N;;;;; +16D44;KIRAT RAI LETTER KA;Lo;0;L;;;;;N;;;;; +16D45;KIRAT RAI LETTER KHA;Lo;0;L;;;;;N;;;;; +16D46;KIRAT RAI LETTER GA;Lo;0;L;;;;;N;;;;; +16D47;KIRAT RAI LETTER GHA;Lo;0;L;;;;;N;;;;; +16D48;KIRAT RAI LETTER NGA;Lo;0;L;;;;;N;;;;; +16D49;KIRAT RAI LETTER CA;Lo;0;L;;;;;N;;;;; +16D4A;KIRAT RAI LETTER CHA;Lo;0;L;;;;;N;;;;; +16D4B;KIRAT RAI LETTER JA;Lo;0;L;;;;;N;;;;; +16D4C;KIRAT RAI LETTER JHA;Lo;0;L;;;;;N;;;;; +16D4D;KIRAT RAI LETTER NYA;Lo;0;L;;;;;N;;;;; +16D4E;KIRAT RAI LETTER TTA;Lo;0;L;;;;;N;;;;; +16D4F;KIRAT RAI LETTER TTHA;Lo;0;L;;;;;N;;;;; +16D50;KIRAT RAI LETTER DDA;Lo;0;L;;;;;N;;;;; +16D51;KIRAT RAI LETTER DDHA;Lo;0;L;;;;;N;;;;; +16D52;KIRAT RAI LETTER TA;Lo;0;L;;;;;N;;;;; +16D53;KIRAT RAI LETTER THA;Lo;0;L;;;;;N;;;;; +16D54;KIRAT RAI LETTER DA;Lo;0;L;;;;;N;;;;; +16D55;KIRAT RAI LETTER DHA;Lo;0;L;;;;;N;;;;; +16D56;KIRAT RAI LETTER NA;Lo;0;L;;;;;N;;;;; +16D57;KIRAT RAI LETTER PA;Lo;0;L;;;;;N;;;;; +16D58;KIRAT RAI LETTER PHA;Lo;0;L;;;;;N;;;;; +16D59;KIRAT RAI LETTER BA;Lo;0;L;;;;;N;;;;; +16D5A;KIRAT RAI LETTER BHA;Lo;0;L;;;;;N;;;;; +16D5B;KIRAT RAI LETTER MA;Lo;0;L;;;;;N;;;;; +16D5C;KIRAT RAI LETTER YA;Lo;0;L;;;;;N;;;;; +16D5D;KIRAT RAI LETTER RA;Lo;0;L;;;;;N;;;;; +16D5E;KIRAT RAI LETTER LA;Lo;0;L;;;;;N;;;;; +16D5F;KIRAT RAI LETTER VA;Lo;0;L;;;;;N;;;;; +16D60;KIRAT RAI LETTER SA;Lo;0;L;;;;;N;;;;; +16D61;KIRAT RAI LETTER SHA;Lo;0;L;;;;;N;;;;; +16D62;KIRAT RAI LETTER HA;Lo;0;L;;;;;N;;;;; +16D63;KIRAT RAI VOWEL SIGN AA;Lo;0;L;;;;;N;;;;; +16D64;KIRAT RAI VOWEL SIGN I;Lo;0;L;;;;;N;;;;; +16D65;KIRAT RAI VOWEL SIGN U;Lo;0;L;;;;;N;;;;; +16D66;KIRAT RAI VOWEL SIGN UE;Lo;0;L;;;;;N;;;;; +16D67;KIRAT RAI VOWEL SIGN E;Lo;0;L;;;;;N;;;;; +16D68;KIRAT RAI VOWEL SIGN AI;Lo;0;L;16D67 16D67;;;;N;;;;; +16D69;KIRAT RAI VOWEL SIGN O;Lo;0;L;16D63 16D67;;;;N;;;;; +16D6A;KIRAT RAI VOWEL SIGN AU;Lo;0;L;16D69 16D67;;;;N;;;;; +16D6B;KIRAT RAI SIGN VIRAMA;Lm;0;L;;;;;N;;;;; +16D6C;KIRAT RAI SIGN SAAT;Lm;0;L;;;;;N;;;;; +16D6D;KIRAT RAI SIGN YUPI;Po;0;L;;;;;N;;;;; +16D6E;KIRAT RAI DANDA;Po;0;L;;;;;N;;;;; +16D6F;KIRAT RAI DOUBLE DANDA;Po;0;L;;;;;N;;;;; +16D70;KIRAT RAI DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +16D71;KIRAT RAI DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +16D72;KIRAT RAI DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +16D73;KIRAT RAI DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +16D74;KIRAT RAI DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +16D75;KIRAT RAI DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +16D76;KIRAT RAI DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +16D77;KIRAT RAI DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +16D78;KIRAT RAI DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +16D79;KIRAT RAI DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; 16E40;MEDEFAIDRIN CAPITAL LETTER M;Lu;0;L;;;;;N;;;;16E60; 16E41;MEDEFAIDRIN CAPITAL LETTER S;Lu;0;L;;;;;N;;;;16E61; 16E42;MEDEFAIDRIN CAPITAL LETTER V;Lu;0;L;;;;;N;;;;16E62; @@ -25721,6 +30288,56 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 16E98;MEDEFAIDRIN FULL STOP;Po;0;L;;;;;N;;;;; 16E99;MEDEFAIDRIN SYMBOL AIVA;Po;0;L;;;;;N;;;;; 16E9A;MEDEFAIDRIN EXCLAMATION OH;Po;0;L;;;;;N;;;;; +16EA0;BERIA ERFE CAPITAL LETTER ARKAB;Lu;0;L;;;;;N;;;;16EBB; +16EA1;BERIA ERFE CAPITAL LETTER BASIGNA;Lu;0;L;;;;;N;;;;16EBC; +16EA2;BERIA ERFE CAPITAL LETTER DARBAI;Lu;0;L;;;;;N;;;;16EBD; +16EA3;BERIA ERFE CAPITAL LETTER EH;Lu;0;L;;;;;N;;;;16EBE; +16EA4;BERIA ERFE CAPITAL LETTER FITKO;Lu;0;L;;;;;N;;;;16EBF; +16EA5;BERIA ERFE CAPITAL LETTER GOWAY;Lu;0;L;;;;;N;;;;16EC0; +16EA6;BERIA ERFE CAPITAL LETTER HIRDEABO;Lu;0;L;;;;;N;;;;16EC1; +16EA7;BERIA ERFE CAPITAL LETTER I;Lu;0;L;;;;;N;;;;16EC2; +16EA8;BERIA ERFE CAPITAL LETTER DJAI;Lu;0;L;;;;;N;;;;16EC3; +16EA9;BERIA ERFE CAPITAL LETTER KOBO;Lu;0;L;;;;;N;;;;16EC4; +16EAA;BERIA ERFE CAPITAL LETTER LAKKO;Lu;0;L;;;;;N;;;;16EC5; +16EAB;BERIA ERFE CAPITAL LETTER MERI;Lu;0;L;;;;;N;;;;16EC6; +16EAC;BERIA ERFE CAPITAL LETTER NINI;Lu;0;L;;;;;N;;;;16EC7; +16EAD;BERIA ERFE CAPITAL LETTER GNA;Lu;0;L;;;;;N;;;;16EC8; +16EAE;BERIA ERFE CAPITAL LETTER NGAY;Lu;0;L;;;;;N;;;;16EC9; +16EAF;BERIA ERFE CAPITAL LETTER OI;Lu;0;L;;;;;N;;;;16ECA; +16EB0;BERIA ERFE CAPITAL LETTER PI;Lu;0;L;;;;;N;;;;16ECB; +16EB1;BERIA ERFE CAPITAL LETTER ERIGO;Lu;0;L;;;;;N;;;;16ECC; +16EB2;BERIA ERFE CAPITAL LETTER ERIGO TAMURA;Lu;0;L;;;;;N;;;;16ECD; +16EB3;BERIA ERFE CAPITAL LETTER SERI;Lu;0;L;;;;;N;;;;16ECE; +16EB4;BERIA ERFE CAPITAL LETTER SHEP;Lu;0;L;;;;;N;;;;16ECF; +16EB5;BERIA ERFE CAPITAL LETTER TATASOUE;Lu;0;L;;;;;N;;;;16ED0; +16EB6;BERIA ERFE CAPITAL LETTER UI;Lu;0;L;;;;;N;;;;16ED1; +16EB7;BERIA ERFE CAPITAL LETTER WASSE;Lu;0;L;;;;;N;;;;16ED2; +16EB8;BERIA ERFE CAPITAL LETTER AY;Lu;0;L;;;;;N;;;;16ED3; +16EBB;BERIA ERFE SMALL LETTER ARKAB;Ll;0;L;;;;;N;;;16EA0;;16EA0 +16EBC;BERIA ERFE SMALL LETTER BASIGNA;Ll;0;L;;;;;N;;;16EA1;;16EA1 +16EBD;BERIA ERFE SMALL LETTER DARBAI;Ll;0;L;;;;;N;;;16EA2;;16EA2 +16EBE;BERIA ERFE SMALL LETTER EH;Ll;0;L;;;;;N;;;16EA3;;16EA3 +16EBF;BERIA ERFE SMALL LETTER FITKO;Ll;0;L;;;;;N;;;16EA4;;16EA4 +16EC0;BERIA ERFE SMALL LETTER GOWAY;Ll;0;L;;;;;N;;;16EA5;;16EA5 +16EC1;BERIA ERFE SMALL LETTER HIRDEABO;Ll;0;L;;;;;N;;;16EA6;;16EA6 +16EC2;BERIA ERFE SMALL LETTER I;Ll;0;L;;;;;N;;;16EA7;;16EA7 +16EC3;BERIA ERFE SMALL LETTER DJAI;Ll;0;L;;;;;N;;;16EA8;;16EA8 +16EC4;BERIA ERFE SMALL LETTER KOBO;Ll;0;L;;;;;N;;;16EA9;;16EA9 +16EC5;BERIA ERFE SMALL LETTER LAKKO;Ll;0;L;;;;;N;;;16EAA;;16EAA +16EC6;BERIA ERFE SMALL LETTER MERI;Ll;0;L;;;;;N;;;16EAB;;16EAB +16EC7;BERIA ERFE SMALL LETTER NINI;Ll;0;L;;;;;N;;;16EAC;;16EAC +16EC8;BERIA ERFE SMALL LETTER GNA;Ll;0;L;;;;;N;;;16EAD;;16EAD +16EC9;BERIA ERFE SMALL LETTER NGAY;Ll;0;L;;;;;N;;;16EAE;;16EAE +16ECA;BERIA ERFE SMALL LETTER OI;Ll;0;L;;;;;N;;;16EAF;;16EAF +16ECB;BERIA ERFE SMALL LETTER PI;Ll;0;L;;;;;N;;;16EB0;;16EB0 +16ECC;BERIA ERFE SMALL LETTER ERIGO;Ll;0;L;;;;;N;;;16EB1;;16EB1 +16ECD;BERIA ERFE SMALL LETTER ERIGO TAMURA;Ll;0;L;;;;;N;;;16EB2;;16EB2 +16ECE;BERIA ERFE SMALL LETTER SERI;Ll;0;L;;;;;N;;;16EB3;;16EB3 +16ECF;BERIA ERFE SMALL LETTER SHEP;Ll;0;L;;;;;N;;;16EB4;;16EB4 +16ED0;BERIA ERFE SMALL LETTER TATASOUE;Ll;0;L;;;;;N;;;16EB5;;16EB5 +16ED1;BERIA ERFE SMALL LETTER UI;Ll;0;L;;;;;N;;;16EB6;;16EB6 +16ED2;BERIA ERFE SMALL LETTER WASSE;Ll;0;L;;;;;N;;;16EB7;;16EB7 +16ED3;BERIA ERFE SMALL LETTER AY;Ll;0;L;;;;;N;;;16EB8;;16EB8 16F00;MIAO LETTER PA;Lo;0;L;;;;;N;;;;; 16F01;MIAO LETTER BA;Lo;0;L;;;;;N;;;;; 16F02;MIAO LETTER YI PA;Lo;0;L;;;;;N;;;;; @@ -25877,8 +30494,13 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 16FE4;KHITAN SMALL SCRIPT FILLER;Mn;0;NSM;;;;;N;;;;; 16FF0;VIETNAMESE ALTERNATE READING MARK CA;Mc;6;L;;;;;N;;;;; 16FF1;VIETNAMESE ALTERNATE READING MARK NHAY;Mc;6;L;;;;;N;;;;; +16FF2;CHINESE SMALL SIMPLIFIED ER;Lm;0;L;;;;;N;;;;; +16FF3;CHINESE SMALL TRADITIONAL ER;Lm;0;L;;;;;N;;;;; +16FF4;YANGQIN SIGN SLOW ONE BEAT;Nl;0;L;;;;1;N;;;;; +16FF5;YANGQIN SIGN SLOW THREE HALF BEATS;Nl;0;L;;;;3/2;N;;;;; +16FF6;YANGQIN SIGN SLOW TWO BEATS;Nl;0;L;;;;2;N;;;;; 17000;;Lo;0;L;;;;;N;;;;; -187F7;;Lo;0;L;;;;;N;;;;; +187FF;;Lo;0;L;;;;;N;;;;; 18800;TANGUT COMPONENT-001;Lo;0;L;;;;;N;;;;; 18801;TANGUT COMPONENT-002;Lo;0;L;;;;;N;;;;; 18802;TANGUT COMPONENT-003;Lo;0;L;;;;;N;;;;; @@ -27117,8 +31739,124 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 18CD3;KHITAN SMALL SCRIPT CHARACTER-18CD3;Lo;0;L;;;;;N;;;;; 18CD4;KHITAN SMALL SCRIPT CHARACTER-18CD4;Lo;0;L;;;;;N;;;;; 18CD5;KHITAN SMALL SCRIPT CHARACTER-18CD5;Lo;0;L;;;;;N;;;;; +18CFF;KHITAN SMALL SCRIPT CHARACTER-18CFF;Lo;0;L;;;;;N;;;;; 18D00;;Lo;0;L;;;;;N;;;;; -18D08;;Lo;0;L;;;;;N;;;;; +18D1E;;Lo;0;L;;;;;N;;;;; +18D80;TANGUT COMPONENT-769;Lo;0;L;;;;;N;;;;; +18D81;TANGUT COMPONENT-770;Lo;0;L;;;;;N;;;;; +18D82;TANGUT COMPONENT-771;Lo;0;L;;;;;N;;;;; +18D83;TANGUT COMPONENT-772;Lo;0;L;;;;;N;;;;; +18D84;TANGUT COMPONENT-773;Lo;0;L;;;;;N;;;;; +18D85;TANGUT COMPONENT-774;Lo;0;L;;;;;N;;;;; +18D86;TANGUT COMPONENT-775;Lo;0;L;;;;;N;;;;; +18D87;TANGUT COMPONENT-776;Lo;0;L;;;;;N;;;;; +18D88;TANGUT COMPONENT-777;Lo;0;L;;;;;N;;;;; +18D89;TANGUT COMPONENT-778;Lo;0;L;;;;;N;;;;; +18D8A;TANGUT COMPONENT-779;Lo;0;L;;;;;N;;;;; +18D8B;TANGUT COMPONENT-780;Lo;0;L;;;;;N;;;;; +18D8C;TANGUT COMPONENT-781;Lo;0;L;;;;;N;;;;; +18D8D;TANGUT COMPONENT-782;Lo;0;L;;;;;N;;;;; +18D8E;TANGUT COMPONENT-783;Lo;0;L;;;;;N;;;;; +18D8F;TANGUT COMPONENT-784;Lo;0;L;;;;;N;;;;; +18D90;TANGUT COMPONENT-785;Lo;0;L;;;;;N;;;;; +18D91;TANGUT COMPONENT-786;Lo;0;L;;;;;N;;;;; +18D92;TANGUT COMPONENT-787;Lo;0;L;;;;;N;;;;; +18D93;TANGUT COMPONENT-788;Lo;0;L;;;;;N;;;;; +18D94;TANGUT COMPONENT-789;Lo;0;L;;;;;N;;;;; +18D95;TANGUT COMPONENT-790;Lo;0;L;;;;;N;;;;; +18D96;TANGUT COMPONENT-791;Lo;0;L;;;;;N;;;;; +18D97;TANGUT COMPONENT-792;Lo;0;L;;;;;N;;;;; +18D98;TANGUT COMPONENT-793;Lo;0;L;;;;;N;;;;; +18D99;TANGUT COMPONENT-794;Lo;0;L;;;;;N;;;;; +18D9A;TANGUT COMPONENT-795;Lo;0;L;;;;;N;;;;; +18D9B;TANGUT COMPONENT-796;Lo;0;L;;;;;N;;;;; +18D9C;TANGUT COMPONENT-797;Lo;0;L;;;;;N;;;;; +18D9D;TANGUT COMPONENT-798;Lo;0;L;;;;;N;;;;; +18D9E;TANGUT COMPONENT-799;Lo;0;L;;;;;N;;;;; +18D9F;TANGUT COMPONENT-800;Lo;0;L;;;;;N;;;;; +18DA0;TANGUT COMPONENT-801;Lo;0;L;;;;;N;;;;; +18DA1;TANGUT COMPONENT-802;Lo;0;L;;;;;N;;;;; +18DA2;TANGUT COMPONENT-803;Lo;0;L;;;;;N;;;;; +18DA3;TANGUT COMPONENT-804;Lo;0;L;;;;;N;;;;; +18DA4;TANGUT COMPONENT-805;Lo;0;L;;;;;N;;;;; +18DA5;TANGUT COMPONENT-806;Lo;0;L;;;;;N;;;;; +18DA6;TANGUT COMPONENT-807;Lo;0;L;;;;;N;;;;; +18DA7;TANGUT COMPONENT-808;Lo;0;L;;;;;N;;;;; +18DA8;TANGUT COMPONENT-809;Lo;0;L;;;;;N;;;;; +18DA9;TANGUT COMPONENT-810;Lo;0;L;;;;;N;;;;; +18DAA;TANGUT COMPONENT-811;Lo;0;L;;;;;N;;;;; +18DAB;TANGUT COMPONENT-812;Lo;0;L;;;;;N;;;;; +18DAC;TANGUT COMPONENT-813;Lo;0;L;;;;;N;;;;; +18DAD;TANGUT COMPONENT-814;Lo;0;L;;;;;N;;;;; +18DAE;TANGUT COMPONENT-815;Lo;0;L;;;;;N;;;;; +18DAF;TANGUT COMPONENT-816;Lo;0;L;;;;;N;;;;; +18DB0;TANGUT COMPONENT-817;Lo;0;L;;;;;N;;;;; +18DB1;TANGUT COMPONENT-818;Lo;0;L;;;;;N;;;;; +18DB2;TANGUT COMPONENT-819;Lo;0;L;;;;;N;;;;; +18DB3;TANGUT COMPONENT-820;Lo;0;L;;;;;N;;;;; +18DB4;TANGUT COMPONENT-821;Lo;0;L;;;;;N;;;;; +18DB5;TANGUT COMPONENT-822;Lo;0;L;;;;;N;;;;; +18DB6;TANGUT COMPONENT-823;Lo;0;L;;;;;N;;;;; +18DB7;TANGUT COMPONENT-824;Lo;0;L;;;;;N;;;;; +18DB8;TANGUT COMPONENT-825;Lo;0;L;;;;;N;;;;; +18DB9;TANGUT COMPONENT-826;Lo;0;L;;;;;N;;;;; +18DBA;TANGUT COMPONENT-827;Lo;0;L;;;;;N;;;;; +18DBB;TANGUT COMPONENT-828;Lo;0;L;;;;;N;;;;; +18DBC;TANGUT COMPONENT-829;Lo;0;L;;;;;N;;;;; +18DBD;TANGUT COMPONENT-830;Lo;0;L;;;;;N;;;;; +18DBE;TANGUT COMPONENT-831;Lo;0;L;;;;;N;;;;; +18DBF;TANGUT COMPONENT-832;Lo;0;L;;;;;N;;;;; +18DC0;TANGUT COMPONENT-833;Lo;0;L;;;;;N;;;;; +18DC1;TANGUT COMPONENT-834;Lo;0;L;;;;;N;;;;; +18DC2;TANGUT COMPONENT-835;Lo;0;L;;;;;N;;;;; +18DC3;TANGUT COMPONENT-836;Lo;0;L;;;;;N;;;;; +18DC4;TANGUT COMPONENT-837;Lo;0;L;;;;;N;;;;; +18DC5;TANGUT COMPONENT-838;Lo;0;L;;;;;N;;;;; +18DC6;TANGUT COMPONENT-839;Lo;0;L;;;;;N;;;;; +18DC7;TANGUT COMPONENT-840;Lo;0;L;;;;;N;;;;; +18DC8;TANGUT COMPONENT-841;Lo;0;L;;;;;N;;;;; +18DC9;TANGUT COMPONENT-842;Lo;0;L;;;;;N;;;;; +18DCA;TANGUT COMPONENT-843;Lo;0;L;;;;;N;;;;; +18DCB;TANGUT COMPONENT-844;Lo;0;L;;;;;N;;;;; +18DCC;TANGUT COMPONENT-845;Lo;0;L;;;;;N;;;;; +18DCD;TANGUT COMPONENT-846;Lo;0;L;;;;;N;;;;; +18DCE;TANGUT COMPONENT-847;Lo;0;L;;;;;N;;;;; +18DCF;TANGUT COMPONENT-848;Lo;0;L;;;;;N;;;;; +18DD0;TANGUT COMPONENT-849;Lo;0;L;;;;;N;;;;; +18DD1;TANGUT COMPONENT-850;Lo;0;L;;;;;N;;;;; +18DD2;TANGUT COMPONENT-851;Lo;0;L;;;;;N;;;;; +18DD3;TANGUT COMPONENT-852;Lo;0;L;;;;;N;;;;; +18DD4;TANGUT COMPONENT-853;Lo;0;L;;;;;N;;;;; +18DD5;TANGUT COMPONENT-854;Lo;0;L;;;;;N;;;;; +18DD6;TANGUT COMPONENT-855;Lo;0;L;;;;;N;;;;; +18DD7;TANGUT COMPONENT-856;Lo;0;L;;;;;N;;;;; +18DD8;TANGUT COMPONENT-857;Lo;0;L;;;;;N;;;;; +18DD9;TANGUT COMPONENT-858;Lo;0;L;;;;;N;;;;; +18DDA;TANGUT COMPONENT-859;Lo;0;L;;;;;N;;;;; +18DDB;TANGUT COMPONENT-860;Lo;0;L;;;;;N;;;;; +18DDC;TANGUT COMPONENT-861;Lo;0;L;;;;;N;;;;; +18DDD;TANGUT COMPONENT-862;Lo;0;L;;;;;N;;;;; +18DDE;TANGUT COMPONENT-863;Lo;0;L;;;;;N;;;;; +18DDF;TANGUT COMPONENT-864;Lo;0;L;;;;;N;;;;; +18DE0;TANGUT COMPONENT-865;Lo;0;L;;;;;N;;;;; +18DE1;TANGUT COMPONENT-866;Lo;0;L;;;;;N;;;;; +18DE2;TANGUT COMPONENT-867;Lo;0;L;;;;;N;;;;; +18DE3;TANGUT COMPONENT-868;Lo;0;L;;;;;N;;;;; +18DE4;TANGUT COMPONENT-869;Lo;0;L;;;;;N;;;;; +18DE5;TANGUT COMPONENT-870;Lo;0;L;;;;;N;;;;; +18DE6;TANGUT COMPONENT-871;Lo;0;L;;;;;N;;;;; +18DE7;TANGUT COMPONENT-872;Lo;0;L;;;;;N;;;;; +18DE8;TANGUT COMPONENT-873;Lo;0;L;;;;;N;;;;; +18DE9;TANGUT COMPONENT-874;Lo;0;L;;;;;N;;;;; +18DEA;TANGUT COMPONENT-875;Lo;0;L;;;;;N;;;;; +18DEB;TANGUT COMPONENT-876;Lo;0;L;;;;;N;;;;; +18DEC;TANGUT COMPONENT-877;Lo;0;L;;;;;N;;;;; +18DED;TANGUT COMPONENT-878;Lo;0;L;;;;;N;;;;; +18DEE;TANGUT COMPONENT-879;Lo;0;L;;;;;N;;;;; +18DEF;TANGUT COMPONENT-880;Lo;0;L;;;;;N;;;;; +18DF0;TANGUT COMPONENT-881;Lo;0;L;;;;;N;;;;; +18DF1;TANGUT COMPONENT-882;Lo;0;L;;;;;N;;;;; +18DF2;TANGUT COMPONENT-883;Lo;0;L;;;;;N;;;;; 1AFF0;KATAKANA LETTER MINNAN TONE-2;Lm;0;L;;;;;N;;;;; 1AFF1;KATAKANA LETTER MINNAN TONE-3;Lm;0;L;;;;;N;;;;; 1AFF2;KATAKANA LETTER MINNAN TONE-4;Lm;0;L;;;;;N;;;;; @@ -27975,6 +32713,735 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1BCA1;SHORTHAND FORMAT CONTINUING OVERLAP;Cf;0;BN;;;;;N;;;;; 1BCA2;SHORTHAND FORMAT DOWN STEP;Cf;0;BN;;;;;N;;;;; 1BCA3;SHORTHAND FORMAT UP STEP;Cf;0;BN;;;;;N;;;;; +1CC00;UP-POINTING GO-KART;So;0;ON;;;;;N;;;;; +1CC01;RIGHT-POINTING GO-KART;So;0;ON;;;;;N;;;;; +1CC02;LEFT-POINTING STICK FIGURE;So;0;ON;;;;;N;;;;; +1CC03;RIGHT-POINTING STICK FIGURE;So;0;ON;;;;;N;;;;; +1CC04;DOWN-POINTING STICK FIGURE;So;0;ON;;;;;N;;;;; +1CC05;LOWER HORIZONTAL RULER SEGMENT;So;0;ON;;;;;N;;;;; +1CC06;RIGHT VERTICAL RULER SEGMENT;So;0;ON;;;;;N;;;;; +1CC07;LOWER RIGHT RULER SEGMENT;So;0;ON;;;;;N;;;;; +1CC08;ANTENNA;So;0;ON;;;;;N;;;;; +1CC09;HORIZONTAL RESISTOR SEGMENT;So;0;ON;;;;;N;;;;; +1CC0A;VERTICAL RESISTOR SEGMENT;So;0;ON;;;;;N;;;;; +1CC0B;LEFT THIRD INDUCTOR;So;0;ON;;;;;N;;;;; +1CC0C;MIDDLE THIRD INDUCTOR;So;0;ON;;;;;N;;;;; +1CC0D;RIGHT THIRD INDUCTOR;So;0;ON;;;;;N;;;;; +1CC0E;LEFT-POINTING DIODE;So;0;ON;;;;;N;;;;; +1CC0F;RIGHT-POINTING DIODE;So;0;ON;;;;;N;;;;; +1CC10;NPN TRANSISTOR;So;0;ON;;;;;N;;;;; +1CC11;PNP TRANSISTOR;So;0;ON;;;;;N;;;;; +1CC12;RECEPTACLE;So;0;ON;;;;;N;;;;; +1CC13;HORIZONTAL CAPACITOR;So;0;ON;;;;;N;;;;; +1CC14;VERTICAL CAPACITOR;So;0;ON;;;;;N;;;;; +1CC15;LOGIC GATE OR;So;0;ON;;;;;N;;;;; +1CC16;LOGIC GATE AND;So;0;ON;;;;;N;;;;; +1CC17;LOGIC GATE INVERTED INPUTS;So;0;ON;;;;;N;;;;; +1CC18;LOGIC GATE INVERTED OUTPUT;So;0;ON;;;;;N;;;;; +1CC19;LOGIC GATE BUFFER;So;0;ON;;;;;N;;;;; +1CC1A;LOGIC GATE BUFFER WITH INVERTED INPUT;So;0;ON;;;;;N;;;;; +1CC1B;BOX DRAWINGS LIGHT HORIZONTAL AND UPPER RIGHT;So;0;ON;;;;;N;;;;; +1CC1C;BOX DRAWINGS LIGHT HORIZONTAL AND LOWER RIGHT;So;0;ON;;;;;N;;;;; +1CC1D;BOX DRAWINGS LIGHT TOP AND UPPER LEFT;So;0;ON;;;;;N;;;;; +1CC1E;BOX DRAWINGS LIGHT BOTTOM AND LOWER LEFT;So;0;ON;;;;;N;;;;; +1CC1F;BOX DRAWINGS DOUBLE DIAGONAL UPPER RIGHT TO LOWER LEFT;So;0;ON;;;;;N;;;;; +1CC20;BOX DRAWINGS DOUBLE DIAGONAL UPPER LEFT TO LOWER RIGHT;So;0;ON;;;;;N;;;;; +1CC21;SEPARATED BLOCK QUADRANT-1;So;0;ON;;;;;N;;;;; +1CC22;SEPARATED BLOCK QUADRANT-2;So;0;ON;;;;;N;;;;; +1CC23;SEPARATED BLOCK QUADRANT-12;So;0;ON;;;;;N;;;;; +1CC24;SEPARATED BLOCK QUADRANT-3;So;0;ON;;;;;N;;;;; +1CC25;SEPARATED BLOCK QUADRANT-13;So;0;ON;;;;;N;;;;; +1CC26;SEPARATED BLOCK QUADRANT-23;So;0;ON;;;;;N;;;;; +1CC27;SEPARATED BLOCK QUADRANT-123;So;0;ON;;;;;N;;;;; +1CC28;SEPARATED BLOCK QUADRANT-4;So;0;ON;;;;;N;;;;; +1CC29;SEPARATED BLOCK QUADRANT-14;So;0;ON;;;;;N;;;;; +1CC2A;SEPARATED BLOCK QUADRANT-24;So;0;ON;;;;;N;;;;; +1CC2B;SEPARATED BLOCK QUADRANT-124;So;0;ON;;;;;N;;;;; +1CC2C;SEPARATED BLOCK QUADRANT-34;So;0;ON;;;;;N;;;;; +1CC2D;SEPARATED BLOCK QUADRANT-134;So;0;ON;;;;;N;;;;; +1CC2E;SEPARATED BLOCK QUADRANT-234;So;0;ON;;;;;N;;;;; +1CC2F;SEPARATED BLOCK QUADRANT-1234;So;0;ON;;;;;N;;;;; +1CC30;UPPER LEFT TWELFTH CIRCLE;So;0;ON;;;;;N;;;;; +1CC31;UPPER CENTRE LEFT TWELFTH CIRCLE;So;0;ON;;;;;N;;;;; +1CC32;UPPER CENTRE RIGHT TWELFTH CIRCLE;So;0;ON;;;;;N;;;;; +1CC33;UPPER RIGHT TWELFTH CIRCLE;So;0;ON;;;;;N;;;;; +1CC34;UPPER MIDDLE LEFT TWELFTH CIRCLE;So;0;ON;;;;;N;;;;; +1CC35;UPPER LEFT QUARTER CIRCLE;So;0;ON;;;;;N;;;;; +1CC36;UPPER RIGHT QUARTER CIRCLE;So;0;ON;;;;;N;;;;; +1CC37;UPPER MIDDLE RIGHT TWELFTH CIRCLE;So;0;ON;;;;;N;;;;; +1CC38;LOWER MIDDLE LEFT TWELFTH CIRCLE;So;0;ON;;;;;N;;;;; +1CC39;LOWER LEFT QUARTER CIRCLE;So;0;ON;;;;;N;;;;; +1CC3A;LOWER RIGHT QUARTER CIRCLE;So;0;ON;;;;;N;;;;; +1CC3B;LOWER MIDDLE RIGHT TWELFTH CIRCLE;So;0;ON;;;;;N;;;;; +1CC3C;LOWER LEFT TWELFTH CIRCLE;So;0;ON;;;;;N;;;;; +1CC3D;LOWER CENTRE LEFT TWELFTH CIRCLE;So;0;ON;;;;;N;;;;; +1CC3E;LOWER CENTRE RIGHT TWELFTH CIRCLE;So;0;ON;;;;;N;;;;; +1CC3F;LOWER RIGHT TWELFTH CIRCLE;So;0;ON;;;;;N;;;;; +1CC40;SPARSE HORIZONTAL FILL;So;0;ON;;;;;N;;;;; +1CC41;SPARSE VERTICAL FILL;So;0;ON;;;;;N;;;;; +1CC42;ORTHOGONAL CROSSHATCH FILL;So;0;ON;;;;;N;;;;; +1CC43;DIAGONAL CROSSHATCH FILL;So;0;ON;;;;;N;;;;; +1CC44;DENSE VERTICAL FILL;So;0;ON;;;;;N;;;;; +1CC45;DENSE HORIZONTAL FILL;So;0;ON;;;;;N;;;;; +1CC46;SPECKLE FILL FRAME-1;So;0;ON;;;;;N;;;;; +1CC47;SPECKLE FILL FRAME-2;So;0;ON;;;;;N;;;;; +1CC48;LEFT-FACING BASSINET;So;0;ON;;;;;N;;;;; +1CC49;RIGHT-FACING BASSINET;So;0;ON;;;;;N;;;;; +1CC4A;FLYING SAUCER WITH BEAMS;So;0;ON;;;;;N;;;;; +1CC4B;FLYING SAUCER WITHOUT BEAMS;So;0;ON;;;;;N;;;;; +1CC4C;ALIEN MONSTER OPEN JAWS;So;0;ON;;;;;N;;;;; +1CC4D;ALIEN MONSTER CLOSED JAWS;So;0;ON;;;;;N;;;;; +1CC4E;ALIEN SQUID OPEN TENTACLES;So;0;ON;;;;;N;;;;; +1CC4F;ALIEN SQUID CLOSED TENTACLES;So;0;ON;;;;;N;;;;; +1CC50;ALIEN CRAB STEPPING RIGHT;So;0;ON;;;;;N;;;;; +1CC51;ALIEN CRAB STEPPING LEFT;So;0;ON;;;;;N;;;;; +1CC52;ALIEN SPIDER CROUCHING;So;0;ON;;;;;N;;;;; +1CC53;ALIEN SPIDER SPREAD;So;0;ON;;;;;N;;;;; +1CC54;ALIEN MONSTER STEP-1;So;0;ON;;;;;N;;;;; +1CC55;ALIEN MONSTER STEP-2;So;0;ON;;;;;N;;;;; +1CC56;LEFT-POINTING ROCKET SHIP;So;0;ON;;;;;N;;;;; +1CC57;UP-POINTING ROCKET SHIP;So;0;ON;;;;;N;;;;; +1CC58;RIGHT-POINTING ROCKET SHIP;So;0;ON;;;;;N;;;;; +1CC59;DOWN-POINTING ROCKET SHIP;So;0;ON;;;;;N;;;;; +1CC5A;TOP HALF LEFT-FACING ROBOT;So;0;ON;;;;;N;;;;; +1CC5B;TOP HALF FORWARD-FACING ROBOT;So;0;ON;;;;;N;;;;; +1CC5C;TOP HALF RIGHT-FACING ROBOT;So;0;ON;;;;;N;;;;; +1CC5D;BOTTOM HALF LEFT-FACING ROBOT;So;0;ON;;;;;N;;;;; +1CC5E;BOTTOM HALF FORWARD-FACING ROBOT;So;0;ON;;;;;N;;;;; +1CC5F;BOTTOM HALF RIGHT-FACING ROBOT;So;0;ON;;;;;N;;;;; +1CC60;LEFT-POINTING ATOMIC BOMB;So;0;ON;;;;;N;;;;; +1CC61;UP-POINTING ATOMIC BOMB;So;0;ON;;;;;N;;;;; +1CC62;RIGHT-POINTING ATOMIC BOMB;So;0;ON;;;;;N;;;;; +1CC63;DOWN-POINTING ATOMIC BOMB;So;0;ON;;;;;N;;;;; +1CC64;MUSHROOM CLOUD;So;0;ON;;;;;N;;;;; +1CC65;LEFT-POINTING RIFLE;So;0;ON;;;;;N;;;;; +1CC66;UP-POINTING RIFLE;So;0;ON;;;;;N;;;;; +1CC67;RIGHT-POINTING RIFLE;So;0;ON;;;;;N;;;;; +1CC68;DOWN-POINTING RIFLE;So;0;ON;;;;;N;;;;; +1CC69;EIGHT RAYS INWARD;So;0;ON;;;;;N;;;;; +1CC6A;EIGHT RAYS OUTWARD;So;0;ON;;;;;N;;;;; +1CC6B;BLACK LARGE CIRCLE MINUS LEFT QUARTER SECTION;So;0;ON;;;;;N;;;;; +1CC6C;BLACK LARGE CIRCLE MINUS UPPER QUARTER SECTION;So;0;ON;;;;;N;;;;; +1CC6D;BLACK LARGE CIRCLE MINUS RIGHT QUARTER SECTION;So;0;ON;;;;;N;;;;; +1CC6E;BLACK LARGE CIRCLE MINUS LOWER QUARTER SECTION;So;0;ON;;;;;N;;;;; +1CC6F;BLACK NEUTRAL FACE;So;0;ON;;;;;N;;;;; +1CC70;LEFT-FACING SNAKE HEAD WITH OPEN MOUTH;So;0;ON;;;;;N;;;;; +1CC71;UP-FACING SNAKE HEAD WITH OPEN MOUTH;So;0;ON;;;;;N;;;;; +1CC72;RIGHT-FACING SNAKE HEAD WITH OPEN MOUTH;So;0;ON;;;;;N;;;;; +1CC73;DOWN-FACING SNAKE HEAD WITH OPEN MOUTH;So;0;ON;;;;;N;;;;; +1CC74;LEFT-FACING SNAKE HEAD WITH CLOSED MOUTH;So;0;ON;;;;;N;;;;; +1CC75;UP-FACING SNAKE HEAD WITH CLOSED MOUTH;So;0;ON;;;;;N;;;;; +1CC76;RIGHT-FACING SNAKE HEAD WITH CLOSED MOUTH;So;0;ON;;;;;N;;;;; +1CC77;DOWN-FACING SNAKE HEAD WITH CLOSED MOUTH;So;0;ON;;;;;N;;;;; +1CC78;LEFT-POINTING ENERGY WAVE;So;0;ON;;;;;N;;;;; +1CC79;UP-POINTING ENERGY WAVE;So;0;ON;;;;;N;;;;; +1CC7A;RIGHT-POINTING ENERGY WAVE;So;0;ON;;;;;N;;;;; +1CC7B;DOWN-POINTING ENERGY WAVE;So;0;ON;;;;;N;;;;; +1CC7C;SQUARE SPIRAL FROM TOP LEFT;So;0;ON;;;;;N;;;;; +1CC7D;SQUARE SPIRAL FROM TOP RIGHT;So;0;ON;;;;;N;;;;; +1CC7E;SQUARE SPIRAL FROM BOTTOM RIGHT;So;0;ON;;;;;N;;;;; +1CC7F;SQUARE SPIRAL FROM BOTTOM LEFT;So;0;ON;;;;;N;;;;; +1CC80;STRIPED LEFT-POINTING TRIANGLE;So;0;ON;;;;;N;;;;; +1CC81;STRIPED UP-POINTING TRIANGLE;So;0;ON;;;;;N;;;;; +1CC82;STRIPED RIGHT-POINTING TRIANGLE;So;0;ON;;;;;N;;;;; +1CC83;STRIPED DOWN-POINTING TRIANGLE;So;0;ON;;;;;N;;;;; +1CC84;VERTICAL LADDER;So;0;ON;;;;;N;;;;; +1CC85;HORIZONTAL LADDER;So;0;ON;;;;;N;;;;; +1CC86;WHITE LOWER LEFT POINTER;So;0;ON;;;;;N;;;;; +1CC87;WHITE LOWER RIGHT POINTER;So;0;ON;;;;;N;;;;; +1CC88;TWO RINGS ALIGNED HORIZONTALLY;So;0;ON;;;;;N;;;;; +1CC89;SQUARE FOUR CORNER SALTIRES;So;0;ON;;;;;N;;;;; +1CC8A;SQUARE FOUR CORNER DIAGONALS;So;0;ON;;;;;N;;;;; +1CC8B;SQUARE FOUR CORNER BLACK TRIANGLES;So;0;ON;;;;;N;;;;; +1CC8C;SQUARE APERTURE;So;0;ON;;;;;N;;;;; +1CC8D;INVERSE BLACK DIAMOND;So;0;ON;;;;;N;;;;; +1CC8E;LEFT AND UPPER ONE EIGHTH BLOCK CONTAINING BLACK SMALL SQUARE;So;0;ON;;;;;N;;;;; +1CC8F;INVERSE BLACK SMALL SQUARE;So;0;ON;;;;;N;;;;; +1CC90;VERTICAL LINE WITH FOUR TICK MARKS;So;0;ON;;;;;N;;;;; +1CC91;HORIZONTAL LINE WITH FOUR TICK MARKS;So;0;ON;;;;;N;;;;; +1CC92;LEFT-FACING FISH;So;0;ON;;;;;N;;;;; +1CC93;RIGHT-FACING FISH;So;0;ON;;;;;N;;;;; +1CC94;LEFT-FACING FISH WITH OPEN MOUTH;So;0;ON;;;;;N;;;;; +1CC95;RIGHT-FACING FISH WITH OPEN MOUTH;So;0;ON;;;;;N;;;;; +1CC96;FLAPPING BIRD;So;0;ON;;;;;N;;;;; +1CC97;LEFT-POINTING RACING CAR;So;0;ON;;;;;N;;;;; +1CC98;UP-POINTING RACING CAR;So;0;ON;;;;;N;;;;; +1CC99;RIGHT-POINTING RACING CAR;So;0;ON;;;;;N;;;;; +1CC9A;DOWN-POINTING RACING CAR;So;0;ON;;;;;N;;;;; +1CC9B;HORIZONTAL RACING CAR;So;0;ON;;;;;N;;;;; +1CC9C;VERTICAL RACING CAR;So;0;ON;;;;;N;;;;; +1CC9D;VERTICAL GO-KART;So;0;ON;;;;;N;;;;; +1CC9E;LEFT-POINTING TANK;So;0;ON;;;;;N;;;;; +1CC9F;RIGHT-POINTING TANK;So;0;ON;;;;;N;;;;; +1CCA0;LEFT-POINTING ROCKET BOOSTER;So;0;ON;;;;;N;;;;; +1CCA1;RIGHT-POINTING ROCKET BOOSTER;So;0;ON;;;;;N;;;;; +1CCA2;LEFT-POINTING ROLLER COASTER CAR;So;0;ON;;;;;N;;;;; +1CCA3;RIGHT-POINTING ROLLER COASTER CAR;So;0;ON;;;;;N;;;;; +1CCA4;LEFT HALF FLYING SAUCER;So;0;ON;;;;;N;;;;; +1CCA5;RIGHT HALF FLYING SAUCER;So;0;ON;;;;;N;;;;; +1CCA6;UPPER LEFT QUADRANT FACE WITH OPEN EYES;So;0;ON;;;;;N;;;;; +1CCA7;UPPER RIGHT QUADRANT FACE WITH OPEN EYES;So;0;ON;;;;;N;;;;; +1CCA8;UPPER LEFT QUADRANT FACE WITH CLOSED EYES;So;0;ON;;;;;N;;;;; +1CCA9;UPPER RIGHT QUADRANT FACE WITH CLOSED EYES;So;0;ON;;;;;N;;;;; +1CCAA;LOWER LEFT QUADRANT SMILING FACE;So;0;ON;;;;;N;;;;; +1CCAB;LOWER RIGHT QUADRANT SMILING FACE;So;0;ON;;;;;N;;;;; +1CCAC;LOWER LEFT QUADRANT NEUTRAL FACE;So;0;ON;;;;;N;;;;; +1CCAD;LOWER RIGHT QUADRANT NEUTRAL FACE;So;0;ON;;;;;N;;;;; +1CCAE;LOWER LEFT QUADRANT FACE WITH OPEN MOUTH;So;0;ON;;;;;N;;;;; +1CCAF;LOWER RIGHT QUADRANT FACE WITH OPEN MOUTH;So;0;ON;;;;;N;;;;; +1CCB0;LOWER LEFT QUADRANT FROWNING FACE;So;0;ON;;;;;N;;;;; +1CCB1;LOWER RIGHT QUADRANT FROWNING FACE;So;0;ON;;;;;N;;;;; +1CCB2;UPPER LEFT QUADRANT TELEVISION;So;0;ON;;;;;N;;;;; +1CCB3;UPPER RIGHT QUADRANT TELEVISION;So;0;ON;;;;;N;;;;; +1CCB4;LOWER LEFT QUADRANT TELEVISION;So;0;ON;;;;;N;;;;; +1CCB5;LOWER RIGHT QUADRANT TELEVISION;So;0;ON;;;;;N;;;;; +1CCB6;UPPER LEFT QUADRANT MICROCOMPUTER;So;0;ON;;;;;N;;;;; +1CCB7;UPPER RIGHT QUADRANT MICROCOMPUTER;So;0;ON;;;;;N;;;;; +1CCB8;LOWER LEFT QUADRANT MICROCOMPUTER;So;0;ON;;;;;N;;;;; +1CCB9;LOWER RIGHT QUADRANT MICROCOMPUTER;So;0;ON;;;;;N;;;;; +1CCBA;UPPER LEFT QUADRANT CHESS KING;So;0;ON;;;;;N;;;;; +1CCBB;UPPER RIGHT QUADRANT CHESS KING;So;0;ON;;;;;N;;;;; +1CCBC;LOWER LEFT QUADRANT CHESS KING;So;0;ON;;;;;N;;;;; +1CCBD;LOWER RIGHT QUADRANT CHESS KING;So;0;ON;;;;;N;;;;; +1CCBE;UPPER LEFT QUADRANT CHESS QUEEN;So;0;ON;;;;;N;;;;; +1CCBF;UPPER RIGHT QUADRANT CHESS QUEEN;So;0;ON;;;;;N;;;;; +1CCC0;LOWER LEFT QUADRANT CHESS QUEEN;So;0;ON;;;;;N;;;;; +1CCC1;LOWER RIGHT QUADRANT CHESS QUEEN;So;0;ON;;;;;N;;;;; +1CCC2;UPPER LEFT QUADRANT CHESS ROOK;So;0;ON;;;;;N;;;;; +1CCC3;UPPER RIGHT QUADRANT CHESS ROOK;So;0;ON;;;;;N;;;;; +1CCC4;LOWER LEFT QUADRANT CHESS ROOK;So;0;ON;;;;;N;;;;; +1CCC5;LOWER RIGHT QUADRANT CHESS ROOK;So;0;ON;;;;;N;;;;; +1CCC6;UPPER LEFT QUADRANT CHESS BISHOP;So;0;ON;;;;;N;;;;; +1CCC7;UPPER RIGHT QUADRANT CHESS BISHOP;So;0;ON;;;;;N;;;;; +1CCC8;LOWER LEFT QUADRANT CHESS BISHOP;So;0;ON;;;;;N;;;;; +1CCC9;LOWER RIGHT QUADRANT CHESS BISHOP;So;0;ON;;;;;N;;;;; +1CCCA;UPPER LEFT QUADRANT CHESS KNIGHT;So;0;ON;;;;;N;;;;; +1CCCB;UPPER RIGHT QUADRANT CHESS KNIGHT;So;0;ON;;;;;N;;;;; +1CCCC;LOWER LEFT QUADRANT CHESS KNIGHT;So;0;ON;;;;;N;;;;; +1CCCD;LOWER RIGHT QUADRANT CHESS KNIGHT;So;0;ON;;;;;N;;;;; +1CCCE;UPPER LEFT QUADRANT CHESS PAWN;So;0;ON;;;;;N;;;;; +1CCCF;UPPER RIGHT QUADRANT CHESS PAWN;So;0;ON;;;;;N;;;;; +1CCD0;LOWER LEFT QUADRANT CHESS PAWN;So;0;ON;;;;;N;;;;; +1CCD1;LOWER RIGHT QUADRANT CHESS PAWN;So;0;ON;;;;;N;;;;; +1CCD2;UPPER LEFT QUADRANT STANDING KNIGHT;So;0;ON;;;;;N;;;;; +1CCD3;UPPER RIGHT QUADRANT STANDING KNIGHT;So;0;ON;;;;;N;;;;; +1CCD4;LOWER LEFT QUADRANT STANDING KNIGHT;So;0;ON;;;;;N;;;;; +1CCD5;LOWER RIGHT QUADRANT STANDING KNIGHT;So;0;ON;;;;;N;;;;; +1CCD6;OUTLINED LATIN CAPITAL LETTER A;So;0;L; 0041;;;;N;;;;; +1CCD7;OUTLINED LATIN CAPITAL LETTER B;So;0;L; 0042;;;;N;;;;; +1CCD8;OUTLINED LATIN CAPITAL LETTER C;So;0;L; 0043;;;;N;;;;; +1CCD9;OUTLINED LATIN CAPITAL LETTER D;So;0;L; 0044;;;;N;;;;; +1CCDA;OUTLINED LATIN CAPITAL LETTER E;So;0;L; 0045;;;;N;;;;; +1CCDB;OUTLINED LATIN CAPITAL LETTER F;So;0;L; 0046;;;;N;;;;; +1CCDC;OUTLINED LATIN CAPITAL LETTER G;So;0;L; 0047;;;;N;;;;; +1CCDD;OUTLINED LATIN CAPITAL LETTER H;So;0;L; 0048;;;;N;;;;; +1CCDE;OUTLINED LATIN CAPITAL LETTER I;So;0;L; 0049;;;;N;;;;; +1CCDF;OUTLINED LATIN CAPITAL LETTER J;So;0;L; 004A;;;;N;;;;; +1CCE0;OUTLINED LATIN CAPITAL LETTER K;So;0;L; 004B;;;;N;;;;; +1CCE1;OUTLINED LATIN CAPITAL LETTER L;So;0;L; 004C;;;;N;;;;; +1CCE2;OUTLINED LATIN CAPITAL LETTER M;So;0;L; 004D;;;;N;;;;; +1CCE3;OUTLINED LATIN CAPITAL LETTER N;So;0;L; 004E;;;;N;;;;; +1CCE4;OUTLINED LATIN CAPITAL LETTER O;So;0;L; 004F;;;;N;;;;; +1CCE5;OUTLINED LATIN CAPITAL LETTER P;So;0;L; 0050;;;;N;;;;; +1CCE6;OUTLINED LATIN CAPITAL LETTER Q;So;0;L; 0051;;;;N;;;;; +1CCE7;OUTLINED LATIN CAPITAL LETTER R;So;0;L; 0052;;;;N;;;;; +1CCE8;OUTLINED LATIN CAPITAL LETTER S;So;0;L; 0053;;;;N;;;;; +1CCE9;OUTLINED LATIN CAPITAL LETTER T;So;0;L; 0054;;;;N;;;;; +1CCEA;OUTLINED LATIN CAPITAL LETTER U;So;0;L; 0055;;;;N;;;;; +1CCEB;OUTLINED LATIN CAPITAL LETTER V;So;0;L; 0056;;;;N;;;;; +1CCEC;OUTLINED LATIN CAPITAL LETTER W;So;0;L; 0057;;;;N;;;;; +1CCED;OUTLINED LATIN CAPITAL LETTER X;So;0;L; 0058;;;;N;;;;; +1CCEE;OUTLINED LATIN CAPITAL LETTER Y;So;0;L; 0059;;;;N;;;;; +1CCEF;OUTLINED LATIN CAPITAL LETTER Z;So;0;L; 005A;;;;N;;;;; +1CCF0;OUTLINED DIGIT ZERO;Nd;0;EN; 0030;0;0;0;N;;;;; +1CCF1;OUTLINED DIGIT ONE;Nd;0;EN; 0031;1;1;1;N;;;;; +1CCF2;OUTLINED DIGIT TWO;Nd;0;EN; 0032;2;2;2;N;;;;; +1CCF3;OUTLINED DIGIT THREE;Nd;0;EN; 0033;3;3;3;N;;;;; +1CCF4;OUTLINED DIGIT FOUR;Nd;0;EN; 0034;4;4;4;N;;;;; +1CCF5;OUTLINED DIGIT FIVE;Nd;0;EN; 0035;5;5;5;N;;;;; +1CCF6;OUTLINED DIGIT SIX;Nd;0;EN; 0036;6;6;6;N;;;;; +1CCF7;OUTLINED DIGIT SEVEN;Nd;0;EN; 0037;7;7;7;N;;;;; +1CCF8;OUTLINED DIGIT EIGHT;Nd;0;EN; 0038;8;8;8;N;;;;; +1CCF9;OUTLINED DIGIT NINE;Nd;0;EN; 0039;9;9;9;N;;;;; +1CCFA;SNAKE SYMBOL;So;0;ON;;;;;N;;;;; +1CCFB;FLYING SAUCER SYMBOL;So;0;ON;;;;;N;;;;; +1CCFC;NOSE SYMBOL;So;0;ON;;;;;N;;;;; +1CD00;BLOCK OCTANT-3;So;0;ON;;;;;N;;;;; +1CD01;BLOCK OCTANT-23;So;0;ON;;;;;N;;;;; +1CD02;BLOCK OCTANT-123;So;0;ON;;;;;N;;;;; +1CD03;BLOCK OCTANT-4;So;0;ON;;;;;N;;;;; +1CD04;BLOCK OCTANT-14;So;0;ON;;;;;N;;;;; +1CD05;BLOCK OCTANT-124;So;0;ON;;;;;N;;;;; +1CD06;BLOCK OCTANT-34;So;0;ON;;;;;N;;;;; +1CD07;BLOCK OCTANT-134;So;0;ON;;;;;N;;;;; +1CD08;BLOCK OCTANT-234;So;0;ON;;;;;N;;;;; +1CD09;BLOCK OCTANT-5;So;0;ON;;;;;N;;;;; +1CD0A;BLOCK OCTANT-15;So;0;ON;;;;;N;;;;; +1CD0B;BLOCK OCTANT-25;So;0;ON;;;;;N;;;;; +1CD0C;BLOCK OCTANT-125;So;0;ON;;;;;N;;;;; +1CD0D;BLOCK OCTANT-135;So;0;ON;;;;;N;;;;; +1CD0E;BLOCK OCTANT-235;So;0;ON;;;;;N;;;;; +1CD0F;BLOCK OCTANT-1235;So;0;ON;;;;;N;;;;; +1CD10;BLOCK OCTANT-45;So;0;ON;;;;;N;;;;; +1CD11;BLOCK OCTANT-145;So;0;ON;;;;;N;;;;; +1CD12;BLOCK OCTANT-245;So;0;ON;;;;;N;;;;; +1CD13;BLOCK OCTANT-1245;So;0;ON;;;;;N;;;;; +1CD14;BLOCK OCTANT-345;So;0;ON;;;;;N;;;;; +1CD15;BLOCK OCTANT-1345;So;0;ON;;;;;N;;;;; +1CD16;BLOCK OCTANT-2345;So;0;ON;;;;;N;;;;; +1CD17;BLOCK OCTANT-12345;So;0;ON;;;;;N;;;;; +1CD18;BLOCK OCTANT-6;So;0;ON;;;;;N;;;;; +1CD19;BLOCK OCTANT-16;So;0;ON;;;;;N;;;;; +1CD1A;BLOCK OCTANT-26;So;0;ON;;;;;N;;;;; +1CD1B;BLOCK OCTANT-126;So;0;ON;;;;;N;;;;; +1CD1C;BLOCK OCTANT-36;So;0;ON;;;;;N;;;;; +1CD1D;BLOCK OCTANT-136;So;0;ON;;;;;N;;;;; +1CD1E;BLOCK OCTANT-236;So;0;ON;;;;;N;;;;; +1CD1F;BLOCK OCTANT-1236;So;0;ON;;;;;N;;;;; +1CD20;BLOCK OCTANT-146;So;0;ON;;;;;N;;;;; +1CD21;BLOCK OCTANT-246;So;0;ON;;;;;N;;;;; +1CD22;BLOCK OCTANT-1246;So;0;ON;;;;;N;;;;; +1CD23;BLOCK OCTANT-346;So;0;ON;;;;;N;;;;; +1CD24;BLOCK OCTANT-1346;So;0;ON;;;;;N;;;;; +1CD25;BLOCK OCTANT-2346;So;0;ON;;;;;N;;;;; +1CD26;BLOCK OCTANT-12346;So;0;ON;;;;;N;;;;; +1CD27;BLOCK OCTANT-56;So;0;ON;;;;;N;;;;; +1CD28;BLOCK OCTANT-156;So;0;ON;;;;;N;;;;; +1CD29;BLOCK OCTANT-256;So;0;ON;;;;;N;;;;; +1CD2A;BLOCK OCTANT-1256;So;0;ON;;;;;N;;;;; +1CD2B;BLOCK OCTANT-356;So;0;ON;;;;;N;;;;; +1CD2C;BLOCK OCTANT-1356;So;0;ON;;;;;N;;;;; +1CD2D;BLOCK OCTANT-2356;So;0;ON;;;;;N;;;;; +1CD2E;BLOCK OCTANT-12356;So;0;ON;;;;;N;;;;; +1CD2F;BLOCK OCTANT-456;So;0;ON;;;;;N;;;;; +1CD30;BLOCK OCTANT-1456;So;0;ON;;;;;N;;;;; +1CD31;BLOCK OCTANT-2456;So;0;ON;;;;;N;;;;; +1CD32;BLOCK OCTANT-12456;So;0;ON;;;;;N;;;;; +1CD33;BLOCK OCTANT-3456;So;0;ON;;;;;N;;;;; +1CD34;BLOCK OCTANT-13456;So;0;ON;;;;;N;;;;; +1CD35;BLOCK OCTANT-23456;So;0;ON;;;;;N;;;;; +1CD36;BLOCK OCTANT-17;So;0;ON;;;;;N;;;;; +1CD37;BLOCK OCTANT-27;So;0;ON;;;;;N;;;;; +1CD38;BLOCK OCTANT-127;So;0;ON;;;;;N;;;;; +1CD39;BLOCK OCTANT-37;So;0;ON;;;;;N;;;;; +1CD3A;BLOCK OCTANT-137;So;0;ON;;;;;N;;;;; +1CD3B;BLOCK OCTANT-237;So;0;ON;;;;;N;;;;; +1CD3C;BLOCK OCTANT-1237;So;0;ON;;;;;N;;;;; +1CD3D;BLOCK OCTANT-47;So;0;ON;;;;;N;;;;; +1CD3E;BLOCK OCTANT-147;So;0;ON;;;;;N;;;;; +1CD3F;BLOCK OCTANT-247;So;0;ON;;;;;N;;;;; +1CD40;BLOCK OCTANT-1247;So;0;ON;;;;;N;;;;; +1CD41;BLOCK OCTANT-347;So;0;ON;;;;;N;;;;; +1CD42;BLOCK OCTANT-1347;So;0;ON;;;;;N;;;;; +1CD43;BLOCK OCTANT-2347;So;0;ON;;;;;N;;;;; +1CD44;BLOCK OCTANT-12347;So;0;ON;;;;;N;;;;; +1CD45;BLOCK OCTANT-157;So;0;ON;;;;;N;;;;; +1CD46;BLOCK OCTANT-257;So;0;ON;;;;;N;;;;; +1CD47;BLOCK OCTANT-1257;So;0;ON;;;;;N;;;;; +1CD48;BLOCK OCTANT-357;So;0;ON;;;;;N;;;;; +1CD49;BLOCK OCTANT-2357;So;0;ON;;;;;N;;;;; +1CD4A;BLOCK OCTANT-12357;So;0;ON;;;;;N;;;;; +1CD4B;BLOCK OCTANT-457;So;0;ON;;;;;N;;;;; +1CD4C;BLOCK OCTANT-1457;So;0;ON;;;;;N;;;;; +1CD4D;BLOCK OCTANT-12457;So;0;ON;;;;;N;;;;; +1CD4E;BLOCK OCTANT-3457;So;0;ON;;;;;N;;;;; +1CD4F;BLOCK OCTANT-13457;So;0;ON;;;;;N;;;;; +1CD50;BLOCK OCTANT-23457;So;0;ON;;;;;N;;;;; +1CD51;BLOCK OCTANT-67;So;0;ON;;;;;N;;;;; +1CD52;BLOCK OCTANT-167;So;0;ON;;;;;N;;;;; +1CD53;BLOCK OCTANT-267;So;0;ON;;;;;N;;;;; +1CD54;BLOCK OCTANT-1267;So;0;ON;;;;;N;;;;; +1CD55;BLOCK OCTANT-367;So;0;ON;;;;;N;;;;; +1CD56;BLOCK OCTANT-1367;So;0;ON;;;;;N;;;;; +1CD57;BLOCK OCTANT-2367;So;0;ON;;;;;N;;;;; +1CD58;BLOCK OCTANT-12367;So;0;ON;;;;;N;;;;; +1CD59;BLOCK OCTANT-467;So;0;ON;;;;;N;;;;; +1CD5A;BLOCK OCTANT-1467;So;0;ON;;;;;N;;;;; +1CD5B;BLOCK OCTANT-2467;So;0;ON;;;;;N;;;;; +1CD5C;BLOCK OCTANT-12467;So;0;ON;;;;;N;;;;; +1CD5D;BLOCK OCTANT-3467;So;0;ON;;;;;N;;;;; +1CD5E;BLOCK OCTANT-13467;So;0;ON;;;;;N;;;;; +1CD5F;BLOCK OCTANT-23467;So;0;ON;;;;;N;;;;; +1CD60;BLOCK OCTANT-123467;So;0;ON;;;;;N;;;;; +1CD61;BLOCK OCTANT-567;So;0;ON;;;;;N;;;;; +1CD62;BLOCK OCTANT-1567;So;0;ON;;;;;N;;;;; +1CD63;BLOCK OCTANT-2567;So;0;ON;;;;;N;;;;; +1CD64;BLOCK OCTANT-12567;So;0;ON;;;;;N;;;;; +1CD65;BLOCK OCTANT-3567;So;0;ON;;;;;N;;;;; +1CD66;BLOCK OCTANT-13567;So;0;ON;;;;;N;;;;; +1CD67;BLOCK OCTANT-23567;So;0;ON;;;;;N;;;;; +1CD68;BLOCK OCTANT-123567;So;0;ON;;;;;N;;;;; +1CD69;BLOCK OCTANT-4567;So;0;ON;;;;;N;;;;; +1CD6A;BLOCK OCTANT-14567;So;0;ON;;;;;N;;;;; +1CD6B;BLOCK OCTANT-24567;So;0;ON;;;;;N;;;;; +1CD6C;BLOCK OCTANT-124567;So;0;ON;;;;;N;;;;; +1CD6D;BLOCK OCTANT-34567;So;0;ON;;;;;N;;;;; +1CD6E;BLOCK OCTANT-134567;So;0;ON;;;;;N;;;;; +1CD6F;BLOCK OCTANT-234567;So;0;ON;;;;;N;;;;; +1CD70;BLOCK OCTANT-1234567;So;0;ON;;;;;N;;;;; +1CD71;BLOCK OCTANT-18;So;0;ON;;;;;N;;;;; +1CD72;BLOCK OCTANT-28;So;0;ON;;;;;N;;;;; +1CD73;BLOCK OCTANT-128;So;0;ON;;;;;N;;;;; +1CD74;BLOCK OCTANT-38;So;0;ON;;;;;N;;;;; +1CD75;BLOCK OCTANT-138;So;0;ON;;;;;N;;;;; +1CD76;BLOCK OCTANT-238;So;0;ON;;;;;N;;;;; +1CD77;BLOCK OCTANT-1238;So;0;ON;;;;;N;;;;; +1CD78;BLOCK OCTANT-48;So;0;ON;;;;;N;;;;; +1CD79;BLOCK OCTANT-148;So;0;ON;;;;;N;;;;; +1CD7A;BLOCK OCTANT-248;So;0;ON;;;;;N;;;;; +1CD7B;BLOCK OCTANT-1248;So;0;ON;;;;;N;;;;; +1CD7C;BLOCK OCTANT-348;So;0;ON;;;;;N;;;;; +1CD7D;BLOCK OCTANT-1348;So;0;ON;;;;;N;;;;; +1CD7E;BLOCK OCTANT-2348;So;0;ON;;;;;N;;;;; +1CD7F;BLOCK OCTANT-12348;So;0;ON;;;;;N;;;;; +1CD80;BLOCK OCTANT-58;So;0;ON;;;;;N;;;;; +1CD81;BLOCK OCTANT-158;So;0;ON;;;;;N;;;;; +1CD82;BLOCK OCTANT-258;So;0;ON;;;;;N;;;;; +1CD83;BLOCK OCTANT-1258;So;0;ON;;;;;N;;;;; +1CD84;BLOCK OCTANT-358;So;0;ON;;;;;N;;;;; +1CD85;BLOCK OCTANT-1358;So;0;ON;;;;;N;;;;; +1CD86;BLOCK OCTANT-2358;So;0;ON;;;;;N;;;;; +1CD87;BLOCK OCTANT-12358;So;0;ON;;;;;N;;;;; +1CD88;BLOCK OCTANT-458;So;0;ON;;;;;N;;;;; +1CD89;BLOCK OCTANT-1458;So;0;ON;;;;;N;;;;; +1CD8A;BLOCK OCTANT-2458;So;0;ON;;;;;N;;;;; +1CD8B;BLOCK OCTANT-12458;So;0;ON;;;;;N;;;;; +1CD8C;BLOCK OCTANT-3458;So;0;ON;;;;;N;;;;; +1CD8D;BLOCK OCTANT-13458;So;0;ON;;;;;N;;;;; +1CD8E;BLOCK OCTANT-23458;So;0;ON;;;;;N;;;;; +1CD8F;BLOCK OCTANT-123458;So;0;ON;;;;;N;;;;; +1CD90;BLOCK OCTANT-168;So;0;ON;;;;;N;;;;; +1CD91;BLOCK OCTANT-268;So;0;ON;;;;;N;;;;; +1CD92;BLOCK OCTANT-1268;So;0;ON;;;;;N;;;;; +1CD93;BLOCK OCTANT-368;So;0;ON;;;;;N;;;;; +1CD94;BLOCK OCTANT-2368;So;0;ON;;;;;N;;;;; +1CD95;BLOCK OCTANT-12368;So;0;ON;;;;;N;;;;; +1CD96;BLOCK OCTANT-468;So;0;ON;;;;;N;;;;; +1CD97;BLOCK OCTANT-1468;So;0;ON;;;;;N;;;;; +1CD98;BLOCK OCTANT-12468;So;0;ON;;;;;N;;;;; +1CD99;BLOCK OCTANT-3468;So;0;ON;;;;;N;;;;; +1CD9A;BLOCK OCTANT-13468;So;0;ON;;;;;N;;;;; +1CD9B;BLOCK OCTANT-23468;So;0;ON;;;;;N;;;;; +1CD9C;BLOCK OCTANT-568;So;0;ON;;;;;N;;;;; +1CD9D;BLOCK OCTANT-1568;So;0;ON;;;;;N;;;;; +1CD9E;BLOCK OCTANT-2568;So;0;ON;;;;;N;;;;; +1CD9F;BLOCK OCTANT-12568;So;0;ON;;;;;N;;;;; +1CDA0;BLOCK OCTANT-3568;So;0;ON;;;;;N;;;;; +1CDA1;BLOCK OCTANT-13568;So;0;ON;;;;;N;;;;; +1CDA2;BLOCK OCTANT-23568;So;0;ON;;;;;N;;;;; +1CDA3;BLOCK OCTANT-123568;So;0;ON;;;;;N;;;;; +1CDA4;BLOCK OCTANT-4568;So;0;ON;;;;;N;;;;; +1CDA5;BLOCK OCTANT-14568;So;0;ON;;;;;N;;;;; +1CDA6;BLOCK OCTANT-24568;So;0;ON;;;;;N;;;;; +1CDA7;BLOCK OCTANT-124568;So;0;ON;;;;;N;;;;; +1CDA8;BLOCK OCTANT-34568;So;0;ON;;;;;N;;;;; +1CDA9;BLOCK OCTANT-134568;So;0;ON;;;;;N;;;;; +1CDAA;BLOCK OCTANT-234568;So;0;ON;;;;;N;;;;; +1CDAB;BLOCK OCTANT-1234568;So;0;ON;;;;;N;;;;; +1CDAC;BLOCK OCTANT-178;So;0;ON;;;;;N;;;;; +1CDAD;BLOCK OCTANT-278;So;0;ON;;;;;N;;;;; +1CDAE;BLOCK OCTANT-1278;So;0;ON;;;;;N;;;;; +1CDAF;BLOCK OCTANT-378;So;0;ON;;;;;N;;;;; +1CDB0;BLOCK OCTANT-1378;So;0;ON;;;;;N;;;;; +1CDB1;BLOCK OCTANT-2378;So;0;ON;;;;;N;;;;; +1CDB2;BLOCK OCTANT-12378;So;0;ON;;;;;N;;;;; +1CDB3;BLOCK OCTANT-478;So;0;ON;;;;;N;;;;; +1CDB4;BLOCK OCTANT-1478;So;0;ON;;;;;N;;;;; +1CDB5;BLOCK OCTANT-2478;So;0;ON;;;;;N;;;;; +1CDB6;BLOCK OCTANT-12478;So;0;ON;;;;;N;;;;; +1CDB7;BLOCK OCTANT-3478;So;0;ON;;;;;N;;;;; +1CDB8;BLOCK OCTANT-13478;So;0;ON;;;;;N;;;;; +1CDB9;BLOCK OCTANT-23478;So;0;ON;;;;;N;;;;; +1CDBA;BLOCK OCTANT-123478;So;0;ON;;;;;N;;;;; +1CDBB;BLOCK OCTANT-578;So;0;ON;;;;;N;;;;; +1CDBC;BLOCK OCTANT-1578;So;0;ON;;;;;N;;;;; +1CDBD;BLOCK OCTANT-2578;So;0;ON;;;;;N;;;;; +1CDBE;BLOCK OCTANT-12578;So;0;ON;;;;;N;;;;; +1CDBF;BLOCK OCTANT-3578;So;0;ON;;;;;N;;;;; +1CDC0;BLOCK OCTANT-13578;So;0;ON;;;;;N;;;;; +1CDC1;BLOCK OCTANT-23578;So;0;ON;;;;;N;;;;; +1CDC2;BLOCK OCTANT-123578;So;0;ON;;;;;N;;;;; +1CDC3;BLOCK OCTANT-4578;So;0;ON;;;;;N;;;;; +1CDC4;BLOCK OCTANT-14578;So;0;ON;;;;;N;;;;; +1CDC5;BLOCK OCTANT-24578;So;0;ON;;;;;N;;;;; +1CDC6;BLOCK OCTANT-124578;So;0;ON;;;;;N;;;;; +1CDC7;BLOCK OCTANT-34578;So;0;ON;;;;;N;;;;; +1CDC8;BLOCK OCTANT-134578;So;0;ON;;;;;N;;;;; +1CDC9;BLOCK OCTANT-234578;So;0;ON;;;;;N;;;;; +1CDCA;BLOCK OCTANT-1234578;So;0;ON;;;;;N;;;;; +1CDCB;BLOCK OCTANT-678;So;0;ON;;;;;N;;;;; +1CDCC;BLOCK OCTANT-1678;So;0;ON;;;;;N;;;;; +1CDCD;BLOCK OCTANT-2678;So;0;ON;;;;;N;;;;; +1CDCE;BLOCK OCTANT-12678;So;0;ON;;;;;N;;;;; +1CDCF;BLOCK OCTANT-3678;So;0;ON;;;;;N;;;;; +1CDD0;BLOCK OCTANT-13678;So;0;ON;;;;;N;;;;; +1CDD1;BLOCK OCTANT-23678;So;0;ON;;;;;N;;;;; +1CDD2;BLOCK OCTANT-123678;So;0;ON;;;;;N;;;;; +1CDD3;BLOCK OCTANT-4678;So;0;ON;;;;;N;;;;; +1CDD4;BLOCK OCTANT-14678;So;0;ON;;;;;N;;;;; +1CDD5;BLOCK OCTANT-24678;So;0;ON;;;;;N;;;;; +1CDD6;BLOCK OCTANT-124678;So;0;ON;;;;;N;;;;; +1CDD7;BLOCK OCTANT-34678;So;0;ON;;;;;N;;;;; +1CDD8;BLOCK OCTANT-134678;So;0;ON;;;;;N;;;;; +1CDD9;BLOCK OCTANT-234678;So;0;ON;;;;;N;;;;; +1CDDA;BLOCK OCTANT-1234678;So;0;ON;;;;;N;;;;; +1CDDB;BLOCK OCTANT-15678;So;0;ON;;;;;N;;;;; +1CDDC;BLOCK OCTANT-25678;So;0;ON;;;;;N;;;;; +1CDDD;BLOCK OCTANT-125678;So;0;ON;;;;;N;;;;; +1CDDE;BLOCK OCTANT-35678;So;0;ON;;;;;N;;;;; +1CDDF;BLOCK OCTANT-235678;So;0;ON;;;;;N;;;;; +1CDE0;BLOCK OCTANT-1235678;So;0;ON;;;;;N;;;;; +1CDE1;BLOCK OCTANT-45678;So;0;ON;;;;;N;;;;; +1CDE2;BLOCK OCTANT-145678;So;0;ON;;;;;N;;;;; +1CDE3;BLOCK OCTANT-1245678;So;0;ON;;;;;N;;;;; +1CDE4;BLOCK OCTANT-1345678;So;0;ON;;;;;N;;;;; +1CDE5;BLOCK OCTANT-2345678;So;0;ON;;;;;N;;;;; +1CDE6;TOP HALF STANDING PERSON;So;0;ON;;;;;N;;;;; +1CDE7;BOTTOM HALF STANDING PERSON;So;0;ON;;;;;N;;;;; +1CDE8;TOP HALF RIGHT-FACING RUNNER FRAME-1;So;0;ON;;;;;N;;;;; +1CDE9;BOTTOM HALF RIGHT-FACING RUNNER FRAME-1;So;0;ON;;;;;N;;;;; +1CDEA;TOP HALF RIGHT-FACING RUNNER FRAME-2;So;0;ON;;;;;N;;;;; +1CDEB;BOTTOM HALF RIGHT-FACING RUNNER FRAME-2;So;0;ON;;;;;N;;;;; +1CDEC;TOP HALF LEFT-FACING RUNNER FRAME-1;So;0;ON;;;;;N;;;;; +1CDED;BOTTOM HALF LEFT-FACING RUNNER FRAME-1;So;0;ON;;;;;N;;;;; +1CDEE;TOP HALF LEFT-FACING RUNNER FRAME-2;So;0;ON;;;;;N;;;;; +1CDEF;BOTTOM HALF LEFT-FACING RUNNER FRAME-2;So;0;ON;;;;;N;;;;; +1CDF0;TOP HALF FORWARD-FACING RUNNER;So;0;ON;;;;;N;;;;; +1CDF1;BOTTOM HALF FORWARD-FACING RUNNER FRAME-1;So;0;ON;;;;;N;;;;; +1CDF2;BOTTOM HALF FORWARD-FACING RUNNER FRAME-2;So;0;ON;;;;;N;;;;; +1CDF3;BOTTOM HALF FORWARD-FACING RUNNER FRAME-3;So;0;ON;;;;;N;;;;; +1CDF4;BOTTOM HALF FORWARD-FACING RUNNER FRAME-4;So;0;ON;;;;;N;;;;; +1CDF5;MOON LANDER;So;0;ON;;;;;N;;;;; +1CDF6;TOP HALF FLAILING ROBOT FRAME-1;So;0;ON;;;;;N;;;;; +1CDF7;TOP HALF FLAILING ROBOT FRAME-2;So;0;ON;;;;;N;;;;; +1CDF8;DOWN-POINTING AIRPLANE;So;0;ON;;;;;N;;;;; +1CDF9;LEFT-POINTING AIRPLANE;So;0;ON;;;;;N;;;;; +1CDFA;SMALL UP-POINTING AIRPLANE;So;0;ON;;;;;N;;;;; +1CDFB;UP-POINTING FROG;So;0;ON;;;;;N;;;;; +1CDFC;DOWN-POINTING FROG;So;0;ON;;;;;N;;;;; +1CDFD;EXPLOSION FRAME-1;So;0;ON;;;;;N;;;;; +1CDFE;EXPLOSION FRAME-2;So;0;ON;;;;;N;;;;; +1CDFF;EXPLOSION FRAME-3;So;0;ON;;;;;N;;;;; +1CE00;RIGHT HALF AND LEFT HALF WHITE CIRCLE;So;0;ON;;;;;N;;;;; +1CE01;LOWER HALF AND UPPER HALF WHITE CIRCLE;So;0;ON;;;;;N;;;;; +1CE02;EXPLOSION AT HORIZON;So;0;ON;;;;;N;;;;; +1CE03;UPPER HALF HEAVY WHITE SQUARE;So;0;ON;;;;;N;;;;; +1CE04;LOWER HALF HEAVY WHITE SQUARE;So;0;ON;;;;;N;;;;; +1CE05;HEAVY WHITE SQUARE CONTAINING BLACK VERY SMALL SQUARE;So;0;ON;;;;;N;;;;; +1CE06;WHITE VERTICAL RECTANGLE WITH HORIZONTAL BAR;So;0;ON;;;;;N;;;;; +1CE07;TOP LEFT BLACK LEFT-POINTING SMALL TRIANGLE;So;0;ON;;;;;N;;;;; +1CE08;FUNNEL;So;0;ON;;;;;N;;;;; +1CE09;BOX DRAWINGS DOUBLE DIAGONAL LOWER LEFT TO MIDDLE CENTRE TO LOWER RIGHT;So;0;ON;;;;;N;;;;; +1CE0A;BOX DRAWINGS DOUBLE DIAGONAL UPPER LEFT TO MIDDLE CENTRE TO UPPER RIGHT;So;0;ON;;;;;N;;;;; +1CE0B;LEFT HALF WHITE ELLIPSE;So;0;ON;;;;;N;;;;; +1CE0C;RIGHT HALF WHITE ELLIPSE;So;0;ON;;;;;N;;;;; +1CE0D;LEFT HALF TRIPLE DASH HORIZONTAL;So;0;ON;;;;;N;;;;; +1CE0E;RIGHT HALF TRIPLE DASH HORIZONTAL;So;0;ON;;;;;N;;;;; +1CE0F;HORIZONTAL LINE WITH TICK MARK;So;0;ON;;;;;N;;;;; +1CE10;LEFT HALF HORIZONTAL LINE WITH THREE TICK MARKS;So;0;ON;;;;;N;;;;; +1CE11;RIGHT HALF HORIZONTAL LINE WITH THREE TICK MARKS;So;0;ON;;;;;N;;;;; +1CE12;HORIZONTAL LINE WITH THREE TICK MARKS;So;0;ON;;;;;N;;;;; +1CE13;LOWER HALF VERTICAL LINE WITH THREE TICK MARKS;So;0;ON;;;;;N;;;;; +1CE14;UPPER HALF VERTICAL LINE WITH THREE TICK MARKS;So;0;ON;;;;;N;;;;; +1CE15;VERTICAL LINE WITH THREE TICK MARKS;So;0;ON;;;;;N;;;;; +1CE16;BOX DRAWINGS LIGHT VERTICAL AND TOP RIGHT;So;0;ON;;;;;N;;;;; +1CE17;BOX DRAWINGS LIGHT VERTICAL AND BOTTOM RIGHT;So;0;ON;;;;;N;;;;; +1CE18;BOX DRAWINGS LIGHT VERTICAL AND TOP LEFT;So;0;ON;;;;;N;;;;; +1CE19;BOX DRAWINGS LIGHT VERTICAL AND BOTTOM LEFT;So;0;ON;;;;;N;;;;; +1CE1A;LARGE TYPE PIECE UPPER LEFT ARC;So;0;ON;;;;;N;;;;; +1CE1B;LARGE TYPE PIECE UPPER LEFT CORNER;So;0;ON;;;;;N;;;;; +1CE1C;LARGE TYPE PIECE UPPER TERMINAL;So;0;ON;;;;;N;;;;; +1CE1D;LARGE TYPE PIECE UPPER LEFT CROTCH;So;0;ON;;;;;N;;;;; +1CE1E;LARGE TYPE PIECE LEFT ARM;So;0;ON;;;;;N;;;;; +1CE1F;LARGE TYPE PIECE CROSSBAR;So;0;ON;;;;;N;;;;; +1CE20;LARGE TYPE PIECE CROSSBAR WITH LOWER STEM;So;0;ON;;;;;N;;;;; +1CE21;LARGE TYPE PIECE UPPER HALF VERTEX OF M;So;0;ON;;;;;N;;;;; +1CE22;LARGE TYPE PIECE DIAGONAL LOWER LEFT;So;0;ON;;;;;N;;;;; +1CE23;LARGE TYPE PIECE SHORT UPPER TERMINAL;So;0;ON;;;;;N;;;;; +1CE24;LARGE TYPE PIECE UPPER RIGHT ARC;So;0;ON;;;;;N;;;;; +1CE25;LARGE TYPE PIECE RIGHT ARM;So;0;ON;;;;;N;;;;; +1CE26;LARGE TYPE PIECE UPPER RIGHT CROTCH;So;0;ON;;;;;N;;;;; +1CE27;LARGE TYPE PIECE UPPER RIGHT CORNER;So;0;ON;;;;;N;;;;; +1CE28;LARGE TYPE PIECE STEM WITH RIGHT CROSSBAR;So;0;ON;;;;;N;;;;; +1CE29;LARGE TYPE PIECE STEM;So;0;ON;;;;;N;;;;; +1CE2A;LARGE TYPE PIECE DIAGONAL UPPER RIGHT AND LOWER RIGHT;So;0;ON;;;;;N;;;;; +1CE2B;LARGE TYPE PIECE DIAGONAL UPPER RIGHT;So;0;ON;;;;;N;;;;; +1CE2C;LARGE TYPE PIECE DIAGONAL LOWER RIGHT;So;0;ON;;;;;N;;;;; +1CE2D;LARGE TYPE PIECE SHORT LOWER TERMINAL;So;0;ON;;;;;N;;;;; +1CE2E;LARGE TYPE PIECE LOWER LEFT AND UPPER LEFT ARC;So;0;ON;;;;;N;;;;; +1CE2F;LARGE TYPE PIECE CENTRE OF K;So;0;ON;;;;;N;;;;; +1CE30;LARGE TYPE PIECE LOWER HALF VERTEX OF M;So;0;ON;;;;;N;;;;; +1CE31;LARGE TYPE PIECE UPPER HALF VERTEX OF W;So;0;ON;;;;;N;;;;; +1CE32;LARGE TYPE PIECE CENTRE OF X;So;0;ON;;;;;N;;;;; +1CE33;LARGE TYPE PIECE CENTRE OF Y;So;0;ON;;;;;N;;;;; +1CE34;LARGE TYPE PIECE CENTRE OF Z WITH CROSSBAR;So;0;ON;;;;;N;;;;; +1CE35;LARGE TYPE PIECE RAISED UPPER LEFT ARC;So;0;ON;;;;;N;;;;; +1CE36;LARGE TYPE PIECE STEM WITH LEFT CROSSBAR;So;0;ON;;;;;N;;;;; +1CE37;LARGE TYPE PIECE LOWER RIGHT AND UPPER RIGHT ARC;So;0;ON;;;;;N;;;;; +1CE38;LARGE TYPE PIECE DIAGONAL UPPER LEFT AND LOWER LEFT;So;0;ON;;;;;N;;;;; +1CE39;LARGE TYPE PIECE STEM WITH LEFT JOINT;So;0;ON;;;;;N;;;;; +1CE3A;LARGE TYPE PIECE STEM WITH CROSSBAR;So;0;ON;;;;;N;;;;; +1CE3B;LARGE TYPE PIECE DIAGONAL UPPER LEFT;So;0;ON;;;;;N;;;;; +1CE3C;LARGE TYPE PIECE LOWER TERMINAL;So;0;ON;;;;;N;;;;; +1CE3D;LARGE TYPE PIECE LOWER LEFT CORNER;So;0;ON;;;;;N;;;;; +1CE3E;LARGE TYPE PIECE LOWER LEFT ARC;So;0;ON;;;;;N;;;;; +1CE3F;LARGE TYPE PIECE LOWER LEFT CROTCH;So;0;ON;;;;;N;;;;; +1CE40;LARGE TYPE PIECE CROSSBAR WITH UPPER STEM;So;0;ON;;;;;N;;;;; +1CE41;LARGE TYPE PIECE VERTEX OF V;So;0;ON;;;;;N;;;;; +1CE42;LARGE TYPE PIECE LOWER HALF VERTEX OF W;So;0;ON;;;;;N;;;;; +1CE43;LARGE TYPE PIECE LOWER RIGHT ARC;So;0;ON;;;;;N;;;;; +1CE44;LARGE TYPE PIECE LOWER RIGHT CORNER;So;0;ON;;;;;N;;;;; +1CE45;LARGE TYPE PIECE LOWER RIGHT ARC WITH TAIL;So;0;ON;;;;;N;;;;; +1CE46;LARGE TYPE PIECE LOWER RIGHT CROTCH;So;0;ON;;;;;N;;;;; +1CE47;LARGE TYPE PIECE STEM-45;So;0;ON;;;;;N;;;;; +1CE48;LARGE TYPE PIECE STEM-2345;So;0;ON;;;;;N;;;;; +1CE49;LARGE TYPE PIECE STEM-4;So;0;ON;;;;;N;;;;; +1CE4A;LARGE TYPE PIECE STEM-34;So;0;ON;;;;;N;;;;; +1CE4B;LARGE TYPE PIECE STEM-234;So;0;ON;;;;;N;;;;; +1CE4C;LARGE TYPE PIECE STEM-1234;So;0;ON;;;;;N;;;;; +1CE4D;LARGE TYPE PIECE STEM-3;So;0;ON;;;;;N;;;;; +1CE4E;LARGE TYPE PIECE STEM-23;So;0;ON;;;;;N;;;;; +1CE4F;LARGE TYPE PIECE STEM-2;So;0;ON;;;;;N;;;;; +1CE50;LARGE TYPE PIECE STEM-12;So;0;ON;;;;;N;;;;; +1CE51;SEPARATED BLOCK SEXTANT-1;So;0;ON;;;;;N;;;;; +1CE52;SEPARATED BLOCK SEXTANT-2;So;0;ON;;;;;N;;;;; +1CE53;SEPARATED BLOCK SEXTANT-12;So;0;ON;;;;;N;;;;; +1CE54;SEPARATED BLOCK SEXTANT-3;So;0;ON;;;;;N;;;;; +1CE55;SEPARATED BLOCK SEXTANT-13;So;0;ON;;;;;N;;;;; +1CE56;SEPARATED BLOCK SEXTANT-23;So;0;ON;;;;;N;;;;; +1CE57;SEPARATED BLOCK SEXTANT-123;So;0;ON;;;;;N;;;;; +1CE58;SEPARATED BLOCK SEXTANT-4;So;0;ON;;;;;N;;;;; +1CE59;SEPARATED BLOCK SEXTANT-14;So;0;ON;;;;;N;;;;; +1CE5A;SEPARATED BLOCK SEXTANT-24;So;0;ON;;;;;N;;;;; +1CE5B;SEPARATED BLOCK SEXTANT-124;So;0;ON;;;;;N;;;;; +1CE5C;SEPARATED BLOCK SEXTANT-34;So;0;ON;;;;;N;;;;; +1CE5D;SEPARATED BLOCK SEXTANT-134;So;0;ON;;;;;N;;;;; +1CE5E;SEPARATED BLOCK SEXTANT-234;So;0;ON;;;;;N;;;;; +1CE5F;SEPARATED BLOCK SEXTANT-1234;So;0;ON;;;;;N;;;;; +1CE60;SEPARATED BLOCK SEXTANT-5;So;0;ON;;;;;N;;;;; +1CE61;SEPARATED BLOCK SEXTANT-15;So;0;ON;;;;;N;;;;; +1CE62;SEPARATED BLOCK SEXTANT-25;So;0;ON;;;;;N;;;;; +1CE63;SEPARATED BLOCK SEXTANT-125;So;0;ON;;;;;N;;;;; +1CE64;SEPARATED BLOCK SEXTANT-35;So;0;ON;;;;;N;;;;; +1CE65;SEPARATED BLOCK SEXTANT-135;So;0;ON;;;;;N;;;;; +1CE66;SEPARATED BLOCK SEXTANT-235;So;0;ON;;;;;N;;;;; +1CE67;SEPARATED BLOCK SEXTANT-1235;So;0;ON;;;;;N;;;;; +1CE68;SEPARATED BLOCK SEXTANT-45;So;0;ON;;;;;N;;;;; +1CE69;SEPARATED BLOCK SEXTANT-145;So;0;ON;;;;;N;;;;; +1CE6A;SEPARATED BLOCK SEXTANT-245;So;0;ON;;;;;N;;;;; +1CE6B;SEPARATED BLOCK SEXTANT-1245;So;0;ON;;;;;N;;;;; +1CE6C;SEPARATED BLOCK SEXTANT-345;So;0;ON;;;;;N;;;;; +1CE6D;SEPARATED BLOCK SEXTANT-1345;So;0;ON;;;;;N;;;;; +1CE6E;SEPARATED BLOCK SEXTANT-2345;So;0;ON;;;;;N;;;;; +1CE6F;SEPARATED BLOCK SEXTANT-12345;So;0;ON;;;;;N;;;;; +1CE70;SEPARATED BLOCK SEXTANT-6;So;0;ON;;;;;N;;;;; +1CE71;SEPARATED BLOCK SEXTANT-16;So;0;ON;;;;;N;;;;; +1CE72;SEPARATED BLOCK SEXTANT-26;So;0;ON;;;;;N;;;;; +1CE73;SEPARATED BLOCK SEXTANT-126;So;0;ON;;;;;N;;;;; +1CE74;SEPARATED BLOCK SEXTANT-36;So;0;ON;;;;;N;;;;; +1CE75;SEPARATED BLOCK SEXTANT-136;So;0;ON;;;;;N;;;;; +1CE76;SEPARATED BLOCK SEXTANT-236;So;0;ON;;;;;N;;;;; +1CE77;SEPARATED BLOCK SEXTANT-1236;So;0;ON;;;;;N;;;;; +1CE78;SEPARATED BLOCK SEXTANT-46;So;0;ON;;;;;N;;;;; +1CE79;SEPARATED BLOCK SEXTANT-146;So;0;ON;;;;;N;;;;; +1CE7A;SEPARATED BLOCK SEXTANT-246;So;0;ON;;;;;N;;;;; +1CE7B;SEPARATED BLOCK SEXTANT-1246;So;0;ON;;;;;N;;;;; +1CE7C;SEPARATED BLOCK SEXTANT-346;So;0;ON;;;;;N;;;;; +1CE7D;SEPARATED BLOCK SEXTANT-1346;So;0;ON;;;;;N;;;;; +1CE7E;SEPARATED BLOCK SEXTANT-2346;So;0;ON;;;;;N;;;;; +1CE7F;SEPARATED BLOCK SEXTANT-12346;So;0;ON;;;;;N;;;;; +1CE80;SEPARATED BLOCK SEXTANT-56;So;0;ON;;;;;N;;;;; +1CE81;SEPARATED BLOCK SEXTANT-156;So;0;ON;;;;;N;;;;; +1CE82;SEPARATED BLOCK SEXTANT-256;So;0;ON;;;;;N;;;;; +1CE83;SEPARATED BLOCK SEXTANT-1256;So;0;ON;;;;;N;;;;; +1CE84;SEPARATED BLOCK SEXTANT-356;So;0;ON;;;;;N;;;;; +1CE85;SEPARATED BLOCK SEXTANT-1356;So;0;ON;;;;;N;;;;; +1CE86;SEPARATED BLOCK SEXTANT-2356;So;0;ON;;;;;N;;;;; +1CE87;SEPARATED BLOCK SEXTANT-12356;So;0;ON;;;;;N;;;;; +1CE88;SEPARATED BLOCK SEXTANT-456;So;0;ON;;;;;N;;;;; +1CE89;SEPARATED BLOCK SEXTANT-1456;So;0;ON;;;;;N;;;;; +1CE8A;SEPARATED BLOCK SEXTANT-2456;So;0;ON;;;;;N;;;;; +1CE8B;SEPARATED BLOCK SEXTANT-12456;So;0;ON;;;;;N;;;;; +1CE8C;SEPARATED BLOCK SEXTANT-3456;So;0;ON;;;;;N;;;;; +1CE8D;SEPARATED BLOCK SEXTANT-13456;So;0;ON;;;;;N;;;;; +1CE8E;SEPARATED BLOCK SEXTANT-23456;So;0;ON;;;;;N;;;;; +1CE8F;SEPARATED BLOCK SEXTANT-123456;So;0;ON;;;;;N;;;;; +1CE90;UPPER LEFT ONE SIXTEENTH BLOCK;So;0;ON;;;;;N;;;;; +1CE91;UPPER CENTRE LEFT ONE SIXTEENTH BLOCK;So;0;ON;;;;;N;;;;; +1CE92;UPPER CENTRE RIGHT ONE SIXTEENTH BLOCK;So;0;ON;;;;;N;;;;; +1CE93;UPPER RIGHT ONE SIXTEENTH BLOCK;So;0;ON;;;;;N;;;;; +1CE94;UPPER MIDDLE LEFT ONE SIXTEENTH BLOCK;So;0;ON;;;;;N;;;;; +1CE95;UPPER MIDDLE CENTRE LEFT ONE SIXTEENTH BLOCK;So;0;ON;;;;;N;;;;; +1CE96;UPPER MIDDLE CENTRE RIGHT ONE SIXTEENTH BLOCK;So;0;ON;;;;;N;;;;; +1CE97;UPPER MIDDLE RIGHT ONE SIXTEENTH BLOCK;So;0;ON;;;;;N;;;;; +1CE98;LOWER MIDDLE LEFT ONE SIXTEENTH BLOCK;So;0;ON;;;;;N;;;;; +1CE99;LOWER MIDDLE CENTRE LEFT ONE SIXTEENTH BLOCK;So;0;ON;;;;;N;;;;; +1CE9A;LOWER MIDDLE CENTRE RIGHT ONE SIXTEENTH BLOCK;So;0;ON;;;;;N;;;;; +1CE9B;LOWER MIDDLE RIGHT ONE SIXTEENTH BLOCK;So;0;ON;;;;;N;;;;; +1CE9C;LOWER LEFT ONE SIXTEENTH BLOCK;So;0;ON;;;;;N;;;;; +1CE9D;LOWER CENTRE LEFT ONE SIXTEENTH BLOCK;So;0;ON;;;;;N;;;;; +1CE9E;LOWER CENTRE RIGHT ONE SIXTEENTH BLOCK;So;0;ON;;;;;N;;;;; +1CE9F;LOWER RIGHT ONE SIXTEENTH BLOCK;So;0;ON;;;;;N;;;;; +1CEA0;RIGHT HALF LOWER ONE QUARTER BLOCK;So;0;ON;;;;;N;;;;; +1CEA1;RIGHT THREE QUARTERS LOWER ONE QUARTER BLOCK;So;0;ON;;;;;N;;;;; +1CEA2;LEFT THREE QUARTERS LOWER ONE QUARTER BLOCK;So;0;ON;;;;;N;;;;; +1CEA3;LEFT HALF LOWER ONE QUARTER BLOCK;So;0;ON;;;;;N;;;;; +1CEA4;LOWER HALF LEFT ONE QUARTER BLOCK;So;0;ON;;;;;N;;;;; +1CEA5;LOWER THREE QUARTERS LEFT ONE QUARTER BLOCK;So;0;ON;;;;;N;;;;; +1CEA6;UPPER THREE QUARTERS LEFT ONE QUARTER BLOCK;So;0;ON;;;;;N;;;;; +1CEA7;UPPER HALF LEFT ONE QUARTER BLOCK;So;0;ON;;;;;N;;;;; +1CEA8;LEFT HALF UPPER ONE QUARTER BLOCK;So;0;ON;;;;;N;;;;; +1CEA9;LEFT THREE QUARTERS UPPER ONE QUARTER BLOCK;So;0;ON;;;;;N;;;;; +1CEAA;RIGHT THREE QUARTERS UPPER ONE QUARTER BLOCK;So;0;ON;;;;;N;;;;; +1CEAB;RIGHT HALF UPPER ONE QUARTER BLOCK;So;0;ON;;;;;N;;;;; +1CEAC;UPPER HALF RIGHT ONE QUARTER BLOCK;So;0;ON;;;;;N;;;;; +1CEAD;UPPER THREE QUARTERS RIGHT ONE QUARTER BLOCK;So;0;ON;;;;;N;;;;; +1CEAE;LOWER THREE QUARTERS RIGHT ONE QUARTER BLOCK;So;0;ON;;;;;N;;;;; +1CEAF;LOWER HALF RIGHT ONE QUARTER BLOCK;So;0;ON;;;;;N;;;;; +1CEB0;HORIZONTAL ZIGZAG LINE;So;0;ON;;;;;N;;;;; +1CEB1;KEYHOLE;So;0;ON;;;;;N;;;;; +1CEB2;OLD PERSONAL COMPUTER WITH MONITOR IN PORTRAIT ORIENTATION;So;0;ON;;;;;N;;;;; +1CEB3;BLACK RIGHT TRIANGLE CARET;So;0;ON;;;;;N;;;;; +1CEBA;FRAGILE SYMBOL;So;0;ON;;;;;N;;;;; +1CEBB;OFFICE BUILDING SYMBOL;So;0;ON;;;;;N;;;;; +1CEBC;TREE SYMBOL;So;0;ON;;;;;N;;;;; +1CEBD;APPLE SYMBOL;So;0;ON;;;;;N;;;;; +1CEBE;CHERRY SYMBOL;So;0;ON;;;;;N;;;;; +1CEBF;STRAWBERRY SYMBOL;So;0;ON;;;;;N;;;;; +1CEC0;HEBE;So;0;ON;;;;;N;;;;; +1CEC1;IRIS;So;0;ON;;;;;N;;;;; +1CEC2;FLORA;So;0;ON;;;;;N;;;;; +1CEC3;METIS;So;0;ON;;;;;N;;;;; +1CEC4;PARTHENOPE;So;0;ON;;;;;N;;;;; +1CEC5;VICTORIA;So;0;ON;;;;;N;;;;; +1CEC6;EGERIA;So;0;ON;;;;;N;;;;; +1CEC7;IRENE;So;0;ON;;;;;N;;;;; +1CEC8;EUNOMIA;So;0;ON;;;;;N;;;;; +1CEC9;PSYCHE;So;0;ON;;;;;N;;;;; +1CECA;THETIS;So;0;ON;;;;;N;;;;; +1CECB;MELPOMENE;So;0;ON;;;;;N;;;;; +1CECC;FORTUNA;So;0;ON;;;;;N;;;;; +1CECD;ASTRONOMICAL SYMBOL FOR ASTEROID PROSERPINA;So;0;ON;;;;;N;;;;; +1CECE;BELLONA;So;0;ON;;;;;N;;;;; +1CECF;AMPHITRITE;So;0;ON;;;;;N;;;;; +1CED0;LEUKOTHEA;So;0;ON;;;;;N;;;;; +1CEE0;GEOMANTIC FIGURE POPULUS;So;0;ON;;;;;N;;;;; +1CEE1;GEOMANTIC FIGURE TRISTITIA;So;0;ON;;;;;N;;;;; +1CEE2;GEOMANTIC FIGURE ALBUS;So;0;ON;;;;;N;;;;; +1CEE3;GEOMANTIC FIGURE FORTUNA MAJOR;So;0;ON;;;;;N;;;;; +1CEE4;GEOMANTIC FIGURE RUBEUS;So;0;ON;;;;;N;;;;; +1CEE5;GEOMANTIC FIGURE ACQUISITIO;So;0;ON;;;;;N;;;;; +1CEE6;GEOMANTIC FIGURE CONJUNCTIO;So;0;ON;;;;;N;;;;; +1CEE7;GEOMANTIC FIGURE CAPUT DRACONIS;So;0;ON;;;;;N;;;;; +1CEE8;GEOMANTIC FIGURE LAETITIA;So;0;ON;;;;;N;;;;; +1CEE9;GEOMANTIC FIGURE CARCER;So;0;ON;;;;;N;;;;; +1CEEA;GEOMANTIC FIGURE AMISSIO;So;0;ON;;;;;N;;;;; +1CEEB;GEOMANTIC FIGURE PUELLA;So;0;ON;;;;;N;;;;; +1CEEC;GEOMANTIC FIGURE FORTUNA MINOR;So;0;ON;;;;;N;;;;; +1CEED;GEOMANTIC FIGURE PUER;So;0;ON;;;;;N;;;;; +1CEEE;GEOMANTIC FIGURE CAUDA DRACONIS;So;0;ON;;;;;N;;;;; +1CEEF;GEOMANTIC FIGURE VIA;So;0;ON;;;;;N;;;;; +1CEF0;MEDIUM SMALL WHITE CIRCLE WITH HORIZONTAL BAR;Sm;0;ON;;;;;N;;;;; 1CF00;ZNAMENNY COMBINING MARK GORAZDO NIZKO S KRYZHEM ON LEFT;Mn;0;NSM;;;;;N;;;;; 1CF01;ZNAMENNY COMBINING MARK NIZKO S KRYZHEM ON LEFT;Mn;0;NSM;;;;;N;;;;; 1CF02;ZNAMENNY COMBINING MARK TSATA ON LEFT;Mn;0;NSM;;;;;N;;;;; @@ -29540,7 +35007,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1D6BE;MATHEMATICAL BOLD CAPITAL CHI;Lu;0;L; 03A7;;;;N;;;;; 1D6BF;MATHEMATICAL BOLD CAPITAL PSI;Lu;0;L; 03A8;;;;N;;;;; 1D6C0;MATHEMATICAL BOLD CAPITAL OMEGA;Lu;0;L; 03A9;;;;N;;;;; -1D6C1;MATHEMATICAL BOLD NABLA;Sm;0;L; 2207;;;;N;;;;; +1D6C1;MATHEMATICAL BOLD NABLA;Sm;0;ON; 2207;;;;N;;;;; 1D6C2;MATHEMATICAL BOLD SMALL ALPHA;Ll;0;L; 03B1;;;;N;;;;; 1D6C3;MATHEMATICAL BOLD SMALL BETA;Ll;0;L; 03B2;;;;N;;;;; 1D6C4;MATHEMATICAL BOLD SMALL GAMMA;Ll;0;L; 03B3;;;;N;;;;; @@ -29598,7 +35065,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1D6F8;MATHEMATICAL ITALIC CAPITAL CHI;Lu;0;L; 03A7;;;;N;;;;; 1D6F9;MATHEMATICAL ITALIC CAPITAL PSI;Lu;0;L; 03A8;;;;N;;;;; 1D6FA;MATHEMATICAL ITALIC CAPITAL OMEGA;Lu;0;L; 03A9;;;;N;;;;; -1D6FB;MATHEMATICAL ITALIC NABLA;Sm;0;L; 2207;;;;N;;;;; +1D6FB;MATHEMATICAL ITALIC NABLA;Sm;0;ON; 2207;;;;N;;;;; 1D6FC;MATHEMATICAL ITALIC SMALL ALPHA;Ll;0;L; 03B1;;;;N;;;;; 1D6FD;MATHEMATICAL ITALIC SMALL BETA;Ll;0;L; 03B2;;;;N;;;;; 1D6FE;MATHEMATICAL ITALIC SMALL GAMMA;Ll;0;L; 03B3;;;;N;;;;; @@ -29656,7 +35123,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1D732;MATHEMATICAL BOLD ITALIC CAPITAL CHI;Lu;0;L; 03A7;;;;N;;;;; 1D733;MATHEMATICAL BOLD ITALIC CAPITAL PSI;Lu;0;L; 03A8;;;;N;;;;; 1D734;MATHEMATICAL BOLD ITALIC CAPITAL OMEGA;Lu;0;L; 03A9;;;;N;;;;; -1D735;MATHEMATICAL BOLD ITALIC NABLA;Sm;0;L; 2207;;;;N;;;;; +1D735;MATHEMATICAL BOLD ITALIC NABLA;Sm;0;ON; 2207;;;;N;;;;; 1D736;MATHEMATICAL BOLD ITALIC SMALL ALPHA;Ll;0;L; 03B1;;;;N;;;;; 1D737;MATHEMATICAL BOLD ITALIC SMALL BETA;Ll;0;L; 03B2;;;;N;;;;; 1D738;MATHEMATICAL BOLD ITALIC SMALL GAMMA;Ll;0;L; 03B3;;;;N;;;;; @@ -29714,7 +35181,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1D76C;MATHEMATICAL SANS-SERIF BOLD CAPITAL CHI;Lu;0;L; 03A7;;;;N;;;;; 1D76D;MATHEMATICAL SANS-SERIF BOLD CAPITAL PSI;Lu;0;L; 03A8;;;;N;;;;; 1D76E;MATHEMATICAL SANS-SERIF BOLD CAPITAL OMEGA;Lu;0;L; 03A9;;;;N;;;;; -1D76F;MATHEMATICAL SANS-SERIF BOLD NABLA;Sm;0;L; 2207;;;;N;;;;; +1D76F;MATHEMATICAL SANS-SERIF BOLD NABLA;Sm;0;ON; 2207;;;;N;;;;; 1D770;MATHEMATICAL SANS-SERIF BOLD SMALL ALPHA;Ll;0;L; 03B1;;;;N;;;;; 1D771;MATHEMATICAL SANS-SERIF BOLD SMALL BETA;Ll;0;L; 03B2;;;;N;;;;; 1D772;MATHEMATICAL SANS-SERIF BOLD SMALL GAMMA;Ll;0;L; 03B3;;;;N;;;;; @@ -29772,7 +35239,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1D7A6;MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL CHI;Lu;0;L; 03A7;;;;N;;;;; 1D7A7;MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL PSI;Lu;0;L; 03A8;;;;N;;;;; 1D7A8;MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL OMEGA;Lu;0;L; 03A9;;;;N;;;;; -1D7A9;MATHEMATICAL SANS-SERIF BOLD ITALIC NABLA;Sm;0;L; 2207;;;;N;;;;; +1D7A9;MATHEMATICAL SANS-SERIF BOLD ITALIC NABLA;Sm;0;ON; 2207;;;;N;;;;; 1D7AA;MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL ALPHA;Ll;0;L; 03B1;;;;N;;;;; 1D7AB;MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL BETA;Ll;0;L; 03B2;;;;N;;;;; 1D7AC;MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL GAMMA;Ll;0;L; 03B3;;;;N;;;;; @@ -30870,6 +36337,105 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1E4F7;NAG MUNDARI DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; 1E4F8;NAG MUNDARI DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; 1E4F9;NAG MUNDARI DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; +1E5D0;OL ONAL LETTER O;Lo;0;L;;;;;N;;;;; +1E5D1;OL ONAL LETTER OM;Lo;0;L;;;;;N;;;;; +1E5D2;OL ONAL LETTER ONG;Lo;0;L;;;;;N;;;;; +1E5D3;OL ONAL LETTER ORR;Lo;0;L;;;;;N;;;;; +1E5D4;OL ONAL LETTER OO;Lo;0;L;;;;;N;;;;; +1E5D5;OL ONAL LETTER OY;Lo;0;L;;;;;N;;;;; +1E5D6;OL ONAL LETTER A;Lo;0;L;;;;;N;;;;; +1E5D7;OL ONAL LETTER AD;Lo;0;L;;;;;N;;;;; +1E5D8;OL ONAL LETTER AB;Lo;0;L;;;;;N;;;;; +1E5D9;OL ONAL LETTER AH;Lo;0;L;;;;;N;;;;; +1E5DA;OL ONAL LETTER AL;Lo;0;L;;;;;N;;;;; +1E5DB;OL ONAL LETTER AW;Lo;0;L;;;;;N;;;;; +1E5DC;OL ONAL LETTER I;Lo;0;L;;;;;N;;;;; +1E5DD;OL ONAL LETTER IT;Lo;0;L;;;;;N;;;;; +1E5DE;OL ONAL LETTER IP;Lo;0;L;;;;;N;;;;; +1E5DF;OL ONAL LETTER ITT;Lo;0;L;;;;;N;;;;; +1E5E0;OL ONAL LETTER ID;Lo;0;L;;;;;N;;;;; +1E5E1;OL ONAL LETTER IN;Lo;0;L;;;;;N;;;;; +1E5E2;OL ONAL LETTER U;Lo;0;L;;;;;N;;;;; +1E5E3;OL ONAL LETTER UK;Lo;0;L;;;;;N;;;;; +1E5E4;OL ONAL LETTER UDD;Lo;0;L;;;;;N;;;;; +1E5E5;OL ONAL LETTER UJ;Lo;0;L;;;;;N;;;;; +1E5E6;OL ONAL LETTER UNY;Lo;0;L;;;;;N;;;;; +1E5E7;OL ONAL LETTER UR;Lo;0;L;;;;;N;;;;; +1E5E8;OL ONAL LETTER E;Lo;0;L;;;;;N;;;;; +1E5E9;OL ONAL LETTER ES;Lo;0;L;;;;;N;;;;; +1E5EA;OL ONAL LETTER EH;Lo;0;L;;;;;N;;;;; +1E5EB;OL ONAL LETTER EC;Lo;0;L;;;;;N;;;;; +1E5EC;OL ONAL LETTER ENN;Lo;0;L;;;;;N;;;;; +1E5ED;OL ONAL LETTER EG;Lo;0;L;;;;;N;;;;; +1E5EE;OL ONAL SIGN MU;Mn;230;NSM;;;;;N;;;;; +1E5EF;OL ONAL SIGN IKIR;Mn;220;NSM;;;;;N;;;;; +1E5F0;OL ONAL SIGN HODDOND;Lo;0;L;;;;;N;;;;; +1E5F1;OL ONAL DIGIT ZERO;Nd;0;L;;0;0;0;N;;;;; +1E5F2;OL ONAL DIGIT ONE;Nd;0;L;;1;1;1;N;;;;; +1E5F3;OL ONAL DIGIT TWO;Nd;0;L;;2;2;2;N;;;;; +1E5F4;OL ONAL DIGIT THREE;Nd;0;L;;3;3;3;N;;;;; +1E5F5;OL ONAL DIGIT FOUR;Nd;0;L;;4;4;4;N;;;;; +1E5F6;OL ONAL DIGIT FIVE;Nd;0;L;;5;5;5;N;;;;; +1E5F7;OL ONAL DIGIT SIX;Nd;0;L;;6;6;6;N;;;;; +1E5F8;OL ONAL DIGIT SEVEN;Nd;0;L;;7;7;7;N;;;;; +1E5F9;OL ONAL DIGIT EIGHT;Nd;0;L;;8;8;8;N;;;;; +1E5FA;OL ONAL DIGIT NINE;Nd;0;L;;9;9;9;N;;;;; +1E5FF;OL ONAL ABBREVIATION SIGN;Po;0;L;;;;;N;;;;; +1E6C0;TAI YO LETTER LOW KO;Lo;0;L;;;;;N;;;;; +1E6C1;TAI YO LETTER HIGH KO;Lo;0;L;;;;;N;;;;; +1E6C2;TAI YO LETTER LOW KHO;Lo;0;L;;;;;N;;;;; +1E6C3;TAI YO LETTER HIGH KHO;Lo;0;L;;;;;N;;;;; +1E6C4;TAI YO LETTER GO;Lo;0;L;;;;;N;;;;; +1E6C5;TAI YO LETTER NGO;Lo;0;L;;;;;N;;;;; +1E6C6;TAI YO LETTER CO;Lo;0;L;;;;;N;;;;; +1E6C7;TAI YO LETTER LOW XO;Lo;0;L;;;;;N;;;;; +1E6C8;TAI YO LETTER HIGH XO;Lo;0;L;;;;;N;;;;; +1E6C9;TAI YO LETTER LOW NYO;Lo;0;L;;;;;N;;;;; +1E6CA;TAI YO LETTER HIGH NYO;Lo;0;L;;;;;N;;;;; +1E6CB;TAI YO LETTER DO;Lo;0;L;;;;;N;;;;; +1E6CC;TAI YO LETTER LOW TO;Lo;0;L;;;;;N;;;;; +1E6CD;TAI YO LETTER HIGH TO;Lo;0;L;;;;;N;;;;; +1E6CE;TAI YO LETTER THO;Lo;0;L;;;;;N;;;;; +1E6CF;TAI YO LETTER NO;Lo;0;L;;;;;N;;;;; +1E6D0;TAI YO LETTER BO;Lo;0;L;;;;;N;;;;; +1E6D1;TAI YO LETTER LOW PO;Lo;0;L;;;;;N;;;;; +1E6D2;TAI YO LETTER HIGH PO;Lo;0;L;;;;;N;;;;; +1E6D3;TAI YO LETTER PHO;Lo;0;L;;;;;N;;;;; +1E6D4;TAI YO LETTER LOW FO;Lo;0;L;;;;;N;;;;; +1E6D5;TAI YO LETTER HIGH FO;Lo;0;L;;;;;N;;;;; +1E6D6;TAI YO LETTER MO;Lo;0;L;;;;;N;;;;; +1E6D7;TAI YO LETTER YO;Lo;0;L;;;;;N;;;;; +1E6D8;TAI YO LETTER LO;Lo;0;L;;;;;N;;;;; +1E6D9;TAI YO LETTER VO;Lo;0;L;;;;;N;;;;; +1E6DA;TAI YO LETTER LOW HO;Lo;0;L;;;;;N;;;;; +1E6DB;TAI YO LETTER HIGH HO;Lo;0;L;;;;;N;;;;; +1E6DC;TAI YO LETTER QO;Lo;0;L;;;;;N;;;;; +1E6DD;TAI YO LETTER LOW KVO;Lo;0;L;;;;;N;;;;; +1E6DE;TAI YO LETTER HIGH KVO;Lo;0;L;;;;;N;;;;; +1E6E0;TAI YO LETTER AA;Lo;0;L;;;;;N;;;;; +1E6E1;TAI YO LETTER I;Lo;0;L;;;;;N;;;;; +1E6E2;TAI YO LETTER UE;Lo;0;L;;;;;N;;;;; +1E6E3;TAI YO SIGN UE;Mn;230;NSM;;;;;N;;;;; +1E6E4;TAI YO LETTER U;Lo;0;L;;;;;N;;;;; +1E6E5;TAI YO LETTER AE;Lo;0;L;;;;;N;;;;; +1E6E6;TAI YO SIGN AU;Mn;230;NSM;;;;;N;;;;; +1E6E7;TAI YO LETTER O;Lo;0;L;;;;;N;;;;; +1E6E8;TAI YO LETTER E;Lo;0;L;;;;;N;;;;; +1E6E9;TAI YO LETTER IA;Lo;0;L;;;;;N;;;;; +1E6EA;TAI YO LETTER UEA;Lo;0;L;;;;;N;;;;; +1E6EB;TAI YO LETTER UA;Lo;0;L;;;;;N;;;;; +1E6EC;TAI YO LETTER OO;Lo;0;L;;;;;N;;;;; +1E6ED;TAI YO LETTER AUE;Lo;0;L;;;;;N;;;;; +1E6EE;TAI YO SIGN AY;Mn;230;NSM;;;;;N;;;;; +1E6EF;TAI YO SIGN ANG;Mn;230;NSM;;;;;N;;;;; +1E6F0;TAI YO LETTER AN;Lo;0;L;;;;;N;;;;; +1E6F1;TAI YO LETTER AM;Lo;0;L;;;;;N;;;;; +1E6F2;TAI YO LETTER AK;Lo;0;L;;;;;N;;;;; +1E6F3;TAI YO LETTER AT;Lo;0;L;;;;;N;;;;; +1E6F4;TAI YO LETTER AP;Lo;0;L;;;;;N;;;;; +1E6F5;TAI YO SIGN OM;Mn;230;NSM;;;;;N;;;;; +1E6FE;TAI YO SYMBOL MUEANG;Lo;0;L;;;;;N;;;;; +1E6FF;TAI YO XAM LAI;Lm;0;L;;;;;N;;;;; 1E7E0;ETHIOPIC SYLLABLE HHYA;Lo;0;L;;;;;N;;;;; 1E7E1;ETHIOPIC SYLLABLE HHYU;Lo;0;L;;;;;N;;;;; 1E7E2;ETHIOPIC SYLLABLE HHYI;Lo;0;L;;;;;N;;;;; @@ -32945,6 +38511,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1F6D5;HINDU TEMPLE;So;0;ON;;;;;N;;;;; 1F6D6;HUT;So;0;ON;;;;;N;;;;; 1F6D7;ELEVATOR;So;0;ON;;;;;N;;;;; +1F6D8;LANDSLIDE;So;0;ON;;;;;N;;;;; 1F6DC;WIRELESS;So;0;ON;;;;;N;;;;; 1F6DD;PLAYGROUND SLIDE;So;0;ON;;;;;N;;;;; 1F6DE;WHEEL;So;0;ON;;;;;N;;;;; @@ -33094,6 +38661,10 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1F774;LOT OF FORTUNE;So;0;ON;;;;;N;;;;; 1F775;OCCULTATION;So;0;ON;;;;;N;;;;; 1F776;LUNAR ECLIPSE;So;0;ON;;;;;N;;;;; +1F777;VESTA FORM TWO;So;0;ON;;;;;N;;;;; +1F778;ASTRAEA FORM TWO;So;0;ON;;;;;N;;;;; +1F779;HYGIEA FORM TWO;So;0;ON;;;;;N;;;;; +1F77A;PARTHENOPE FORM TWO;So;0;ON;;;;;N;;;;; 1F77B;HAUMEA;So;0;ON;;;;;N;;;;; 1F77C;MAKEMAKE;So;0;ON;;;;;N;;;;; 1F77D;GONGGONG;So;0;ON;;;;;N;;;;; @@ -33352,6 +38923,27 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1F8AD;WHITE ARROW SHAFT WIDTH TWO THIRDS;So;0;ON;;;;;N;;;;; 1F8B0;ARROW POINTING UPWARDS THEN NORTH WEST;So;0;ON;;;;;N;;;;; 1F8B1;ARROW POINTING RIGHTWARDS THEN CURVING SOUTH WEST;So;0;ON;;;;;N;;;;; +1F8B2;RIGHTWARDS ARROW WITH LOWER HOOK;So;0;ON;;;;;N;;;;; +1F8B3;DOWNWARDS BLACK ARROW TO BAR;So;0;ON;;;;;N;;;;; +1F8B4;NEGATIVE SQUARED LEFTWARDS ARROW;So;0;ON;;;;;N;;;;; +1F8B5;NEGATIVE SQUARED UPWARDS ARROW;So;0;ON;;;;;N;;;;; +1F8B6;NEGATIVE SQUARED RIGHTWARDS ARROW;So;0;ON;;;;;N;;;;; +1F8B7;NEGATIVE SQUARED DOWNWARDS ARROW;So;0;ON;;;;;N;;;;; +1F8B8;NORTH WEST ARROW FROM BAR;So;0;ON;;;;;N;;;;; +1F8B9;NORTH EAST ARROW FROM BAR;So;0;ON;;;;;N;;;;; +1F8BA;SOUTH EAST ARROW FROM BAR;So;0;ON;;;;;N;;;;; +1F8BB;SOUTH WEST ARROW FROM BAR;So;0;ON;;;;;N;;;;; +1F8C0;LEFTWARDS ARROW FROM DOWNWARDS ARROW;So;0;ON;;;;;N;;;;; +1F8C1;RIGHTWARDS ARROW FROM DOWNWARDS ARROW;So;0;ON;;;;;N;;;;; +1F8D0;LONG RIGHTWARDS ARROW OVER LONG LEFTWARDS ARROW;Sm;0;ON;;;;;N;;;;; +1F8D1;LONG RIGHTWARDS HARPOON OVER LONG LEFTWARDS HARPOON;Sm;0;ON;;;;;N;;;;; +1F8D2;LONG RIGHTWARDS HARPOON ABOVE SHORT LEFTWARDS HARPOON;Sm;0;ON;;;;;N;;;;; +1F8D3;SHORT RIGHTWARDS HARPOON ABOVE LONG LEFTWARDS HARPOON;Sm;0;ON;;;;;N;;;;; +1F8D4;LONG LEFTWARDS HARPOON ABOVE SHORT RIGHTWARDS HARPOON;Sm;0;ON;;;;;N;;;;; +1F8D5;SHORT LEFTWARDS HARPOON ABOVE LONG RIGHTWARDS HARPOON;Sm;0;ON;;;;;N;;;;; +1F8D6;LONG RIGHTWARDS ARROW THROUGH X;Sm;0;ON;;;;;N;;;;; +1F8D7;LONG RIGHTWARDS ARROW WITH DOUBLE SLASH;Sm;0;ON;;;;;N;;;;; +1F8D8;LONG LEFT RIGHT ARROW WITH DEPENDENT LOBE;Sm;0;ON;;;;;N;;;;; 1F900;CIRCLED CROSS FORMEE WITH FOUR DOTS;So;0;ON;;;;;N;;;;; 1F901;CIRCLED CROSS FORMEE WITH TWO DOTS;So;0;ON;;;;;N;;;;; 1F902;CIRCLED CROSS FORMEE;So;0;ON;;;;;N;;;;; @@ -33692,6 +39284,10 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1FA51;BLACK CHESS KNIGHT-QUEEN;So;0;ON;;;;;N;;;;; 1FA52;BLACK CHESS KNIGHT-ROOK;So;0;ON;;;;;N;;;;; 1FA53;BLACK CHESS KNIGHT-BISHOP;So;0;ON;;;;;N;;;;; +1FA54;WHITE CHESS FERZ;So;0;ON;;;;;N;;;;; +1FA55;WHITE CHESS ALFIL;So;0;ON;;;;;N;;;;; +1FA56;BLACK CHESS FERZ;So;0;ON;;;;;N;;;;; +1FA57;BLACK CHESS ALFIL;So;0;ON;;;;;N;;;;; 1FA60;XIANGQI RED GENERAL;So;0;ON;;;;;N;;;;; 1FA61;XIANGQI RED MANDARIN;So;0;ON;;;;;N;;;;; 1FA62;XIANGQI RED ELEPHANT;So;0;ON;;;;;N;;;;; @@ -33728,6 +39324,10 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1FA86;NESTING DOLLS;So;0;ON;;;;;N;;;;; 1FA87;MARACAS;So;0;ON;;;;;N;;;;; 1FA88;FLUTE;So;0;ON;;;;;N;;;;; +1FA89;HARP;So;0;ON;;;;;N;;;;; +1FA8A;TROMBONE;So;0;ON;;;;;N;;;;; +1FA8E;TREASURE CHEST;So;0;ON;;;;;N;;;;; +1FA8F;SHOVEL;So;0;ON;;;;;N;;;;; 1FA90;RINGED PLANET;So;0;ON;;;;;N;;;;; 1FA91;CHAIR;So;0;ON;;;;;N;;;;; 1FA92;RAZOR;So;0;ON;;;;;N;;;;; @@ -33774,6 +39374,7 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1FABB;HYACINTH;So;0;ON;;;;;N;;;;; 1FABC;JELLYFISH;So;0;ON;;;;;N;;;;; 1FABD;WING;So;0;ON;;;;;N;;;;; +1FABE;LEAFLESS TREE;So;0;ON;;;;;N;;;;; 1FABF;GOOSE;So;0;ON;;;;;N;;;;; 1FAC0;ANATOMICAL HEART;So;0;ON;;;;;N;;;;; 1FAC1;LUNGS;So;0;ON;;;;;N;;;;; @@ -33781,6 +39382,9 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1FAC3;PREGNANT MAN;So;0;ON;;;;;N;;;;; 1FAC4;PREGNANT PERSON;So;0;ON;;;;;N;;;;; 1FAC5;PERSON WITH CROWN;So;0;ON;;;;;N;;;;; +1FAC6;FINGERPRINT;So;0;ON;;;;;N;;;;; +1FAC8;HAIRY CREATURE;So;0;ON;;;;;N;;;;; +1FACD;ORCA;So;0;ON;;;;;N;;;;; 1FACE;MOOSE;So;0;ON;;;;;N;;;;; 1FACF;DONKEY;So;0;ON;;;;;N;;;;; 1FAD0;BLUEBERRIES;So;0;ON;;;;;N;;;;; @@ -33795,6 +39399,8 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1FAD9;JAR;So;0;ON;;;;;N;;;;; 1FADA;GINGER ROOT;So;0;ON;;;;;N;;;;; 1FADB;PEA POD;So;0;ON;;;;;N;;;;; +1FADC;ROOT VEGETABLE;So;0;ON;;;;;N;;;;; +1FADF;SPLATTER;So;0;ON;;;;;N;;;;; 1FAE0;MELTING FACE;So;0;ON;;;;;N;;;;; 1FAE1;SALUTING FACE;So;0;ON;;;;;N;;;;; 1FAE2;FACE WITH OPEN EYES AND HAND OVER MOUTH;So;0;ON;;;;;N;;;;; @@ -33804,6 +39410,9 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1FAE6;BITING LIP;So;0;ON;;;;;N;;;;; 1FAE7;BUBBLES;So;0;ON;;;;;N;;;;; 1FAE8;SHAKING FACE;So;0;ON;;;;;N;;;;; +1FAE9;FACE WITH BAGS UNDER EYES;So;0;ON;;;;;N;;;;; +1FAEA;DISTORTED FACE;So;0;ON;;;;;N;;;;; +1FAEF;FIGHT CLOUD;So;0;ON;;;;;N;;;;; 1FAF0;HAND WITH INDEX FINGER AND THUMB CROSSED;So;0;ON;;;;;N;;;;; 1FAF1;RIGHTWARDS HAND;So;0;ON;;;;;N;;;;; 1FAF2;LEFTWARDS HAND;So;0;ON;;;;;N;;;;; @@ -34015,6 +39624,43 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1FBC8;STICK FIGURE LEANING RIGHT;So;0;ON;;;;;N;;;;; 1FBC9;STICK FIGURE WITH DRESS;So;0;ON;;;;;N;;;;; 1FBCA;WHITE UP-POINTING CHEVRON;So;0;ON;;;;;N;;;;; +1FBCB;WHITE CROSS MARK;So;0;ON;;;;;N;;;;; +1FBCC;RAISED SMALL LEFT SQUARE BRACKET;So;0;ON;;;;;N;;;;; +1FBCD;BLACK SMALL UP-POINTING CHEVRON;So;0;ON;;;;;N;;;;; +1FBCE;LEFT TWO THIRDS BLOCK;So;0;ON;;;;;N;;;;; +1FBCF;LEFT ONE THIRD BLOCK;So;0;ON;;;;;N;;;;; +1FBD0;BOX DRAWINGS LIGHT DIAGONAL MIDDLE RIGHT TO LOWER LEFT;So;0;ON;;;;;N;;;;; +1FBD1;BOX DRAWINGS LIGHT DIAGONAL UPPER RIGHT TO MIDDLE LEFT;So;0;ON;;;;;N;;;;; +1FBD2;BOX DRAWINGS LIGHT DIAGONAL UPPER LEFT TO MIDDLE RIGHT;So;0;ON;;;;;N;;;;; +1FBD3;BOX DRAWINGS LIGHT DIAGONAL MIDDLE LEFT TO LOWER RIGHT;So;0;ON;;;;;N;;;;; +1FBD4;BOX DRAWINGS LIGHT DIAGONAL UPPER LEFT TO LOWER CENTRE;So;0;ON;;;;;N;;;;; +1FBD5;BOX DRAWINGS LIGHT DIAGONAL UPPER CENTRE TO LOWER RIGHT;So;0;ON;;;;;N;;;;; +1FBD6;BOX DRAWINGS LIGHT DIAGONAL UPPER RIGHT TO LOWER CENTRE;So;0;ON;;;;;N;;;;; +1FBD7;BOX DRAWINGS LIGHT DIAGONAL UPPER CENTRE TO LOWER LEFT;So;0;ON;;;;;N;;;;; +1FBD8;BOX DRAWINGS LIGHT DIAGONAL UPPER LEFT TO MIDDLE CENTRE TO UPPER RIGHT;So;0;ON;;;;;N;;;;; +1FBD9;BOX DRAWINGS LIGHT DIAGONAL UPPER RIGHT TO MIDDLE CENTRE TO LOWER RIGHT;So;0;ON;;;;;N;;;;; +1FBDA;BOX DRAWINGS LIGHT DIAGONAL LOWER LEFT TO MIDDLE CENTRE TO LOWER RIGHT;So;0;ON;;;;;N;;;;; +1FBDB;BOX DRAWINGS LIGHT DIAGONAL UPPER LEFT TO MIDDLE CENTRE TO LOWER LEFT;So;0;ON;;;;;N;;;;; +1FBDC;BOX DRAWINGS LIGHT DIAGONAL UPPER LEFT TO LOWER CENTRE TO UPPER RIGHT;So;0;ON;;;;;N;;;;; +1FBDD;BOX DRAWINGS LIGHT DIAGONAL UPPER RIGHT TO MIDDLE LEFT TO LOWER RIGHT;So;0;ON;;;;;N;;;;; +1FBDE;BOX DRAWINGS LIGHT DIAGONAL LOWER LEFT TO UPPER CENTRE TO LOWER RIGHT;So;0;ON;;;;;N;;;;; +1FBDF;BOX DRAWINGS LIGHT DIAGONAL UPPER LEFT TO MIDDLE RIGHT TO LOWER LEFT;So;0;ON;;;;;N;;;;; +1FBE0;TOP JUSTIFIED LOWER HALF WHITE CIRCLE;So;0;ON;;;;;N;;;;; +1FBE1;RIGHT JUSTIFIED LEFT HALF WHITE CIRCLE;So;0;ON;;;;;N;;;;; +1FBE2;BOTTOM JUSTIFIED UPPER HALF WHITE CIRCLE;So;0;ON;;;;;N;;;;; +1FBE3;LEFT JUSTIFIED RIGHT HALF WHITE CIRCLE;So;0;ON;;;;;N;;;;; +1FBE4;UPPER CENTRE ONE QUARTER BLOCK;So;0;ON;;;;;N;;;;; +1FBE5;LOWER CENTRE ONE QUARTER BLOCK;So;0;ON;;;;;N;;;;; +1FBE6;MIDDLE LEFT ONE QUARTER BLOCK;So;0;ON;;;;;N;;;;; +1FBE7;MIDDLE RIGHT ONE QUARTER BLOCK;So;0;ON;;;;;N;;;;; +1FBE8;TOP JUSTIFIED LOWER HALF BLACK CIRCLE;So;0;ON;;;;;N;;;;; +1FBE9;RIGHT JUSTIFIED LEFT HALF BLACK CIRCLE;So;0;ON;;;;;N;;;;; +1FBEA;BOTTOM JUSTIFIED UPPER HALF BLACK CIRCLE;So;0;ON;;;;;N;;;;; +1FBEB;LEFT JUSTIFIED RIGHT HALF BLACK CIRCLE;So;0;ON;;;;;N;;;;; +1FBEC;TOP RIGHT JUSTIFIED LOWER LEFT QUARTER BLACK CIRCLE;So;0;ON;;;;;N;;;;; +1FBED;BOTTOM LEFT JUSTIFIED UPPER RIGHT QUARTER BLACK CIRCLE;So;0;ON;;;;;N;;;;; +1FBEE;BOTTOM RIGHT JUSTIFIED UPPER LEFT QUARTER BLACK CIRCLE;So;0;ON;;;;;N;;;;; +1FBEF;TOP LEFT JUSTIFIED LOWER RIGHT QUARTER BLACK CIRCLE;So;0;ON;;;;;N;;;;; 1FBF0;SEGMENTED DIGIT ZERO;Nd;0;EN; 0030;0;0;0;N;;;;; 1FBF1;SEGMENTED DIGIT ONE;Nd;0;EN; 0031;1;1;1;N;;;;; 1FBF2;SEGMENTED DIGIT TWO;Nd;0;EN; 0032;2;2;2;N;;;;; @@ -34025,16 +39671,19 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 1FBF7;SEGMENTED DIGIT SEVEN;Nd;0;EN; 0037;7;7;7;N;;;;; 1FBF8;SEGMENTED DIGIT EIGHT;Nd;0;EN; 0038;8;8;8;N;;;;; 1FBF9;SEGMENTED DIGIT NINE;Nd;0;EN; 0039;9;9;9;N;;;;; +1FBFA;ALARM BELL SYMBOL;So;0;ON;;;;;N;;;;; 20000;;Lo;0;L;;;;;N;;;;; 2A6DF;;Lo;0;L;;;;;N;;;;; 2A700;;Lo;0;L;;;;;N;;;;; -2B739;;Lo;0;L;;;;;N;;;;; +2B73F;;Lo;0;L;;;;;N;;;;; 2B740;;Lo;0;L;;;;;N;;;;; 2B81D;;Lo;0;L;;;;;N;;;;; 2B820;;Lo;0;L;;;;;N;;;;; -2CEA1;;Lo;0;L;;;;;N;;;;; +2CEAD;;Lo;0;L;;;;;N;;;;; 2CEB0;;Lo;0;L;;;;;N;;;;; 2EBE0;;Lo;0;L;;;;;N;;;;; +2EBF0;;Lo;0;L;;;;;N;;;;; +2EE5D;;Lo;0;L;;;;;N;;;;; 2F800;CJK COMPATIBILITY IDEOGRAPH-2F800;Lo;0;L;4E3D;;;;N;;;;; 2F801;CJK COMPATIBILITY IDEOGRAPH-2F801;Lo;0;L;4E38;;;;N;;;;; 2F802;CJK COMPATIBILITY IDEOGRAPH-2F802;Lo;0;L;4E41;;;;N;;;;; @@ -34581,6 +40230,8 @@ FFFD;REPLACEMENT CHARACTER;So;0;ON;;;;;N;;;;; 3134A;;Lo;0;L;;;;;N;;;;; 31350;;Lo;0;L;;;;;N;;;;; 323AF;;Lo;0;L;;;;;N;;;;; +323B0;;Lo;0;L;;;;;N;;;;; +33479;;Lo;0;L;;;;;N;;;;; E0001;LANGUAGE TAG;Cf;0;BN;;;;;N;;;;; E0020;TAG SPACE;Cf;0;BN;;;;;N;;;;; E0021;TAG EXCLAMATION MARK;Cf;0;BN;;;;;N;;;;; diff --git a/src/adt/Makefile b/src/adt/Makefile index 05199f2dc..6fae4e7ca 100644 --- a/src/adt/Makefile +++ b/src/adt/Makefile @@ -2,6 +2,7 @@ SRC += src/adt/alloc.c SRC += src/adt/bitmap.c +SRC += src/adt/idmap.c SRC += src/adt/internedstateset.c SRC += src/adt/priq.c SRC += src/adt/path.c @@ -19,12 +20,10 @@ CFLAGS.${src} += -I src # XXX: for internal.h DFLAGS.${src} += -I src # XXX: for internal.h .endfor -# not all concrete set interfaces use all static functions from set.inc -.if ${CC:T:Mgcc*} || ${CC:T:Mclang*} -.for src in ${SRC:Msrc/adt/stateset.c} ${SRC:Msrc/adt/tupleset.c} ${SRC:Msrc/adt/edgeset.c} -CFLAGS.${src} += -Wno-unused-function +.for src in ${SRC:Msrc/adt/siphash.c} ${SRC:Msrc/adt/edgeset.c} ${SRC:Msrc/adt/idmap.c} ${SRC:Msrc/adt/ipriq.c} ${SRC:Msrc/adt/internedstateset.c} +CFLAGS.${src} += -std=c99 # XXX: for internal.h +DFLAGS.${src} += -std=c99 # XXX: for internal.h .endfor -.endif # I want to assert on things which are currently true for this platform, # but not true in general. diff --git a/src/adt/bitmap.c b/src/adt/bitmap.c index 434dfc222..1fb016083 100644 --- a/src/adt/bitmap.c +++ b/src/adt/bitmap.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -34,6 +35,15 @@ bm_set(struct bm *bm, size_t i) u64bitset_set(bm->map, i); } +void +bm_unset(struct bm *bm, size_t i) +{ + assert(bm != NULL); + assert(i <= UCHAR_MAX); + + u64bitset_clear(bm->map, i); +} + uint64_t * bm_nth_word(struct bm *bm, size_t n) { @@ -325,3 +335,34 @@ bm_snprint(const struct bm *bm, const struct fsm_options *opt, return -1; } + +void +bm_copy(struct bm *dst, const struct bm *src) +{ + memcpy(dst, src, sizeof(*src)); +} + +void +bm_intersect(struct bm *dst, const struct bm *src) +{ + for (size_t i = 0; i < sizeof(src->map)/sizeof(src->map[0]); i++) { + dst->map[i] &= src->map[i]; + } +} + +void +bm_union(struct bm *dst, const struct bm *src) +{ + for (size_t i = 0; i < sizeof(src->map)/sizeof(src->map[0]); i++) { + dst->map[i] |= src->map[i]; + } +} + +int +bm_any(const struct bm *bm) +{ + for (size_t i = 0; i < sizeof(bm->map)/sizeof(bm->map[0]); i++) { + if (bm->map[i]) { return 1; } + } + return 0; +} diff --git a/src/adt/edgeset.c b/src/adt/edgeset.c index c718727ca..178ddbc2b 100644 --- a/src/adt/edgeset.c +++ b/src/adt/edgeset.c @@ -11,6 +11,7 @@ #include #define LOG_BITSET 0 +#define LOG_BSEARCH 0 #include "libfsm/internal.h" /* XXX: for allocating struct fsm_edge, and the edges array */ @@ -34,8 +35,8 @@ struct edge_set { size_t ceil; /* nonzero */ size_t count; /* <= ceil */ struct edge_group { - fsm_state_t to; /* distinct */ uint64_t symbols[256/64]; + fsm_state_t to; /* distinct */ } *groups; /* sorted by .to */ }; @@ -120,7 +121,7 @@ dump_edge_set(const struct edge_set *set) static struct edge_set * init_empty(const struct fsm_alloc *alloc) { - struct edge_set *set = f_calloc(alloc, 1, sizeof(*set)); + struct edge_set *set = f_malloc(alloc, sizeof(*set)); if (set == NULL) { return NULL; } @@ -184,6 +185,100 @@ edge_set_advise_growth(struct edge_set **pset, const struct fsm_alloc *alloc, return 1; } +enum fsp_res { + FSP_FOUND_INSERT_POSITION, + FSP_FOUND_VALUE_PRESENT, +}; + +/* Use binary search to find the first position N where set->groups[N].to >= state, + * which includes the position immediately following the last entry. Return an enum + * which indicates whether state is already present. */ +static enum fsp_res +find_state_position(const struct edge_set *set, fsm_state_t state, size_t *dst) +{ + size_t lo = 0, hi = set->count; + if (LOG_BSEARCH) { + fprintf(stderr, "%s: looking for %d in %p (count %zu)\n", + __func__, state, (void *)set, set->count); + } + +#if EXPENSIVE_CHECKS + /* invariant: input is unique and sorted */ + for (size_t i = 1; i < set->count; i++) { + assert(set->groups[i - 1].to < set->groups[i].to); + } +#endif + + if (set->count == 0) { + if (LOG_BSEARCH) { + fprintf(stderr, "%s: empty, returning 0\n", __func__); + } + *dst = 0; + return FSP_FOUND_INSERT_POSITION; + } else { + if (LOG_BSEARCH) { + fprintf(stderr, "%s: fast path: looking for %d, set->groups[last].to %d\n", + __func__, state, set->groups[hi - 1].to); + } + + /* Check the last entry so we can append in constant time. */ + const fsm_state_t last = set->groups[hi - 1].to; + if (state > last) { + *dst = hi; + return FSP_FOUND_INSERT_POSITION; + } else if (state == last) { + *dst = hi - 1; + return FSP_FOUND_VALUE_PRESENT; + } + } + + size_t mid; + while (lo < hi) { /* lo <= mid < hi */ + mid = lo + (hi - lo)/2; /* avoid overflow */ + const struct edge_group *eg = &set->groups[mid]; + const fsm_state_t cur = eg->to; + if (LOG_BSEARCH) { + fprintf(stderr, "%s: lo %zu, hi %zu, mid %zu, cur %d, looking for %d\n", + __func__, lo, hi, mid, cur, state); + } + + if (state == cur) { + *dst = mid; + return FSP_FOUND_VALUE_PRESENT; + } else if (state > cur) { + lo = mid + 1; + if (LOG_BSEARCH) { + fprintf(stderr, "%s: new lo %zd\n", __func__, lo); + } + + /* Update mid if we're about to halt, because we're looking + * for the first position >= state, not the last position <=. */ + if (lo == hi) { + mid = lo; + if (LOG_BSEARCH) { + fprintf(stderr, "%s: special case, updating mid to %zd\n", __func__, mid); + } + } + } else if (state < cur) { + hi = mid; + if (LOG_BSEARCH) { + fprintf(stderr, "%s: new hi %zd\n", __func__, hi); + } + } + } + + if (LOG_BSEARCH) { + fprintf(stderr, "%s: halting at %zd (looking for %d, cur %d)\n", + __func__, mid, state, set->groups[mid].to); + } + + /* dst is now the first position > state (== case is handled above), + * which may be one past the end of the array. */ + assert(mid == set->count || set->groups[mid].to > state); + *dst = mid; + return FSP_FOUND_INSERT_POSITION; +} + int edge_set_add_bulk(struct edge_set **pset, const struct fsm_alloc *alloc, uint64_t symbols[256/64], fsm_state_t state) @@ -223,30 +318,24 @@ edge_set_add_bulk(struct edge_set **pset, const struct fsm_alloc *alloc, assert(set->count <= set->ceil); #if LOG_BITSET - fprintf(stderr, " -- edge_set_add: symbols [0x%lx, 0x%lx, 0x%lx, 0x%lx] -> state %d on %p\n", - symbols[0], symbols[1], symbols[2], symbols[3], - state, (void *)set); + fprintf(stderr, " -- edge_set_add: symbols [0x%lx, 0x%lx, 0x%lx, 0x%lx] -> state %d on %p\n", + symbols[0], symbols[1], symbols[2], symbols[3], + state, (void *)set); #endif - /* Linear search for a group with the same destination - * state, or the position where that group would go. */ - for (i = 0; i < set->count; i++) { + switch (find_state_position(set, state, &i)) { + case FSP_FOUND_VALUE_PRESENT: + assert(i < set->count); eg = &set->groups[i]; - - if (eg->to == state) { - /* This API does not indicate whether that - * symbol -> to edge was already present. */ - size_t i; - for (i = 0; i < 256/64; i++) { - eg->symbols[i] |= symbols[i]; - } - dump_edge_set(set); - return 1; - } else if (eg->to > state) { - break; /* will shift down and insert below */ - } else { - continue; + for (i = 0; i < 256/64; i++) { + eg->symbols[i] |= symbols[i]; } + dump_edge_set(set); + return 1; + + break; + case FSP_FOUND_INSERT_POSITION: + break; /* continue below */ } /* insert/append at i */ @@ -630,7 +719,7 @@ edge_set_copy(struct edge_set **dst, const struct fsm_alloc *alloc, return 1; } - set = f_calloc(alloc, 1, sizeof(*set)); + set = f_malloc(alloc, sizeof(*set)); if (set == NULL) { return 0; } diff --git a/src/adt/hashrec.c b/src/adt/hashrec.c index 6a341710b..7348cbc38 100644 --- a/src/adt/hashrec.c +++ b/src/adt/hashrec.c @@ -23,7 +23,7 @@ static const unsigned char hashk[] = { 0x14, 0xa8, 0xff, 0x36, 0x15, 0x16, 0x2c, 0xf7, 0xf4, 0xce, 0xb8, 0x66, 0x74, 0xf4, 0x3d, 0x64, }; -unsigned long +uint64_t hashrec(const void *p, size_t n) { uint64_t h = 0; diff --git a/src/adt/idmap.c b/src/adt/idmap.c new file mode 100644 index 000000000..8d4004e87 --- /dev/null +++ b/src/adt/idmap.c @@ -0,0 +1,412 @@ +/* + * Copyright 2021 Scott Vokes + * + * See LICENCE for the full copyright terms. + */ + +#include "adt/idmap.h" + +#include "adt/alloc.h" +#include "adt/hash.h" +#include "adt/u64bitset.h" + +#include +#include +#include + +#define NO_STATE ((fsm_state_t)-1) + +#define DEF_BUCKET_COUNT 4 + +struct idmap { + const struct fsm_alloc *alloc; + unsigned bucket_count; + unsigned buckets_used; + + /* All buckets' values are assumed to be large + * enough to store this value, and they will all + * grow as necessary. */ + unsigned max_value; + + /* Basic linear-probing, add-only hash table. */ + struct idmap_bucket { + fsm_state_t state; /* Key. NO_STATE when empty. */ + + /* values[] is always either NULL or has at least + * max_value + 1 bits; all grow on demand. */ + uint64_t *values; + } *buckets; +}; + +static unsigned +value_words(unsigned max_value) { + if (max_value == 0) { + /* Still allocate one word, for storing 0. */ + return 1; + } else { + return u64bitset_words(max_value); + } +} + +struct idmap * +idmap_new(const struct fsm_alloc *alloc) +{ + struct idmap *res = NULL; + struct idmap_bucket *buckets = NULL; + + res = f_malloc(alloc, sizeof(*res)); + if (res == NULL) { + goto cleanup; + } + + buckets = f_calloc(alloc, + DEF_BUCKET_COUNT, sizeof(buckets[0])); + if (buckets == NULL) { + goto cleanup; + } + + for (size_t i = 0; i < DEF_BUCKET_COUNT; i++) { + buckets[i].state = NO_STATE; + } + + res->alloc = alloc; + res->buckets_used = 0; + res->bucket_count = DEF_BUCKET_COUNT; + res->max_value = 0; + res->buckets = buckets; + + return res; + +cleanup: + f_free(alloc, res); + f_free(alloc, buckets); + return NULL; +} + +void +idmap_free(struct idmap *m) +{ + if (m == NULL) { + return; + } + + for (size_t i = 0; i < m->bucket_count; i++) { + if (m->buckets[i].state == NO_STATE) { + continue; + } + f_free(m->alloc, m->buckets[i].values); + } + + f_free(m->alloc, m->buckets); + f_free(m->alloc, m); +} + +static int +grow_bucket_values(struct idmap *m, unsigned old_words, unsigned new_words) +{ + assert(new_words > old_words); + + for (size_t b_i = 0; b_i < m->bucket_count; b_i++) { + struct idmap_bucket *b = &m->buckets[b_i]; + if (b->state == NO_STATE) { + assert(b->values == NULL); + continue; + } + + uint64_t *nv = f_calloc(m->alloc, + new_words, sizeof(nv[0])); + if (nv == NULL) { + return 0; + } + + // TODO: memcpy + for (size_t w_i = 0; w_i < old_words; w_i++) { + nv[w_i] = b->values[w_i]; + } + f_free(m->alloc, b->values); + b->values = nv; + } + return 1; +} + +static int +grow_buckets(struct idmap *m) +{ + const size_t ocount = m->bucket_count; + const size_t ncount = 2*ocount; + assert(ncount > m->bucket_count); + + struct idmap_bucket *nbuckets = f_calloc(m->alloc, + ncount, sizeof(nbuckets[0])); + if (nbuckets == NULL) { + return 0; + } + for (size_t nb_i = 0; nb_i < ncount; nb_i++) { + nbuckets[nb_i].state = NO_STATE; + } + + const size_t nmask = ncount - 1; + + for (size_t ob_i = 0; ob_i < ocount; ob_i++) { + const struct idmap_bucket *ob = &m->buckets[ob_i]; + if (ob->state == NO_STATE) { + continue; + } + + const uint64_t h = hash_id(ob->state); + for (size_t nb_i = 0; nb_i < ncount; nb_i++) { + struct idmap_bucket *nb = &nbuckets[(h + nb_i) & nmask]; + if (nb->state == NO_STATE) { + nb->state = ob->state; + nb->values = ob->values; + break; + } else { + assert(nb->state != ob->state); + /* collision */ + continue; + } + } + } + + f_free(m->alloc, m->buckets); + + m->buckets = nbuckets; + m->bucket_count = ncount; + + return 1; +} + +int +idmap_set(struct idmap *m, fsm_state_t state_id, + unsigned value) +{ + assert(state_id != NO_STATE); + + const uint64_t h = hash_id(state_id); + if (value > m->max_value) { + const unsigned ovw = value_words(m->max_value); + const unsigned nvw = value_words(value); + /* If this value won't fit in the existing value + * arrays, then grow them all. We do not track the + * number of bits in each individual array. */ + if (nvw > ovw && !grow_bucket_values(m, ovw, nvw)) { + return 0; + } + m->max_value = value; + } + + assert(m->max_value >= value); + + if (m->buckets_used >= m->bucket_count/2) { + if (!grow_buckets(m)) { + return 0; + } + } + + const uint64_t mask = m->bucket_count - 1; + int res = 0; + size_t b_i; + for (b_i = 0; b_i < m->bucket_count; b_i++) { + struct idmap_bucket *b = &m->buckets[(h + b_i) & mask]; + if (b->state == state_id) { + assert(b->values != NULL); + u64bitset_set(b->values, value); + res = 1; + goto done; + } else if (b->state == NO_STATE) { + b->state = state_id; + assert(b->values == NULL); + + const unsigned vw = value_words(m->max_value); + b->values = f_calloc(m->alloc, + vw, sizeof(b->values[0])); + if (b->values == NULL) { + goto done; + } + m->buckets_used++; + + u64bitset_set(b->values, value); + res = 1; + goto done; + } else { + continue; /* collision */ + } + } + +done: +#if HASH_LOG_PROBES + fprintf(stderr, "%s: res %d after %zu probes (%u buckets)\n", + __func__, res, b_i, m->bucket_count); +#endif + +#ifdef HASH_PROBE_LIMIT + assert(b_i < HASH_PROBE_LIMIT); +#endif + assert(b_i < m->bucket_count); + + return res; +} + +static const struct idmap_bucket * +get_bucket(const struct idmap *m, fsm_state_t state_id) +{ + const uint64_t h = hash_id(state_id); + const uint64_t mask = m->bucket_count - 1; + for (size_t b_i = 0; b_i < m->bucket_count; b_i++) { + const struct idmap_bucket *b = &m->buckets[(h + b_i) & mask]; + if (b->state == NO_STATE) { + return NULL; + } else if (b->state == state_id) { + return b; + } + } + + return NULL; +} + +size_t +idmap_get_value_count(const struct idmap *m, fsm_state_t state_id) +{ + const struct idmap_bucket *b = get_bucket(m, state_id); + if (b == NULL) { + return 0; + } + assert(b->values != NULL); + + size_t res = 0; + const size_t words = value_words(m->max_value); + for (size_t w_i = 0; w_i < words; w_i++) { + const uint64_t w = b->values[w_i]; + /* This could use popcount64(w). */ + if (w == 0) { + continue; + } + for (uint64_t bit = 1; bit; bit <<= 1) { + if (w & bit) { + res++; + } + } + } + + return res; +} + +int +idmap_get(const struct idmap *m, fsm_state_t state_id, + size_t buf_size, unsigned *buf, size_t *written) +{ + const struct idmap_bucket *b = get_bucket(m, state_id); + if (b == NULL) { + if (written != NULL) { + *written = 0; + } + return 1; + } + + size_t buf_offset = 0; + const size_t words = value_words(m->max_value); + for (size_t w_i = 0; w_i < words; w_i++) { + const uint64_t w = b->values[w_i]; + if (w == 0) { + continue; + } + + for (uint64_t b_i = 0; b_i < 64; b_i++) { + if (w & ((uint64_t)1 << b_i)) { + if (buf_offset * sizeof(buf[0]) >= buf_size) { + return 0; + } + buf[buf_offset] = 64*w_i + b_i; + buf_offset++; + } + } + } + + if (written != NULL) { + *written = buf_offset; + } + return 1; +} + +void +idmap_iter(const struct idmap *m, + idmap_iter_fun *cb, void *opaque) +{ + const size_t words = value_words(m->max_value); + + for (size_t b_i = 0; b_i < m->bucket_count; b_i++) { + const struct idmap_bucket *b = &m->buckets[b_i]; + if (b->state == NO_STATE) { + continue; + } + + for (size_t w_i = 0; w_i < words; w_i++) { + const uint64_t w = b->values[w_i]; + if (w == 0) { + continue; + } + for (uint64_t b_i = 0; b_i < 64; b_i++) { + if (w & ((uint64_t)1 << b_i)) { + const unsigned v = 64*w_i + b_i; + if (!cb(b->state, v, opaque)) { + return; + } + } + } + } + } +} + +void +idmap_iter_for_state(const struct idmap *m, fsm_state_t state_id, + idmap_iter_fun *cb, void *opaque) +{ + const size_t words = value_words(m->max_value); + const struct idmap_bucket *b = get_bucket(m, state_id); + if (b == NULL) { + return; + } + + for (size_t w_i = 0; w_i < words; w_i++) { + const uint64_t w = b->values[w_i]; + if (w == 0) { + continue; + } + /* if N contiguous bits are all zero, skip them all at once */ +#define BLOCK_BITS 16 + uint64_t block = ((uint64_t)1 << BLOCK_BITS) - 1; + size_t block_count = 0; + + uint64_t b_i = 0; + while (b_i < 64) { + if ((w & block) == 0) { + block <<= BLOCK_BITS; + b_i += BLOCK_BITS; + continue; + } + + if (w & ((uint64_t)1 << b_i)) { + const unsigned v = 64*w_i + b_i; + if (!cb(b->state, v, opaque)) { + return; + } + block_count++; + } + b_i++; + block <<= 1; + } + +#define CHECK 0 +#if CHECK + size_t check_count = 0; + for (uint64_t b_i = 0; b_i < 64; b_i++) { + if (w & ((uint64_t)1 << b_i)) { + check_count++; + } + } + assert(block_count == check_count); +#else + (void)block_count; +#endif + } +} diff --git a/src/adt/internedstateset.c b/src/adt/internedstateset.c index e6e3a2b8c..990d9c98f 100644 --- a/src/adt/internedstateset.c +++ b/src/adt/internedstateset.c @@ -77,7 +77,7 @@ interned_state_set_pool_alloc(const struct fsm_alloc *a) fsm_state_t *buf = NULL; uint32_t *buckets = NULL; - res = f_calloc(a, 1, sizeof(*res)); + res = f_malloc(a, sizeof(*res)); if (res == NULL) { goto cleanup; } sets = f_malloc(a, DEF_SETS * sizeof(sets[0])); @@ -93,7 +93,7 @@ interned_state_set_pool_alloc(const struct fsm_alloc *a) buckets[i] = NO_ID; } - struct interned_state_set_pool p = { + *res = (struct interned_state_set_pool) { .alloc = a, .sets = { .ceil = DEF_SETS, @@ -108,7 +108,7 @@ interned_state_set_pool_alloc(const struct fsm_alloc *a) .buckets = buckets, }, }; - memcpy(res, &p, sizeof(p)); + return res; cleanup: @@ -197,15 +197,10 @@ dump_tables(FILE *f, const struct interned_state_set_pool *pool) #endif } -SUPPRESS_EXPECTED_UNSIGNED_INTEGER_OVERFLOW() static uint64_t hash_state_ids(size_t count, const fsm_state_t *ids) { - uint64_t h = 0; - for (size_t i = 0; i < count; i++) { - h = hash_id(ids[i]) + (FSM_PHI_64 * h); - } - return h; + return hash_ids(count, ids); } static bool @@ -329,8 +324,13 @@ interned_state_set_intern_set(struct interned_state_set_pool *pool, fprintf(stderr, "%s: htab[(0x%lx + %lu) & 0x%lx => %d\n", __func__, h, b_i, mask, *b); #endif + +#ifdef HASH_PROBE_LIMIT + assert(probes < HASH_PROBE_LIMIT); +#endif + if (*b == NO_ID) { -#if LOG_ISS > 3 +#if LOG_ISS > 3 || HASH_LOG_PROBES fprintf(stderr, "%s: empty bucket (%zd probes)\n", __func__, probes); #endif dst_bucket = b; @@ -351,7 +351,7 @@ interned_state_set_intern_set(struct interned_state_set_pool *pool, if (0 == memcmp(states, buf, s->length * sizeof(buf[0]))) { *result = id; -#if LOG_ISS > 3 +#if LOG_ISS > 3 || HASH_LOG_PROBES fprintf(stderr, "%s: reused %u (%zd probes)\n", __func__, id, probes); #endif return true; @@ -362,8 +362,10 @@ interned_state_set_intern_set(struct interned_state_set_pool *pool, } assert(dst_bucket != NULL); -#if LOG_ISS > 3 +#if LOG_ISS > 3 || HASH_LOG_PROBES fprintf(stderr, "%s: miss after %zd probes\n", __func__, probes); +#else + (void)probes; #endif diff --git a/src/adt/queue.c b/src/adt/queue.c index 95f059ce7..9f3507f69 100644 --- a/src/adt/queue.c +++ b/src/adt/queue.c @@ -30,11 +30,13 @@ queue_new(const struct fsm_alloc *a, size_t max_capacity) if (max_capacity == 0) { return NULL; } alloc_size = sizeof(*q) + (max_capacity - 1) * sizeof(q->q[0]); - q = f_calloc(a, 1, alloc_size); + q = f_malloc(a, alloc_size); if (q == NULL) { return NULL; } q->alloc = a; q->capacity = max_capacity; + q->rd = 0; + q->wr = 0; return q; } @@ -51,7 +53,7 @@ queue_push(struct queue *q, fsm_state_t state) * memmove everything after reading 1. */ if (q->rd > 0 && q->rd < q->wr) { memmove(&q->q[0], &q->q[q->rd], - q->rd * sizeof(q->q[0])); + (q->capacity - q->rd) * sizeof(q->q[0])); q->wr -= q->rd; q->rd = 0; } else { diff --git a/src/adt/stateset.c b/src/adt/stateset.c index c1cff9933..75dc934df 100644 --- a/src/adt/stateset.c +++ b/src/adt/stateset.c @@ -15,6 +15,11 @@ #include #include +/* This is used here because the calls to + * state_set_contains change the order of growth. */ +#include + + /* * TODO: now fsm_state_t is a numeric index, this could be a dynamically * allocated bitmap, instead of a set.inc's array of items. @@ -44,8 +49,8 @@ struct state_set { const struct fsm_alloc *alloc; fsm_state_t *a; - size_t i; - size_t n; + size_t i; /* used */ + size_t n; /* ceil */ }; int @@ -138,7 +143,8 @@ state_set_cmp(const struct state_set *a, const struct state_set *b) } /* - * Return where an item would be, if it were inserted + * Return where an item would be, if it were inserted. + * When insertion would append this returns one past the array. */ static size_t state_set_search(const struct state_set *set, fsm_state_t state) @@ -150,6 +156,11 @@ state_set_search(const struct state_set *set, fsm_state_t state) assert(!IS_SINGLETON(set)); assert(set->a != NULL); + /* fast path: append case */ + if (set->i > 0 && state > set->a[set->i - 1]) { + return set->i; + } + start = mid = 0; end = set->i; @@ -161,6 +172,12 @@ state_set_search(const struct state_set *set, fsm_state_t state) end = mid; } else if (r > 0) { start = mid + 1; + /* update mid if we're about to halt, because + * we're looking for the first position >= state, + * not the last position <= */ + if (start == end) { + mid = start; + } } else { return mid; } @@ -242,7 +259,7 @@ state_set_add(struct state_set **setp, const struct fsm_alloc *alloc, */ if (!state_set_empty(set)) { i = state_set_search(set, state); - if (set->a[i] == state) { + if (i < set->i && set->a[i] == state) { return 1; } } @@ -261,11 +278,7 @@ state_set_add(struct state_set **setp, const struct fsm_alloc *alloc, set->n *= 2; } - if (state_set_cmpval(state, set->a[i]) > 0) { - i++; - } - - if (i <= set->i) { + if (i < set->i) { memmove(&set->a[i + 1], &set->a[i], (set->i - i) * (sizeof *set->a)); } @@ -276,6 +289,8 @@ state_set_add(struct state_set **setp, const struct fsm_alloc *alloc, set->i = 1; } + /* This assert can be pretty expensive in -O0 but in -O3 it has very + * little impact on the overall runtime. */ assert(state_set_contains(set, state)); return 1; @@ -470,7 +485,7 @@ state_set_remove(struct state_set **setp, fsm_state_t state) } i = state_set_search(set, state); - if (set->a[i] == state) { + if (i < set->i && set->a[i] == state) { if (i < set->i) { memmove(&set->a[i], &set->a[i + 1], (set->i - i - 1) * (sizeof *set->a)); } @@ -478,7 +493,9 @@ state_set_remove(struct state_set **setp, fsm_state_t state) set->i--; } +#if EXPENSIVE_CHECKS assert(!state_set_contains(set, state)); +#endif } int @@ -524,7 +541,7 @@ state_set_contains(const struct state_set *set, fsm_state_t state) } i = state_set_search(set, state); - if (set->a[i] == state) { + if (i < set->i && set->a[i] == state) { return 1; } @@ -658,19 +675,3 @@ state_set_replace(struct state_set **setp, fsm_state_t old, fsm_state_t new) } } } - -unsigned long -state_set_hash(const struct state_set *set) -{ - if (set == NULL) { - return 0; /* empty */ - } - - if (IS_SINGLETON(set)) { - fsm_state_t state; - state = SINGLETON_DECODE(set); - return hashrec(&state, sizeof state); - } - - return hashrec(set->a, set->i * sizeof *set->a); -} diff --git a/src/adt/xalloc.c b/src/adt/xalloc.c index b80723ffd..3c9f0dd98 100644 --- a/src/adt/xalloc.c +++ b/src/adt/xalloc.c @@ -1,9 +1,11 @@ /* - * Copyright 2008-2017 Katherine Flavel + * Copyright 2008-2024 Katherine Flavel * * See LICENCE for the full copyright terms. */ +#define _POSIX_C_SOURCE 200809L + #include #include #include @@ -16,12 +18,31 @@ xstrdup(const char *s) { char *new; - new = malloc(strlen(s) + 1); + assert(s != NULL); + + new = strdup(s); + if (new == NULL) { + perror("strdup"); + return NULL; + } + + return new; +} + +char * +xstrndup(const char *s, size_t n) +{ + char *new; + + assert(s != NULL); + + new = strndup(s, n); if (new == NULL) { + perror("strndup"); return NULL; } - return strcpy(new, s); + return new; } void * @@ -57,11 +78,8 @@ xrealloc(void *p, size_t sz) { void *q; - /* This is legal and frees p, but confusing; use free() instead */ - assert(sz != 0); - q = realloc(p, sz); - if (q == NULL) { + if (sz > 0 && q == NULL) { perror("realloc"); abort(); } diff --git a/src/fsm/main.c b/src/fsm/main.c index 8e668b779..770fea2ac 100644 --- a/src/fsm/main.c +++ b/src/fsm/main.c @@ -25,6 +25,7 @@ #include #include /* XXX */ +#include #include "libfsm/internal.h" /* XXX */ @@ -60,8 +61,6 @@ static int clock_gettime(int clk_id, struct timespec *ts) extern int optind; extern char *optarg; -static struct fsm_options opt; - #define OP_ARITY 0x1 enum op { @@ -103,6 +102,16 @@ query_epsilonclosure(const struct fsm *fsm, fsm_state_t state) abort(); } +static int +query_required_chars(const struct fsm *fsm, fsm_state_t state) +{ + (void) fsm; + (void) state; + + /* never called */ + abort(); +} + static void usage(void) { @@ -148,39 +157,40 @@ io(const char *name) exit(EXIT_FAILURE); } -static fsm_print * -print_name(const char *name) +static enum fsm_print_lang +lang_name(const char *name) { size_t i; struct { const char *name; - fsm_print *f; + enum fsm_print_lang lang; } a[] = { - { "api", fsm_print_api }, - { "awk", fsm_print_awk }, - { "c", fsm_print_c }, - { "dot", fsm_print_dot }, - { "fsm", fsm_print_fsm }, - { "ir", fsm_print_ir }, - { "json", fsm_print_json }, - { "vmc", fsm_print_vmc }, - { "vmdot", fsm_print_vmdot }, - { "rust", fsm_print_rust }, - { "sh", fsm_print_sh }, - { "go", fsm_print_go }, - - { "amd64", fsm_print_vmasm }, - { "amd64_att", fsm_print_vmasm_amd64_att }, - { "amd64_nasm", fsm_print_vmasm_amd64_nasm }, - { "amd64_go", fsm_print_vmasm_amd64_go } + { "api", FSM_PRINT_API }, + { "awk", FSM_PRINT_AWK }, + { "c", FSM_PRINT_C }, + { "dot", FSM_PRINT_DOT }, + { "fsm", FSM_PRINT_FSM }, + { "ir", FSM_PRINT_IR }, + { "json", FSM_PRINT_JSON }, + { "vmc", FSM_PRINT_VMC }, + { "vmdot", FSM_PRINT_VMDOT }, + { "rust", FSM_PRINT_RUST }, + { "llvm", FSM_PRINT_LLVM }, + { "sh", FSM_PRINT_SH }, + { "go", FSM_PRINT_GO }, + + { "amd64", FSM_PRINT_AMD64_NASM }, + { "amd64_att", FSM_PRINT_AMD64_ATT }, + { "amd64_nasm", FSM_PRINT_AMD64_NASM }, + { "amd64_go", FSM_PRINT_AMD64_GO } }; assert(name != NULL); for (i = 0; i < sizeof a / sizeof *a; i++) { if (0 == strcmp(a[i].name, name)) { - return a[i].f; + return a[i].lang; } } @@ -228,7 +238,9 @@ static int { "hasambiguity", fsm_has, fsm_hasnondeterminism }, { "hasnondeterminism", fsm_has, fsm_hasnondeterminism }, { "hasepsilons", fsm_has, fsm_hasepsilons }, - { "epsilons", fsm_has, fsm_hasepsilons } + { "epsilons", fsm_has, fsm_hasepsilons }, + { "requiredchars", NULL, query_required_chars }, + { "chars", NULL, query_required_chars }, }; assert(name != NULL); @@ -364,14 +376,22 @@ do_fsm_cleanup(void) int main(int argc, char *argv[]) { + static const struct fsm_options zero_options; + + /* TODO: use alloc hooks for -Q accounting */ + struct fsm_alloc *alloc = NULL; + unsigned iterations, i; - double elapsed; - fsm_print *print; + enum fsm_print_lang lang; enum op op; + const char *charset; + struct fsm_options opt; struct fsm *fsm; + double elapsed; int xfiles; int r; size_t generate_bounds = 0; + size_t step_limit = 0; int (*query)(const struct fsm *, fsm_state_t); int (*walk )(const struct fsm *, @@ -379,23 +399,26 @@ main(int argc, char *argv[]) atexit(do_fsm_cleanup); + opt = zero_options; + opt.ambig = AMBIG_MULTIPLE; opt.comments = 1; opt.io = FSM_IO_GETC; xfiles = 0; - print = NULL; + lang = FSM_PRINT_NONE; query = NULL; walk = NULL; op = OP_IDENTITY; iterations = 1; fsm = NULL; + charset = NULL; r = 0; { int c; - while (c = getopt(argc, argv, "h" "aCcgwXe:k:i:" "xpq:l:dG:mrt:EW:"), c != -1) { + while (c = getopt(argc, argv, "h" "aCcgwXe:k:i:" "xpq:l:dG:mrt:ES:U:W:"), c != -1) { switch (c) { case 'a': opt.anonymous_states = 1; break; case 'c': opt.consolidate_edges = 1; break; @@ -412,8 +435,8 @@ main(int argc, char *argv[]) break; case 'x': xfiles = 1; break; - case 'l': print = print_name(optarg); break; - case 'p': print = fsm_print_fsm; break; + case 'l': lang = lang_name(optarg); break; + case 'p': lang = FSM_PRINT_FSM; break; case 'q': query = query_name(optarg, &walk); break; case 'd': op = op_name("determinise"); break; @@ -421,6 +444,11 @@ main(int argc, char *argv[]) case 'r': op = op_name("reverse"); break; case 't': op = op_name(optarg); break; case 'E': op = op_name("remove_epsilons"); break; + + case 'U': + charset = optarg; + break; + case 'W': /* print = gen_words; */ /* num_words = strtoul(optarg, NULL, 10); */ @@ -428,6 +456,7 @@ main(int argc, char *argv[]) fprintf(stderr, "not yet implemented.\n"); exit(EXIT_FAILURE); break; + case 'G': generate_bounds = strtoul(optarg, NULL, 10); if (generate_bounds == 0) { @@ -436,6 +465,10 @@ main(int argc, char *argv[]) } break; + case 'S': + step_limit = strtoul(optarg, NULL, 10); + break; /* can be 0 */ + case 'h': usage(); exit(EXIT_SUCCESS); @@ -456,7 +489,7 @@ main(int argc, char *argv[]) return EXIT_FAILURE; } - if ((op == OP_EQUAL) + !!print > 1) { + if ((op == OP_EQUAL) + (lang != FSM_PRINT_NONE) > 1) { fprintf(stderr, "-t equal and -p are mutually exclusive\n"); return EXIT_FAILURE; } @@ -469,27 +502,24 @@ main(int argc, char *argv[]) struct fsm *q; if ((op & OP_ARITY) == 1) { - if (argc > 1) { - usage(); - exit(EXIT_FAILURE); - } + /* argc < 1 is okay */ - q = fsm_parse((argc == 0) ? stdin : xopen(argv[0]), &opt); + q = fsm_parse((argc == 0) ? stdin : xopen(argv[0]), alloc); if (q == NULL) { exit(EXIT_FAILURE); } } else { - if (argc != 2) { + if (argc < 2) { usage(); exit(EXIT_FAILURE); } - a = fsm_parse(xopen(argv[0]), &opt); + a = fsm_parse(xopen(argv[0]), alloc); if (a == NULL) { exit(EXIT_FAILURE); } - b = fsm_parse(xopen(argv[1]), &opt); + b = fsm_parse(xopen(argv[1]), alloc); if (b == NULL) { exit(EXIT_FAILURE); } @@ -548,6 +578,29 @@ main(int argc, char *argv[]) q = NULL; } + /* + * It might be more efficient to intersect the character set for each + * operand, but that gives a different result for some operations + * (complement especially). So since we'd also need to intersect the + * result here too, I'm just doing it in the one place for simplicity. + * + * Passing a NULL charset is a no-op, so the default charset is a byte. + * We can't include \0 here because optarg is a string, and I don't + * want to invent a syntax for character sets. + */ + if (charset != NULL) { + if (!fsm_determinise(q)) { + perror("fsm_determinise"); + exit(EXIT_FAILURE); + } + + q = fsm_intersect_charset(q, strlen(charset), charset); + if (q == NULL) { + perror("fsm_intersect_charset"); + exit(EXIT_FAILURE); + } + } + fsm_to_cleanup = q; if (-1 == clock_gettime(CLOCK_MONOTONIC, &post)) { @@ -579,6 +632,17 @@ main(int argc, char *argv[]) printf("=> total %g ms (avg %g ms)\n", elapsed, elapsed / iterations); } + /* we're done consuming filenames, remaining argv is text to match */ + if ((op & OP_ARITY) == 1) { + if (argc > 0) { + argc -= 1; + argv += 1; + } + } else { + argc -= 2; + argv += 2; + } + /* henceforth, r is $?-convention (0 for success) */ if (fsm == NULL) { @@ -599,7 +663,7 @@ main(int argc, char *argv[]) size_t n; struct state_iter it; - closures = epsilon_closure(fsm); + closures = fsm_epsilon_closure(fsm); if (closures == NULL) { return -1; } @@ -620,9 +684,37 @@ main(int argc, char *argv[]) printf("\n"); } - closure_free(closures, fsm->statecount); + fsm_closure_free(fsm, closures, fsm->statecount); return 0; + } else if (query == query_required_chars) { + assert(walk == NULL); + uint64_t charmap[4]; + size_t count; + enum fsm_detect_required_characters_res res; + res = fsm_detect_required_characters(fsm, step_limit, charmap, &count); + if (res == FSM_DETECT_REQUIRED_CHARACTERS_STEP_LIMIT_REACHED) { + fprintf(stderr, "fsm_detect_required_characters: step limit reached (%zd)\n", step_limit); + exit(EXIT_FAILURE); + } else { + assert(res == FSM_DETECT_REQUIRED_CHARACTERS_WRITTEN); + char buf[257] = {0}; + size_t used = 0; + for (size_t i = 0; i < 256; i++) { + if (u64bitset_get(charmap, i)) { + buf[used++] = (char)i; + } + } + printf("%zd ", count); + for (size_t i = 0; i < used; i++) { + c_escputc_str(stdout, &opt, buf[i]); + } + printf("\n"); + + fsm_free(fsm); + fsm_to_cleanup = NULL; + return EXIT_SUCCESS; + } } else { assert(walk != NULL); r |= !walk(fsm, query); @@ -630,14 +722,15 @@ main(int argc, char *argv[]) } } - /* TODO: optional -- to delimit texts as opposed to .fsm filenames */ - if (op == OP_IDENTITY && argc > 0) { + /* match text */ + if (argc > 0) { int i; /* TODO: option to print input texts which match. like grep(1) does. * This is not the same as printing patterns which match (by associating * a pattern to the end state), like lx(1) does */ + /* TODO: optional -- to delimit texts as opposed to .fsm filenames */ for (i = 0; i < argc; i++) { fsm_state_t state; int e; @@ -647,7 +740,7 @@ main(int argc, char *argv[]) f = xopen(argv[0]); - e = fsm_exec(fsm, fsm_fgetc, f, &state, NULL); + e = fsm_exec(fsm, fsm_fgetc, f, &state); fclose(f); } else { @@ -655,7 +748,7 @@ main(int argc, char *argv[]) s = argv[i]; - e = fsm_exec(fsm, fsm_sgetc, &s, &state, NULL); + e = fsm_exec(fsm, fsm_sgetc, &s, &state); } if (e != 1) { @@ -663,23 +756,21 @@ main(int argc, char *argv[]) continue; } - /* TODO: option to print state number? */ + /* TODO: option to print matching end-ids */ } } - if (print != NULL) { - if (-1 == print(stdout, fsm)) { - if (errno == ENOTSUP) { - fprintf(stderr, "unsupported IO API\n"); - } else { - perror("print_fsm"); - } - exit(EXIT_FAILURE); + if (-1 == fsm_print(stdout, fsm, &opt, NULL, lang)) { + if (errno == ENOTSUP) { + fprintf(stderr, "unsupported IO API\n"); + } else { + perror("fsm_print"); } + exit(EXIT_FAILURE); } if (generate_bounds > 0) { - r = fsm_generate_matches(fsm, generate_bounds, fsm_generate_cb_printf_escaped, &opt); + r = fsm_generate_matches(fsm, generate_bounds, 0, fsm_generate_cb_printf_escaped, &opt); } fsm_free(fsm); diff --git a/src/libfsm/Makefile b/src/libfsm/Makefile index 9af51a5a4..a29627d3d 100644 --- a/src/libfsm/Makefile +++ b/src/libfsm/Makefile @@ -1,11 +1,16 @@ .include "../../share/mk/top.mk" SRC += src/libfsm/capture.c +SRC += src/libfsm/capture_vm.c +SRC += src/libfsm/capture_vm_exec.c +SRC += src/libfsm/charset.c SRC += src/libfsm/collate.c SRC += src/libfsm/complete.c SRC += src/libfsm/consolidate.c SRC += src/libfsm/clone.c SRC += src/libfsm/closure.c +SRC += src/libfsm/detect_required.c +SRC += src/libfsm/eager_output.c SRC += src/libfsm/edge.c SRC += src/libfsm/empty.c SRC += src/libfsm/end.c @@ -15,12 +20,14 @@ SRC += src/libfsm/exec.c SRC += src/libfsm/fsm.c SRC += src/libfsm/gen.c SRC += src/libfsm/mode.c +SRC += src/libfsm/print.c SRC += src/libfsm/start.c SRC += src/libfsm/state.c SRC += src/libfsm/trim.c SRC += src/libfsm/example.c SRC += src/libfsm/getc.c SRC += src/libfsm/shuffle.c +SRC += src/libfsm/vacuum.c SRC += src/libfsm/vm.c # graph things diff --git a/src/libfsm/capture.c b/src/libfsm/capture.c index 806bb3b12..89f4d3a37 100644 --- a/src/libfsm/capture.c +++ b/src/libfsm/capture.c @@ -6,30 +6,89 @@ #include -#include "capture_internal.h" +#include +#include +#include + +#include +#include +#include +#include + +#include +#include + +#include +#include +#include + +#include "internal.h" +#include "capture.h" +#include "capture_vm_program.h" +#include "capture_log.h" +#include "capture_vm.h" +#include "endids.h" + +#define DEF_PROGRAMS_CEIL 4 + +struct fsm_capture_info { + unsigned max_capture_id; + + /* For particular end states, which captures are active? */ + struct idmap *end_capture_map; + + /* Set of capture resolution programs associated with specific + * end states. */ + struct capvm_program_set { + uint32_t ceil; + uint32_t used; + struct capvm_program **set; + } programs; + + /* For particular end states, which capture programs are + * associtaed with them? */ + struct idmap *end_capvm_program_map; +}; int fsm_capture_init(struct fsm *fsm) { struct fsm_capture_info *ci = NULL; - size_t i; + struct idmap *end_capture_map = NULL; + struct idmap *end_capvm_program_map = NULL; - ci = f_calloc(fsm->opt->alloc, - 1, sizeof(*ci)); + ci = f_calloc(fsm->alloc, 1, sizeof(*ci)); if (ci == NULL) { goto cleanup; } + end_capture_map = idmap_new(fsm->alloc); + if (end_capture_map == NULL) { + goto cleanup; + } + + ci->max_capture_id = 0; + fsm->capture_info = ci; - for (i = 0; i < fsm->statealloc; i++) { - fsm->states[i].has_capture_actions = 0; + ci->end_capture_map = end_capture_map; + + end_capvm_program_map = idmap_new(fsm->alloc); + if (end_capvm_program_map == NULL) { + goto cleanup; } + ci->end_capvm_program_map = end_capvm_program_map; + + fsm->capture_info = ci; return 1; cleanup: + f_free(fsm->alloc, ci); + idmap_free(end_capture_map); + idmap_free(end_capvm_program_map); + if (ci != NULL) { - f_free(fsm->opt->alloc, ci); + f_free(fsm->alloc, ci); } return 0; } @@ -41,799 +100,579 @@ fsm_capture_free(struct fsm *fsm) if (ci == NULL) { return; } - f_free(fsm->opt->alloc, ci->buckets); - f_free(fsm->opt->alloc, ci); + idmap_free(ci->end_capture_map); + idmap_free(ci->end_capvm_program_map); + + for (size_t p_i = 0; p_i < ci->programs.used; p_i++) { + fsm_capvm_program_free(fsm->alloc, ci->programs.set[p_i]); + } + f_free(fsm->alloc, ci->programs.set); + + f_free(fsm->alloc, ci); fsm->capture_info = NULL; } unsigned -fsm_countcaptures(const struct fsm *fsm) +fsm_capture_ceiling(const struct fsm *fsm) { - (void)fsm; if (fsm->capture_info == NULL) { return 0; } - if (fsm->capture_info->buckets_used == 0) { - return 0; - } - /* check actual */ #if EXPENSIVE_CHECKS - { - struct fsm_capture_info *ci = fsm->capture_info; - size_t i; - for (i = 0; i < ci->bucket_count; i++) { - struct fsm_capture_action_bucket *b = &ci->buckets[i]; - if (b->state == CAPTURE_NO_STATE) { /* empty */ - continue; - } - assert(ci->max_capture_id >= b->action.id); + /* check actual */ + unsigned res = 0; + for (size_t i = 0; i < fsm->capture_info->programs.used; i++) { + const unsigned id = fsm_capvm_program_get_max_capture_id(fsm->capture_info->programs.set[i]); + if (id > res) { + res = id; } } + assert(res == fsm->capture_info->max_capture_id); #endif return fsm->capture_info->max_capture_id + 1; } +struct fsm_capture * +fsm_capture_alloc_capture_buffer(const struct fsm *fsm) +{ + assert(fsm != NULL); + const size_t len = fsm_capture_ceiling(fsm); + struct fsm_capture *res = f_malloc(fsm->alloc, + len * sizeof(res[0])); + return res; +} + +void +fsm_capture_free_capture_buffer(const struct fsm *fsm, + struct fsm_capture *capture_buffer) +{ + assert(fsm != NULL); + f_free(fsm->alloc, capture_buffer); +} + + int fsm_capture_has_captures(const struct fsm *fsm) { return fsm->capture_info - ? fsm->capture_info->buckets_used > 0 + ? fsm->capture_info->programs.used > 0 : 0; } -int -fsm_capture_has_capture_actions(const struct fsm *fsm, fsm_state_t state) +void +fsm_capture_dump_programs(FILE *f, const struct fsm *fsm) { - assert(state < fsm->statecount); - return fsm->states[state].has_capture_actions; + fprintf(f, "\n==== %s:\n", __func__); + struct fsm_capture_info *ci = fsm->capture_info; + for (uint32_t i = 0; i < ci->programs.used; i++) { + const struct capvm_program *p = ci->programs.set[i]; + fprintf(f, "# program %u, capture_count %u, base %u\n", + i, p->capture_count, p->capture_base); + fsm_capvm_program_dump(f, p); + fprintf(f, "\n"); + } } +#if EXPENSIVE_CHECKS +#include +#endif + int -fsm_capture_set_path(struct fsm *fsm, unsigned capture_id, - fsm_state_t start, fsm_state_t end) +fsm_capture_set_active_for_end(struct fsm *fsm, + unsigned capture_id, fsm_state_t end_state) { - struct fsm_capture_info *ci; - struct capture_set_path_env env; - size_t seen_words; - int res = 0; - - assert(fsm != NULL); - assert(start < fsm->statecount); - assert(end < fsm->statecount); - - ci = fsm->capture_info; + struct fsm_capture_info *ci = fsm->capture_info; assert(ci != NULL); + struct idmap *m = ci->end_capture_map; + assert(m != NULL); - /* captures should no longer be stored as paths -- instead, set - * the info on the states _here_, and convert it as necessary. */ - -#if LOG_CAPTURE > 0 - fprintf(stderr, "fsm_capture_set_path: capture %u: <%u, %u>\n", - capture_id, start, end); -#endif - - if (capture_id > FSM_CAPTURE_MAX) { - return 0; /* ID out of range */ - } - - if (!init_capture_action_htab(fsm, ci)) { - return 0; + // FIXME: failing currently + #if EXPENSIVE_CHECKS + if (!fsm_isend(fsm, end_state)) { + fprintf(stderr, "FAILING: %s: capture_id %u, end_state %d\n", + __func__, capture_id, end_state); + fsm_dump(stderr, fsm); } + assert(fsm_isend(fsm, end_state)); + #endif - /* This will create a trail and do a depth-first search from the - * start state, marking every unique path to the end state. */ - env.fsm = fsm; - env.capture_id = capture_id; - env.start = start; - env.end = end; - - env.trail_ceil = 0; - env.trail = NULL; - env.seen = NULL; - - env.trail = f_malloc(fsm->opt->alloc, - DEF_TRAIL_CEIL * sizeof(env.trail[0])); - if (env.trail == NULL) { - goto cleanup; - } - env.trail_ceil = DEF_TRAIL_CEIL; - - seen_words = fsm->statecount/64 + 1; - env.seen = f_malloc(fsm->opt->alloc, - seen_words * sizeof(env.seen[0])); + return idmap_set(m, end_state, capture_id); +} - if (!mark_capture_path(&env)) { - goto cleanup; - } +void +fsm_capture_iter_active_for_end_state(const struct fsm *fsm, fsm_state_t state, + fsm_capture_iter_active_for_end_cb *cb, void *opaque) +{ + /* These types should be the same. */ + idmap_iter_fun *idmap_cb = cb; + idmap_iter_for_state(fsm->capture_info->end_capture_map, state, + idmap_cb, opaque); +} - if (capture_id >= ci->max_capture_id) { - ci->max_capture_id = capture_id; - } +void +fsm_capture_iter_active_for_all_end_states(const struct fsm *fsm, + fsm_capture_iter_active_for_end_cb *cb, void *opaque) +{ + /* These types should be the same. */ + idmap_iter_fun *idmap_cb = cb; + idmap_iter(fsm->capture_info->end_capture_map, + idmap_cb, opaque); +} - res = 1; - /* fall through */ +void +fsm_capture_iter_program_ids_for_end_state(const struct fsm *fsm, fsm_state_t state, + fsm_capture_iter_program_ids_for_end_state_cb *cb, void *opaque) +{ + /* These types should be the same. */ + idmap_iter_fun *idmap_cb = cb; + idmap_iter_for_state(fsm->capture_info->end_capvm_program_map, state, + idmap_cb, opaque); +} -cleanup: - f_free(fsm->opt->alloc, env.trail); - f_free(fsm->opt->alloc, env.seen); - return res; +void +fsm_capture_iter_program_ids_for_all_end_states(const struct fsm *fsm, + fsm_capture_iter_program_ids_for_end_state_cb *cb, void *opaque) +{ + /* These types should be the same. */ + idmap_iter_fun *idmap_cb = cb; + idmap_iter(fsm->capture_info->end_capvm_program_map, + idmap_cb, opaque); } static int -init_capture_action_htab(struct fsm *fsm, struct fsm_capture_info *ci) +dump_active_for_ends_cb(fsm_state_t state_id, unsigned value, void *opaque) { - size_t count, i; - assert(fsm != NULL); - assert(ci != NULL); + FILE *f = opaque; + fprintf(f, " -- state %d: value %u\n", state_id, value); + return 1; +} - if (ci->bucket_count > 0) { - assert(ci->buckets != NULL); - return 1; /* done */ - } +void +fsm_capture_dump_active_for_ends(FILE *f, const struct fsm *fsm) +{ + fprintf(f, "%s:\n", __func__); + idmap_iter(fsm->capture_info->end_capture_map, dump_active_for_ends_cb, f); +} - assert(ci->buckets == NULL); - assert(ci->buckets_used == 0); +void +fsm_capture_dump_program_end_mapping(FILE *f, const struct fsm *fsm) +{ + fprintf(f, "%s:\n", __func__); + idmap_iter(fsm->capture_info->end_capvm_program_map, dump_active_for_ends_cb, f); +} - count = DEF_CAPTURE_ACTION_BUCKET_COUNT; - ci->buckets = f_malloc(fsm->opt->alloc, - count * sizeof(ci->buckets[0])); - if (ci->buckets == NULL) { - return 0; - } +/* Dump capture metadata about an FSM. */ +void +fsm_capture_dump(FILE *f, const char *tag, const struct fsm *fsm) +{ + struct fsm_capture_info *ci; - /* Init buckets to CAPTURE_NO_STATE -> empty. */ - for (i = 0; i < count; i++) { - ci->buckets[i].state = CAPTURE_NO_STATE; + assert(fsm != NULL); + ci = fsm->capture_info; + if (ci == NULL) { + fprintf(f, "==== %s -- no captures\n", tag); + return; } - ci->bucket_count = count; - return 1; + fsm_endid_dump(f, fsm); + fsm_capture_dump_active_for_ends(f, fsm); + fsm_capture_dump_programs(f, fsm); + fsm_capture_dump_program_end_mapping(f, fsm); } +struct carry_active_captures_env { + fsm_state_t dst; + struct idmap *dst_m; + int ok; +}; + static int -mark_capture_path(struct capture_set_path_env *env) +copy_active_captures_cb(fsm_state_t state_id, unsigned value, void *opaque) { - const size_t seen_words = env->fsm->statecount/64 + 1; - -#if LOG_CAPTURE > 0 - fprintf(stderr, "mark_capture_path: path [id %u, %u - %u]\n", - env->capture_id, env->start, env->end); -#endif - - if (env->start == env->end) { - struct fsm_capture_action action; - action.type = CAPTURE_ACTION_COMMIT_ZERO_STEP; - action.id = env->capture_id; - action.to = CAPTURE_NO_STATE; - if (!add_capture_action(env->fsm, env->fsm->capture_info, - env->start, &action)) { - return 0; - } - return 1; - } + (void)state_id; - memset(env->seen, 0x00, - seen_words * sizeof(env->seen[0])); - - /* initialize to starting node */ - env->trail_i = 1; - env->trail[0].state = env->start; - env->trail[0].step = TRAIL_STEP_START; - env->trail[0].has_self_edge = 0; - - while (env->trail_i > 0) { - const enum trail_step step = env->trail[env->trail_i - 1].step; -#if LOG_CAPTURE > 0 - fprintf(stderr, "mark_capture_path: trail %u/%u, cur %u, step %d\n", - env->trail_i, env->trail_ceil, - env->trail[env->trail_i - 1].state, - step); -#endif - - switch (step) { - case TRAIL_STEP_START: - if (!step_trail_start(env)) { - return 0; - } - break; - case TRAIL_STEP_ITER_EDGES: - if (!step_trail_iter_edges(env)) { - return 0; - } - break; - case TRAIL_STEP_ITER_EPSILONS: - if (!step_trail_iter_epsilons(env)) { - return 0; - } - break; - case TRAIL_STEP_DONE: - if (!step_trail_done(env)) { - return 0; - } - break; - default: - assert(!"match fail"); - } + struct carry_active_captures_env *env = opaque; + if (!idmap_set(env->dst_m, env->dst, value)) { + env->ok = false; + return 0; } - return 1; } static int -cmp_action(const struct fsm_capture_action *a, - const struct fsm_capture_action *b) { - /* could use memcmp here, provided padding is always zeroed. */ - return a->id < b->id ? -1 - : a->id > b->id ? 1 - : a->type < b->type ? -1 - : a->type > b->type ? 1 - : a->to < b->to ? -1 - : a->to > b->to ? 1 - : 0; -} - -int -fsm_capture_add_action(struct fsm *fsm, - fsm_state_t state, enum capture_action_type type, - unsigned id, fsm_state_t to) +copy_program_associations_cb(fsm_state_t state_id, unsigned value, void *opaque) { - struct fsm_capture_action action; - assert(fsm->capture_info != NULL); + (void)state_id; - action.type = type; - action.id = id; - action.to = to; - return add_capture_action(fsm, fsm->capture_info, - state, &action); + struct carry_active_captures_env *env = opaque; + if (!idmap_set(env->dst_m, env->dst, value)) { + env->ok = false; + return 0; + } + return 1; } -static int -add_capture_action(struct fsm *fsm, struct fsm_capture_info *ci, - fsm_state_t state, const struct fsm_capture_action *action) +int +fsm_capture_copy_active_for_ends(const struct fsm *src_fsm, + const struct state_set *states, + struct fsm *dst_fsm, fsm_state_t dst_state) { - uint64_t h; - size_t b_i, mask; - - assert(state < fsm->statecount); - assert(action->to == CAPTURE_NO_STATE || action->to < fsm->statecount); - -#if LOG_CAPTURE > 0 - fprintf(stderr, "add_capture_action: state %u, type %s, ID %u, TO %d\n", - state, fsm_capture_action_type_name[action->type], - action->id, action->to); -#endif - - if (ci->bucket_count == 0) { - if (!init_capture_action_htab(fsm, ci)) { - return 0; - } - } else if (ci->buckets_used >= ci->bucket_count/2) { /* grow */ - if (!grow_capture_action_buckets(fsm->opt->alloc, ci)) { - return 0; + struct state_iter it; + fsm_state_t s; + + assert(src_fsm != NULL); + assert(src_fsm->capture_info != NULL); + assert(src_fsm->capture_info->end_capture_map != NULL); + assert(dst_fsm != NULL); + assert(dst_fsm->capture_info != NULL); + assert(dst_fsm->capture_info->end_capture_map != NULL); + struct idmap *src_m = src_fsm->capture_info->end_capture_map; + struct idmap *dst_m = dst_fsm->capture_info->end_capture_map; + + struct carry_active_captures_env env = { + .dst_m = dst_m, + .dst = dst_state, + .ok = true, + }; + + state_set_reset(states, &it); + while (state_set_next(&it, &s)) { + if (!fsm_isend(src_fsm, s)) { + continue; } - } - h = hash_id(state); - mask = ci->bucket_count - 1; - - for (b_i = 0; b_i < ci->bucket_count; b_i++) { - struct fsm_capture_action_bucket *b = &ci->buckets[(h + b_i) & mask]; - if (b->state == CAPTURE_NO_STATE) { /* empty */ - b->state = state; - memcpy(&b->action, action, sizeof(*action)); - ci->buckets_used++; - fsm->states[state].has_capture_actions = 1; - if (action->id > ci->max_capture_id) { - ci->max_capture_id = action->id; - } - return 1; - } else if (b->state == state && - 0 == cmp_action(action, &b->action)) { - /* already present, ignore duplicate */ - assert(fsm->states[state].has_capture_actions); - assert(ci->max_capture_id >= action->id); - return 1; - } else { - continue; /* skip past collision */ + idmap_iter_for_state(src_m, s, copy_active_captures_cb, &env); + if (!env.ok) { + goto cleanup; } } - assert(!"unreachable"); - return 0; +cleanup: + return env.ok; } -static int -grow_capture_action_buckets(const struct fsm_alloc *alloc, - struct fsm_capture_info *ci) +int +fsm_capture_copy_program_end_state_associations(const struct fsm *src_fsm, + const struct state_set *states, + struct fsm *dst_fsm, fsm_state_t dst_state) { - const size_t ncount = 2 * ci->bucket_count; - struct fsm_capture_action_bucket *nbuckets; - size_t nused = 0; - size_t i; + struct state_iter it; + fsm_state_t s; + + assert(src_fsm != NULL); + assert(src_fsm->capture_info != NULL); + assert(src_fsm->capture_info->end_capvm_program_map != NULL); + assert(dst_fsm != NULL); + assert(dst_fsm->capture_info != NULL); + assert(dst_fsm->capture_info->end_capvm_program_map != NULL); + struct idmap *src_m = src_fsm->capture_info->end_capvm_program_map; + struct idmap *dst_m = dst_fsm->capture_info->end_capvm_program_map; + + struct carry_active_captures_env env = { + .dst_m = dst_m, + .dst = dst_state, + .ok = true, + }; + + state_set_reset(states, &it); + while (state_set_next(&it, &s)) { + if (!fsm_isend(src_fsm, s)) { + continue; + } - assert(ncount != 0); - nbuckets = f_malloc(alloc, ncount * sizeof(nbuckets[0])); - if (nbuckets == NULL) { - return 0; - } + LOG(5 - LOG_CAPTURE_COMBINING_ANALYSIS, + "%s: dst_state %d, state_set_next => %d\n", + __func__, dst_state, s); - for (i = 0; i < ncount; i++) { - nbuckets[i].state = CAPTURE_NO_STATE; + idmap_iter_for_state(src_m, s, copy_program_associations_cb, &env); + if (!env.ok) { + goto cleanup; + } } - for (i = 0; i < ci->bucket_count; i++) { - const struct fsm_capture_action_bucket *src_b = &ci->buckets[i]; - uint64_t h; - const size_t mask = ncount - 1; - size_t b_i; +cleanup: + return env.ok; +} - if (src_b->state == CAPTURE_NO_STATE) { - continue; +int +fsm_capture_copy_programs(const struct fsm *src_fsm, + struct fsm *dst_fsm) +{ + const struct fsm_alloc *alloc = src_fsm->alloc; + assert(alloc == dst_fsm->alloc); + const struct fsm_capture_info *src_ci = src_fsm->capture_info; + + for (uint32_t p_i = 0; p_i < src_ci->programs.used; p_i++) { + const struct capvm_program *p = src_ci->programs.set[p_i]; + struct capvm_program *cp = capvm_program_copy(alloc, p); + if (cp == NULL) { + return 0; } - h = hash_id(src_b->state); - for (b_i = 0; b_i < ncount; b_i++) { - struct fsm_capture_action_bucket *dst_b; - dst_b = &nbuckets[(h + b_i) & mask]; - if (dst_b->state == CAPTURE_NO_STATE) { - memcpy(dst_b, src_b, sizeof(*src_b)); - nused++; - break; - } else { - continue; - } + /* unused: because this is an in-order copy, it's assumed + * the programs will retain their order. */ + uint32_t prog_id; + if (!fsm_capture_add_program(dst_fsm, cp, &prog_id)) { + return 0; } } - - assert(nused == ci->buckets_used); - f_free(alloc, ci->buckets); - ci->buckets = nbuckets; - ci->bucket_count = ncount; return 1; } -static int -grow_trail(struct capture_set_path_env *env) +size_t +fsm_capture_program_count(const struct fsm *fsm) { - struct trail_cell *ntrail; - unsigned nceil; - assert(env != NULL); - - nceil = 2 * env->trail_ceil; - assert(nceil > env->trail_ceil); + return fsm->capture_info->programs.used; +} - ntrail = f_realloc(env->fsm->opt->alloc, env->trail, - nceil * sizeof(env->trail[0])); - if (ntrail == NULL) { - return 0; - } +struct check_program_mappings_env { + const struct fsm *fsm; +}; - env->trail = ntrail; - env->trail_ceil = nceil; +static int +check_program_mappings_cb(fsm_state_t state_id, unsigned value, void *opaque) +{ + const uint32_t prog_id = (uint32_t)value; + struct check_program_mappings_env *env = opaque; + assert(state_id < env->fsm->statecount); + assert(prog_id < env->fsm->capture_info->programs.used); return 1; } -static int -step_trail_start(struct capture_set_path_env *env) -{ - struct trail_cell *tc = &env->trail[env->trail_i - 1]; - const fsm_state_t cur = tc->state; - size_t i; - struct edge_set *edge_set = NULL; - - /* check if node is endpoint, if so mark trail, - * then pop trail and continue */ - if (cur == env->end) { - struct fsm_capture_action action; -#if LOG_CAPTURE > 0 - fprintf(stderr, " -- GOT END at %u\n", cur); -#endif - action.id = env->capture_id; +void +fsm_capture_integrity_check(const struct fsm *fsm) +{ + if (!EXPENSIVE_CHECKS) { return; } - for (i = 0; i < env->trail_i; i++) { - fsm_state_t state = env->trail[i].state; -#if LOG_CAPTURE > 0 - fprintf(stderr, " -- %lu: %d\n", - i, state); -#endif + /* check that all program mappings are in range */ + struct check_program_mappings_env env = { + .fsm = fsm, + }; + idmap_iter(fsm->capture_info->end_capvm_program_map, check_program_mappings_cb, &env); +} - /* Special case: if this is marked as having - * a self-edge on the path, then also add an - * extend for that. */ - if (env->trail[i].has_self_edge) { - struct fsm_capture_action self_action; - self_action.type = CAPTURE_ACTION_EXTEND; - self_action.id = env->capture_id; - self_action.to = state; - - if (!add_capture_action(env->fsm, - env->fsm->capture_info, - state, &self_action)) { - return 0; - } - } - - - if (i == 0) { - action.type = CAPTURE_ACTION_START; - } else { - action.type = (i < env->trail_i - 1 - ? CAPTURE_ACTION_EXTEND - : CAPTURE_ACTION_COMMIT); - } - - if (i < env->trail_i - 1) { - action.to = env->trail[i + 1].state; - } else { - action.to = CAPTURE_NO_STATE; - } - - if (!add_capture_action(env->fsm, - env->fsm->capture_info, - state, &action)) { - return 0; - } - } +struct capture_idmap_compact_env { + int ok; + struct idmap *dst; + const fsm_state_t *mapping; + size_t orig_statecount; +}; - tc->step = TRAIL_STEP_DONE; - return 1; - } +static int +copy_with_mapping_cb(fsm_state_t state_id, unsigned value, void *opaque) +{ + fsm_state_t dst_id; + struct capture_idmap_compact_env *env = opaque; -#if LOG_CAPTURE > 0 - fprintf(stderr, " -- resetting edge iterator\n"); -#endif - edge_set = env->fsm->states[cur].edges; + assert(state_id < env->orig_statecount); + dst_id = env->mapping[state_id]; - MARK_SEEN(env, cur); -#if LOG_CAPTURE > 0 - fprintf(stderr, " -- marking %u as seen\n", cur); -#endif + if (dst_id == FSM_STATE_REMAP_NO_STATE) { + return 1; /* discard */ + } + + if (!idmap_set(env->dst, dst_id, value)) { + env->ok = 0; + return 0; + } - edge_set_reset(edge_set, &tc->iter); - tc->step = TRAIL_STEP_ITER_EDGES; return 1; } -static int -step_trail_iter_edges(struct capture_set_path_env *env) +int +fsm_capture_id_compact(struct fsm *fsm, const fsm_state_t *mapping, + size_t orig_statecount) { - struct trail_cell *tc = &env->trail[env->trail_i - 1]; - struct trail_cell *next_tc = NULL; + struct capture_idmap_compact_env env; + struct idmap *old_idmap = fsm->capture_info->end_capture_map; + struct idmap *new_idmap = idmap_new(fsm->alloc); - struct fsm_edge e; - - if (!edge_set_next(&tc->iter, &e)) { -#if LOG_CAPTURE > 0 - fprintf(stderr, " -- ITER_EDGE_NEXT: DONE %u\n", tc->state); -#endif - tc->step = TRAIL_STEP_ITER_EPSILONS; - return 1; + if (new_idmap == NULL) { + return 0; } -#if LOG_CAPTURE > 0 - fprintf(stderr, " -- ITER_EDGE_NEXT: %u -- NEXT %u\n", - tc->state, e.state); -#endif + env.ok = 1; + env.dst = new_idmap; + env.mapping = mapping; + env.orig_statecount = orig_statecount; - if (tc->state == e.state) { -#if LOG_CAPTURE > 0 - fprintf(stderr, " -- special case, self-edge\n"); -#endif - /* Mark this state as having a self-edge, then continue - * the iterator. An EXTEND action will be added for the - * self-edge later, if necessary. */ - tc->has_self_edge = 1; - return 1; - } else if (CHECK_SEEN(env, e.state)) { -#if LOG_CAPTURE > 0 - fprintf(stderr, " -- seen, skipping\n"); -#endif - return 1; /* continue */ - } - - if (env->trail_i == env->trail_ceil) { - if (!grow_trail(env)) { - return 0; - } + idmap_iter(old_idmap, copy_with_mapping_cb, &env); + if (!env.ok) { + idmap_free(new_idmap); + return 0; } -#if LOG_CAPTURE > 0 - fprintf(stderr, " -- marking %u as seen\n", e.state); -#endif - MARK_SEEN(env, e.state); + idmap_free(old_idmap); + fsm->capture_info->end_capture_map = new_idmap; -#if LOG_CAPTURE > 0 - fprintf(stderr, " -- not seen (%u), exploring\n", e.state); -#endif - env->trail_i++; - next_tc = &env->trail[env->trail_i - 1]; - next_tc->state = e.state; - next_tc->step = TRAIL_STEP_START; - next_tc->has_self_edge = 0; return 1; } -static int -step_trail_iter_epsilons(struct capture_set_path_env *env) +int +fsm_capture_program_association_compact(struct fsm *fsm, const fsm_state_t *mapping, + size_t orig_statecount) { - struct trail_cell *tc = &env->trail[env->trail_i - 1]; - - /* skipping this for now */ - -#if LOG_CAPTURE > 0 - fprintf(stderr, " -- ITER_EPSILONS: %u\n", tc->state); -#endif + struct capture_idmap_compact_env env; + struct idmap *old_idmap = fsm->capture_info->end_capvm_program_map; + struct idmap *new_idmap = idmap_new(fsm->alloc); - tc->step = TRAIL_STEP_DONE; - return 1; -} + if (new_idmap == NULL) { + return 0; + } -static int -step_trail_done(struct capture_set_path_env *env) -{ - struct trail_cell *tc; + env.ok = 1; + env.dst = new_idmap; + env.mapping = mapping; + env.orig_statecount = orig_statecount; - /* 0-step paths already handled outside loop */ - assert(env->trail_i > 0); + idmap_iter(old_idmap, copy_with_mapping_cb, &env); + if (!env.ok) { + idmap_free(new_idmap); + return 0; + } - tc = &env->trail[env->trail_i - 1]; -#if LOG_CAPTURE > 0 - fprintf(stderr, " -- DONE: %u\n", tc->state); -#endif - CLEAR_SEEN(env, tc->state); + idmap_free(old_idmap); + fsm->capture_info->end_capvm_program_map = new_idmap; - env->trail_i--; return 1; } void -fsm_capture_rebase_capture_id(struct fsm *fsm, unsigned base) +fsm_capture_update_max_capture_id(struct fsm_capture_info *ci, + unsigned capture_id) { - size_t i; - struct fsm_capture_info *ci = fsm->capture_info; assert(ci != NULL); - - for (i = 0; i < ci->bucket_count; i++) { - struct fsm_capture_action_bucket *b = &ci->buckets[i]; - if (b->state == CAPTURE_NO_STATE) { - continue; - } - - b->action.id += base; - if (b->action.id > ci->max_capture_id) { - ci->max_capture_id = b->action.id; - } + if (capture_id >= ci->max_capture_id) { + ci->max_capture_id = capture_id; } } -void -fsm_capture_rebase_capture_action_states(struct fsm *fsm, fsm_state_t base) +int +fsm_capture_add_program(struct fsm *fsm, + struct capvm_program *program, uint32_t *prog_id) { - size_t i; + assert(program != NULL); + assert(prog_id != NULL); + struct fsm_capture_info *ci = fsm->capture_info; - assert(ci != NULL); - for (i = 0; i < ci->bucket_count; i++) { - struct fsm_capture_action_bucket *b = &ci->buckets[i]; - if (b->state == CAPTURE_NO_STATE) { - continue; + if (ci->programs.used == ci->programs.ceil) { + const size_t nceil = (ci->programs.ceil == 0 + ? DEF_PROGRAMS_CEIL + : 2*ci->programs.ceil); + assert(nceil > ci->programs.ceil); + struct capvm_program **nset = f_realloc(fsm->alloc, + ci->programs.set, nceil * sizeof(nset[0])); + if (nset == NULL) { + return 0; } - b->state += base; - if (b->action.to != CAPTURE_NO_STATE) { - b->action.to += base; - } + ci->programs.ceil = nceil; + ci->programs.set = nset; } -} - -struct fsm_capture * -fsm_capture_alloc(const struct fsm *fsm) -{ - (void)fsm; - assert(!"todo"); - return NULL; -} - -void -fsm_capture_update_captures(const struct fsm *fsm, - fsm_state_t cur_state, fsm_state_t next_state, size_t offset, - struct fsm_capture *captures) -{ - const struct fsm_capture_info *ci; - uint64_t h; - size_t b_i, mask; - - assert(cur_state < fsm->statecount); - assert(fsm->states[cur_state].has_capture_actions); - - ci = fsm->capture_info; - assert(ci != NULL); - - h = hash_id(cur_state); - mask = ci->bucket_count - 1; - -#if LOG_CAPTURE > 0 - fprintf(stderr, "-- updating captures at state %u, to %d, offset %lu\n", - cur_state, next_state, offset); -#endif - - for (b_i = 0; b_i < ci->bucket_count; b_i++) { - const size_t b_id = (h + b_i) & mask; - struct fsm_capture_action_bucket *b = &ci->buckets[b_id]; - unsigned capture_id; - -#if LOG_CAPTURE > 3 - fprintf(stderr, " -- update_captures: bucket %lu, state %d\n", b_id, b->state); -#endif - - - if (b->state == CAPTURE_NO_STATE) { -#if LOG_CAPTURE > 3 - fprintf(stderr, " -- no more actions for this state\n"); -#endif - break; /* no more for this state */ - } else if (b->state != cur_state) { - continue; /* skip collision */ - } - - assert(b->state == cur_state); - capture_id = b->action.id; - - switch (b->action.type) { - case CAPTURE_ACTION_START: -#if LOG_CAPTURE > 0 - fprintf(stderr, "START [%u, %u]\n", - b->action.id, b->action.to); -#endif - if (next_state == b->action.to && captures[capture_id].pos[0] == FSM_CAPTURE_NO_POS) { - captures[capture_id].pos[0] = offset; -#if LOG_CAPTURE > 0 - fprintf(stderr, " -- set capture[%u].[0] to %lu\n", b->action.id, offset); -#endif - } else { - /* filtered, ignore */ - } - break; - case CAPTURE_ACTION_EXTEND: -#if LOG_CAPTURE > 0 - fprintf(stderr, "EXTEND [%u, %u]\n", - b->action.id, b->action.to); -#endif - if (captures[capture_id].pos[0] != FSM_CAPTURE_NO_POS - && (0 == (captures[capture_id].pos[1] & COMMITTED_CAPTURE_FLAG))) { - if (next_state == b->action.to) { - captures[capture_id].pos[1] = offset; -#if LOG_CAPTURE > 0 - fprintf(stderr, " -- set capture[%u].[1] to %lu\n", b->action.id, offset); -#endif - } else { - /* filtered, ignore */ - } - } - break; - case CAPTURE_ACTION_COMMIT_ZERO_STEP: -#if LOG_CAPTURE > 0 - fprintf(stderr, "COMMIT_ZERO_STEP [%u]\n", - b->action.id); -#endif + assert(ci->programs.used < ci->programs.ceil); - if (captures[capture_id].pos[0] == FSM_CAPTURE_NO_POS) { - captures[capture_id].pos[0] = offset; - captures[capture_id].pos[1] = offset | COMMITTED_CAPTURE_FLAG; - } else { /* extend */ - captures[capture_id].pos[1] = offset | COMMITTED_CAPTURE_FLAG; - } - -#if LOG_CAPTURE > 0 - fprintf(stderr, " -- set capture[%u].[0] and [1] to %lu (with COMMIT flag)\n", b->action.id, offset); -#endif - break; - case CAPTURE_ACTION_COMMIT: -#if LOG_CAPTURE > 0 - fprintf(stderr, "COMMIT [%u]\n", - b->action.id); -#endif - captures[capture_id].pos[1] = offset | COMMITTED_CAPTURE_FLAG; -#if LOG_CAPTURE > 0 - fprintf(stderr, " -- set capture[%u].[1] to %lu (with COMMIT flag)\n", b->action.id, offset); -#endif - break; - default: - assert(!"matchfail"); - } + const unsigned max_prog_capture_id = fsm_capvm_program_get_max_capture_id(program); + if (max_prog_capture_id > ci->max_capture_id) { + fsm_capture_update_max_capture_id(ci, max_prog_capture_id); } -} -void -fsm_capture_finalize_captures(const struct fsm *fsm, - size_t capture_count, struct fsm_capture *captures) -{ - size_t i; - - /* If either pos[] is FSM_CAPTURE_NO_POS or the - * COMMITTED_CAPTURE_FLAG isn't set on pos[1], then the capture - * wasn't finalized; clear it. Otherwise, clear that bit so the - * pos[1] offset is meaningful. */ - - /* FIXME: this should also take the end state(s) associated - * with a capture into account, when that information is available; - * otherwise there will be false positives for zero-width captures - * where the paths have a common prefix. */ - (void)fsm; - - for (i = 0; i < capture_count; i++) { -#if LOG_CAPTURE > 1 - fprintf(stderr, "finalize[%lu]: pos[0]: %ld, pos[1]: %ld\n", - i, captures[i].pos[0], captures[i].pos[1]); -#endif + *prog_id = ci->programs.used; + ci->programs.set[ci->programs.used] = program; + ci->programs.used++; + return 1; +} - if (captures[i].pos[0] == FSM_CAPTURE_NO_POS - || captures[i].pos[1] == FSM_CAPTURE_NO_POS - || (0 == (captures[i].pos[1] & COMMITTED_CAPTURE_FLAG))) { - captures[i].pos[0] = FSM_CAPTURE_NO_POS; - captures[i].pos[1] = FSM_CAPTURE_NO_POS; -#if LOG_CAPTURE > 1 - fprintf(stderr, "finalize: discard %lu\n", i); -#endif - } else if (captures[i].pos[1] & COMMITTED_CAPTURE_FLAG) { - captures[i].pos[1] &=~ COMMITTED_CAPTURE_FLAG; - } +const struct capvm_program * +fsm_capture_get_program_by_id(const struct fsm *fsm, uint32_t prog_id) +{ + struct fsm_capture_info *ci = fsm->capture_info; + if (prog_id >= ci->programs.used) { + return NULL; } + return ci->programs.set[prog_id]; } -void -fsm_capture_action_iter(const struct fsm *fsm, - fsm_capture_action_iter_cb *cb, void *opaque) +int +fsm_capture_associate_program_with_end_state(struct fsm *fsm, + uint32_t prog_id, fsm_state_t end_state) { - size_t i; struct fsm_capture_info *ci = fsm->capture_info; - assert(ci != NULL); - - for (i = 0; i < ci->bucket_count; i++) { - struct fsm_capture_action_bucket *b = &ci->buckets[i]; - if (b->state == CAPTURE_NO_STATE) { - continue; - } + assert(end_state < fsm->statecount); + assert(prog_id < ci->programs.used); - if (!cb(b->state, b->action.type, - b->action.id, b->action.to, opaque)) { - break; - } + if (!idmap_set(ci->end_capvm_program_map, end_state, prog_id)) { + return 0; } + return 1; } -const char *fsm_capture_action_type_name[] = { - "START", "EXTEND", - "COMMIT_ZERO_STEP", "COMMIT" +struct capture_resolve_env { + const struct fsm_capture_info *ci; + const unsigned char *input; + const size_t length; + + int res; + struct fsm_capture *captures; + size_t captures_len; }; static int -dump_iter_cb(fsm_state_t state, - enum capture_action_type type, unsigned capture_id, fsm_state_t to, - void *opaque) +exec_capvm_program_cb(fsm_state_t state_id, unsigned prog_id, void *opaque) { - FILE *f = opaque; - fprintf(f, " - state %u, %s [capture_id: %u, to: %d]\n", - state, fsm_capture_action_type_name[type], capture_id, to); + struct capture_resolve_env *env = opaque; + (void)state_id; + + /* TODO: idmap_iter could take a halt return value */ + if (env->res != 1) { return 0; } + + assert(prog_id < env->ci->programs.used); + struct capvm_program *p = env->ci->programs.set[prog_id]; + + LOG(5 - LOG_EVAL, "%s: evaluating prog_id %u for state %d\n", + __func__, prog_id, state_id); + +#define EXEC_COUNT 1 /* can be increased for benchmarking */ + + for (size_t i = 0; i < EXEC_COUNT; i++) { + const enum fsm_capvm_program_exec_res exec_res = + fsm_capvm_program_exec(p, + (const uint8_t *)env->input, env->length, + env->captures, env->captures_len); + if (exec_res != FSM_CAPVM_PROGRAM_EXEC_SOLUTION_WRITTEN) { + env->res = 0; + return 0; + } + } return 1; } -/* Dump capture metadata about an FSM. */ -void -fsm_capture_dump(FILE *f, const char *tag, const struct fsm *fsm) +int +fsm_capture_resolve_during_exec(const struct fsm *fsm, + fsm_state_t end_state, const unsigned char *input, size_t input_offset, + struct fsm_capture *captures, size_t captures_len) { - struct fsm_capture_info *ci; - assert(fsm != NULL); - ci = fsm->capture_info; - if (ci == NULL || ci->bucket_count == 0) { - fprintf(f, "==== %s -- no captures\n", tag); - return; - } - - fprintf(f, "==== %s -- capture action hash table (%u buckets)\n", - tag, ci->bucket_count); - fsm_capture_action_iter(fsm, dump_iter_cb, f); + assert(input != NULL); + assert(captures != NULL); + + const struct fsm_capture_info *ci = fsm->capture_info; + + struct capture_resolve_env capture_env = { + .res = 1, + .ci = ci, + .input = input, + .length = input_offset, + .captures = captures, + .captures_len = captures_len, + }; + + LOG(5 - LOG_EVAL, "%s: ended on state %d\n", + __func__, end_state); + idmap_iter_for_state(ci->end_capvm_program_map, + end_state, exec_capvm_program_cb, &capture_env); + + return capture_env.res; } diff --git a/src/libfsm/capture.h b/src/libfsm/capture.h index 4c0ba4722..16588060c 100644 --- a/src/libfsm/capture.h +++ b/src/libfsm/capture.h @@ -2,28 +2,26 @@ #define LIBFSM_CAPTURE_H #include +#include #include #include +#include -#define NEXT_STATE_END ((fsm_state_t)-1) - +/* Internal state IDs that are out of range for valid state IDs. + * + * CAPTURE_NO_STATE is used to represent the absence of a state, such as + * when remapping a state to a dead state (removing it) or empty + * hash table buckets. + * + * NEXT_STATE_END is used as a destination for capture actions that + * trigger when ending on a state. */ #define CAPTURE_NO_STATE ((fsm_state_t)-1) /* Capture interface -- functions internal to libfsm. * The public interface should not depend on any of these details. */ -enum capture_action_type { - /* Start an active capture if transitioning to TO. */ - CAPTURE_ACTION_START, - /* Continue an active capture if transitioning to TO, - * otherwise deactivate it. */ - CAPTURE_ACTION_EXTEND, - /* Write a zero-step capture (i.e., the start and - * end state are the same). */ - CAPTURE_ACTION_COMMIT_ZERO_STEP, - /* Write an active capture's endpoints. */ - CAPTURE_ACTION_COMMIT -}; +struct fsm_capture_info; +struct capvm_program; int fsm_capture_init(struct fsm *fsm); @@ -31,41 +29,110 @@ fsm_capture_init(struct fsm *fsm); void fsm_capture_free(struct fsm *fsm); +void +fsm_capture_dump_active_for_ends(FILE *f, const struct fsm *fsm); + +void +fsm_capture_dump_program_end_mapping(FILE *f, const struct fsm *fsm); + /* Does the FSM have captures? */ int fsm_capture_has_captures(const struct fsm *fsm); -/* Update captures, called when exiting or ending on a state. - * If ending on a state, use NEXT_STATE_END for next_state. */ void -fsm_capture_update_captures(const struct fsm *fsm, - fsm_state_t cur_state, fsm_state_t next_state, size_t offset, - struct fsm_capture *captures); +fsm_capture_dump(FILE *f, const char *tag, const struct fsm *fsm); + +void +fsm_capture_dump_programs(FILE *f, const struct fsm *fsm); +/* If EXPENSIVE_CHECKS is non-zero, assert that all capture metadata on + * an FSM is internally consistent. */ void -fsm_capture_finalize_captures(const struct fsm *fsm, - size_t capture_count, struct fsm_capture *captures); +fsm_capture_integrity_check(const struct fsm *fsm); + +int +fsm_capture_id_compact(struct fsm *fsm, const fsm_state_t *mapping, + size_t orig_statecount); -/* Add a capture action. This is used to update capture actions - * in the destination FSM when combining/transforming other FSMs. */ int -fsm_capture_add_action(struct fsm *fsm, - fsm_state_t state, enum capture_action_type type, - unsigned id, fsm_state_t to); - -/* Callback for iterating over capture actions. - * Return 1 to continue, return 0 to halt. - * If TO is not meaningful for a particular type, it will be - * set to NEXT_STATE_END. */ +fsm_capture_program_association_compact(struct fsm *fsm, const fsm_state_t *mapping, + size_t orig_statecount); + +/* Iterator callback for capture IDs that are active for a particular + * end state. Returns whether iteration should continue. */ typedef int -fsm_capture_action_iter_cb(fsm_state_t state, - enum capture_action_type type, unsigned capture_id, fsm_state_t to, +fsm_capture_iter_active_for_end_cb(fsm_state_t state, unsigned capture_id, void *opaque); void -fsm_capture_action_iter(const struct fsm *fsm, - fsm_capture_action_iter_cb *cb, void *opaque); +fsm_capture_iter_active_for_end_state(const struct fsm *fsm, fsm_state_t state, + fsm_capture_iter_active_for_end_cb *cb, void *opaque); + +void +fsm_capture_iter_active_for_all_end_states(const struct fsm *fsm, + fsm_capture_iter_active_for_end_cb *cb, void *opaque); -extern const char *fsm_capture_action_type_name[]; +/* Iterator callback for program IDs that are active for a particular + * end state. Returns whether iteration should continue. */ +typedef int +fsm_capture_iter_program_ids_for_end_state_cb(fsm_state_t state, unsigned prog_id, + void *opaque); + +void +fsm_capture_iter_program_ids_for_end_state(const struct fsm *fsm, fsm_state_t state, + fsm_capture_iter_program_ids_for_end_state_cb *cb, void *opaque); +void +fsm_capture_iter_program_ids_for_all_end_states(const struct fsm *fsm, + fsm_capture_iter_program_ids_for_end_state_cb *cb, void *opaque); + +/* TODO: combine/rename */ +int +fsm_capture_copy_active_for_ends(const struct fsm *src_fsm, + const struct state_set *states, + struct fsm *dst_fsm, fsm_state_t dst_state); +int +fsm_capture_copy_program_end_state_associations(const struct fsm *src_fsm, + const struct state_set *states, + struct fsm *dst_fsm, fsm_state_t dst_state); + +int +fsm_capture_copy_programs(const struct fsm *src_fsm, + struct fsm *dst_fsm); + +size_t +fsm_capture_program_count(const struct fsm *fsm); + +void +fsm_capture_update_max_capture_id(struct fsm_capture_info *ci, + unsigned capture_id); + +int +fsm_capture_add_program(struct fsm *fsm, + struct capvm_program *program, uint32_t *prog_id); + +const struct capvm_program * +fsm_capture_get_program_by_id(const struct fsm *fsm, uint32_t prog_id); + +int +fsm_capture_associate_program_with_end_state(struct fsm *fsm, + uint32_t prog_id, fsm_state_t end_state); + +/* Resolve captures. + * + * FIXME: With the current implementation, if enough memory + * was passed in then it couldn't fail, but it may be worth + * changing the interface so that it doesn't assume there was + * already a successful match in order to support one-pass + * matching & capture resolution attempts from a stream. + * + * TODO: This should pass in a size for captures[]. + * TODO: An alternate interface that allows passing in + * preallocated buffers for working memory. + * + * TODO: describe return value. */ +int +fsm_capture_resolve_during_exec(const struct fsm *fsm, + fsm_state_t end_state, const unsigned char *input, size_t input_offset, + struct fsm_capture *captures, size_t captures_len); #endif diff --git a/src/libfsm/capture_internal.h b/src/libfsm/capture_internal.h deleted file mode 100644 index 70418b988..000000000 --- a/src/libfsm/capture_internal.h +++ /dev/null @@ -1,115 +0,0 @@ -#ifndef CAPTURE_INTERNAL_H -#define CAPTURE_INTERNAL_H - -#include -#include - -#include -#include -#include - -#include -#include - -#include -#include -#include - -#include "internal.h" -#include "capture.h" - -/* Bucket count for capture action hash table. - * Must be a power of 2. */ - -#define DEF_CAPTURE_ACTION_BUCKET_COUNT 32 -#define DEF_TRAIL_CEIL 8 - -#define LOG_CAPTURE 0 - -/* Most significant bit of a size_t. */ -#define COMMITTED_CAPTURE_FLAG ((SIZE_MAX) ^ (SIZE_MAX >> 1)) - -struct fsm_capture_info { - unsigned max_capture_id; - - /* Add-only hash table. */ - unsigned bucket_count; - unsigned buckets_used; /* grow if >= 1/2 used */ - - /* Hash buckets. If state is CAPTURE_NO_STATE, - * the bucket is empty. */ - struct fsm_capture_action_bucket { - fsm_state_t state; /* key */ - struct fsm_capture_action { - enum capture_action_type type; - unsigned id; - /* only used by START and EXTEND */ - fsm_state_t to; - } action; - } *buckets; -}; - -enum trail_step { - TRAIL_STEP_START, - TRAIL_STEP_ITER_EDGES, - TRAIL_STEP_ITER_EPSILONS, - TRAIL_STEP_DONE -}; - -/* env->seen is used as a bit set for tracking which states have already - * been processed. These macros set/check/clear the bits. */ -#define SEEN_BITOP(ENV, STATE, OP) ENV->seen[STATE/64] OP ((uint64_t)1 << (STATE&63)) -#define MARK_SEEN(ENV, STATE) SEEN_BITOP(ENV, STATE, |=) -#define CHECK_SEEN(ENV, STATE) SEEN_BITOP(ENV, STATE, &) -#define CLEAR_SEEN(ENV, STATE) SEEN_BITOP(ENV, STATE, &=~) - -struct capture_set_path_env { - struct fsm *fsm; - unsigned capture_id; - fsm_state_t start; - fsm_state_t end; - - unsigned trail_i; - unsigned trail_ceil; - struct trail_cell { - fsm_state_t state; - enum trail_step step; - char has_self_edge; - struct edge_iter iter; - } *trail; - - /* bitset for which states have already been seen. */ - uint64_t *seen; -}; - -static int -init_capture_action_htab(struct fsm *fsm, struct fsm_capture_info *ci); - -static int -mark_capture_path(struct capture_set_path_env *env); - -static int -add_capture_action(struct fsm *fsm, struct fsm_capture_info *ci, - fsm_state_t state, const struct fsm_capture_action *action); - -static int -grow_capture_action_buckets(const struct fsm_alloc *alloc, - struct fsm_capture_info *ci); - -static int -grow_trail(struct capture_set_path_env *env); - -static int -step_trail_start(struct capture_set_path_env *env); -static int -step_trail_iter_edges(struct capture_set_path_env *env); -static int -step_trail_iter_epsilons(struct capture_set_path_env *env); -static int -step_trail_done(struct capture_set_path_env *env); - -static int -cmp_action(const struct fsm_capture_action *a, - const struct fsm_capture_action *b); - -#endif diff --git a/src/libfsm/capture_log.h b/src/libfsm/capture_log.h new file mode 100644 index 000000000..c850460bb --- /dev/null +++ b/src/libfsm/capture_log.h @@ -0,0 +1,27 @@ +/* + * Copyright 2020 Scott Vokes + * + * See LICENCE for the full copyright terms. + */ + +#ifndef CAPTURE_LOG_H +#define CAPTURE_LOG_H + +/* Log levels */ +#define LOG_CAPTURE 0 +#define LOG_CAPTURE_COMBINING_ANALYSIS 0 +#define LOG_EVAL 0 +#define LOG_APPEND_ACTION 0 +#define LOG_PRINT_FSM 0 +#define LOG_MARK_PATH 0 + +#include + +#define LOG(LEVEL, ...) \ + do { \ + if ((LEVEL) <= LOG_CAPTURE) { \ + fprintf(stderr, __VA_ARGS__); \ + } \ + } while(0) + +#endif diff --git a/src/libfsm/capture_vm.c b/src/libfsm/capture_vm.c new file mode 100644 index 000000000..e6a1f0539 --- /dev/null +++ b/src/libfsm/capture_vm.c @@ -0,0 +1,194 @@ +/* + * Copyright 2022 Scott Vokes + * + * See LICENCE for the full copyright terms. + */ + +/* Virtual machine for resolving captures while executing regular + * expressions from a subset of PCRE. This is based on the approach + * described in Russ Cox's "Regular Expression Matching: the Virtual + * Machine Approach" (https://swtch.com/~rsc/regexp/regexp2.html), but + * has a couple major modifications, mainly to keep memory usage low and + * predictable, and to be more consistent (arguably, bug-compatible...) + * with PCRE's behavior for libfsm's supported subset of PCRE. + * + * Instead of giving each green thread its own copy of the capture + * buffers, which uses a prohibitive amount of memory when combining DFAs + * with several captures each, operate in two passes: + * + * In the first pass, each thread keeps track of its execution path, + * appending a bit for each branch: 1 for the greedy option, 0 for the + * non-greedy. Since there can be at most one live thread per program + * instruction, and all of them are either on the current or next input + * character, there's a bounded window for diverging paths during execution. + * After a certain distance back all paths either have a common prefix + * or consist entirely of 0 bits (for continually looping at an unanchored + * start). The path bits are stored in chunks in a backwards linked list, + * so nodes for common path prefixes can be shared by multiple threads, + * and the prefix of all 0 bits is instead stored as a counter. This + * keeps memory usage substantially lower. This search runs threads in + * parallel, breadth-first, halting any threads that duplicate work of + * a greedier search path (since PCRE's results match the greediest). + * + * In the second pass, replay the execution path for just the single + * greediest thread, which represents the "correct" match (according to + * PCRE semantics), and write capture offsets into buffers passed in by + * the caller. + * + * Most of the other differences have to do with matching PCRE + * edge cases, particularly interactions between newlines and start/end + * anchors. */ + +#include "capture_vm.h" +#include "capture_vm_program.h" + +#include + +#include +#include +#include + +void +fsm_capvm_program_free(const struct fsm_alloc *alloc, + struct capvm_program *program) +{ + if (program == NULL) { return; } + f_free(alloc, program->ops); + f_free(alloc, program->char_classes.sets); + f_free(alloc, program); +} + +struct capvm_program * +capvm_program_copy(const struct fsm_alloc *alloc, + const struct capvm_program *src) +{ + assert(src != NULL); + struct capvm_program *p = NULL; + struct capvm_opcode *ops = NULL; + struct capvm_char_class *sets = NULL; + + p = f_calloc(alloc, 1, sizeof(*p)); + if (p == NULL) { goto cleanup; } + + /* This allocates exactly as many instructions and char_classes + * as necessary, rather than a power-of-2 buffer, because + * they are only added during compilation in libre. */ + + ops = f_calloc(alloc, src->used, sizeof(ops[0])); + if (ops == NULL) { goto cleanup; } + + sets = f_calloc(alloc, + /* do non-zero allocation to silence EFENCE */ + src->char_classes.count == 0 ? 1 : src->char_classes.count, + sizeof(src->char_classes.sets[0])); + if (sets == NULL) { goto cleanup; } + + memcpy(ops, src->ops, src->used * sizeof(src->ops[0])); + + assert(src->char_classes.sets != NULL || src->char_classes.count == 0); + if (src->char_classes.count > 0) { + memcpy(sets, src->char_classes.sets, + src->char_classes.count * sizeof(src->char_classes.sets[0])); + } + + struct capvm_program np = { + .capture_count = src->capture_count, + .capture_base = src->capture_base, + + .used = src->used, + .ceil = src->used, + .ops = ops, + + .char_classes = { + .count = src->char_classes.count, + .ceil = src->char_classes.count, + .sets = sets, + }, + }; + memcpy(p, &np, sizeof(np)); + return p; + +cleanup: + f_free(alloc, p); + f_free(alloc, ops); + f_free(alloc, sets); + return NULL; +} + +void +capvm_program_rebase(struct capvm_program *program, unsigned capture_offset) +{ + assert(program->capture_base + capture_offset > program->capture_base); + program->capture_base += capture_offset; +} + +void +fsm_capvm_program_dump(FILE *f, + const struct capvm_program *p) +{ + for (size_t i = 0; i < p->used; i++) { + const struct capvm_opcode *op = &p->ops[i]; + switch (op->t) { + case CAPVM_OP_CHAR: + fprintf(f, "%zu: char 0x%02x (%c)\n", + i, op->u.chr, isprint(op->u.chr) ? op->u.chr : '.'); + break; + case CAPVM_OP_CHARCLASS: + { + const uint32_t id = op->u.charclass_id; + assert(id < p->char_classes.count); + const struct capvm_char_class *cc = &p->char_classes.sets[id]; + fprintf(f, "%zu: charclass %u -> [", i, id); + for (size_t i = 0; i < 4; i++) { + fprintf(f, "%016lx", cc->octets[i]); + } + fprintf(f, "]\n"); + break; + } + case CAPVM_OP_MATCH: + fprintf(f, "%zu: match\n", i); + break; + case CAPVM_OP_JMP: + fprintf(f, "%zu: jmp %u\n", i, op->u.jmp); + break; + case CAPVM_OP_JMP_ONCE: + fprintf(f, "%zu: jmp_once %u\n", i, op->u.jmp_once); + break; + case CAPVM_OP_SPLIT: + fprintf(f, "%zu: split greedy %u nongreedy %u\n", i, op->u.split.greedy, op->u.split.nongreedy); + break; + case CAPVM_OP_SAVE: + fprintf(f, "%zu: save %u (cap %u, %s)\n", + i, op->u.save, + op->u.save / 2, (op->u.save & (uint32_t)0x01) ? "end" : "start"); + break; + case CAPVM_OP_ANCHOR: + fprintf(f, "%zu: anchor %s\n", i, + op->u.anchor == CAPVM_ANCHOR_START ? "start" : "end"); + break; + default: + assert(!"matchfail"); + } + } + for (size_t i = 0; i < p->char_classes.count; i++) { + const uint64_t *octets = p->char_classes.sets[i].octets; + fprintf(f, "char_class %zu: 0x%016lx 0x%016lx 0x%016lx 0x%016lx\n", + i, octets[0], octets[1], octets[2], octets[3]); + } +} + +unsigned +fsm_capvm_program_get_capture_count(const struct capvm_program *program) +{ + assert(program != NULL); + return program->capture_count; +} + +unsigned +fsm_capvm_program_get_max_capture_id(const struct capvm_program *program) +{ + assert(program != NULL); + return (program->capture_count == 0 + ? 0 + : program->capture_base + program->capture_count - 1); +} diff --git a/src/libfsm/capture_vm.h b/src/libfsm/capture_vm.h new file mode 100644 index 000000000..02c198dab --- /dev/null +++ b/src/libfsm/capture_vm.h @@ -0,0 +1,68 @@ +/* + * Copyright 2022 Scott Vokes + * + * See LICENCE for the full copyright terms. + */ + +#ifndef CAPTURE_VM_H +#define CAPTURE_VM_H + +#include +#include + +#include +#include + +/* Interface for the virtual machine used to resolve captures. + * These interfaces are exposed to libre but should not be + * used directly. */ + +/* Opaque struct, details in capture_vm_program.h. */ +struct capvm_program; + +void +fsm_capvm_program_free(const struct fsm_alloc *alloc, + struct capvm_program *program); + +struct capvm_program * +capvm_program_copy(const struct fsm_alloc *alloc, + const struct capvm_program *program); + +/* Add an offset to the capture ID base for a program. + * Used when FSMs are merged, one of the source FSMs' capture IDs + * will be shifted to appear after the others. */ +void +capvm_program_rebase(struct capvm_program *program, unsigned capture_offset); + +void +fsm_capvm_program_dump(FILE *f, + const struct capvm_program *program); + +enum fsm_capvm_program_exec_res { + FSM_CAPVM_PROGRAM_EXEC_SOLUTION_WRITTEN, + FSM_CAPVM_PROGRAM_EXEC_NO_SOLUTION_FOUND, + FSM_CAPVM_PROGRAM_EXEC_STEP_LIMIT_REACHED, + FSM_CAPVM_PROGRAM_EXEC_ERROR_ALLOC = -1, +}; + +/* Execute a capture program with the given input and populate + * the capture buffer. + * + * This asserts that the capture buffer is at least as large as + * necessary. This is an internal interface, and the buffer size + * should have already been checked by the caller. */ +enum fsm_capvm_program_exec_res +fsm_capvm_program_exec(const struct capvm_program *program, + const uint8_t *input, size_t length, + struct fsm_capture *capture_buf, size_t capture_buf_length); + +/* Get the capture count from the program. */ +unsigned +fsm_capvm_program_get_capture_count(const struct capvm_program *program); + +/* Get the max capture ID from the program. + * If there are no captures (which is pointless) it will return 0. */ +unsigned +fsm_capvm_program_get_max_capture_id(const struct capvm_program *program); + +#endif diff --git a/src/libfsm/capture_vm_exec.c b/src/libfsm/capture_vm_exec.c new file mode 100644 index 000000000..9d4be066a --- /dev/null +++ b/src/libfsm/capture_vm_exec.c @@ -0,0 +1,2076 @@ +/* + * Copyright 2022 Scott Vokes + * + * See LICENCE for the full copyright terms. + */ + +#include "capture_vm.h" +#include "capture_vm_program.h" +#include "capture_vm_log.h" + +#include +#include +#include +#include + +#include + +/* for EXPENSIVE_CHECKS and TRACK_TIMES */ +#include "internal.h" + +#if EXPENSIVE_CHECKS +#include +#endif + +/* Special handling for a path node that has a long prefix of all 0 + * bits, as is common when the regex is unanchored at the start. */ +#define USE_COLLAPSED_ZERO_PREFIX 1 + +/* Special out-of-range NONE values. */ +#define NO_POS ((uint32_t)-1) +#define NO_ID ((uint32_t)-1) +#define COLLAPSED_ZERO_PREFIX_ID ((uint32_t)-2) +#define NO_POS_SIZE_T ((size_t)-1) + +/* Max number of bits each path link can store. + * This value cannot be changed without reworking the data structures. */ +#define PATH_LINK_BITS 32 + +/* This enables extra debugging/testing output in an easily scraped format */ +#ifndef TESTING_OPTIONS +#define TESTING_OPTIONS 0 +#endif + +/* Write the solution to stdout (used for testing). */ +#define LOG_SOLUTION_TO_STDOUT (0 || TESTING_OPTIONS) + +/* Enable extra fields for debugging/performance tuning, most notably + * a 'uniq_id' field that helps to see the various execution paths. */ +#define CAPVM_STATS (0 || TESTING_OPTIONS) +#define CAPVM_PATH_STATS (0 && CAPVM_STATS) + +/* Allow the path table to grow on demand. + * In theory it should be possible to determine the worst case + * based on compile-time analysis and the input length; if an + * appropriately sized buffer was passed in capture resolution + * would not need dynamic allocation at all. */ +#define ALLOW_PATH_TABLE_RESIZING 1 + +/* Set to non-zero to trap runaway path table growth */ +#define PATH_TABLE_CEIL_LIMIT 0 + +/* Specialized logging that can be scraped to reconstruct non-interleaved + * execution paths per thread. */ +#define LOG_EXECUTION 0 +#define LOG_EXECUTION_FILE stderr +#if LOG_EXECUTION + +#if CAPVM_STATS == 0 +#error CAPVM_STATS must be 1 for uniq_id +#endif + +/* Various execution log messages, in an easily scraped format */ +#define LOG_EXEC_OP(UNIQ_ID, INPUT_POS, OP_ID, OP_NAME) \ + fprintf(LOG_EXECUTION_FILE, \ + "LOG_EXEC OP %u %u %u %s\n", \ + UNIQ_ID, INPUT_POS, OP_ID, OP_NAME) + +#define LOG_EXEC_CHAR(UNIQ_ID, CHAR) \ + fprintf(LOG_EXECUTION_FILE, \ + "LOG_EXEC CHAR %u %c 0x%02x\n", UNIQ_ID, isprint(CHAR) ? CHAR : '.', CHAR) + +#define LOG_EXEC_HALT(UNIQ_ID) \ + fprintf(LOG_EXECUTION_FILE, \ + "LOG_EXEC HALT %u\n", UNIQ_ID) + +#define LOG_EXEC_PATH_FIND_SOLUTION(UNIQ_ID, BIT) \ + fprintf(LOG_EXECUTION_FILE, \ + "LOG_EXEC PATH_FIND_SOLUTION %u %u\n", UNIQ_ID, BIT) + +#define LOG_EXEC_PATH_SAVE_CAPTURES(UNIQ_ID, BIT) \ + fprintf(LOG_EXECUTION_FILE, \ + "LOG_EXEC PATH_SAVE_CAPTURES %u %u\n", UNIQ_ID, BIT) + +#define LOG_EXEC_SPLIT(PARENT_UNIQ_ID, CHILD_UNIQ_ID) \ + fprintf(LOG_EXECUTION_FILE, \ + "LOG_EXEC SPLIT %u %u\n", PARENT_UNIQ_ID, CHILD_UNIQ_ID) +#else +#define LOG_EXEC_OP(UNIQ_ID, INPUT_POS, OP_ID, OP_NAME) /* no-op */ +#define LOG_EXEC_CHAR(UNIQ_ID, CHAR) /* no-op */ +#define LOG_EXEC_HALT(UNIQ_ID) /* no-op */ +#define LOG_EXEC_PATH_FIND_SOLUTION(UNIQ_ID, BIT) /* no-op */ +#define LOG_EXEC_PATH_SAVE_CAPTURES(UNIQ_ID, BIT) /* no-op */ +#define LOG_EXEC_SPLIT(PARENT_UNIQ_ID, CHILD_UNIQ_ID) /* no-op */ +#endif + +/* Bitset backed by an array of 32-bit words */ +#define GET_BIT32(BITARRAY, BIT) (BITARRAY[BIT/32] & ((uint32_t)1 << (BIT & 31))) +#define SET_BIT32(BITARRAY, BIT) (BITARRAY[BIT/32] |= ((uint32_t)1 << (BIT & 31))) + +static const char * +op_name[] = { + [CAPVM_OP_CHAR] = "CHAR", + [CAPVM_OP_CHARCLASS] = "CHARCLASS", + [CAPVM_OP_MATCH] = "MATCH", + [CAPVM_OP_JMP] = "JMP", + [CAPVM_OP_JMP_ONCE] = "JMP_ONCE", + [CAPVM_OP_SPLIT] = "SPLIT", + [CAPVM_OP_SAVE] = "SAVE", + [CAPVM_OP_ANCHOR] = "ANCHOR", +}; + +enum pair_id { PAIR_ID_CURRENT = 0, PAIR_ID_NEXT = 1 }; + +struct capvm { + const struct capvm_program *p; + const uint8_t *input; + const uint32_t input_len; + struct fsm_capture *capture_buf; + const size_t capture_buf_length; + size_t step_limit; + +#if CAPVM_STATS + uint32_t uniq_id_counter; +#endif + + /* Two stacks, used to track which execution instruction should + * be advanced next. The current stack is + * run_stacks[PAIR_ID_CURRENT], run_stacks[PAIR_ID_NEXT] is the + * stack for the next input position, and when the current stack + * is completed the next stack is copied over (and reversed, so + * the greediest threads end up on top and resume first). + * Same with run_stacks_h, the height for each stack, and the + * other fields with [2] below. */ + uint32_t *run_stacks[2]; + uint32_t run_stacks_h[2]; + + /* Similarly, two columns of bits and two arrays of path_info + * node IDs and uniq_ids for the execution at a particular + * opcode. + * + * evaluated bit array[]: Has the instruction n already been + * evaluated at the current input position? */ + uint32_t *evaluated[2]; + uint32_t *path_info_heads[2]; /* path for thread on instruction */ +#if CAPVM_STATS + uint32_t *uniq_ids[2]; +#endif + + struct capvm_thread_stats { + uint32_t live; + uint32_t max_live; + } threads; + + /* Pool of nodes for linked lists of path segments. */ + struct capvm_path_info_pool { + uint32_t ceil; + uint32_t live; + uint32_t max_live; + uint32_t freelist_head; + struct capvm_path_info { + union { + struct capvm_path_freelist_link { + uint16_t refcount; /* == 0: tag for freelist node */ + uint32_t freelist; + } freelist_node; + struct capvm_path_info_link { + /* refcount: When > 0 this is a path node. + * This could be sticky at UINT16_MAX, but in order + * to get there it would need a regex whose compiled + * program has well over 2**16 instructions that all + * share the same path info node. */ + uint16_t refcount; + uint8_t used; /* .bits used, <= PATH_LINK_BITS */ + uint32_t bits; /* buffer for this link's path bits */ + uint32_t offset; /* offset into the path bit array */ + /* Linked list to earlier path nodes, with common + * nodes shared until paths diverge. + * + * This can be either a valid path node ID, NO_ID + * for end of list, or COLLAPSED_ZERO_PREFIX_ID + * to indicate that the node is preceded by + * (offset) zero bits. */ + uint32_t backlink; +#if CAPVM_PATH_STATS + uint32_t bits_added_per_input_character; +#endif + } path; + } u; + } *pool; + } paths; + + struct capvm_solution_info { + uint32_t best_path_id; +#if CAPVM_STATS + uint32_t best_path_uniq_id; +#endif + uint32_t zeros_evaluated_up_to; + } solution; + + struct { + size_t steps; +#if CAPVM_STATS + uint32_t matches; + uint32_t path_prefixes_shared; + uint32_t collapsed_zero_prefixes; +#endif +#if CAPVM_PATH_STATS + uint32_t max_bits_added_per_input_character; + uint32_t max_path_length_memory; +#endif + } stats; + + enum fsm_capvm_program_exec_res res; +}; + +/* Type identifier macros */ +#define IS_THREAD_FREELIST(T) (T->u.thread.path_info_head == NO_ID) +#define IS_PATH_FREELIST(P) (P->u.path.refcount == 0) +#define IS_PATH_NODE(P) (P->u.path.refcount > 0 && P->u.path.used <= PATH_LINK_BITS) + +static void +release_path_info_link(struct capvm *vm, uint32_t *pi_id); + +static void +dump_path_table(FILE *f, const struct capvm *vm); + +static void +set_max_threads_live(struct capvm *vm, uint32_t new_max_live) +{ + vm->threads.max_live = new_max_live; + if (LOG_CAPVM >= 6) { + LOG(0, "==== new vm->threads.max_live: %u\n", vm->threads.max_live); + dump_path_table(stderr, vm); + } +} + + +/*********************** + * path_info functions * + ***********************/ + +static void +set_max_paths_live(struct capvm *vm) +{ + vm->paths.max_live = vm->paths.live; + if (LOG_CAPVM >= 6) { + LOG(0, "==== new vm->paths.max_live: %u\n", vm->paths.max_live); + dump_path_table(stderr, vm); + } +} + +static uint32_t +get_path_node_refcount(const struct capvm *vm, uint32_t p_id) +{ + assert(p_id < vm->paths.ceil); + const struct capvm_path_info *pi = &vm->paths.pool[p_id]; + if (IS_PATH_FREELIST(pi)) { + return pi->u.freelist_node.refcount; + } else { + assert(IS_PATH_NODE(pi)); + return pi->u.path.refcount; + } +} + +static void +inc_path_node_refcount(struct capvm *vm, uint32_t p_id) +{ + /* TODO: sticky refcount handling is not currently implemented */ + if (p_id == COLLAPSED_ZERO_PREFIX_ID) { return; } + assert(p_id < vm->paths.ceil); + struct capvm_path_info *pi = &vm->paths.pool[p_id]; + assert(IS_PATH_NODE(pi)); + LOG(5, "%s: p_id %u: refcnt %u -> %u\n", + __func__, p_id, pi->u.path.refcount, pi->u.path.refcount + 1); + pi->u.path.refcount++; +} + +static uint32_t +get_path_node_offset(const struct capvm *vm, uint32_t p_id) +{ + assert(p_id < vm->paths.ceil); + const struct capvm_path_info *pi = &vm->paths.pool[p_id]; + assert(IS_PATH_NODE(pi)); + return pi->u.path.offset; +} + +static uint32_t +get_path_node_backlink(const struct capvm *vm, uint32_t p_id) +{ + assert(p_id < vm->paths.ceil); + const struct capvm_path_info *pi = &vm->paths.pool[p_id]; + if (IS_PATH_FREELIST(pi)) { + return pi->u.freelist_node.freelist; + } else { + assert(IS_PATH_NODE(pi)); + return pi->u.path.backlink; + } +} + +static void +set_path_node_backlink(struct capvm *vm, uint32_t p_id, uint32_t backlink) +{ + assert(p_id < vm->paths.ceil); + assert(backlink < vm->paths.ceil || (backlink == NO_ID || backlink == COLLAPSED_ZERO_PREFIX_ID)); + struct capvm_path_info *pi = &vm->paths.pool[p_id]; + assert(IS_PATH_NODE(pi)); + pi->u.path.backlink = backlink; +} + +static void +dump_path_table(FILE *f, const struct capvm *vm) +{ + fprintf(f, "=== path table, %u/%u live\n", + vm->paths.live, vm->paths.ceil); + for (uint32_t i = 0; i < vm->paths.ceil; i++) { + struct capvm_path_info *pi = &vm->paths.pool[i]; + if (IS_PATH_FREELIST(pi)) { + if (LOG_CAPVM >= 5) { + fprintf(f, "paths[%u]: freelist -> %d\n", + i, (int)pi->u.freelist_node.freelist); + } + } else { + assert(IS_PATH_NODE(pi)); + fprintf(f, "paths[%u]: refcount %u, used %u, bits 0x%08x, offset %u, backlink %d%s\n", + i, pi->u.path.refcount, pi->u.path.used, pi->u.path.bits, + pi->u.path.offset, (int)pi->u.path.backlink, + pi->u.path.backlink == COLLAPSED_ZERO_PREFIX_ID + ? " (collapsed zero prefix)" + : pi->u.path.backlink == NO_ID + ? " (none)" + : " (link)"); + } + } +} + +static void +check_path_table(const struct capvm *vm) +{ +#if EXPENSIVE_CHECKS + uint32_t *refcounts = calloc(vm->paths.ceil, sizeof(refcounts[0])); + assert(refcounts); + + if (LOG_CAPVM >= 4) { + dump_path_table(stderr, vm); + } + + LOG(4, "%s: stack heights %u, %u\n", __func__, + vm->run_stacks_h[PAIR_ID_CURRENT], vm->run_stacks_h[PAIR_ID_NEXT]); + + for (uint32_t pair_id = 0; pair_id < 2; pair_id++) { + for (uint32_t h = 0; h < vm->run_stacks_h[pair_id]; h++) { + const uint32_t op_id = vm->run_stacks[pair_id][h]; + if (op_id == NO_ID) { continue; } +#if CAPVM_STATS + const uint32_t uniq_id = vm->uniq_ids[pair_id][op_id]; +#else + const uint32_t uniq_id = 0; +#endif + + LOG(4, "%s: run_stacks[%u][%u/%u]: op_id %u (uniq_id %u) -> path_info_head %u\n", + __func__, pair_id, h, vm->run_stacks_h[pair_id], op_id, + uniq_id, vm->path_info_heads[pair_id][op_id]); + if (op_id == NO_ID) { continue; } + const uint32_t p_id = vm->path_info_heads[pair_id][op_id]; + if (p_id != NO_ID) { + refcounts[p_id]++; + } + } + } + + for (uint32_t p_id = 0; p_id < vm->paths.ceil; p_id++) { + const struct capvm_path_info *pi = &vm->paths.pool[p_id]; + if (IS_PATH_FREELIST(pi)) { + continue; + } + const uint32_t backlink = get_path_node_backlink(vm, p_id); + if (backlink != NO_ID && backlink != COLLAPSED_ZERO_PREFIX_ID) { + assert(backlink < vm->paths.ceil); + refcounts[backlink]++; + } + } + + if (vm->solution.best_path_id != NO_ID) { + assert(vm->solution.best_path_id < vm->paths.ceil); + refcounts[vm->solution.best_path_id]++; + } + + for (uint32_t p_id = 0; p_id < vm->paths.ceil; p_id++) { + const struct capvm_path_info *pi = &vm->paths.pool[p_id]; + if (IS_PATH_FREELIST(pi)) { continue; } + bool ok; + const uint32_t refcount = get_path_node_refcount(vm, p_id); + ok = refcounts[p_id] == refcount; + + if (!ok) { + dump_path_table(stderr, vm); + + fprintf(stderr, "BAD REFCOUNT: pi[%u], expected %u, got %u\n", + p_id, refcounts[p_id], refcount); + assert(ok); + } + } + + free(refcounts); + LOG(6, "%s: passed\n", __func__); +#else + (void)vm; +#endif +} + +static bool +reserve_path_info_link(struct capvm *vm, uint32_t *pi_id) +{ + if (vm->paths.live == vm->paths.ceil) { +#if ALLOW_PATH_TABLE_RESIZING + if (LOG_CAPVM >= 4) { + fprintf(stderr, "\n"); + dump_path_table(stderr, vm); + check_path_table(vm); + fprintf(stderr, "\n"); + } + + const uint32_t nceil = 2*vm->paths.ceil; + LOG(1, "%s: growing path table %u -> %u\n", + __func__, vm->paths.ceil, nceil); + + /* This can legitimitely be reached with very long inputs, but + * if PATH_TABLE_CEIL_LIMIT is non-zero and this is hit then + * it's most likely a sign of an infinite loop. */ + if (PATH_TABLE_CEIL_LIMIT != 0 && nceil > PATH_TABLE_CEIL_LIMIT) { + assert(!"reached PATH_TABLE_CEIL_LIMIT"); + } + + assert(nceil > vm->paths.ceil); + struct capvm_path_info *npool = realloc(vm->paths.pool, + nceil * sizeof(npool[0])); + if (npool == NULL) { + return false; + } + + for (size_t i = vm->paths.ceil; i < nceil; i++) { + npool[i].u.freelist_node.refcount = 0; + npool[i].u.freelist_node.freelist = i + 1; + } + npool[nceil - 1].u.freelist_node.refcount = 0; + npool[nceil - 1].u.freelist_node.freelist = NO_POS; + vm->paths.freelist_head = vm->paths.ceil; + vm->paths.ceil = nceil; + vm->paths.pool = npool; +#else + assert(!"shouldn't need to grow path pool"); +#endif + } + + assert(vm->paths.live < vm->paths.ceil); + assert(vm->paths.freelist_head != NO_POS); + + *pi_id = vm->paths.freelist_head; + LOG(3, "%s: returning %u\n", __func__, *pi_id); + return true; +} + +/* Release a reference to a path_info_link. Consume the argument. + * If the reference count reaches 0, repool the node and release + * its backlink. */ +static void +release_path_info_link(struct capvm *vm, uint32_t *pi_id) +{ +#define LOG_RELEASE_PI 0 + size_t count = 0; + assert(pi_id != NULL); + uint32_t cur_id = *pi_id; + LOG(4 - LOG_RELEASE_PI, "%s: pi_id %u\n", __func__, cur_id); + *pi_id = NO_ID; + + while (cur_id != NO_ID) { + struct capvm_path_info *pi = &vm->paths.pool[cur_id]; + uint32_t refcount = get_path_node_refcount(vm, cur_id); + LOG(4 - LOG_RELEASE_PI, "-- checking path_info[%u]: refcount %u\n", + cur_id, refcount); + assert(refcount > 0); + LOG(4 - LOG_RELEASE_PI, "release: pi[%u] refcount %u -> %u\n", + cur_id, refcount, refcount - 1); + + const uint32_t backlink = get_path_node_backlink(vm, cur_id); + assert(IS_PATH_NODE(pi)); + pi->u.path.refcount--; + refcount = pi->u.path.refcount; + + if (refcount > 0) { + break; + } + + count++; + LOG(3 - LOG_RELEASE_PI, "-- repooling path_info %u, now %u live\n", + cur_id, vm->paths.live - 1); + LOG(3 - LOG_RELEASE_PI, "-- backlink: %d\n", backlink); + + pi->u.freelist_node.freelist = vm->paths.freelist_head; + vm->paths.freelist_head = cur_id; + assert(vm->paths.live > 0); + vm->paths.live--; + + cur_id = backlink; + if (cur_id == COLLAPSED_ZERO_PREFIX_ID) { + break; + } + } +} + +static void +print_path(FILE *f, const struct capvm *vm, uint32_t p_id) +{ + if (p_id == NO_ID) { + fprintf(f, "/0"); + return; + } + + /* reverse links to the root node */ + uint32_t zero_prefix = 0; + uint32_t next = NO_ID; + uint32_t first = NO_ID; + uint32_t prev; + + while (p_id != NO_ID) { + assert(p_id < vm->paths.ceil); + struct capvm_path_info *pi = &vm->paths.pool[p_id]; + assert(!IS_PATH_FREELIST(pi)); + + uint32_t bl; + assert(IS_PATH_NODE(pi)); + bl = pi->u.path.backlink; + pi->u.path.backlink = next; + + if (bl == NO_ID) { + prev = bl; + first = p_id; + break; + } else if (bl == COLLAPSED_ZERO_PREFIX_ID) { + prev = bl; + first = p_id; + zero_prefix = pi->u.path.offset; + break; + } + + next = p_id; + p_id = bl; + } + + if (zero_prefix > 0) { + fprintf(f, "0/%u", zero_prefix); + } + + /* iterate forward, printing and restoring link order */ + p_id = first; + while (p_id != NO_ID) { + assert(p_id < vm->paths.ceil); + struct capvm_path_info *pi = &vm->paths.pool[p_id]; + assert(!IS_PATH_FREELIST(pi)); + + uint32_t fl; /* now a forward link */ + assert(IS_PATH_NODE(pi)); + fl = pi->u.path.backlink; + pi->u.path.backlink = prev; + prev = p_id; + fprintf(f, "%s%08x/%u", prev == NO_ID ? "" : " ", + pi->u.path.bits, pi->u.path.used); + + p_id = fl; + } +} + +#if EXPENSIVE_CHECKS +SUPPRESS_EXPECTED_UNSIGNED_INTEGER_OVERFLOW() +#endif +static int +cmp_paths(struct capvm *vm, uint32_t p_a, uint32_t p_b) +{ +#if EXPENSIVE_CHECKS + /* When EXPENSIVE_CHECKS is set, walk the chains + * before and after and compare incremental hashing of node IDs, + * to ensure the chains are restored properly. */ + uint64_t hash_a_before = 0; + uint64_t hash_b_before = 0; +#endif + +#define LOG_CMP_PATHS 0 + LOG(3 - LOG_CMP_PATHS, "%s: p_a %d, p_b %d\n", __func__, p_a, p_b); + + if (p_a == NO_ID) { + return p_b == NO_ID ? 0 : -1; + } else if (p_b == NO_ID) { + return 1; + } + + assert(p_a != p_b); + + if (LOG_CAPVM >= 5) { + fprintf(stderr, "A: "); + print_path(stderr, vm, p_a); + fprintf(stderr, "\n"); + + fprintf(stderr, "B: "); + print_path(stderr, vm, p_b); + fprintf(stderr, "\n"); + } + + /* walk both paths backward until they reach a beginning + * or the common prefix node, reversing links along the + * way, then compare forward and restore link order. */ + uint32_t link_a = p_a; + uint32_t link_b = p_b; + + uint32_t fwd_a = NO_ID; + uint32_t fwd_b = NO_ID; + + /* Walk both paths backward, individually until reaching a + * common offset, then back until reaching a common prefix + * (including the start). While iterating backward, replace + * the .backlink field with a forward link, which will be + * reverted when iterating forward and comparing from the + * common prefix. */ + bool common_prefix_found = false; + uint32_t first_a = NO_ID; + uint32_t first_b = NO_ID; + uint32_t common_prefix_link; /* can be NO_ID */ + +#if EXPENSIVE_CHECKS + uint32_t hash_step = 0; /* added so ordering matters */ + while (link_a != NO_ID) { + assert(link_a < vm->paths.ceil); + const uint32_t prev = get_path_node_backlink(vm, link_a); + hash_a_before += hash_id(link_a + hash_step); + link_a = prev; + hash_step++; + } + hash_step = 0; + while (link_b != NO_ID) { + assert(link_b < vm->paths.ceil); + const uint32_t prev = get_path_node_backlink(vm, link_b); + hash_b_before += hash_id(link_b + hash_step); + link_b = prev; + hash_step++; + } + + link_a = p_a; + link_b = p_b; +#endif + + while (!common_prefix_found) { + assert(link_a != NO_ID); + assert(link_b != NO_ID); + assert(link_a < vm->paths.ceil); + assert(link_b < vm->paths.ceil); + + const uint32_t prev_a = get_path_node_backlink(vm, link_a); + const uint32_t prev_b = get_path_node_backlink(vm, link_b); + const uint32_t offset_a = get_path_node_offset(vm, link_a); + const uint32_t offset_b = get_path_node_offset(vm, link_b); + const uint32_t backlink_a = get_path_node_backlink(vm, link_a); + const uint32_t backlink_b = get_path_node_backlink(vm, link_b); + + /* These are only used for logging, which may compile out. */ + (void)backlink_a; + (void)backlink_b; + + LOG(3 - LOG_CMP_PATHS, + "%s: backward loop: link_a %d (offset %u, prev %d), link_b %d (offset %u, prev %d)\n", + __func__, link_a, offset_a, prev_a, link_b, offset_b, prev_b); + + assert((offset_a & (PATH_LINK_BITS - 1)) == 0); /* multiple of 32 */ + assert((offset_b & (PATH_LINK_BITS - 1)) == 0); /* multiple of 32 */ + if (offset_a > offset_b) { + LOG(3 - LOG_CMP_PATHS, "%s: backward loop: a longer than b\n", __func__); + set_path_node_backlink(vm, link_a, fwd_a); + fwd_a = link_a; + link_a = prev_a; + } else if (offset_b > offset_a) { + LOG(3 - LOG_CMP_PATHS, "%s: backward loop: b longer than a\n", __func__); + set_path_node_backlink(vm, link_b, fwd_b); + fwd_b = link_b; + link_b = prev_b; + } else { + assert(offset_b == offset_a); + LOG(3 - LOG_CMP_PATHS, "%s: backward loop: comparing backlinks: a: %d, b: %d\n", + __func__, backlink_a, backlink_b); + assert(fwd_a != link_a); + set_path_node_backlink(vm, link_a, fwd_a); + assert(fwd_b != link_b); + set_path_node_backlink(vm, link_b, fwd_b); + + if (prev_a == prev_b) { + /* if == NO_ID, empty prefix */ + common_prefix_found = true; + common_prefix_link = prev_a; + first_a = link_a; + first_b = link_b; + + LOG(3 - LOG_CMP_PATHS, "%s: backward loop: common_prefix_found: %d\n", + __func__, common_prefix_link); + } else { + fwd_a = link_a; + fwd_b = link_b; + + link_a = prev_a; + link_b = prev_b; + } + } + } + + assert(first_a != NO_ID); + assert(first_b != NO_ID); + link_a = first_a; + link_b = first_b; + + bool cmp_done = false; + int res; + bool done_restoring_link_order = false; + uint32_t prev_a = common_prefix_link; + uint32_t prev_b = common_prefix_link; + while (!done_restoring_link_order) { + LOG(3 - LOG_CMP_PATHS, + "%s: fwd loop, link_a %d, link_b %d, cmp_done %d\n", + __func__, link_a, link_b, cmp_done); + if (!cmp_done) { + if (link_a == NO_ID) { /* b is longer */ + cmp_done = true; + if (link_b == NO_ID) { + res = 0; + LOG(3 - LOG_CMP_PATHS, + "%s: fwd loop, equal length, res %d\n", __func__, res); + } else { + res = -1; + LOG(3 - LOG_CMP_PATHS, + "%s: fwd loop, b is longer, res %d\n", __func__, res); + } + } else if (link_b == NO_ID) { /* a is longer */ + cmp_done = true; + res = 1; + LOG(3 - LOG_CMP_PATHS, + "%s: fwd loop, a is longer, res %d\n", __func__, res); + } else { + assert(link_a < vm->paths.ceil); + assert(link_b < vm->paths.ceil); + struct capvm_path_info *pi_a = &vm->paths.pool[link_a]; + struct capvm_path_info *pi_b = &vm->paths.pool[link_b]; + + const uint32_t offset_a = get_path_node_offset(vm, link_a); + const uint32_t offset_b = get_path_node_offset(vm, link_b); + + assert(offset_a == offset_b); + + if (pi_a->u.path.bits != pi_b->u.path.bits) { + res = pi_a->u.path.bits < pi_b->u.path.bits ? -1 : 1; + cmp_done = true; + LOG(3 - LOG_CMP_PATHS, + "%s: fwd loop, different bits (0x%08x, 0x%08x) => res %d\n", + __func__, pi_a->u.path.bits, pi_b->u.path.bits, res); + } + } + } + + /* Check if both have reached the original head node. */ + if (link_a == NO_ID && link_b == NO_ID) { + done_restoring_link_order = true; + LOG(3 - LOG_CMP_PATHS, + "%s: fwd loop: reached end of both paths, prev_a %d (p_a %d), prev_b %d (p_b %d)\n", + __func__, prev_a, p_a, prev_b, p_b); + assert(prev_a == p_a); + assert(prev_b == p_b); + } + + /* Whether or not comparison has finished, iterate forward, + * restoring forward links. */ + if (link_a != NO_ID) { + assert(link_a < vm->paths.ceil); + const uint32_t fwd_a = get_path_node_backlink(vm, link_a); + LOG(3 - LOG_CMP_PATHS, "%s: fwd loop: link_a %d, fwd_a %d\n", + __func__, link_a, fwd_a); + assert(fwd_a != link_a); + + LOG(3 - LOG_CMP_PATHS, + "%s: fwd loop, restoring a's backlink: pi[%u].backlink <- %d\n", + __func__, link_a, prev_a); + set_path_node_backlink(vm, link_a, prev_a); + prev_a = link_a; + link_a = fwd_a; + } + + if (link_b != NO_ID) { + assert(link_b < vm->paths.ceil); + const uint32_t fwd_b = get_path_node_backlink(vm, link_b); + LOG(3 - LOG_CMP_PATHS, "%s: fwd loop: link_b %d, fwd_b %d\n", + __func__, link_b, fwd_b); + assert(fwd_b != link_b); + + LOG(3 - LOG_CMP_PATHS, + "%s: fwd loop, restoring b's backlink: pi[%u].backlink <- %d\n", + __func__, link_b, prev_b); + set_path_node_backlink(vm, link_b, prev_b); + prev_b = link_b; + link_b = fwd_b; + } + } + + LOG(3 - LOG_CMP_PATHS, "%s: res %d\n", __func__, res); + +#if EXPENSIVE_CHECKS + uint64_t hash_a_after = 0; + uint64_t hash_b_after = 0; + hash_step = 0; + link_a = p_a; + while (link_a != NO_ID) { + assert(link_a < vm->paths.ceil); + const uint32_t prev = get_path_node_backlink(vm, link_a); + hash_a_after += hash_id(link_a + hash_step); + link_a = prev; + hash_step++; + } + link_b = p_b; + hash_step = 0; + while (link_b != NO_ID) { + assert(link_b < vm->paths.ceil); + const uint32_t prev = get_path_node_backlink(vm, link_b); + hash_b_after += hash_id(link_b + hash_step); + link_b = prev; + hash_step++; + } + + assert(hash_a_after == hash_a_before); + assert(hash_b_after == hash_b_before); +#endif + + return res; +#undef LOG_CMP_PATHS +} + +static void +handle_possible_matching_path(struct capvm *vm, uint32_t path_info_head, uint32_t uniq_id); + +static bool +copy_path_info(struct capvm *vm, uint32_t path_info_head, + uint32_t *new_path_info_head); + +static bool +extend_path_info(struct capvm *vm, uint32_t path_info_head, bool greedy, uint32_t uniq_id, + uint32_t *new_path_info_head); + +/* Push the next execution step onto the stack, if it hasn't already + * been run by a greedier path. Calling this hands off ownership of the + * path_info_head, so it is released if execution will not be resumed + * later. */ +static void +schedule_possible_next_step(struct capvm *vm, enum pair_id pair_id, + uint32_t input_pos, uint32_t op_id, + uint32_t path_info_head, uint32_t uniq_id) +{ + assert(path_info_head != NO_ID); + uint32_t *stack = vm->run_stacks[pair_id]; + uint32_t *stack_h = &vm->run_stacks_h[pair_id]; + assert((*stack_h) < vm->p->used); + + /* If that instruction has already been evaluated, skip the + * redundant execution by a less greedy path. */ + const uint32_t *evaluated = vm->evaluated[pair_id]; + const bool already_evaluated = GET_BIT32(evaluated, op_id) != 0; + LOG(3, "%s: pair_id %u, input_pos %u, op_id %u, path_info_head %u, uniq_id %u, already_evaluated %d, stack_h %u\n", + __func__, pair_id, input_pos, op_id, path_info_head, uniq_id, already_evaluated, *stack_h); + + if (already_evaluated) { + LOG_EXEC_HALT(uniq_id); + release_path_info_link(vm, &path_info_head); + assert(vm->threads.live > 0); + vm->threads.live--; + } else { + + /* If the work being scheduled by the current greediest + * thread pre-empts work scheduled by a less greedy + * thread, release that thread's path link and clear its + * op ID on the run stack. + * + * TODO: Ideally, avoid the linear scan here, but the + * total stack height is bounded by the generated program size + * and should be fairly small in practice. Wait to change this + * untill there are benchmarks in place showing it's necessary. + * + * An extra two bits per opcode (one for each stack) could + * be used to track whether the stack already contains + * op_id, so the linear scan could be avoided except when + * actually necessary. */ + uint32_t cur_pih = vm->path_info_heads[pair_id][op_id]; + if (cur_pih != NO_ID) { + release_path_info_link(vm, &cur_pih); + vm->path_info_heads[pair_id][op_id] = NO_ID; + const size_t h = *stack_h; + for (size_t i = 0; i < h; i++) { + if (stack[i] == op_id) { + stack[i] = NO_ID; /* cancel thread */ + vm->threads.live--; + } + } + } + stack[(*stack_h)++] = op_id; + vm->path_info_heads[pair_id][op_id] = path_info_head; +#if CAPVM_STATS + vm->uniq_ids[pair_id][op_id] = uniq_id; +#endif + + if (*stack_h > vm->threads.max_live) { + vm->threads.max_live = *stack_h; + if (LOG_CAPVM >= 6) { + LOG(0, "==== new vm->threads.max_live: %u\n", vm->threads.max_live); + dump_path_table(stderr, vm); + } + } + } +} + +/* returns whether the vm should continue. */ +static bool +eval_vm_advance_greediest(struct capvm *vm, uint32_t input_pos, + uint32_t path_info_head, uint32_t uniq_id, uint32_t op_id) +{ + LOG(5, "%s: input_pos %u, input_len %u, op_id %u, threads_live %u\n", + __func__, input_pos, vm->input_len, op_id, vm->threads.live); + + assert(op_id < vm->p->used); + + if (vm->stats.steps == vm->step_limit) { + LOG(1, "%s: halting, steps == step_limit %zu\n", + __func__, vm->step_limit); + vm->res = FSM_CAPVM_PROGRAM_EXEC_STEP_LIMIT_REACHED; + return false; + } + vm->stats.steps++; + + const struct capvm_opcode *op = &vm->p->ops[op_id]; + LOG(2, "%s: op_id[%u]: input_pos %u, path_info_head %u, uniq_id %u, op %s\n", + __func__, op_id, input_pos, path_info_head, uniq_id, op_name[op->t]); + LOG_EXEC_OP(uniq_id, input_pos, op_id, op_name[op->t]); + + switch (op->t) { + case CAPVM_OP_CHAR: + if (input_pos == vm->input_len) { + goto halt_thread; /* past end of input */ + } + + LOG(3, "OP_CHAR: input_pos %u, exp char '%c', got '%c'\n", + input_pos, op->u.chr, vm->input[input_pos]); + + if (vm->input[input_pos] != op->u.chr) { + goto halt_thread; /* character mismatch */ + } + LOG_EXEC_CHAR(uniq_id, vm->input[input_pos]); + + schedule_possible_next_step(vm, PAIR_ID_NEXT, input_pos + 1, op_id + 1, + path_info_head, uniq_id); + break; + + case CAPVM_OP_CHARCLASS: + { + if (input_pos == vm->input_len) { + goto halt_thread; /* past end of input */ + } + + const uint8_t c = vm->input[input_pos]; + const uint32_t cc_id = op->u.charclass_id; + assert(cc_id < vm->p->char_classes.count); + const struct capvm_char_class *class = &vm->p->char_classes.sets[cc_id]; + + if (!(class->octets[c/64] & ((uint64_t)1 << (c&63)))) { + goto halt_thread; /* character not in class */ + } + LOG_EXEC_CHAR(uniq_id, vm->input[input_pos]); + + schedule_possible_next_step(vm, PAIR_ID_NEXT, input_pos + 1, op_id + 1, + path_info_head, uniq_id); + break; + } + + case CAPVM_OP_MATCH: + if (input_pos == vm->input_len) { + handle_possible_matching_path(vm, path_info_head, uniq_id); + } else if (vm->input_len > 0 && input_pos == vm->input_len - 1 + && vm->input[input_pos] == '\n') { + LOG(3, "OP_MATCH: special case for trailing newline\n"); + handle_possible_matching_path(vm, path_info_head, uniq_id); + } + goto halt_thread; + + case CAPVM_OP_JMP: + schedule_possible_next_step(vm, PAIR_ID_CURRENT, input_pos, op->u.jmp, + path_info_head, uniq_id); + break; + + case CAPVM_OP_JMP_ONCE: + { + /* If the destination for this jump has already been visited + * without advancing input, then skip the jump. This is necessary + * for edge cases like the first branch in `^(^|.$)*`, which would + * otherwise have a backward jump to before the first case, due to + * the repetition, and would effectively be treated as an infinite + * loop and ignored, leading to incorrect match bounds for "x". + * + * Replaying the capture path does not track what has been evaluated, + * so this needs to record the branch in the path. This will make + * repetition more expensive in some cases, but compilation could + * emit a JMP when it's safe to do so. */ + const bool greedy = GET_BIT32(vm->evaluated[PAIR_ID_CURRENT], op->u.jmp_once); + if (greedy) { + /* non-greedy branch -- fall through */ + uint32_t new_path_info_head = NO_ID; + if (!extend_path_info(vm, path_info_head, 0, uniq_id, &new_path_info_head)) { + release_path_info_link(vm, &path_info_head); + goto alloc_error; + } + schedule_possible_next_step(vm, PAIR_ID_CURRENT, input_pos, op_id + 1, + new_path_info_head, uniq_id); + } else { + /* greedy branch -- loop back and potentially match more */ + uint32_t new_path_info_head = NO_ID; + if (!extend_path_info(vm, path_info_head, 1, uniq_id, &new_path_info_head)) { + release_path_info_link(vm, &path_info_head); + goto alloc_error; + } + schedule_possible_next_step(vm, PAIR_ID_CURRENT, input_pos, op->u.jmp_once, + new_path_info_head, uniq_id); + } + break; + } + + case CAPVM_OP_SPLIT: + { + const uint32_t dst_greedy = op->u.split.greedy; + const uint32_t dst_nongreedy = op->u.split.nongreedy; + + /* destinations must be in range and not self-referential */ + assert(dst_greedy < vm->p->used); + assert(dst_nongreedy < vm->p->used); + assert(dst_greedy != op_id); + assert(dst_nongreedy != op_id); + + uint32_t nongreedy_path_info_head; + if (!copy_path_info(vm, path_info_head, &nongreedy_path_info_head)) { + goto alloc_error; + } + + if (!extend_path_info(vm, path_info_head, 1, uniq_id, &path_info_head)) { + release_path_info_link(vm, &path_info_head); + goto alloc_error; + } + + /* nongreedy is the non-greedy branch */ + if (!extend_path_info(vm, nongreedy_path_info_head, 0, uniq_id, &nongreedy_path_info_head)) { + release_path_info_link(vm, &path_info_head); + goto alloc_error; + } + +#if CAPVM_STATS + const uint32_t nongreedy_uniq_id = ++vm->uniq_id_counter; +#else + const uint32_t nongreedy_uniq_id = 0; +#endif + + vm->threads.live++; + if (vm->threads.live > vm->threads.max_live) { + set_max_threads_live(vm, vm->threads.live); + } + + /* Push the split.nongreedy destination, and then the + * split.greedy destination on top of it, so that the + * greedier branch will be fully evaluated first. */ + schedule_possible_next_step(vm, PAIR_ID_CURRENT, input_pos, dst_nongreedy, + nongreedy_path_info_head, nongreedy_uniq_id); + schedule_possible_next_step(vm, PAIR_ID_CURRENT, input_pos, dst_greedy, + path_info_head, uniq_id); + LOG_EXEC_SPLIT(uniq_id, nongreedy_uniq_id); + + break; + } + + case CAPVM_OP_SAVE: + /* no-op, during this stage */ + schedule_possible_next_step(vm, PAIR_ID_CURRENT, input_pos, op_id + 1, + path_info_head, uniq_id); + break; + + case CAPVM_OP_ANCHOR: + if (op->u.anchor == CAPVM_ANCHOR_START) { + LOG(3, "%s: ^ anchor\n", __func__); + /* ignore a single trailing newline, because PCRE does. + * For ^ this affects the capture position. */ + if (input_pos == 0 + && vm->input_len == 1 + && vm->input[0] == '\n') { + /* allowed */ + LOG(3, "%s: special case: ^ ignoring trailing newline\n", __func__); + schedule_possible_next_step(vm, PAIR_ID_CURRENT, input_pos, op_id + 1, + path_info_head, uniq_id); + break; + } else if (input_pos == 1 + && vm->input_len == 1 + && vm->input[0] == '\n') { + /* allowed */ + } else if (input_pos != 0) { goto halt_thread; } + } else { + assert(op->u.anchor == CAPVM_ANCHOR_END); + LOG(3, "%s: $ anchor: input_len %u, input_pos %u\n", + __func__, vm->input_len, input_pos); + + /* ignore a single trailing newline, because PCRE does */ + if (vm->input_len > 0 && input_pos == vm->input_len - 1) { + if (vm->input[input_pos] != '\n') { + goto halt_thread; + } + LOG(3, "%s: special case: $ allowing trailing newline\n", __func__); + schedule_possible_next_step(vm, PAIR_ID_CURRENT, input_pos, op_id + 1, + path_info_head, uniq_id); + break; + } else if (input_pos != vm->input_len) { + goto halt_thread; + } + } + + schedule_possible_next_step(vm, PAIR_ID_CURRENT, input_pos, op_id + 1, + path_info_head, uniq_id); + break; + + default: + assert(!"unreachable"); + return false; + } + + if (EXPENSIVE_CHECKS) { /* postcondition */ + check_path_table(vm); + } + + return true; + +halt_thread: + /* do not push further execution on the run stack */ + LOG_EXEC_HALT(uniq_id); + + release_path_info_link(vm, &path_info_head); + assert(vm->threads.live > 0); + vm->threads.live--; + return true; + +alloc_error: + release_path_info_link(vm, &path_info_head); + vm->res = FSM_CAPVM_PROGRAM_EXEC_ERROR_ALLOC; + return false; +} + +static void +handle_possible_matching_path(struct capvm *vm, uint32_t pi_id, uint32_t uniq_id) +{ + LOG(3, "\n%s: HIT, pi_id %u, uniq_id %u\n", __func__, pi_id, uniq_id); + + if (LOG_CAPVM >= 3) { + LOG(3, "--- current_live: %u, max_live: %u\n", + vm->threads.live, vm->threads.max_live); + dump_path_table(stderr, vm); + LOG(3, "=====\n"); + } + +#if CAPVM_STATS + vm->stats.matches++; +#endif + + assert(pi_id < vm->paths.ceil); + + if (vm->solution.best_path_id == NO_ID) { + struct capvm_path_info *pi = &vm->paths.pool[pi_id]; + assert(!IS_PATH_FREELIST(pi)); + if (LOG_CAPVM >= 5) { + const uint32_t refcount = get_path_node_refcount(vm, pi_id); + LOG(5, "MATCH: pi_id %u refcount %u -> %u\n", + pi_id, refcount, refcount + 1); + } + inc_path_node_refcount(vm, pi_id); + vm->solution.best_path_id = pi_id; +#if CAPVM_STATS + vm->solution.best_path_uniq_id = uniq_id; +#endif + LOG(3, "MATCH: saved current best solution path (pi_id %u)\n", pi_id); + } else { + /* Compare path info and only keep the path associated + * with the greediest match so far. */ + const int res = cmp_paths(vm, pi_id, vm->solution.best_path_id); + if (res > 0) { + /* replace current best solution */ + struct capvm_path_info *pi = &vm->paths.pool[pi_id]; + assert(!IS_PATH_FREELIST(pi)); + if (LOG_CAPVM >= 5) { + const uint32_t refcount = get_path_node_refcount(vm, pi_id); + LOG(5, "MATCH: pi_id %u refcount %u -> %u\n", + pi_id, refcount, refcount + 1); + } + inc_path_node_refcount(vm, pi_id); + + LOG(3, "MATCH: replacing current best solution path %u with %u\n", + vm->solution.best_path_id, pi_id); + + release_path_info_link(vm, &vm->solution.best_path_id); + vm->solution.best_path_id = pi_id; +#if CAPVM_STATS + vm->solution.best_path_uniq_id = uniq_id; +#endif + } else { + /* keep the current best solution */ + LOG(3, "MATCH: ignoring new solution path %u, keeping %u\n", + pi_id, vm->solution.best_path_id); + } + } +} + +static bool +eval_vm(struct capvm *vm) +{ + uint32_t i_i; + + /* init the path_info_heads tables to NO_ID, except for cell 0 + * in next, which contains the starting point. */ + for (size_t op_i = 0; op_i < vm->p->used; op_i++) { + vm->path_info_heads[PAIR_ID_CURRENT][op_i] = NO_ID; +#if CAPVM_STATS + vm->uniq_ids[PAIR_ID_CURRENT][op_i] = NO_ID; +#endif + } + for (size_t op_i = 1; op_i < vm->p->used; op_i++) { + vm->path_info_heads[PAIR_ID_NEXT][op_i] = NO_ID; +#if CAPVM_STATS + vm->uniq_ids[PAIR_ID_NEXT][op_i] = NO_ID; +#endif + } + + for (i_i = 0; i_i <= vm->input_len; i_i++) { + if (vm->threads.live == 0 + || vm->stats.steps == vm->step_limit) { + LOG(3, "%s: breaking, live %u, steps %zu/%zd\n", + __func__, vm->threads.live, vm->stats.steps, vm->step_limit); + break; + } + LOG(3, "\n###### i_i %u\n", i_i); + + LOG(4, "-- clearing evaluated\n"); + const size_t evaluated_bit_words = vm->p->used/32 + 1; + for (size_t i = 0; i < evaluated_bit_words; i++) { + vm->evaluated[PAIR_ID_CURRENT][i] = 0; + vm->evaluated[PAIR_ID_NEXT][i] = 0; + } + + uint32_t *stack_h = &vm->run_stacks_h[PAIR_ID_CURRENT]; + uint32_t *run_stack = vm->run_stacks[PAIR_ID_CURRENT]; + + /* Copy everything from the next run stack to the + * current. Copy in reverse, so items that were pushed + * earlier by greedier paths end up on the top of the + * stack and evalated first, preserving greedy + * ordering. */ + { + const uint32_t next_stack_h = vm->run_stacks_h[PAIR_ID_NEXT]; + const uint32_t *next_stack = vm->run_stacks[PAIR_ID_NEXT]; + uint32_t *next_path_info_heads = vm->path_info_heads[PAIR_ID_NEXT]; + uint32_t *cur_path_info_heads = vm->path_info_heads[PAIR_ID_CURRENT]; + + uint32_t discarded = 0; + for (size_t i = 0; i < next_stack_h; i++) { + const uint32_t op_id = next_stack[i]; + if (op_id == NO_ID) { + assert(!"unreachable"); + discarded++; + continue; + } + + cur_path_info_heads[op_id] = next_path_info_heads[op_id]; + LOG(3, "%s: run_stack[%zd] <- %u, path_info_head %u\n", + __func__, i, op_id, cur_path_info_heads[op_id]); + assert(next_path_info_heads[op_id] < vm->paths.ceil); + next_path_info_heads[op_id] = NO_ID; /* move reference */ +#if CAPVM_STATS + vm->uniq_ids[PAIR_ID_CURRENT][op_id] = + vm->uniq_ids[PAIR_ID_NEXT][op_id]; +#endif + run_stack[next_stack_h - i - 1 - discarded] = op_id; + } + *stack_h = next_stack_h - discarded; + vm->run_stacks_h[PAIR_ID_NEXT] = 0; + +#if CAPVM_PATH_STATS + /* reset counters */ + for (size_t i = 0; i < vm->paths.ceil; i++) { + struct capvm_path_info *pi = &vm->paths.pool[i]; + if (IS_PATH_NODE(pi)) { + pi->u.path.bits_added_per_input_character = 0; + } + } +#endif + } + + uint32_t *path_info_heads = vm->path_info_heads[PAIR_ID_CURRENT]; + while (vm->run_stacks_h[PAIR_ID_CURRENT] > 0) { + /* Do this here, before popping, so that the reference + * on the stack can be counted properly. */ + if (EXPENSIVE_CHECKS) { + check_path_table(vm); + } + + const uint32_t h = --(*stack_h); + assert(h < vm->p->used); + const uint32_t op_id = run_stack[h]; + LOG(4, "%s: popped op_id %d off stack\n", __func__, op_id); + if (op_id == NO_ID) { + LOG(4, "%s: ignoring halted pending execution\n", __func__); + continue; + } + assert(op_id < vm->p->used); + + if (GET_BIT32(vm->evaluated[PAIR_ID_CURRENT], op_id)) { + LOG(2, "%s: evaluated[current] already set for op_id %u (popped off stack), skipping\n", + __func__, op_id); + assert(!"unreachable"); + continue; + } + + LOG(4, "%s: setting evaluated[current] for op_id %u (popped off stack)\n", __func__, op_id); + SET_BIT32(vm->evaluated[PAIR_ID_CURRENT], op_id); + + const uint32_t path_info_head = path_info_heads[op_id]; + LOG(4, "%s: op_id %d's path_info_head: %d\n", __func__, op_id, path_info_head); + path_info_heads[op_id] = NO_ID; + + +#if CAPVM_STATS + const uint32_t uniq_id = vm->uniq_ids[PAIR_ID_CURRENT][op_id]; + assert(uniq_id != NO_ID); +#else + const uint32_t uniq_id = 0; +#endif + if (!eval_vm_advance_greediest(vm, i_i, path_info_head, uniq_id, op_id)) { + return false; + } + } + + +#if CAPVM_PATH_STATS + uint32_t max_path_bits_added = 0; + for (size_t i = 0; i < vm->paths.ceil; i++) { + const struct capvm_path_info *pi = &vm->paths.pool[i]; + if (IS_PATH_NODE(pi)) { + if (pi->u.path.bits_added_per_input_character > max_path_bits_added) { + max_path_bits_added = pi->u.path.bits_added_per_input_character; + } + } + } + LOG(2, "%s: input_i %u: max_path_bits_added: %u\n", + __func__, i_i, max_path_bits_added); + if (max_path_bits_added > vm->stats.max_bits_added_per_input_character) { + vm->stats.max_bits_added_per_input_character = max_path_bits_added; + } + + if (CAPVM_PATH_STATS > 1) { + dump_path_table(stderr, vm); + } +#endif + } + + return vm->solution.best_path_id != NO_ID; +} + +static bool +copy_path_info(struct capvm *vm, uint32_t path_info_head, + uint32_t *new_path_info_head) +{ + if (!reserve_path_info_link(vm, new_path_info_head)) { + return false; + } + + assert(path_info_head != NO_ID); + assert(path_info_head < vm->paths.ceil); + assert(*new_path_info_head < vm->paths.ceil); + assert(*new_path_info_head != path_info_head); + + /* Since this is the path head, it can never be a collapsed + * zero prefix node. */ + const struct capvm_path_info *pi = &vm->paths.pool[path_info_head]; + assert(IS_PATH_NODE(pi)); + + struct capvm_path_info *npi = &vm->paths.pool[*new_path_info_head]; + assert(IS_PATH_FREELIST(npi)); + + /* unlink from freelist */ + vm->paths.freelist_head = npi->u.freelist_node.freelist; + vm->paths.live++; + if (vm->paths.live > vm->paths.max_live) { + set_max_paths_live(vm); + } + + *npi = (struct capvm_path_info){ + .u.path = { + .refcount = 1, + .used = pi->u.path.used, + .bits = pi->u.path.bits, + .offset = pi->u.path.offset, + .backlink = pi->u.path.backlink, + } + }; + if (pi->u.path.backlink != NO_ID) { + inc_path_node_refcount(vm, pi->u.path.backlink); + } + return true; +} + +#if CAPVM_PATH_STATS +static void +update_max_path_length_memory(struct capvm *vm, const struct capvm_path_info *pi) +{ + const uint32_t len = pi->u.path.used + + (pi->u.path.backlink == COLLAPSED_ZERO_PREFIX_ID + ? 0 /* not actually stored, so don't count it */ + : pi->u.path.offset); + + if (len > vm->stats.max_path_length_memory) { + vm->stats.max_path_length_memory = len; + } +} +#endif + +static bool +extend_path_info(struct capvm *vm, uint32_t pi_id, bool greedy, uint32_t uniq_id, + uint32_t *new_path_info_head) +{ + assert(pi_id < vm->paths.ceil); + struct capvm_path_info *pi = &vm->paths.pool[pi_id]; + assert(IS_PATH_NODE(pi)); + + (void)uniq_id; + LOG_EXEC_PATH_FIND_SOLUTION(uniq_id, greedy); + +#define LOG_EPI 0 + LOG(5 - LOG_EPI, "%s: pi_id %u, greedy %d\n", + __func__, pi_id, greedy); + + + if (pi->u.path.used == PATH_LINK_BITS) { /* full */ + uint32_t npi_id; + if (!reserve_path_info_link(vm, &npi_id)) { + assert(!"alloc fail"); + } + pi = &vm->paths.pool[pi_id]; /* refresh stale pointer */ + LOG(5 - LOG_EPI, "%s: new head at %u (%u is full)\n", __func__, npi_id, pi_id); + assert(npi_id < vm->paths.ceil); + struct capvm_path_info *npi = &vm->paths.pool[npi_id]; + vm->paths.freelist_head = npi->u.freelist_node.freelist; + vm->paths.live++; + if (vm->paths.live > vm->paths.max_live) { + set_max_paths_live(vm); + } + + LOG(5 - LOG_EPI, "%s: npi_id %u refcount 1 (new link)\n", + __func__, npi_id); + npi->u.path.refcount = 1; + npi->u.path.offset = pi->u.path.offset + pi->u.path.used; + npi->u.path.bits = (greedy ? ((uint32_t)1 << 31) : 0); + LOG(5 - LOG_EPI, "%s: bits after: 0x%08x\n", __func__, npi->u.path.bits); + npi->u.path.used = 1; + +#if CAPVM_PATH_STATS + npi->u.path.bits_added_per_input_character = pi->u.path.bits_added_per_input_character + 1; +#endif + + /* If the path node is full of zero bits and it's either at the start, + * or its backlink is a COLLAPSED_ZERO_PREFIX_ID, then extend the + * backlink to a collapsed run of zeroes. The node's offset field + * indicates the prefix length. Long prefixes of zero bits tend to + * occur with an unanchored start loop. */ + if (pi->u.path.bits == (uint32_t)0 && USE_COLLAPSED_ZERO_PREFIX + && (pi->u.path.offset == 0 || pi->u.path.backlink == COLLAPSED_ZERO_PREFIX_ID)) { + release_path_info_link(vm, &pi_id); + pi_id = COLLAPSED_ZERO_PREFIX_ID; + +#if CAPVM_STATS + vm->stats.collapsed_zero_prefixes++; +#endif + } else { + /* Check if there's an existing full path node with + * exactly the same bits. If so, link backward to that + * and free the old full one, rather than saving it as + * a duplicate. */ + const uint32_t old_path_bits = pi->u.path.bits; + const uint32_t old_path_offset = pi->u.path.offset; + const uint32_t old_path_backlink = pi->u.path.backlink; + + for (uint32_t epi_id = 0; epi_id < vm->paths.ceil; epi_id++) { + if (epi_id == pi_id) { continue; } + struct capvm_path_info *epi = &vm->paths.pool[epi_id]; + if (IS_PATH_FREELIST(epi)) { + continue; + } + + assert(IS_PATH_NODE(epi)); + if (epi->u.path.used == PATH_LINK_BITS + && epi->u.path.bits == old_path_bits + && epi->u.path.offset == old_path_offset + && epi->u.path.backlink == old_path_backlink) { + + if (LOG_CAPVM >= 4 || 1) { + const uint32_t refcount = get_path_node_refcount(vm, epi_id); + (void)refcount; + LOG(4 - LOG_EPI, "%s: pi[%u] refcount %u -> %u (reusing identical path backlink %u instead of %u)\n", + __func__, epi_id, refcount, refcount + 1, + epi_id, pi_id); + } + inc_path_node_refcount(vm, epi_id); + release_path_info_link(vm, &pi_id); + pi_id = epi_id; +#if CAPVM_STATS + vm->stats.path_prefixes_shared++; +#endif + break; + } + } + } + + assert(IS_PATH_NODE(npi)); + npi->u.path.backlink = pi_id; + /* transfer pi_id's reference to npi_id */ + *new_path_info_head = npi_id; + +#if CAPVM_PATH_STATS + update_max_path_length_memory(vm, npi); +#endif + + return true; + } else { + assert(IS_PATH_NODE(pi)); + assert(pi->u.path.used < PATH_LINK_BITS); + + LOG(5 - LOG_EPI, "%s: appending to head node %u, %u -> %u used\n", + __func__, pi_id, pi->u.path.used, pi->u.path.used + 1); + assert(pi->u.path.used < PATH_LINK_BITS); + if (greedy) { + LOG(5 - LOG_EPI, "%s: bits before: 0x%08x (greedy: %d)\n", + __func__, pi->u.path.bits, greedy); + pi->u.path.bits |= (uint32_t)1 << (31 - pi->u.path.used); + LOG(5 - LOG_EPI, "%s: bits after: 0x%08x\n", + __func__, pi->u.path.bits); + } + pi->u.path.used++; +#if CAPVM_PATH_STATS + pi->u.path.bits_added_per_input_character++; +#endif + +#if CAPVM_PATH_STATS + update_max_path_length_memory(vm, pi); +#endif + + *new_path_info_head = pi_id; + return true; + } +#undef LOG_EPI +} + +static void +populate_solution(struct capvm *vm) +{ + if (LOG_CAPVM >= 3) { + fsm_capvm_program_dump(stderr, vm->p); + LOG(0, "%s: best_path_id %d, tables:\n", __func__, vm->solution.best_path_id); + dump_path_table(stderr, vm); + check_path_table(vm); + fprintf(stderr, "SOLUTION_PATH: "); + print_path(stderr, vm, vm->solution.best_path_id); + fprintf(stderr, "\n"); + } + +#if CAPVM_PATH_STATS + LOG(1, "%s: prog_size %u, max_path_length_memory %u (bits)\n", + __func__, vm->p->used, vm->stats.max_path_length_memory); + const uint32_t uniq_id = vm->solution.best_path_uniq_id; +#else + const uint32_t uniq_id = NO_ID; +#endif + (void)uniq_id; + + /* Interpret the program again, but rather than using the input to + * drive execution, use the saved path for the primary solution. */ + + /* Walk the solution path, reversing the edges temporarily so it + * can be executed start to finish, and truncate any bits appended + * after branches on the path. */ + assert(vm->solution.best_path_id != NO_ID); + assert(vm->solution.best_path_id < vm->paths.ceil); + + uint32_t path_link = vm->solution.best_path_id; + uint32_t next_link = NO_ID; + uint32_t first_link = NO_ID; + + size_t split_count = 0; + uint32_t zero_prefix_length = 0; + + if (LOG_CAPVM >= 3) { + const struct capvm_path_info *pi = &vm->paths.pool[path_link]; + assert(!IS_PATH_FREELIST(pi)); + LOG(3, "%s: best_path %d, path_length %u\n", + __func__, vm->solution.best_path_id, pi->u.path.offset + pi->u.path.used); + if (LOG_CAPVM > 4) { + dump_path_table(stderr, vm); + } + } + + uint32_t prev; + do { + struct capvm_path_info *pi = &vm->paths.pool[path_link]; + assert(!IS_PATH_FREELIST(pi)); + const uint32_t prev_link = get_path_node_backlink(vm, path_link); + + if (LOG_CAPVM >= 3) { + if (IS_PATH_NODE(pi)) { + LOG(3, "%s (moving back), node %u: refcount %u, used %u, offset %u, backlink %d, bits '", + __func__, path_link, pi->u.path.refcount, pi->u.path.used, + pi->u.path.offset, pi->u.path.backlink); + for (uint8_t i = 0; i < pi->u.path.used; i++) { + const uint32_t bit = (pi->u.path.bits & ((uint32_t)1 << (31 - i))); + LOG(3, "%c", bit ? '1' : '0'); + } + LOG(3, "'\n"); + } + } + + split_count += pi->u.path.used; + + if (next_link != NO_ID) { + LOG(3, "-- setting backlink to %d\n", next_link); + set_path_node_backlink(vm, path_link, next_link); /* point fwd */ + } else { + LOG(3, "-- setting backlink to %d\n", NO_ID); + set_path_node_backlink(vm, path_link, NO_ID); /* now EOL */ + } + + if (prev_link == NO_ID) { + first_link = path_link; + prev = prev_link; + } else if (prev_link == COLLAPSED_ZERO_PREFIX_ID) { + first_link = path_link; + split_count += pi->u.path.offset; + zero_prefix_length = pi->u.path.offset; + prev = prev_link; + } + + next_link = path_link; + assert(path_link != prev_link); + path_link = prev_link; + } while (path_link != NO_ID && path_link != COLLAPSED_ZERO_PREFIX_ID); + + /* iter forward */ + if (LOG_CAPVM >= 3) { + uint32_t cur = first_link; + do { + struct capvm_path_info *pi = &vm->paths.pool[cur]; + + assert(IS_PATH_NODE(pi)); + LOG(3, "%s (moving fwd): node %u: refcount %u, used %u, offset %u, fwdlink %d, bits '", + __func__, cur, get_path_node_refcount(vm, cur), + pi->u.path.used, + get_path_node_offset(vm, cur), + get_path_node_backlink(vm, cur)); + for (uint8_t i = 0; i < pi->u.path.used; i++) { + const uint32_t bit = (pi->u.path.bits & ((uint32_t)1 << (31 - i))); + LOG(3, "%c", bit ? '1' : '0'); + } + LOG(3, "'\n"); + + const uint32_t next_cur = get_path_node_backlink(vm, cur); + assert(cur != next_cur); + cur = next_cur; /* fwd link */ + } while (cur != NO_ID); + } + + /* evaluate program with forward path */ + LOG(3, "%s: split_count %zu\n", __func__, split_count); + size_t split_i = 0; + uint32_t prog_i = 0; + uint32_t input_i = 0; + size_t capture_lookup_steps = 0; + bool done = false; + + /* This flag tracks whether an explicit newline was matched at + * the end of input. Normally a trailing newline is implicitly + * ignored in the bounds for captures, but when the regex + * matches a newline at the end, it must still be included. An + * example case where this matters is `^[^x]$` for "\n", because + * the character class matches the newline this should capture + * as (0,1). */ + bool explicitly_matched_nl_at_end = false; + + uint32_t cur = first_link; + while (split_i < split_count || !done) { + assert(prog_i < vm->p->used); + const uint32_t cur_prog_i = prog_i; + const struct capvm_opcode *op = &vm->p->ops[cur_prog_i]; + LOG(3, "%s: i_i %u, p_i %u, s_i %zu/%zu, op %s\n", + __func__, input_i, cur_prog_i, split_i, split_count, op_name[op->t]); + + prog_i++; + capture_lookup_steps++; + switch (op->t) { + case CAPVM_OP_CHAR: + assert(input_i < vm->input_len); + LOG(3, "OP_CHAR: input_i %u, exp char '%c', got '%c'\n", + input_i, op->u.chr, vm->input[input_i]); + assert(vm->input[input_i] == op->u.chr); + if (vm->input_len > 0 + && input_i == vm->input_len - 1 + && vm->input[input_i] == '\n') { + explicitly_matched_nl_at_end = true; + } + input_i++; + break; + case CAPVM_OP_CHARCLASS: + assert(input_i < vm->input_len); + if (vm->input_len > 0 + && input_i == vm->input_len - 1 + && vm->input[input_i] == '\n') { + explicitly_matched_nl_at_end = true; + } + input_i++; + break; + case CAPVM_OP_MATCH: + LOG(2, "split_i %zu, split_count %zu\n", split_i, split_count); + assert(split_i == split_count); + done = true; + break; + case CAPVM_OP_JMP: + prog_i = op->u.jmp; + break; + case CAPVM_OP_JMP_ONCE: + { + /* look at next bit of path and jmp or fall through */ + const uint32_t offset = get_path_node_offset(vm, cur); + const struct capvm_path_info *pi = &vm->paths.pool[cur]; + + assert(IS_PATH_NODE(pi)); + bool next_bit; + LOG(3, "%s: OP_JMP_ONCE: split_i %zu, zpl %u, offset %u, pi->u.path.used %u\n", + __func__, split_i, zero_prefix_length, offset, pi->u.path.used); + if (split_i < zero_prefix_length) { + next_bit = 0; + } else { + assert(split_i >= offset && + split_i <= offset + pi->u.path.used); + const uint32_t shift = 31 - (split_i & 31); + assert(shift < PATH_LINK_BITS); + next_bit = (pi->u.path.bits & ((uint32_t)1 << shift)) != 0; + } + LOG(3, "jmp_once: next_bit %d\n", next_bit); + LOG_EXEC_PATH_SAVE_CAPTURES(uniq_id, next_bit); + if (next_bit) { /* greedy edge */ + prog_i = op->u.jmp_once; + } else { /* non-greedy edge */ + /* fall through */ + } + split_i++; + if (split_i >= offset && + split_i - offset == pi->u.path.used && split_i < split_count) { + const uint32_t backlink = get_path_node_backlink(vm, cur); + assert(backlink != NO_ID); + cur = backlink; + } + LOG(3, "%s: prog_i now %u, split_i %zu/%zu\n", + __func__, prog_i, split_i, split_count); + assert(split_i <= split_count); + break; + } + case CAPVM_OP_SPLIT: + { + /* look at next bit of path and act accordingly */ + const uint32_t offset = get_path_node_offset(vm, cur); + const struct capvm_path_info *pi = &vm->paths.pool[cur]; + + const uint32_t dst_greedy = op->u.split.greedy; + const uint32_t dst_nongreedy = op->u.split.nongreedy; + + assert(IS_PATH_NODE(pi)); + bool next_bit; + LOG(3, "%s: OP_SPLIT_CONT: split_i %zu, zpl %u, offset %u, pi->u.path.used %u\n", + __func__, split_i, zero_prefix_length, offset, pi->u.path.used); + if (split_i < zero_prefix_length) { + next_bit = 0; + } else { + assert(split_i >= offset && + split_i <= offset + pi->u.path.used); + const uint32_t shift = 31 - (split_i & 31); + assert(shift < PATH_LINK_BITS); + next_bit = (pi->u.path.bits & ((uint32_t)1 << shift)) != 0; + } + LOG(3, "split: next_bit %d\n", next_bit); + LOG_EXEC_PATH_SAVE_CAPTURES(uniq_id, next_bit); + if (next_bit) { /* greedy edge */ + prog_i = dst_greedy; + } else { /* non-greedy edge */ + prog_i = dst_nongreedy; + } + split_i++; + if (split_i >= offset && + split_i - offset == pi->u.path.used && split_i < split_count) { + const uint32_t backlink = get_path_node_backlink(vm, cur); + assert(backlink != NO_ID); + cur = backlink; + } + LOG(3, "%s: prog_i now %u, split_i %zu/%zu\n", + __func__, prog_i, split_i, split_count); + assert(split_i <= split_count); + + break; + } + case CAPVM_OP_SAVE: + { + const unsigned capture_id = op->u.save/2; + const bool is_end = (op->u.save & 1) == 1; + + LOG(5, "%s: input_i %u, save %d -> capture %d pos %d, cur value %zd, prev char 0x%02x\n", + __func__, + input_i, op->u.save, + capture_id, is_end, + vm->capture_buf[op->u.save/2].pos[op->u.save & 1], + input_i > 0 ? vm->input[input_i - 1] : 0xff); + + /* Special case to ignore a trailing + * newline when capturing, unless the + * newline was explicitly matched as the + * last character of input. */ + if (input_i > 0 + && !explicitly_matched_nl_at_end + && input_i == vm->input_len + && vm->input[input_i - 1] == '\n') { + LOG(3, "%s: updating capture[%u].pos[1] to ignore trailing '\\n' at %u\n", + __func__, capture_id, input_i); + vm->capture_buf[capture_id].pos[is_end] = input_i - 1; + } else { + /* Save current position to appropriate capture buffer endpoint */ + vm->capture_buf[op->u.save/2].pos[op->u.save & 1] = input_i; + LOG(3, "%s: saved capture[%d].pos[%d] <- %u\n", + __func__, op->u.save/2, op->u.save&1, input_i); + } + break; + } + case CAPVM_OP_ANCHOR: + if (op->u.anchor == CAPVM_ANCHOR_START) { + assert(input_i == 0 + || (input_i == 1 + && vm->input_len == 1 + && vm->input[0] == '\n')); + } else { + assert(op->u.anchor == CAPVM_ANCHOR_END); + LOG(3, "%s: $ anchor: input_len %u, input_i %u\n", + __func__, vm->input_len, input_i); + + if (vm->input_len > 0 && input_i == vm->input_len - 1) { + /* special hack to not include trailing newline + * in match group zero */ + if (vm->p->capture_count > 0) { + vm->capture_buf[0].pos[1] = input_i; + } + + assert(vm->input[input_i] == '\n'); + input_i++; + } else { + assert(input_i == vm->input_len); + } + } + break; + + default: + assert(!"match fail"); + } + } + + /* write solution into caller's buffers and print */ + if (LOG_SOLUTION_TO_STDOUT) { + /* fprintf(stderr, "capture_count %u\n", vm->p->capture_count); */ + printf("HIT:"); + for (unsigned i = 0; i < vm->p->capture_count; i++) { + printf(" %zd %zd", + vm->capture_buf[i].pos[0], vm->capture_buf[i].pos[1]); + } + printf("\n"); + } + + /* restore original link order */ + cur = first_link; + do { + struct capvm_path_info *pi = &vm->paths.pool[cur]; + assert(!IS_PATH_FREELIST(pi)); + const uint32_t backlink = get_path_node_backlink(vm, cur); + + LOG(3, "%s (moving fwd again): node %u: refcount %u, used %u, offset %u, fwdlink %d, bits '", + __func__, cur, get_path_node_refcount(vm, cur), + pi->u.path.used, + get_path_node_offset(vm, cur), + backlink); + for (uint8_t i = 0; i < pi->u.path.used; i++) { + const uint32_t bit = (pi->u.path.bits & ((uint32_t)1 << (31 - i))); + LOG(3, "%c", (pi->u.path.bits & bit) ? '1' : '0'); + (void)bit; + } + LOG(3, "'\n"); + + LOG(3, "-- setting node %u's backlink to %d\n", cur, prev); + const uint32_t next = backlink; + set_path_node_backlink(vm, cur, prev); + + prev = cur; + cur = next; /* fwd link */ + } while (cur != NO_ID); +} + +/* TODO: It should be possible to avoid dynamic allocation here + * by calculating the max space needed upfront and passing in a + * uint32_t or uint64_t-aligned array for working space. */ + +enum fsm_capvm_program_exec_res +fsm_capvm_program_exec(const struct capvm_program *program, + const uint8_t *input, size_t length, + struct fsm_capture *capture_buf, size_t capture_buf_length) +{ + assert(program != NULL); + assert(input != NULL || length == 0); + assert(capture_buf != NULL); + + const size_t thread_max = program->used; + + /* FIXME: The path node table can grow beyond this, but in + * practice will usually stay fairly small. The worst case + * should be decidable based on the compiled program and input + * length. */ +#if ALLOW_PATH_TABLE_RESIZING + const size_t path_info_max = thread_max; +#else + const size_t path_info_max = 3 * thread_max; +#endif + + struct capvm_path_info *path_info_pool = malloc(path_info_max + * sizeof(path_info_pool[0])); + if (path_info_pool == NULL) { + return FSM_CAPVM_PROGRAM_EXEC_ERROR_ALLOC; + } + assert(path_info_pool != NULL); + + /* link path_info freelist */ + for (size_t i = 1; i < path_info_max - 1; i++) { + struct capvm_path_info *pi = &path_info_pool[i]; + pi->u.freelist_node.refcount = 0; + pi->u.freelist_node.freelist = i + 1; + } + struct capvm_path_info *piZ = &path_info_pool[path_info_max - 1]; + piZ->u.freelist_node.refcount = 0; + piZ->u.freelist_node.freelist = NO_ID; + + /* init an empty path descriptor for initial execution */ + struct capvm_path_info *pi0 = &path_info_pool[0]; + pi0->u.path.refcount = 1; + pi0->u.path.used = 0; + pi0->u.path.bits = 0; + pi0->u.path.offset = 0; + pi0->u.path.backlink = NO_ID; + + uint32_t stack_a[thread_max]; + uint32_t stack_b[thread_max]; + + const size_t evaluated_bit_words = program->used/32 + 1; + uint32_t evaluated_a[evaluated_bit_words]; + uint32_t evaluated_b[evaluated_bit_words]; + uint32_t path_info_head_a[thread_max]; + uint32_t path_info_head_b[thread_max]; +#if CAPVM_STATS + uint32_t uniq_ids_a[thread_max]; + uint32_t uniq_ids_b[thread_max]; +#endif + + assert(capture_buf_length >= program->capture_base + program->capture_count); + + struct fsm_capture *offset_capture_buf = &capture_buf[program->capture_base]; + + struct capvm vm = { + .res = FSM_CAPVM_PROGRAM_EXEC_NO_SOLUTION_FOUND, + .p = program, + .input = input, + .input_len = length, + .capture_buf = offset_capture_buf, + .capture_buf_length = capture_buf_length, + .step_limit = SIZE_MAX, +#if CAPVM_STATS + .uniq_id_counter = 0, +#endif + + .run_stacks = { stack_a, stack_b }, + .evaluated = { evaluated_a, evaluated_b }, + .path_info_heads = { path_info_head_a, path_info_head_b }, +#if CAPVM_STATS + .uniq_ids = { uniq_ids_a, uniq_ids_b }, +#endif + + .paths = { + .ceil = path_info_max, + .live = 1, + .max_live = 1, + .freelist_head = 1, + .pool = path_info_pool, + }, + .solution = { + .best_path_id = NO_ID, + }, + }; + + /* enqueue execution at first opcode */ + vm.run_stacks[PAIR_ID_NEXT][0] = 0; + vm.run_stacks_h[PAIR_ID_NEXT] = 1; + vm.threads.live = 1; + vm.threads.max_live = 1; + vm.path_info_heads[PAIR_ID_NEXT][0] = 0; + +#if CAPVM_STATS + vm.uniq_ids[PAIR_ID_NEXT][0] = 0; +#endif + + INIT_TIMERS(); + TIME(&pre); + if (eval_vm(&vm)) { + assert(vm.threads.live == 0); + assert(vm.paths.live > 0); + + populate_solution(&vm); + release_path_info_link(&vm, &vm.solution.best_path_id); + vm.res = FSM_CAPVM_PROGRAM_EXEC_SOLUTION_WRITTEN; + + /* TODO: This assert will not work if refcounts are + * sticky at the max value, but if the number of paths + * and threads is bounded then it shouldn't be possible + * to overflow the refcount anyway. If sticky refcounts + * are used then reaching one should probably set a + * flag, which would skip this assertion. */ + assert(vm.paths.live == 0); + } else { + assert(vm.res != FSM_CAPVM_PROGRAM_EXEC_SOLUTION_WRITTEN); + } + + TIME(&post); + DIFF_MSEC(__func__, pre, post, NULL); + +#if CAPVM_STATS + LOG(2, "%s: %zu steps, max_threads %u, max_paths %u, matches %u, path_prefixes_shared %u, collapsed_zero_prefixes %u\n", + __func__, vm.stats.steps, vm.threads.max_live, vm.paths.max_live, vm.stats.matches, + vm.stats.path_prefixes_shared, vm.stats.collapsed_zero_prefixes); +#if CAPVM_PATH_STATS + LOG(2, "%s: prog_size %u, max_path_length_memory %u (bits), input length %zu, max_paths * %zu bytes/path => %zu bytes\n", + __func__, vm.p->used, vm.stats.max_path_length_memory, length, + sizeof(vm.paths.pool[0]), + vm.paths.max_live * sizeof(vm.paths.pool[0])); +#endif +#endif + + free(vm.paths.pool); + return vm.res; +} diff --git a/src/libfsm/capture_vm_log.h b/src/libfsm/capture_vm_log.h new file mode 100644 index 000000000..8ff51d8b4 --- /dev/null +++ b/src/libfsm/capture_vm_log.h @@ -0,0 +1,21 @@ +/* + * Copyright 2022 Scott Vokes + * + * See LICENCE for the full copyright terms. + */ + +#ifndef CAPTURE_VM_LOG_H +#define CAPTURE_VM_LOG_H + +#include + +#define LOG_CAPVM (1+0) +#define LOG(LEVEL, ...) \ + do { \ + if ((LEVEL) <= LOG_CAPVM) { \ + fprintf(stderr, __VA_ARGS__); \ + } \ + } while(0) + + +#endif diff --git a/src/libfsm/capture_vm_program.h b/src/libfsm/capture_vm_program.h new file mode 100644 index 000000000..0b24ffb5b --- /dev/null +++ b/src/libfsm/capture_vm_program.h @@ -0,0 +1,74 @@ +/* + * Copyright 2022 Scott Vokes + * + * See LICENCE for the full copyright terms. + */ + +#ifndef CAPTURE_VM_PROGRAM_H +#define CAPTURE_VM_PROGRAM_H + +#include + +struct capvm_program { + uint32_t capture_count; + uint32_t capture_base; + + uint32_t used; + uint32_t ceil; + struct capvm_opcode { + enum capvm_opcode_type { + /* Next character of input == .u.chr */ + CAPVM_OP_CHAR, + /* Next character of input is in char class */ + CAPVM_OP_CHARCLASS, + /* Input has matched */ + CAPVM_OP_MATCH, + /* Unconditional jump */ + CAPVM_OP_JMP, + /* If destination has already been evaulated + * since advancing the input position, fall + * through to next instruction, otherwise jmp. */ + CAPVM_OP_JMP_ONCE, + /* Split execution to two paths, where .cont + * offset is greedier than .new's offset. */ + CAPVM_OP_SPLIT, + /* Save current input position as capture bound */ + CAPVM_OP_SAVE, + /* Check if current input position is at start/end + * of input, after accounting for PCRE's special + * cases for a trailing newline. */ + CAPVM_OP_ANCHOR, + } t; + union { + uint8_t chr; + uint32_t charclass_id; + uint32_t jmp; /* absolute */ + uint32_t jmp_once; /* absolute */ + struct { + uint32_t greedy; + uint32_t nongreedy; + } split; + /* (save >> 1): capture ID, + * (save & 0x01): save pos to start (0b0) or end (0b1). */ + uint32_t save; + enum capvm_anchor_type { + CAPVM_ANCHOR_START, + CAPVM_ANCHOR_END, + } anchor; + } u; + } *ops; + + /* Most compiled programs only use a few distinct character + * classes (if any), and the data is much larger than the + * other instructions, so they are stored in a separate + * table and referred to by op->u.charclass_id. */ + struct capvm_char_classes { + uint32_t count; + uint32_t ceil; + struct capvm_char_class { + uint64_t octets[4]; /* 256-bitset */ + } *sets; + } char_classes; +}; + +#endif diff --git a/src/libfsm/charset.c b/src/libfsm/charset.c new file mode 100644 index 000000000..3f987eaa8 --- /dev/null +++ b/src/libfsm/charset.c @@ -0,0 +1,93 @@ +/* + * Copyright 2024 Katherine Flavel + * + * See LICENCE for the full copyright terms. + */ + +#include +#include +#include + +#include +#include +#include +#include + +#include "internal.h" +#include "walk2.h" + +/* ^[abc..]*$ */ +struct fsm * +fsm_intersect_charset(struct fsm *a, size_t n, const char *charset) +{ + struct fsm *b, *q; + + assert(a != NULL); + + if (charset == NULL) { + return a; + } + + /* + * Since intersection is destructive, there's no point in making + * the charset FSM in advance and then fsm_clone()ing it. + * We may as well just make a new one each time. + */ + { + fsm_state_t state; + + b = fsm_new_statealloc(a->alloc, 1); + if (b == NULL) { + return NULL; + } + + if (!fsm_addstate(b, &state)) { + goto error; + } + + for (size_t i = 0; i < n; i++) { + if (!fsm_addedge_literal(b, state, state, charset[i])) { + goto error; + } + } + + fsm_setend(b, state, true); + fsm_setstart(b, state); + } + + assert(fsm_all(a, fsm_isdfa)); + assert(fsm_all(b, fsm_isdfa)); + + /* + * This is equivalent to fsm_intersect(). fsm_intersect() asserts that + * both operands are DFA at runtime. But we know this empirically from + * our callers. So we call fsm_walk2() directly to avoid needlessly + * running the DFA predicate in DNDEBUG builds. + * + * This is intersection implemented by walking sets of states through + * both FSM simultaneously, as described by Hopcroft, Motwani and Ullman + * (2001, 2nd ed.) 4.2, Closure under Intersection. + * + * The intersection of two FSM consists of only those items which + * are present in _BOTH. + */ + q = fsm_walk2(a, b, FSM_WALK2_BOTH, FSM_WALK2_BOTH); + if (q == NULL) { + return NULL; + } + + fsm_free(a); + fsm_free(b); + + /* walking two DFA produces a DFA */ + assert(fsm_all(q, fsm_isdfa)); + + return q; + +error: + + fsm_free(b); + + return NULL; +} + diff --git a/src/libfsm/clone.c b/src/libfsm/clone.c index bec16bb0f..cc43f4e59 100644 --- a/src/libfsm/clone.c +++ b/src/libfsm/clone.c @@ -6,6 +6,8 @@ #include #include +#include +#include #include #include @@ -17,14 +19,15 @@ #include "internal.h" #include "capture.h" #include "endids.h" +#include "eager_output.h" #define LOG_CLONE_ENDIDS 0 static int -copy_capture_actions(struct fsm *dst, const struct fsm *src); +copy_end_metadata(struct fsm *dst, const struct fsm *src); static int -copy_end_ids(struct fsm *dst, const struct fsm *src); +copy_eager_output_ids(struct fsm *dst, const struct fsm *src); struct fsm * fsm_clone(const struct fsm *fsm) @@ -33,9 +36,8 @@ fsm_clone(const struct fsm *fsm) size_t i; assert(fsm != NULL); - assert(fsm->opt != NULL); - new = fsm_new(fsm->opt); + new = fsm_new_statealloc(fsm->alloc, fsm->statecount); if (new == NULL) { return NULL; } @@ -50,12 +52,12 @@ fsm_clone(const struct fsm *fsm) fsm_setend(new, i, 1); } - if (!state_set_copy(&new->states[i].epsilons, new->opt->alloc, fsm->states[i].epsilons)) { + if (!state_set_copy(&new->states[i].epsilons, new->alloc, fsm->states[i].epsilons)) { fsm_free(new); return NULL; } - if (!edge_set_copy(&new->states[i].edges, new->opt->alloc, fsm->states[i].edges)) { + if (!edge_set_copy(&new->states[i].edges, new->alloc, fsm->states[i].edges)) { fsm_free(new); return NULL; } @@ -70,12 +72,13 @@ fsm_clone(const struct fsm *fsm) } { - if (!copy_capture_actions(new, fsm)) { + if (!copy_end_metadata(new, fsm)) { fsm_free(new); return NULL; } - if (!copy_end_ids(new, fsm)) { + /* does not copy callback */ + if (!copy_eager_output_ids(new, fsm)) { fsm_free(new); return NULL; } @@ -84,75 +87,111 @@ fsm_clone(const struct fsm *fsm) return new; } -struct copy_capture_actions_env { +struct copy_end_ids_env { +#ifndef NDEBUG + char tag; +#endif struct fsm *dst; - int ok; + bool ok; }; static int -copy_capture_actions_cb(fsm_state_t state, - enum capture_action_type type, unsigned capture_id, fsm_state_t to, - void *opaque) +copy_end_ids_cb(const struct fsm *fsm, fsm_state_t state, + size_t nth, const fsm_end_id_t id, void *opaque) { - struct copy_capture_actions_env *env = opaque; - assert(env->dst); + struct copy_end_ids_env *env = opaque; + assert(env->tag == 'c'); + (void)fsm; + (void)nth; - if (!fsm_capture_add_action(env->dst, - state, type, capture_id, to)) { - env->ok = 0; +#if LOG_CLONE_ENDIDS + fprintf(stderr, "clone[%d] <- %d\n", state, id); +#endif + + if (fsm_endid_set(env->dst, state, id) < 0) { + env->ok = false; + return 0; } - return env->ok; + return 1; } static int -copy_capture_actions(struct fsm *dst, const struct fsm *src) +copy_active_capture_ids_cb(fsm_state_t state, unsigned capture_id, void *opaque) { - struct copy_capture_actions_env env = { NULL, 1 }; - env.dst = dst; + struct copy_end_ids_env *env = opaque; - fsm_capture_action_iter(src, - copy_capture_actions_cb, &env); - return env.ok; + if (!fsm_capture_set_active_for_end(env->dst, + capture_id, + state)) { + env->ok = 0; + return 0; + } + return 1; } -struct copy_end_ids_env { - char tag; - struct fsm *dst; - const struct fsm *src; - int ok; -}; - static int -copy_end_ids_cb(fsm_state_t state, const fsm_end_id_t id, void *opaque) +associate_capture_programs_cb(fsm_state_t state, unsigned prog_id, void *opaque) { struct copy_end_ids_env *env = opaque; - enum fsm_endid_set_res sres; - assert(env->tag == 'c'); - -#if LOG_CLONE_ENDIDS - fprintf(stderr, "clone[%d] <- %d\n", state, id); -#endif - sres = fsm_endid_set(env->dst, state, id); - if (sres == FSM_ENDID_SET_ERROR_ALLOC_FAIL) { + if (!fsm_capture_associate_program_with_end_state(env->dst, + prog_id, state)) { env->ok = 0; return 0; } - return 1; } static int -copy_end_ids(struct fsm *dst, const struct fsm *src) +copy_end_metadata(struct fsm *dst, const struct fsm *src) { struct copy_end_ids_env env; +#ifndef NDEBUG env.tag = 'c'; /* for clone */ +#endif env.dst = dst; - env.src = src; - env.ok = 1; + env.ok = true; fsm_endid_iter(src, copy_end_ids_cb, &env); + fsm_capture_iter_active_for_all_end_states(src, + copy_active_capture_ids_cb, &env); + + if (!fsm_capture_copy_programs(src, dst)) { + return 0; + } + + fsm_capture_iter_program_ids_for_all_end_states(src, + associate_capture_programs_cb, &env); + + return env.ok; +} + +struct copy_eager_output_ids_env { + bool ok; + struct fsm *dst; +}; + +static int +copy_eager_output_ids_cb(fsm_state_t state, fsm_output_id_t id, void *opaque) +{ + struct copy_eager_output_ids_env *env = opaque; + if (!fsm_eager_output_set(env->dst, state, id)) { + env->ok = false; + return 0; + } + + return 1; +} + +static int +copy_eager_output_ids(struct fsm *dst, const struct fsm *src) +{ + struct copy_eager_output_ids_env env; + env.dst = dst; + env.ok = true; + + fsm_eager_output_iter_all(src, copy_eager_output_ids_cb, &env); return env.ok; } diff --git a/src/libfsm/closure.c b/src/libfsm/closure.c index fa2d0c783..606f03e22 100644 --- a/src/libfsm/closure.c +++ b/src/libfsm/closure.c @@ -9,10 +9,10 @@ #include #include +#include #include #include #include -#include #include #include @@ -32,7 +32,7 @@ epsilon_closure_single(const struct fsm *fsm, struct state_set **closures, fsm_s assert(closures != NULL); assert(s < fsm->statecount); - q = queue_new(fsm->opt->alloc, fsm->statecount); + q = queue_new(fsm->alloc, fsm->statecount); if (q == NULL) { return 1; } @@ -72,7 +72,7 @@ epsilon_closure_single(const struct fsm *fsm, struct state_set **closures, fsm_s * TODO: use partially-constructed (i.e. not readable) sets, and finalize later */ if (p != s && closures[p] != NULL) { - if (!state_set_copy(&closures[s], fsm->opt->alloc, closures[p])) { + if (!state_set_copy(&closures[s], fsm->alloc, closures[p])) { goto error; } } @@ -89,7 +89,7 @@ epsilon_closure_single(const struct fsm *fsm, struct state_set **closures, fsm_s * multiple threads repeating some of the same work here. */ - if (!state_set_add(&closures[s], fsm->opt->alloc, p)) { + if (!state_set_add(&closures[s], fsm->alloc, p)) { goto error; } @@ -128,7 +128,7 @@ epsilon_closure_single(const struct fsm *fsm, struct state_set **closures, fsm_s } struct state_set ** -epsilon_closure(struct fsm *fsm) +fsm_epsilon_closure(const struct fsm *fsm) { struct state_set **closures; fsm_state_t s; @@ -136,7 +136,7 @@ epsilon_closure(struct fsm *fsm) assert(fsm != NULL); assert(fsm->statecount > 0); - closures = f_malloc(fsm->opt->alloc, fsm->statecount * sizeof *closures); + closures = f_malloc(fsm->alloc, fsm->statecount * sizeof *closures); if (closures == NULL) { return NULL; } @@ -153,7 +153,7 @@ epsilon_closure(struct fsm *fsm) * avoid the bookkeeping overhead needed for general case. */ if (fsm->states[s].epsilons == NULL) { - if (!state_set_add(&closures[s], fsm->opt->alloc, s)) { + if (!state_set_add(&closures[s], fsm->alloc, s)) { goto error; } } @@ -184,13 +184,13 @@ epsilon_closure(struct fsm *fsm) state_set_free(closures[s]); } - f_free(fsm->opt->alloc, closures); + f_free(fsm->alloc, closures); return NULL; } void -closure_free(struct state_set **closures, size_t n) +fsm_closure_free(const struct fsm *fsm, struct state_set **closures, size_t n) { fsm_state_t s; @@ -198,6 +198,6 @@ closure_free(struct state_set **closures, size_t n) state_set_free(closures[s]); } - free(closures); + f_free(fsm->alloc, closures); } diff --git a/src/libfsm/concat.c b/src/libfsm/concat.c index 1892ae4ed..9f8780247 100644 --- a/src/libfsm/concat.c +++ b/src/libfsm/concat.c @@ -13,7 +13,6 @@ #include #include #include -#include #include "internal.h" @@ -33,7 +32,7 @@ fsm_concat(struct fsm *a, struct fsm *b, assert(a != NULL); assert(b != NULL); - if (a->opt != b->opt) { + if (a->alloc != b->alloc) { errno = EINVAL; return NULL; } diff --git a/src/libfsm/consolidate.c b/src/libfsm/consolidate.c index 4518d3926..b7832e05b 100644 --- a/src/libfsm/consolidate.c +++ b/src/libfsm/consolidate.c @@ -9,6 +9,8 @@ #include #include #include +#include +#include #include #include @@ -19,34 +21,28 @@ #include #include #include +#include #include "internal.h" #include "capture.h" #include "endids.h" +#include "eager_output.h" #define LOG_MAPPING 0 #define LOG_CONSOLIDATE_CAPTURES 0 -#define LOG_CONSOLIDATE_ENDIDS 0 +#define LOG_CONSOLIDATE_END_METADATA 0 struct mapping_closure { size_t count; const fsm_state_t *mapping; }; -struct consolidate_copy_capture_actions_env { - char tag; - struct fsm *dst; - size_t mapping_count; - const fsm_state_t *mapping; - int ok; -}; - static int -consolidate_copy_capture_actions(struct fsm *dst, const struct fsm *src, +consolidate_end_metadata(struct fsm *dst, const struct fsm *src, const fsm_state_t *mapping, size_t mapping_count); static int -consolidate_end_ids(struct fsm *dst, const struct fsm *src, +consolidate_eager_output_ids(struct fsm *dst, const struct fsm *src, const fsm_state_t *mapping, size_t mapping_count); static fsm_state_t @@ -67,21 +63,31 @@ fsm_consolidate(const struct fsm *src, struct mapping_closure closure; size_t max_used = 0; +#if LOG_CONSOLIDATE_END_METADATA > 1 + fprintf(stderr, "==== fsm_consolidate -- endid_info before:\n"); + fsm_endid_dump(stderr, src); + fsm_capture_dump_active_for_ends(stderr, src); +#endif + assert(src != NULL); - assert(src->opt != NULL); + if (mapping_count == 0) { + return fsm_clone(src); + } - dst = fsm_new(src->opt); + dst = fsm_new(src->alloc); if (dst == NULL) { goto cleanup; } for (src_i = 0; src_i < mapping_count; src_i++) { + const fsm_state_t dst_i = mapping[src_i]; #if LOG_MAPPING fprintf(stderr, "consolidate_mapping[%u]: %u\n", src_i, mapping[src_i]); #endif - if (mapping[src_i] >= max_used) { - max_used = mapping[src_i]; + if (dst_i > max_used) { + assert(dst_i != FSM_STATE_REMAP_NO_STATE); + max_used = dst_i; } } @@ -90,14 +96,14 @@ fsm_consolidate(const struct fsm *src, } assert(dst->statecount == max_used + 1); - seen = f_calloc(src->opt->alloc, + seen = f_calloc(src->alloc, mapping_count/64 + 1, sizeof(seen[0])); if (seen == NULL) { goto cleanup; } -#define DST_SEEN(I) (seen[I/64] & ((uint64_t)1 << (I&63))) -#define SET_DST_SEEN(I) (seen[I/64] |= ((uint64_t)1 << (I&63))) +#define DST_SEEN(I) u64bitset_get(seen, I) +#define SET_DST_SEEN(I) u64bitset_set(seen, I) /* map N states to M states, where N >= M. * if it's the first time state[M] is seen, @@ -110,23 +116,26 @@ fsm_consolidate(const struct fsm *src, for (src_i = 0; src_i < mapping_count; src_i++) { const fsm_state_t dst_i = mapping[src_i]; + /* fsm_consolidate does not currently support discarding states. */ + assert(dst_i != FSM_STATE_REMAP_NO_STATE); + if (!DST_SEEN(dst_i)) { SET_DST_SEEN(dst_i); if (!state_set_copy(&dst->states[dst_i].epsilons, - dst->opt->alloc, src->states[src_i].epsilons)) { + dst->alloc, src->states[src_i].epsilons)) { goto cleanup; } state_set_compact(&dst->states[dst_i].epsilons, mapping_cb, &closure); if (!edge_set_copy(&dst->states[dst_i].edges, - dst->opt->alloc, + dst->alloc, src->states[src_i].edges)) { goto cleanup; } edge_set_compact(&dst->states[dst_i].edges, - dst->opt->alloc, mapping_cb, &closure); + dst->alloc, mapping_cb, &closure); if (fsm_isend(src, src_i)) { fsm_setend(dst, dst_i, 1); @@ -134,11 +143,11 @@ fsm_consolidate(const struct fsm *src, } } - if (!consolidate_copy_capture_actions(dst, src, mapping, mapping_count)) { + if (!fsm_capture_copy_programs(src, dst)) { goto cleanup; } - if (!consolidate_end_ids(dst, src, mapping, mapping_count)) { + if (!consolidate_end_metadata(dst, src, mapping, mapping_count)) { goto cleanup; } @@ -151,120 +160,174 @@ fsm_consolidate(const struct fsm *src, } } - f_free(src->opt->alloc, seen); + if (!consolidate_eager_output_ids(dst, src, mapping, mapping_count)) { + goto cleanup; + } + + f_free(src->alloc, seen); return dst; cleanup: - if (seen != NULL) { f_free(src->opt->alloc, seen); } + if (seen != NULL) { f_free(src->alloc, seen); } return NULL; } +struct consolidate_end_ids_env { + char tag; + struct fsm *dst; + const struct fsm *src; + const fsm_state_t *mapping; + size_t mapping_count; + int ok; +}; + static int -consolidate_copy_capture_actions_cb(fsm_state_t state, - enum capture_action_type type, unsigned capture_id, fsm_state_t to, +consolidate_active_captures_cb(fsm_state_t state, unsigned capture_id, void *opaque) { - struct consolidate_copy_capture_actions_env *env = opaque; - fsm_state_t s, t; - + struct consolidate_end_ids_env *env = opaque; + fsm_state_t dst_s; assert(env->tag == 'C'); -#if LOG_CONSOLIDATE_CAPTURES - fprintf(stderr, "consolidate_copy_capture_actions_cb: state %u, type %s, ID %u, TO %d\n", - state, - fsm_capture_action_type_name[type], - capture_id, to); -#endif - assert(state < env->mapping_count); - assert(to == CAPTURE_NO_STATE || to < env->mapping_count); - s = env->mapping[state]; - t = to == CAPTURE_NO_STATE - ? CAPTURE_NO_STATE : env->mapping[to]; + dst_s = env->mapping[state]; - if (!fsm_capture_add_action(env->dst, - s, type, capture_id, t)) { +#if LOG_CONSOLIDATE_END_METADATA + fprintf(stderr, "consolidate_active_captures_cb: state %d -> dst_s %d, capture_id %u\n", + state, dst_s, capture_id); +#endif + + if (!fsm_capture_set_active_for_end(env->dst, capture_id, dst_s)) { env->ok = 0; return 0; } - return 1; } static int -consolidate_copy_capture_actions(struct fsm *dst, const struct fsm *src, - const fsm_state_t *mapping, size_t mapping_count) +consolidate_capture_programs_cb(fsm_state_t state, unsigned program_id, + void *opaque) { - size_t i; + struct consolidate_end_ids_env *env = opaque; + fsm_state_t dst_s; + assert(env->tag == 'C'); - struct consolidate_copy_capture_actions_env env; - env.tag = 'C'; - env.dst = dst; - env.mapping_count = mapping_count; - env.mapping = mapping; - env.ok = 1; + assert(state < env->mapping_count); + dst_s = env->mapping[state]; -#if LOG_MAPPING - for (i = 0; i < mapping_count; i++) { - fprintf(stderr, "mapping[%lu]: %u\n", i, mapping[i]); + if (!fsm_capture_associate_program_with_end_state(env->dst, + (uint32_t)program_id, dst_s)) { + env->ok = 0; } -#else - (void)i; -#endif - fsm_capture_action_iter(src, - consolidate_copy_capture_actions_cb, &env); - return env.ok; + return 1; } -struct consolidate_end_ids_env { - char tag; - struct fsm *dst; - const struct fsm *src; - const fsm_state_t *mapping; - size_t mapping_count; -}; - static int consolidate_end_ids_cb(fsm_state_t state, const fsm_end_id_t *ids, size_t num_ids, void *opaque) { struct consolidate_end_ids_env *env = opaque; - enum fsm_endid_set_res sres; - fsm_state_t s; + fsm_state_t dst_s; assert(env->tag == 'C'); assert(state < env->mapping_count); - s = env->mapping[state]; + dst_s = env->mapping[state]; + +#if LOG_CONSOLIDATE_END_METADATA > 1 + fprintf(stderr, "consolidate_end_ids_cb: state %u, dst %u, IDs [", + state, dst_s); + for (size_t i = 0; i < num_ids; i++) { + fprintf(stderr, "%s%d", i > 0 ? " " : "", ids[i]); + } + fprintf(stderr, "]\n"); +#endif - sres = fsm_endid_set_bulk(env->dst, s, num_ids, ids, FSM_ENDID_BULK_APPEND); + enum fsm_endid_set_res sres = fsm_endid_set_bulk(env->dst, + dst_s, num_ids, ids, FSM_ENDID_BULK_APPEND); if (sres == FSM_ENDID_SET_ERROR_ALLOC_FAIL) { return 0; } - return 1; } static int -consolidate_end_ids(struct fsm *dst, const struct fsm *src, +consolidate_end_metadata(struct fsm *dst, const struct fsm *src, const fsm_state_t *mapping, size_t mapping_count) { struct consolidate_end_ids_env env; - int ret; +#ifndef NDEBUG env.tag = 'C'; /* for Consolidate */ +#endif env.dst = dst; env.src = src; env.mapping = mapping; env.mapping_count = mapping_count; - ret = fsm_endid_iter_bulk(src, consolidate_end_ids_cb, &env); + env.ok = fsm_endid_iter_bulk(src, consolidate_end_ids_cb, &env); -#if LOG_CONSOLIDATE_ENDIDS > 1 + if (env.ok) { + fsm_state_t s; + const size_t src_state_count = fsm_countstates(src); + for (s = 0; s < src_state_count; s++) { + fsm_capture_iter_active_for_end_state(src, s, + consolidate_active_captures_cb, &env); + if (!env.ok) { + break; + } + + fsm_capture_iter_program_ids_for_end_state(src, s, + consolidate_capture_programs_cb, &env); + if (!env.ok) { + break; + } + } + } + +#if LOG_CONSOLIDATE_END_METADATA > 1 fprintf(stderr, "==== fsm_consolidate -- endid_info after:\n"); fsm_endid_dump(stderr, dst); + fsm_capture_dump_active_for_ends(stderr, dst); #endif - return ret; + return env.ok; } + +struct consolidate_eager_output_ids_env { + bool ok; + struct fsm *dst; + const fsm_state_t *mapping; + size_t mapping_count; +}; + +static int +consolidate_eager_output_ids_cb(fsm_state_t state, fsm_output_id_t id, void *opaque) +{ + struct consolidate_eager_output_ids_env *env = opaque; + assert(state < env->mapping_count); + const fsm_state_t dst_state = env->mapping[state]; + + if (!fsm_eager_output_set(env->dst, dst_state, id)) { + env->ok = false; + return 0; + } + + return 1; +} + +static int +consolidate_eager_output_ids(struct fsm *dst, const struct fsm *src, + const fsm_state_t *mapping, size_t mapping_count) +{ + struct consolidate_eager_output_ids_env env = { + .ok = true, + .dst = dst, + .mapping = mapping, + .mapping_count = mapping_count, + }; + fsm_eager_output_iter_all(src, consolidate_eager_output_ids_cb, &env); + return env.ok; +} + diff --git a/src/libfsm/detect_required.c b/src/libfsm/detect_required.c new file mode 100644 index 000000000..8488376fc --- /dev/null +++ b/src/libfsm/detect_required.c @@ -0,0 +1,275 @@ +/* + * Copyright 2024 Scott Vokes + * + * See LICENCE for the full copyright terms. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include + +#include "internal.h" + +#define LOG_BASE 0 +#define LOG_PROGRESS (LOG_BASE + 0) +#define LOG_STEPS (LOG_BASE + 0) + +/* More than one label */ +#define LABEL_GROUP ((uint16_t)-1) + +struct dr_env { + const struct fsm *dfa; + size_t steps; + + /* Number of times a unique label has been required -- this is a count so that going + * from 0 <-> 1 can set/clear the accumulator, but going from 1 -> 2 etc. does not. */ + size_t counts[256]; + struct bm current; + bool first_end_state; + struct bm overall; + + struct dr_stack { + size_t used; + size_t ceil; + struct stack_frame { + fsm_state_t state; + uint16_t label; /* unique label followed to get here, or LABEL_GROUP */ + struct edge_group_iter iter; + } *frames; + } stack; +}; + +#define DEF_STACK_FRAMES 16 + +/* Check symbols[]: if there's more than one bit set, then set label to + * LABEL_GROUP, otherwise set it to the single bit's character value. + * At least one bit must be set. */ +static void check_symbols(const struct edge_group_iter_info *info, uint16_t *label) +{ + bool any = false; + + for (size_t i = 0; i < 256/64; i++) { + uint64_t w = info->symbols[i]; + if (w == 0) { continue; } + + /* get position of lowest set bit */ + for (size_t b = 0; b < 64; b++) { + const uint64_t bit = 1ULL << b; + if (w & bit) { + if (any) { + *label = LABEL_GROUP; + return; + } + + /* clear it, check if there's anything else set */ + w &= ~bit; + if (w != 0) { + *label = LABEL_GROUP; + return; + } + + *label = 64*i + b; + any = true; + break; + } + } + } + + /* there must be at least one bit set */ + assert(any); +} + +/* Walk a DFA and attempt to detect which characters must appear in any input to match. + * This finds the intersection of characters required on any start->end paths (tracking + * edges with only one label that must be followed by all matches), so it can take + * prohibitively long for large/complex DFAs. */ +enum fsm_detect_required_characters_res +fsm_detect_required_characters(const struct fsm *dfa, size_t step_limit, + uint64_t charmap[4], size_t *count) +{ + assert(dfa != NULL); + assert(charmap != NULL); + + #if EXPENSIVE_CHECKS + if (!fsm_all(dfa, fsm_isdfa)) { + return FSM_DETECT_REQUIRED_CHARACTERS_ERROR_MISUSE; + } + #endif + + enum fsm_detect_required_characters_res res = FSM_DETECT_REQUIRED_CHARACTERS_ERROR_ALLOC; + + struct dr_env env = { + .dfa = dfa, + .first_end_state = true, + }; + + assert(env.counts[0] == 0); + + const size_t state_count = fsm_countstates(dfa); + fsm_state_t start_state; + if (!fsm_getstart(dfa, &start_state)) { + res = FSM_DETECT_REQUIRED_CHARACTERS_ERROR_MISUSE; + goto cleanup; + } + + #if EXPENSIVE_CHECKS + for (fsm_state_t s = 0; s < state_count; s++) { + assert(!dfa->states[s].visited); + } + #endif + + for (size_t i = 0; i < 4; i++) { charmap[i] = 0; } + + /* If the start state is also an end state, then + * it matches the empty string, so we're done. */ + if (fsm_isend(dfa, start_state)) { + res = FSM_DETECT_REQUIRED_CHARACTERS_WRITTEN; + if (count != NULL) { + *count = 0; + } + goto cleanup; + } + + env.stack.frames = f_malloc(dfa->alloc, DEF_STACK_FRAMES * sizeof(env.stack.frames[0])); + if (env.stack.frames == NULL) { goto cleanup; } + env.stack.ceil = DEF_STACK_FRAMES; + + { /* set up start state's stack frame */ + struct stack_frame *sf0 = &env.stack.frames[0]; + sf0->state = start_state; + sf0->label = LABEL_GROUP; + + dfa->states[start_state].visited = true; + + edge_set_group_iter_reset(dfa->states[start_state].edges, + EDGE_GROUP_ITER_ALL, &sf0->iter); + env.stack.used = 1; + } + + while (env.stack.used > 0) { + struct stack_frame *sf = &env.stack.frames[env.stack.used - 1]; + struct edge_group_iter_info info; + env.steps++; + if (LOG_STEPS > 1) { + fprintf(stderr, "-- steps %zu/%zu\n", env.steps, step_limit); + } + if (env.steps == step_limit) { + res = FSM_DETECT_REQUIRED_CHARACTERS_STEP_LIMIT_REACHED; + goto cleanup; + } + + if (edge_set_group_iter_next(&sf->iter, &info)) { + assert(info.to < state_count); + if (dfa->states[info.to].visited) { + continue; /* skip visited state */ + } + + if (env.stack.used == env.stack.ceil) { /* grow stack */ + const size_t nceil = 2*env.stack.ceil; + assert(nceil > env.stack.ceil); + struct stack_frame *nframes = f_realloc(dfa->alloc, + env.stack.frames, nceil * sizeof(nframes[0])); + if (nframes == NULL) { + return FSM_DETECT_REQUIRED_CHARACTERS_ERROR_ALLOC; + } + + env.stack.frames = nframes; + env.stack.ceil = nceil; + } + + /* enter state */ + dfa->states[info.to].visited = true; + + struct stack_frame *nsf = &env.stack.frames[env.stack.used]; + nsf->state = info.to; + check_symbols(&info, &nsf->label); + + if (nsf->label != LABEL_GROUP) { + size_t offset = (nsf->label & 0xff); + const size_t label_count = ++env.counts[offset]; + if (label_count == 1) { + bm_set(&env.current, offset); + } + } + + edge_set_group_iter_reset(dfa->states[info.to].edges, + EDGE_GROUP_ITER_ALL, &nsf->iter); + env.stack.used++; + + if (fsm_isend(dfa, info.to)) { + if (env.first_end_state) { + bm_copy(&env.overall, &env.current); + env.first_end_state = false; + } else { /* intersect */ + bm_intersect(&env.overall, &env.current); + } + + if (LOG_PROGRESS) { + fprintf(stderr, "-- current: "); + bm_print(stderr, NULL, &env.current, 0, fsm_escputc); + fprintf(stderr, ", overall: "); + bm_print(stderr, NULL, &env.overall, 0, fsm_escputc); + fprintf(stderr, "\n"); + } + + /* Intersecting with the empty set will always be empty, so + * further exploration is unnecessary. */ + if (!bm_any(&env.overall)) { + res = FSM_DETECT_REQUIRED_CHARACTERS_WRITTEN; + break; + } + } + + } else { /* done with state */ + /* If this state was reached via a unique label, then + * reduce the count. If the count returns to 0, remove + * it from the constraint set. */ + if (sf->label != LABEL_GROUP) { + size_t offset = (sf->label & 0xff); + const size_t label_count = --env.counts[offset]; + if (label_count == 0) { + bm_unset(&env.current, offset); + } + } + + /* clear visited */ + dfa->states[sf->state].visited = false; + + env.stack.used--; + } + } + + if (LOG_STEPS) { + fprintf(stderr, "%s: finished in %zu/%zu steps\n", __func__, env.steps, step_limit); + } + + for (size_t i = 0; i < 4; i++) { + charmap[i] = *bm_nth_word(&env.overall, i); + } + + if (count != NULL) { + *count = bm_count(&env.overall); + } + + res = FSM_DETECT_REQUIRED_CHARACTERS_WRITTEN; + +cleanup: + f_free(dfa->alloc, env.stack.frames); + + for (fsm_state_t s = 0; s < state_count; s++) { + dfa->states[s].visited = false; + } + + return res; +} diff --git a/src/libfsm/determinise.c b/src/libfsm/determinise.c index 56e135afd..f2f3a39fb 100644 --- a/src/libfsm/determinise.c +++ b/src/libfsm/determinise.c @@ -6,6 +6,11 @@ #include "determinise_internal.h" +#include +#include + +#define LOG_DETERMINISATION_COUNTERS 0 + static void dump_labels(FILE *f, const uint64_t labels[4]) { @@ -17,21 +22,27 @@ dump_labels(FILE *f, const uint64_t labels[4]) } } -int -fsm_determinise(struct fsm *nfa) +enum fsm_determinise_with_config_res +fsm_determinise_with_config(struct fsm *nfa, + const struct fsm_determinise_config *config) { - int res = 0; + enum fsm_determinise_with_config_res res = FSM_DETERMINISE_WITH_CONFIG_ERRNO; struct mappingstack *stack = NULL; struct interned_state_set_pool *issp = NULL; struct map map = { NULL, 0, 0, NULL }; struct mapping *curr = NULL; size_t dfacount = 0; + const size_t state_limit = config == NULL + ? 0 + : config->state_limit; struct analyze_closures_env ac_env = { 0 }; + INIT_TIMERS(); + INIT_TIMERS_NAMED(overall); assert(nfa != NULL); - map.alloc = nfa->opt->alloc; + map.alloc = nfa->alloc; /* * This NFA->DFA implementation is for epsilon-free NFA only. This keeps @@ -39,20 +50,29 @@ fsm_determinise(struct fsm *nfa) * faster where we can start with an epsilon-free NFA in the first place. */ if (fsm_has(nfa, fsm_hasepsilons)) { + TIME(&pre); if (!fsm_remove_epsilons(nfa)) { - return 0; + return FSM_DETERMINISE_WITH_CONFIG_ERRNO; } + TIME(&post); + DIFF_MSEC("det_remove_eps", pre, post, NULL); } #if LOG_DETERMINISE_CAPTURES || LOG_INPUT fprintf(stderr, "# post_remove_epsilons, pre_determinise\n"); - fsm_print_fsm(stderr, nfa); + fsm_dump(stderr, nfa); fsm_capture_dump(stderr, "#### post_remove_epsilons", nfa); #endif + TIME(&overall_pre); - issp = interned_state_set_pool_alloc(nfa->opt->alloc); + issp = interned_state_set_pool_alloc(nfa->alloc); if (issp == NULL) { - return 0; + return FSM_DETERMINISE_WITH_CONFIG_ERRNO; + } + + if (state_limit != 0 && fsm_countstates(nfa) > state_limit) { + res = FSM_DETERMINISE_WITH_CONFIG_STATE_LIMIT_REACHED; + goto cleanup; } { @@ -74,7 +94,7 @@ fsm_determinise(struct fsm *nfa) */ if (!fsm_getstart(nfa, &start)) { - res = 1; + res = FSM_DETERMINISE_WITH_CONFIG_OK; goto cleanup; } @@ -95,15 +115,26 @@ fsm_determinise(struct fsm *nfa) * Our "todo" list. It needn't be a stack; we treat it as an unordered * set where we can consume arbitrary items in turn. */ - stack = stack_init(nfa->opt->alloc); + stack = stack_init(nfa->alloc); if (stack == NULL) { goto cleanup; } - ac_env.alloc = nfa->opt->alloc; + ac_env.alloc = nfa->alloc; ac_env.fsm = nfa; ac_env.issp = issp; +#if LOG_DETERMINISATION_STATS + fprintf(stderr, "%s: determinising FSM with %d states\n", __func__, fsm_countstates(nfa)); +#endif + + INIT_TIMERS_NAMED(iss); + size_t iss_accum = 0; + size_t iss_calls = 0; + size_t stack_pushes = 0; + size_t inner_steps = 0; + + TIME(&pre); do { size_t o_i; @@ -114,18 +145,25 @@ fsm_determinise(struct fsm *nfa) assert(curr != NULL); + TIME(&iss_pre); if (!analyze_closures__pairwise_grouping(&ac_env, curr->iss)) { goto cleanup; } + TIME(&iss_post); + DIFF_MSEC("det_iss", iss_pre, iss_post, &iss_accum); + (void)iss_accum; + iss_calls++; - if (!edge_set_advise_growth(&curr->edges, nfa->opt->alloc, ac_env.output_count)) { + if (!edge_set_advise_growth(&curr->edges, nfa->alloc, ac_env.output_count)) { goto cleanup; } + /* each output is an outgoing (label set) -> interned_state_set pair */ for (o_i = 0; o_i < ac_env.output_count; o_i++) { struct mapping *m; struct ac_output *output = &ac_env.outputs[o_i]; interned_state_set_id iss = output->iss; + inner_steps++; #if LOG_DETERMINISE_CLOSURES fprintf(stderr, "fsm_determinise: output %zu/%zu: cur (dfa %zu) label [", @@ -150,6 +188,11 @@ fsm_determinise(struct fsm *nfa) assert(m->dfastate < dfacount); } else { /* not found -- add a new one and push it to the stack for processing */ + + if (state_limit != 0 && dfacount > state_limit) { + res = FSM_DETERMINISE_WITH_CONFIG_STATE_LIMIT_REACHED; + goto cleanup; + } if (!map_add(&map, dfacount, iss, &m)) { goto cleanup; } @@ -157,6 +200,7 @@ fsm_determinise(struct fsm *nfa) if (!stack_push(stack, m)) { goto cleanup; } + stack_pushes++; } #if LOG_SYMBOL_CLOSURE @@ -165,26 +209,38 @@ fsm_determinise(struct fsm *nfa) fprintf(stderr, "] -> dfastate %zu on output state %zu\n", m->dfastate, curr->dfastate); #endif - if (!edge_set_add_bulk(&curr->edges, nfa->opt->alloc, output->labels, m->dfastate)) { + if (!edge_set_add_bulk(&curr->edges, nfa->alloc, output->labels, m->dfastate)) { goto cleanup; } } ac_env.output_count = 0; - - /* All elements in sclosures[] are interned, so they will be freed later. */ } while ((curr = stack_pop(stack))); + TIME(&post); + DIFF_MSEC("det_stack_loop", pre, post, NULL); + + if (LOG_DETERMINISATION_COUNTERS) { + fprintf(stderr, "%s: iss_accum total %zu (%zu calls, %g usec avg.), %zu stack pushes, %zu iterations, %zu inner_steps\n", + __func__, iss_accum, iss_calls, iss_accum / (1.0 * iss_calls), stack_pushes, iss_calls, inner_steps); + } { struct map_iter it; struct mapping *m; struct fsm *dfa; - dfa = fsm_new(nfa->opt); + dfa = fsm_new(nfa->alloc); if (dfa == NULL) { goto cleanup; } + TIME(&pre); + if (!fsm_capture_copy_programs(nfa, dfa)) { + goto cleanup; + } + TIME(&post); + DIFF_MSEC("det_copy_captures", pre, post, NULL); + #if DUMP_MAPPING { fprintf(stderr, "#### fsm_determinise: mapping\n"); @@ -192,10 +248,10 @@ fsm_determinise(struct fsm *nfa) /* build reverse mappings table: for every NFA state X, if X is part * of the new DFA state Y, then add Y to a list for X */ for (m = map_first(&map, &it); m != NULL; m = map_next(&it)) { - struct state_iter si; interned_state_set_id iss_id = m->iss; + struct state_iter si; fsm_state_t state; - struct state_set *ss = interned_state_set_get_state_set(ac_env.issp, iss_id); + struct state_set *ss = interned_state_set_get_state_set(issp, iss_id); fprintf(stderr, "%zu:", m->dfastate); for (state_set_reset(ss, &si); state_set_next(&si, &state); ) { @@ -225,6 +281,7 @@ fsm_determinise(struct fsm *nfa) assert(dfa->states[m->dfastate].edges == NULL); dfa->states[m->dfastate].edges = m->edges; + m->edges = NULL; /* transfer ownership */ /* * The current DFA state is an end state if any of its associated NFA @@ -238,29 +295,53 @@ fsm_determinise(struct fsm *nfa) fsm_setend(dfa, m->dfastate, 1); /* - * Carry through end IDs, if present. This isn't anything to do - * with the DFA conversion; it's meaningful only to the caller. + * Copy over metadata associated with end + * states, if present. This isn't anything to do + * with the DFA conversion; it's meaningful only + * to the caller. * * The closure may contain non-end states, but at least one state is * known to have been an end state. */ - if (!fsm_endid_carry(nfa, ss, dfa, m->dfastate)) { + if (!remap_end_metadata(nfa, ss, dfa, m->dfastate)) { goto cleanup; } } + TIME(&post); + DIFF_MSEC("det_map_loop", pre, post, NULL); + + fsm_capture_integrity_check(dfa); - if (!remap_capture_actions(&map, issp, dfa, nfa)) { + if (!remap_eager_outputs(&map, issp, dfa, nfa)) { goto cleanup; } fsm_move(nfa, dfa); } +#if LOG_DETERMINISE_CAPTURES + fprintf(stderr, "# post_determinise\n"); + fsm_print_fsm(stderr, nfa); + fsm_capture_dump(stderr, "#### post_determinise", nfa); +#endif + + TIME(&overall_post); + DIFF_MSEC("det_overall", overall_pre, overall_post, NULL); + +#if LOG_DETERMINISATION_STATS + fprintf(stderr, "%s: created DFA with %d states\n", __func__, fsm_countstates(nfa)); + fprintf(stderr, "%s: analyze_closures_env.analyze_usec: %zu\n", + __func__, ac_env.analyze_usec); +#endif + #if EXPENSIVE_CHECKS assert(fsm_all(nfa, fsm_isdfa)); #endif - res = 1; + /* This should not be carried over from the NFA. */ + assert(nfa->linkage_info == NULL); + + res = FSM_DETERMINISE_WITH_CONFIG_OK; cleanup: map_free(&map); @@ -311,83 +392,20 @@ fsm_determinise(struct fsm *nfa) return res; } -/* Add DFA_state to the list for NFA_state. */ -static int -add_reverse_mapping(const struct fsm_alloc *alloc, - struct reverse_mapping *reverse_mappings, - fsm_state_t dfastate, fsm_state_t nfa_state) -{ - struct reverse_mapping *rm = &reverse_mappings[nfa_state]; - if (rm->count == rm->ceil) { - const unsigned nceil = (rm->ceil ? 2*rm->ceil : 2); - fsm_state_t *nlist = f_realloc(alloc, - rm->list, nceil * sizeof(rm->list)); - if (nlist == NULL) { - return 0; - } - rm->list = nlist; - rm->ceil = nceil; - } - - rm->list[rm->count] = dfastate; - rm->count++; - return 1; -} - -static int -det_copy_capture_actions_cb(fsm_state_t state, - enum capture_action_type type, unsigned capture_id, fsm_state_t to, - void *opaque) +int +fsm_determinise(struct fsm *nfa) { - struct reverse_mapping *rm_s; - size_t s_i, t_i; - struct det_copy_capture_actions_env *env = opaque; - assert(env->tag == 'D'); - -#if LOG_DETERMINISE_CAPTURES - fprintf(stderr, "det_copy_capture_actions_cb: state %u, type %s, ID %u, TO %d\n", - state, fsm_capture_action_type_name[type], - capture_id, to); -#endif - - rm_s = &env->reverse_mappings[state]; - - for (s_i = 0; s_i < rm_s->count; s_i++) { - const fsm_state_t s = rm_s->list[s_i]; - - if (to == CAPTURE_NO_STATE) { - if (!fsm_capture_add_action(env->dst, - s, type, capture_id, CAPTURE_NO_STATE)) { - env->ok = 0; - return 0; - } - } else { - struct reverse_mapping *rm_t = &env->reverse_mappings[to]; - for (t_i = 0; t_i < rm_t->count; t_i++) { - const fsm_state_t t = rm_t->list[t_i]; - - if (!fsm_capture_add_action(env->dst, - s, type, capture_id, t)) { - env->ok = 0; - return 0; - } - } - } + enum fsm_determinise_with_config_res res = fsm_determinise_with_config(nfa, NULL); + switch (res) { + case FSM_DETERMINISE_WITH_CONFIG_OK: + return 1; + case FSM_DETERMINISE_WITH_CONFIG_STATE_LIMIT_REACHED: + /* unreachable */ + return 0; + case FSM_DETERMINISE_WITH_CONFIG_ERRNO: + default: + return 0; } - - return 1; -} - -static int -det_copy_capture_actions(struct reverse_mapping *reverse_mappings, - struct fsm *dst, struct fsm *src) -{ - struct det_copy_capture_actions_env env = { 'D', NULL, NULL, 1 }; - env.dst = dst; - env.reverse_mappings = reverse_mappings; - - fsm_capture_action_iter(src, det_copy_capture_actions_cb, &env); - return env.ok; } SUPPRESS_EXPECTED_UNSIGNED_INTEGER_OVERFLOW() @@ -396,11 +414,11 @@ hash_iss(interned_state_set_id iss) { /* Just hashing the ID directly is fine here -- since they're * interned, they're identified by pointer equality. */ - return FSM_PHI_64 * (uintptr_t)iss; + return hash_id((uintptr_t)iss); } static struct mapping * -map_first(struct map *map, struct map_iter *iter) +map_first(const struct map *map, struct map_iter *iter) { iter->m = map; iter->i = 0; @@ -560,6 +578,8 @@ map_free(struct map *map) if (b == NULL) { continue; } + /* free any edge sets that didn't get transferred */ + edge_set_free(map->alloc, b->edges); f_free(map->alloc, b); } @@ -636,83 +656,6 @@ stack_pop(struct mappingstack *stack) return item; } -static int -remap_capture_actions(struct map *map, struct interned_state_set_pool *issp, - struct fsm *dst_dfa, struct fsm *src_nfa) -{ - struct map_iter it; - struct state_iter si; - struct mapping *m; - struct reverse_mapping *reverse_mappings; - fsm_state_t state; - const size_t capture_count = fsm_countcaptures(src_nfa); - size_t i, j; - int res = 0; - - if (capture_count == 0) { - return 1; - } - - /* This is not 1 to 1 -- if state X is now represented by multiple - * states Y in the DFA, and state X has action(s) when transitioning - * to state Z, this needs to be added on every Y, for every state - * representing Z in the DFA. - * - * We could probably filter this somehow, at the very least by - * checking reachability from every X, but the actual path - * handling later will also check reachability. */ - reverse_mappings = f_calloc(dst_dfa->opt->alloc, src_nfa->statecount, sizeof(reverse_mappings[0])); - if (reverse_mappings == NULL) { - return 0; - } - - /* build reverse mappings table: for every NFA state X, if X is part - * of the new DFA state Y, then add Y to a list for X */ - for (m = map_first(map, &it); m != NULL; m = map_next(&it)) { - struct state_set *ss; - interned_state_set_id iss_id = m->iss; - assert(m->dfastate < dst_dfa->statecount); - ss = interned_state_set_get_state_set(issp, iss_id); - - for (state_set_reset(ss, &si); state_set_next(&si, &state); ) { - if (!add_reverse_mapping(dst_dfa->opt->alloc, - reverse_mappings, - m->dfastate, state)) { - goto cleanup; - } - } - } - -#if LOG_DETERMINISE_CAPTURES - fprintf(stderr, "#### reverse mapping for %zu states\n", src_nfa->statecount); - for (i = 0; i < src_nfa->statecount; i++) { - struct reverse_mapping *rm = &reverse_mappings[i]; - fprintf(stderr, "%lu:", i); - for (j = 0; j < rm->count; j++) { - fprintf(stderr, " %u", rm->list[j]); - } - fprintf(stderr, "\n"); - } -#else - (void)j; -#endif - - if (!det_copy_capture_actions(reverse_mappings, dst_dfa, src_nfa)) { - goto cleanup; - } - - res = 1; -cleanup: - for (i = 0; i < src_nfa->statecount; i++) { - if (reverse_mappings[i].list != NULL) { - f_free(dst_dfa->opt->alloc, reverse_mappings[i].list); - } - } - f_free(dst_dfa->opt->alloc, reverse_mappings); - - return res; -} - static int group_labels_overlap(const struct ac_group *a, const struct ac_group *b) { @@ -730,6 +673,25 @@ group_labels_overlap(const struct ac_group *a, const struct ac_group *b) return 0; } +static int +remap_end_metadata(const struct fsm *src_fsm, const struct state_set *src_set, + struct fsm *dst_fsm, fsm_state_t dst_state) +{ + if (!fsm_endid_carry(src_fsm, src_set, dst_fsm, dst_state)) { + return 0; + } + + if (!fsm_capture_copy_active_for_ends(src_fsm, src_set, dst_fsm, dst_state)) { + return 0; + } + + if (!fsm_capture_copy_program_end_state_associations(src_fsm, src_set, dst_fsm, dst_state)) { + return 0; + } + + return 1; +} + static void intersect_with(uint64_t *a, const uint64_t *b) { @@ -1309,15 +1271,10 @@ build_output_from_cached_analysis(struct analyze_closures_env *env, fsm_state_t #define LOG_TO_SET_HTAB 0 -SUPPRESS_EXPECTED_UNSIGNED_INTEGER_OVERFLOW() static uint64_t to_set_hash(size_t count, const fsm_state_t *ids) { - uint64_t h = hash_id(count); - for (size_t i = 0; i < count; i++) { - h += hash_id(ids[i]); - } - return h; + return hash_ids(count, ids); } static int @@ -1333,18 +1290,27 @@ to_set_htab_check(struct analyze_closures_env *env, return 0; } +#if LOG_TO_SET_HTAB || HASH_LOG_PROBES || defined(HASH_PROBE_LIMIT) + size_t probes = 0; +#endif + int res = 0; + const uint64_t mask = bcount - 1; for (size_t b_i = 0; b_i < bcount; b_i++) { +#if LOG_TO_SET_HTAB || HASH_LOG_PROBES || defined(HASH_PROBE_LIMIT) + probes++; +#endif const struct to_set_bucket *b = &htab->buckets[(h + b_i) & mask]; if (b->count == 0) { - return 0; /* empty bucket -> not found */ + goto done; /* empty bucket -> not found */ } else if (b->count == count) { + assert(env->to_sets.buf != NULL); assert(b->offset + count <= env->to_sets.used); const fsm_state_t *ids = &env->to_sets.buf[b->offset]; if (0 == memcmp(ids, dst, count * sizeof(dst[0]))) { *out_offset = b->offset; - - return 1; /* cache hit */ + res = 1; /* cache hit */ + goto done; } else { continue; /* collision */ } @@ -1353,7 +1319,19 @@ to_set_htab_check(struct analyze_closures_env *env, } } - return 0; +done: +#if LOG_TO_SET_HTAB || HASH_LOG_PROBES + fprintf(stderr, "%s: result %d, %zu probes, htab: used %zu/%zu ceil\n", + __func__, res, probes, htab->buckets_used, htab->bucket_count); +#endif +#ifdef HASH_PROBE_LIMIT + if (probes >= HASH_PROBE_LIMIT) { + fprintf(stderr, "-- %zd probes, limit exceeded\n", probes); + } + assert(probes < HASH_PROBE_LIMIT); +#endif + + return res; } static int @@ -1415,6 +1393,22 @@ to_set_htab_save(struct analyze_closures_env *env, b->count = count; b->offset = offset; htab->buckets_used++; +#if HASH_LOG_PROBES + fprintf(stderr, "%s: [", __func__); + const fsm_state_t *ids = &env->to_sets.buf[offset]; + for (size_t i = 0; i < count; i++) { + fprintf(stderr, "%s%d", + i > 0 ? " " : "", ids[i]); + } + + fprintf(stderr, "] -> hash %lx -> b %zu (%zu/%zu used), %zd probes\n", + hash, (hash + b_i) & mask, + htab->buckets_used, htab->bucket_count, b_i); +#endif +#if HASH_PROBE_LIMIT + assert(b_i < HASH_PROBE_LIMIT); +#endif + return 1; } else if (b->count == count) { assert(b->offset != offset); /* no duplicates */ @@ -1465,6 +1459,7 @@ save_to_set(struct analyze_closures_env *env, env->to_sets.ceil = nceil; env->to_sets.buf = nbuf; } + assert(env->to_sets.buf != NULL); #if LOG_TO_SET static size_t to_set_id; @@ -1569,8 +1564,8 @@ commit_buffered_result(struct analyze_closures_env *env, uint32_t *cache_result_ memcpy(&nr->entries[0], env->results.buffer.entries, nr->count * sizeof(env->results.buffer.entries[0])); #if LOG_GROUPING || LOG_COMMIT_BUFFERED_GROUP - fprintf(stderr, "%s: alloc %zu, hash 0x%016lx\n", - __func__, alloc_sz, hash_fnv1a_64((const uint8_t *)nr, alloc_sz)); + fprintf(stderr, "%s: alloc %zu, count %u\n", + __func__, alloc_sz, nr->count)); #endif } @@ -1626,22 +1621,28 @@ hash_pair(fsm_state_t a, fsm_state_t b) assert(a != b); assert(a & RESULT_BIT); assert(b & RESULT_BIT); - a &=~ RESULT_BIT; - b &=~ RESULT_BIT; - - /* Don't hash a and b separately and combine them with - * hash_id, because it's common to have adjacent pairs of - * result IDs, and with how hash_id works that leads to - * multiples of similar hash values bunching up. - * - * This could be replaced with a better hash function later, - * but use LOG_CACHE_HTAB to ensure there aren't visually obvious - * runs of collisions appearing in the tables. */ - const uint64_t res = hash_id(a + b); + const uint64_t ma = (uint64_t)(a & ~RESULT_BIT); /* m: masked */ + const uint64_t mb = (uint64_t)(b & ~RESULT_BIT); + assert(ma != mb); + + /* Left-shift the smaller ID, so the pair order is consistent for hashing. */ + const uint64_t ab = (ma < mb) + ? ((ma << 32) | mb) + : ((mb << 32) | ma); + const uint64_t res = hash_id(ab); /* fprintf(stderr, "%s: a %d, b %d -> %016lx\n", __func__, a, b, res); */ return res; } +static int eq_bucket_pair(const struct result_pair_bucket *b, fsm_state_t pa, fsm_state_t pb) +{ + if ((pa & ~RESULT_BIT) < (pb & ~RESULT_BIT)) { + return b->ids[0] == pa && b->ids[1] == pb; + } else { + return b->ids[0] == pb && b->ids[1] == pa; + } +} + static int analysis_cache_check_pair(struct analyze_closures_env *env, fsm_state_t pa, fsm_state_t pb, fsm_state_t *result_id) @@ -1667,7 +1668,7 @@ analysis_cache_check_pair(struct analyze_closures_env *env, fsm_state_t pa, fsm_ struct result_pair_bucket *b = &htab->buckets[(h + i) & mask]; if (b->t == RPBT_UNUSED) { break; /* not found */ - } else if (b->t == RPBT_PAIR && b->ids[0] == pa && b->ids[1] == pb) { + } else if (b->t == RPBT_PAIR && eq_bucket_pair(b, pa, pb)) { *result_id = b->result_id; /* hit */ #if LOG_GROUPING > 1 fprintf(stderr, "%s: cache hit for pair { %d_R, %d_R } => %d\n", @@ -1801,6 +1802,7 @@ pair_cache_save(struct analyze_closures_env *env, assert(pa != pb); assert(pa & RESULT_BIT); assert(pb & RESULT_BIT); + assert((pa & ~RESULT_BIT) < (pb & ~RESULT_BIT)); } #if LOG_GROUPING > 1 @@ -1838,6 +1840,8 @@ pair_cache_save(struct analyze_closures_env *env, : hash_pair(ob->ids[0], ob->ids[1])); if (ob->t == RPBT_SINGLE_STATE) { assert(ob->ids[0] == ob->ids[1]); + } else { + assert((ob->ids[0] & ~RESULT_BIT) < (ob->ids[1] & ~RESULT_BIT)); } for (size_t nb_i = 0; nb_i < ncount; nb_i++) { @@ -1872,8 +1876,9 @@ pair_cache_save(struct analyze_closures_env *env, #endif if (b->t == RPBT_UNUSED) { /* empty */ - b->ids[0] = pa; - b->ids[1] = pb; + /* Ensure pair ordering is consistent */ + b->ids[0] = pa < pb ? pa : pb; + b->ids[1] = pa < pb ? pb : pa; b->t = type; b->result_id = result_id; htab->buckets_used++; @@ -1917,6 +1922,16 @@ analysis_cache_save_pair(struct analyze_closures_env *env, fsm_state_t a, fsm_state_t b, fsm_state_t result_id) { assert(a != b); + assert((a & RESULT_BIT) == 0); + assert((b & RESULT_BIT) == 0); + + if (a > b) { /* if necessary, swap to ensure a < b */ + const fsm_state_t tmp = a; + a = b; + b = tmp; + } + assert(a < b); + return pair_cache_save(env, RPBT_PAIR, a | RESULT_BIT, b | RESULT_BIT, result_id); } @@ -2012,32 +2027,179 @@ cmp_fsm_state_t(const void *pa, const void *pb) return a < b ? -1 : a > b ? 1 : 0; } +#define SMALL_INPUT_LIMIT 255 +#define SMALL_INPUT_CHECK 0 + +#ifdef SMALL_INPUT_LIMIT +static void +small_input_sort_and_dedup(fsm_state_t *buf, size_t *pused) +{ + /* Alternate sort_and_dedup_dst_buf implementation for when + * the input is small enough to fit in a few cache lines. */ + + const size_t orig_used = *pused; + assert(orig_used > 0 && orig_used <= SMALL_INPUT_LIMIT); + size_t used = 0; + fsm_state_t tmp[SMALL_INPUT_LIMIT]; + + const fsm_state_t MOVED = (fsm_state_t)-1; + + tmp[used++] = buf[0]; + buf[0] = MOVED; + + /* First, insert unique ascending values into a tmp buffer. + * If the input is already mostly sorted, great. */ + for (size_t i = 1; i < orig_used; i++) { + fsm_state_t cur = buf[i]; + if (cur > tmp[used - 1]) { + tmp[used++] = cur; + buf[i] = MOVED; + } + } + + /* Then, do insertion sort for entries that haven't already been + * moved into tmp. They must go somewhere in the middle, because + * entries > the other entries in tmp would have already been + * appended in the first pass. */ + for (size_t i = 1; i < orig_used; i++) { + const fsm_state_t cur = buf[i]; + if (cur == MOVED) { continue; } + for (size_t j = 0; j < used; j++) { + const fsm_state_t other = tmp[j]; + if (cur < other) { /* shift the rest down */ + const size_t to_move = used - j; + memmove(&tmp[j+1], &tmp[j], to_move * sizeof(tmp[j])); + tmp[j] = cur; + used++; + break; + } else if (cur == other) { + break; /* discard duplicate */ + } + } + } + + /* Finally, copy the sorted/dedup'd input back into the buffer. */ + for (size_t i = 0; i < used; i++) { + buf[i] = tmp[i]; + } + *pused = used; + + if (SMALL_INPUT_CHECK) { + for (size_t i = 1; i < used; i++) { + assert(buf[i - 1] < buf[i]); + } + } +} +#endif + static void sort_and_dedup_dst_buf(fsm_state_t *buf, size_t *used) { const size_t orig_used = *used; - qsort(buf, orig_used, sizeof(buf[0]), cmp_fsm_state_t); - - /* squash out duplicates */ - size_t rd = 1; - size_t wr = 1; - while (rd < orig_used) { - if (buf[rd - 1] == buf[rd]) { - rd++; /* skip */ - } else { - buf[wr] = buf[rd]; - rd++; - wr++; + + if (orig_used <= 1) { + return; /* no change */ + } + +#ifdef SMALL_INPUT_LIMIT + if (orig_used <= SMALL_INPUT_LIMIT) { + small_input_sort_and_dedup(buf, used); + return; + } +#endif + + /* Figure out what the min and max values are, because + * when the difference between them is not too large it + * can be significantly faster to avoid qsort here. */ + fsm_state_t min = (fsm_state_t)-1; + fsm_state_t max = 0; + fsm_state_t prev; + int already_sorted_and_unique = 1; + + for (size_t i = 0; i < orig_used; i++) { + const fsm_state_t cur = buf[i]; + if (cur < min) { min = cur; } + if (cur > max) { max = cur; } + + if (i > 0) { + if (cur <= prev) { + already_sorted_and_unique = 0; + } } + prev = cur; } - *used = wr; -#if EXPENSIVE_CHECKS - assert(wr <= orig_used); - for (size_t i = 1; i < *used; i++) { - assert(buf[i - 1] < buf[i]); + /* If the buffer is already sorted and unique, we're done. */ + if (already_sorted_and_unique) { + *used = orig_used; + return; } + + /* If there's only one unique value, then we're done. */ + if (min == max) { + buf[0] = min; + *used = 1; + return; + } + +/* 81920 = 10 KB buffer on the stack. This must be divisible by 64. + * Set to 0 to disable. */ +#define QSORT_CUTOFF 81920 + + if (QSORT_CUTOFF == 0 || max - min > QSORT_CUTOFF) { + /* If the bitset would be very large but sparse due to + * extreme values, then fall back on using qsort and + * then sweeping over the array to squash out + * duplicates. */ + qsort(buf, orig_used, sizeof(buf[0]), cmp_fsm_state_t); + + /* squash out duplicates */ + size_t rd = 1; + size_t wr = 1; + while (rd < orig_used) { + if (buf[rd - 1] == buf[rd]) { + rd++; /* skip */ + } else { + buf[wr] = buf[rd]; + rd++; + wr++; + } + } + + *used = wr; +#if EXPENSIVE_CHECKS + assert(wr <= orig_used); + for (size_t i = 1; i < *used; i++) { + assert(buf[i - 1] < buf[i]); + } #endif + } else { + /* Convert the array into a bitset and back, which sorts + * and deduplicates in the process. Add 1 to avoid a zero- + * zero-length array error if QSORT_CUTOFF is 0. */ + uint64_t bitset[QSORT_CUTOFF/64 + 1]; + const size_t words = u64bitset_words(max - min + 1); + memset(bitset, 0x00, words * sizeof(bitset[0])); + + for (size_t i = 0; i < orig_used; i++) { + u64bitset_set(bitset, buf[i] - min); + } + + size_t dst = 0; + for (size_t i = 0; i < words; i++) { + const uint64_t w = bitset[i]; + if (w != 0) { /* skip empty words */ + uint64_t bit = 0x1; + for (size_t b_i = 0; b_i < 64; b_i++, bit <<= 1) { + if (w & bit) { + buf[dst] = 64*i + b_i + min; + dst++; + } + } + } + } + *used = dst; + } } static int @@ -2323,3 +2485,50 @@ analyze_closures__grow_outputs(struct analyze_closures_env *env) env->output_ceil = nceil; return 1; } + +struct remap_eager_output_env { + bool ok; + struct fsm *dst; + fsm_state_t dst_state; +}; + +static int +remap_eager_output_cb(fsm_state_t state, fsm_output_id_t id, void *opaque) +{ + (void)state; + struct remap_eager_output_env *env = opaque; + if (!fsm_eager_output_set(env->dst, env->dst_state, id)) { + env->ok = false; + return 0; + } + + return 1; +} + +static int +remap_eager_outputs(const struct map *map, struct interned_state_set_pool *issp, + struct fsm *dst_dfa, const struct fsm *src_nfa) +{ + /* For each DFA state, get the set of NFA states corresponding to it from the + * map and issp, then copy every eager output ID over. */ + struct map_iter iter; + for (struct mapping *b = map_first(map, &iter); b != NULL; b = map_next(&iter)) { + struct state_set *ss = interned_state_set_get_state_set(issp, b->iss); + assert(ss != NULL); + + struct state_iter it; + fsm_state_t s; + state_set_reset(ss, &it); + while (state_set_next(&it, &s)) { + struct remap_eager_output_env env = { + .ok = true, + .dst = dst_dfa, + .dst_state = b->dfastate, + }; + fsm_eager_output_iter_state(src_nfa, s, remap_eager_output_cb, &env); + if (!env.ok) { return 0; } + } + } + + return 1; +} diff --git a/src/libfsm/determinise_internal.h b/src/libfsm/determinise_internal.h index 8fe35fcd8..aa3a2cd80 100644 --- a/src/libfsm/determinise_internal.h +++ b/src/libfsm/determinise_internal.h @@ -3,6 +3,7 @@ #include #include +#include #include #include @@ -22,6 +23,7 @@ #include "internal.h" #include "capture.h" #include "endids.h" +#include "eager_output.h" #include @@ -34,6 +36,7 @@ #define LOG_AC 0 #define LOG_GROUPING 0 #define LOG_ANALYSIS_STATS 0 +#define LOG_BUILD_REVERSE_MAPPING 0 #if LOG_DETERMINISE_CAPTURES || LOG_INPUT #include @@ -71,7 +74,7 @@ struct map { }; struct map_iter { - struct map *m; + const struct map *m; size_t i; }; @@ -82,10 +85,12 @@ struct reverse_mapping { }; struct det_copy_capture_actions_env { +#ifndef NDEBUG char tag; +#endif struct fsm *dst; struct reverse_mapping *reverse_mappings; - int ok; + bool ok; }; #define MAPPINGSTACK_DEF_CEIL 16 @@ -289,6 +294,10 @@ analyze_closures__grow_dst(struct analyze_closures_env *env); static int analyze_closures__grow_outputs(struct analyze_closures_env *env); +static int +remap_end_metadata(const struct fsm *src_fsm, const struct state_set *src_set, + struct fsm *dst_fsm, fsm_state_t dst_state); + static int map_add(struct map *map, fsm_state_t dfastate, interned_state_set_id iss, struct mapping **new_mapping); @@ -301,26 +310,17 @@ static void map_free(struct map *map); static struct mapping * -map_first(struct map *map, struct map_iter *iter); +map_first(const struct map *map, struct map_iter *iter); static struct mapping * map_next(struct map_iter *iter); -static int -add_reverse_mapping(const struct fsm_alloc *alloc, - struct reverse_mapping *reverse_mappings, - fsm_state_t dfastate, fsm_state_t nfa_state); - -static int -det_copy_capture_actions(struct reverse_mapping *reverse_mappings, - struct fsm *dst, struct fsm *src); - static int grow_map(struct map *map); static int -remap_capture_actions(struct map *map, struct interned_state_set_pool *issp, - struct fsm *dst_dfa, struct fsm *src_nfa); +remap_eager_outputs(const struct map *map, struct interned_state_set_pool *issp, + struct fsm *dst_dfa, const struct fsm *src_nfa); static struct mappingstack * stack_init(const struct fsm_alloc *alloc); diff --git a/src/libfsm/eager_output.c b/src/libfsm/eager_output.c new file mode 100644 index 000000000..a6dcdc89f --- /dev/null +++ b/src/libfsm/eager_output.c @@ -0,0 +1,435 @@ +/* + * Copyright 2024 Scott Vokes + * + * See LICENCE for the full copyright terms. + */ + +#include +#include + +#include "internal.h" + +#include +#include + +#include +#include +#include + +#include "eager_output.h" + +#define LOG_LEVEL 0 + +/* must be a power of 2 */ +#define DEF_BUCKET_COUNT 4 +#define DEF_ENTRY_CEIL 2 + +struct eager_output_info { + fsm_eager_output_cb *cb; + void *opaque; + + struct eager_output_htab { + size_t bucket_count; + size_t buckets_used; + /* empty if entry is NULL, otherwise keyed by state */ + struct eager_output_bucket { + fsm_state_t state; + struct eager_output_entry { + unsigned used; + unsigned ceil; + fsm_end_id_t ids[]; + } *entry; + } *buckets; + } htab; +}; + +void +fsm_eager_output_set_cb(struct fsm *fsm, fsm_eager_output_cb *cb, void *opaque) +{ +#if LOG_LEVEL > 2 + fprintf(stderr, "-- fsm_eager_output_set_cb %p\n", (void *)fsm); +#endif + assert(fsm != NULL); + assert(fsm->eager_output_info != NULL); + fsm->eager_output_info->cb = cb; + fsm->eager_output_info->opaque = opaque; +} + +void +fsm_eager_output_get_cb(const struct fsm *fsm, fsm_eager_output_cb **cb, void **opaque) +{ + *cb = fsm->eager_output_info->cb; + *opaque = fsm->eager_output_info->opaque; +} + +int +fsm_eager_output_init(struct fsm *fsm) +{ + struct eager_output_info *ei = f_calloc(fsm->alloc, 1, sizeof(*ei)); + + if (ei == NULL) { return 0; } + + struct eager_output_bucket *buckets = f_calloc(fsm->alloc, + DEF_BUCKET_COUNT, sizeof(buckets[0])); + if (buckets == NULL) { + f_free(fsm->alloc, ei); + return 0; + } + +#if LOG_LEVEL > 2 + fprintf(stderr, "-- fsm_eager_output_init %p\n", (void *)fsm); +#endif + + ei->htab.buckets = buckets; + ei->htab.bucket_count = DEF_BUCKET_COUNT; + + fsm->eager_output_info = ei; + return 1; +} + +void +fsm_eager_output_free(struct fsm *fsm) +{ + if (fsm == NULL || fsm->eager_output_info == NULL) { return; } + + for (size_t i = 0; i < fsm->eager_output_info->htab.bucket_count; i++) { + struct eager_output_bucket *b = &fsm->eager_output_info->htab.buckets[i]; + if (b->entry == NULL) { continue; } + f_free(fsm->alloc, b->entry); + } + f_free(fsm->alloc, fsm->eager_output_info->htab.buckets); + + f_free(fsm->alloc, fsm->eager_output_info); +#if LOG_LEVEL > 2 + fprintf(stderr, "-- fsm_eager_output_free %p\n", (void *)fsm); +#endif + fsm->eager_output_info = NULL; +} + +int +fsm_eager_output_set_on_ends(struct fsm *fsm, fsm_output_id_t id) +{ + assert(fsm != NULL); + const size_t count = fsm_countstates(fsm); + for (size_t i = 0; i < count; i++) { + if (fsm_isend(fsm, i)) { + if (!fsm_eager_output_set(fsm, i, id)) { return 0; } + } + } + return 1; +} + +static bool +grow_htab(const struct fsm_alloc *alloc, struct eager_output_htab *htab) +{ + const size_t nbucket_count = 2*htab->bucket_count; + assert(nbucket_count != 0); + + struct eager_output_bucket *nbuckets = f_calloc(alloc, nbucket_count, + sizeof(nbuckets[0])); + if (nbuckets == NULL) { return false; } + + const uint64_t nmask = nbucket_count - 1; + assert((nmask & nbucket_count) == 0); /* power of 2 */ + + for (size_t ob_i = 0; ob_i < htab->bucket_count; ob_i++) { + struct eager_output_bucket *ob = &htab->buckets[ob_i]; + if (ob->entry == NULL) { continue; } + + const uint64_t hash = hash_id(ob->state); + for (size_t probes = 0; probes < nbucket_count; probes++) { + const size_t nb_i = (hash + probes) & nmask; + struct eager_output_bucket *nb = &nbuckets[nb_i]; + if (nb->entry == NULL) { + nb->state = ob->state; + nb->entry = ob->entry; + break; + } else { + assert(nb->state != ob->state); + } + } + } + + f_free(alloc, htab->buckets); + htab->bucket_count = nbucket_count; + htab->buckets = nbuckets; + return true; +} + +int +fsm_eager_output_set(struct fsm *fsm, fsm_state_t state, fsm_output_id_t id) +{ + assert(fsm != NULL); + + struct eager_output_info *info = fsm->eager_output_info; + assert(info->htab.bucket_count > 0); + + if (info->htab.buckets_used >= info->htab.bucket_count/2) { + if (!grow_htab(fsm->alloc, &info->htab)) { return 0; } + } + + const uint64_t hash = hash_id(state); + const uint64_t mask = info->htab.bucket_count - 1; + assert((mask & info->htab.bucket_count) == 0); /* power of 2 */ + + for (size_t probes = 0; probes < info->htab.bucket_count; probes++) { + const size_t b_i = (hash + probes) & mask; + struct eager_output_bucket *b = &info->htab.buckets[b_i]; + struct eager_output_entry *e = b->entry; + if (e == NULL) { /* empty, add */ + const size_t alloc_sz = sizeof(*e) + + DEF_ENTRY_CEIL * sizeof(e->ids[0]); + e = f_calloc(fsm->alloc, 1, alloc_sz); + if (e == NULL) { + return 0; + } + e->ceil = DEF_ENTRY_CEIL; + b->state = state; + b->entry = e; + info->htab.buckets_used++; + } else if (b->state != state) { /* collision */ + continue; + } + + if (e->used == e->ceil) { + const size_t nceil = 2 * e->ceil; + const size_t nsize = sizeof(*e) + + nceil * sizeof(e->ids[0]); + struct eager_output_entry *nentry = f_realloc(fsm->alloc, e, nsize); + if (nentry == NULL) { return 0; } + nentry->ceil = nceil; + b->entry = nentry; + e = b->entry; + } + + /* ignore duplicates */ + for (size_t i = 0; i < e->used; i++) { + if (e->ids[i] == id) { return 1; } + } + + e->ids[e->used++] = id; + fsm->states[state].has_eager_outputs = 1; + return 1; + } + + return 1; +} + +bool +fsm_eager_output_has_eager_output(const struct fsm *fsm) +{ + assert(fsm->eager_output_info != NULL); + const struct eager_output_htab *htab = &fsm->eager_output_info->htab; + + for (size_t b_i = 0; b_i < htab->bucket_count; b_i++) { + struct eager_output_bucket *b = &htab->buckets[b_i]; + if (b->entry == NULL) { continue; } + if (b->entry->used > 0) { return 1; } + } + return 0; +} + +bool +fsm_eager_output_state_has_eager_output(const struct fsm *fsm, fsm_state_t state) +{ + assert(state < fsm->statecount); + return fsm->states[state].has_eager_outputs; +} + +void +fsm_eager_output_iter_state(const struct fsm *fsm, + fsm_state_t state, fsm_eager_output_iter_cb *cb, void *opaque) +{ + assert(fsm != NULL); + assert(cb != NULL); + + const uint64_t hash = hash_id(state); + + struct eager_output_info *info = fsm->eager_output_info; + const uint64_t mask = info->htab.bucket_count - 1; + assert((mask & info->htab.bucket_count) == 0); /* power of 2 */ + + for (size_t probes = 0; probes < info->htab.bucket_count; probes++) { + const size_t b_i = (hash + probes) & mask; + struct eager_output_bucket *b = &info->htab.buckets[b_i]; + struct eager_output_entry *e = b->entry; + if (e == NULL) { /* empty */ + return; + } else if (b->state != state) { /* collision */ + continue; + } + + assert(e->used == 0 || fsm->states[state].has_eager_outputs); + + for (size_t i = 0; i < e->used; i++) { + if (!cb(state, e->ids[i], opaque)) { return; } + } + } +} + +static int +inc_cb(fsm_state_t state, fsm_output_id_t id, void *opaque) +{ + (void)state; + (void)id; + size_t *count = opaque; + (*count)++; + return 1; +} + +/* Get the number of eager output IDs associated with a state. */ +size_t +fsm_eager_output_count(const struct fsm *fsm, fsm_state_t state) +{ + size_t res = 0; + fsm_eager_output_iter_state(fsm, state, inc_cb, (void *)&res); + return res; +} + +struct get_env { + bool ok; + size_t count; + size_t ceil; + fsm_output_id_t *buf; +}; + +static int +append_cb(fsm_state_t state, fsm_output_id_t id, void *opaque) +{ + struct get_env *env = opaque; + (void)state; + if (env->count == env->ceil) { + env->ok = false; + return 0; + } + env->buf[env->count++] = id; + return 1; +} + +static int +cmp_fsm_output_id_t(const void *pa, const void *pb) +{ + const fsm_output_id_t a = *(fsm_output_id_t *)pa; + const fsm_output_id_t b = *(fsm_output_id_t *)pb; + return a < b ? -1 : a > b ? 1 : 0; +} + +int +fsm_eager_output_get(const struct fsm *fsm, fsm_state_t state, + size_t buf_count, fsm_output_id_t *id_buf) +{ + struct get_env env = { + .ok = true, + .buf = id_buf, + .ceil = buf_count, + }; + fsm_eager_output_iter_state(fsm, state, append_cb, &env); + qsort(id_buf, env.count, sizeof(id_buf[0]), cmp_fsm_output_id_t); + return env.ok ? 1 : 0; +} + +void +fsm_eager_output_iter_all(const struct fsm *fsm, + fsm_eager_output_iter_cb *cb, void *opaque) +{ + assert(fsm != NULL); + assert(cb != NULL); + assert(fsm->eager_output_info != NULL); + + struct eager_output_info *info = fsm->eager_output_info; + + for (size_t b_i = 0; b_i < info->htab.bucket_count; b_i++) { + struct eager_output_bucket *b = &info->htab.buckets[b_i]; + struct eager_output_entry *e = b->entry; + if (e == NULL) { /* empty */ + continue; + } + assert(e->used == 0 || fsm->states[b->state].has_eager_outputs); + + for (size_t i = 0; i < e->used; i++) { + if (!cb(b->state, e->ids[i], opaque)) { return; } + } + } +} + +struct dump_env { + FILE *f; + size_t count; +}; + +static int +dump_cb(fsm_state_t state, fsm_end_id_t id, void *opaque) + +{ + struct dump_env *env = opaque; + fprintf(env->f, "-- %d: id %d\n", state, id); + env->count++; + return 1; +} + +void +fsm_eager_output_dump(FILE *f, const struct fsm *fsm) +{ + struct dump_env env = { .f = f }; + fprintf(f, "%s:\n", __func__); + fsm_eager_output_iter_all(fsm, dump_cb, (void *)&env); + fprintf(f, "== %zu total\n", env.count); +} + +int +fsm_eager_output_compact(struct fsm *fsm, fsm_state_t *mapping, size_t mapping_count) +{ + /* Don't reallocate unless something has actually changed. */ + bool changes = false; + for (size_t i = 0; i < mapping_count; i++) { + if (mapping[i] != i) { + changes = true; + break; + } + } + + /* nothing to do */ + if (!changes) { return 1; } + + struct eager_output_info *eoi = fsm->eager_output_info; + + struct eager_output_bucket *nbuckets = f_calloc(fsm->alloc, + eoi->htab.bucket_count, sizeof(nbuckets[0])); + if (nbuckets == NULL) { + return 0; + } + + const uint64_t mask = eoi->htab.bucket_count - 1; + assert((eoi->htab.bucket_count & mask) == 0); + + for (size_t ob_i = 0; ob_i < eoi->htab.bucket_count; ob_i++) { + const struct eager_output_bucket *ob = &eoi->htab.buckets[ob_i]; + if (ob->entry == NULL) { continue; } + + assert(ob->state < mapping_count); + const fsm_state_t nstate = mapping[ob->state]; + if (nstate == FSM_STATE_REMAP_NO_STATE) { + f_free(fsm->alloc, ob->entry); + continue; + } + + const uint64_t hash = hash_id(nstate); + + bool placed = false; + for (size_t probes = 0; probes < eoi->htab.bucket_count; probes++) { + const size_t nb_i = (hash + probes) & mask; + struct eager_output_bucket *nb = &nbuckets[nb_i]; + if (nb->entry == NULL) { + nb->state = nstate; + nb->entry = ob->entry; + placed = true; + break; + } + } + assert(placed); + } + + f_free(fsm->alloc, eoi->htab.buckets); + eoi->htab.buckets = nbuckets; + return 1; +} diff --git a/src/libfsm/eager_output.h b/src/libfsm/eager_output.h new file mode 100644 index 000000000..b90da935c --- /dev/null +++ b/src/libfsm/eager_output.h @@ -0,0 +1,54 @@ +/* + * Copyright 2024 Scott Vokes + * + * See LICENCE for the full copyright terms. + */ + +#ifndef EAGER_OUTPUT_H +#define EAGER_OUTPUT_H + +#include +#include +#include + +struct eager_output_info; + +int +fsm_eager_output_init(struct fsm *fsm); + +void +fsm_eager_output_free(struct fsm *fsm); + +/* Does an FSM have eager outputs? */ +bool +fsm_eager_output_has_eager_output(const struct fsm *fsm); + +/* Does a particular state have eager outputs? */ +bool +fsm_eager_output_state_has_eager_output(const struct fsm *fsm, fsm_state_t state); + +/* Dump eager outputs on an FSM. (For debugging.) */ +void +fsm_eager_output_dump(FILE *f, const struct fsm *fsm); + +/* Callback for fsm_eager_output_iter_*. + * The return value indicates whether iteration should continue. + * The results may not be sorted in any particular order. */ +typedef int +fsm_eager_output_iter_cb(fsm_state_t state, fsm_output_id_t id, void *opaque); + +/* Iterate over eager outputs on a state. */ +void +fsm_eager_output_iter_state(const struct fsm *fsm, + fsm_state_t state, fsm_eager_output_iter_cb *cb, void *opaque); + +/* Iterate over all eager outputs on an FSM. */ +void +fsm_eager_output_iter_all(const struct fsm *fsm, + fsm_eager_output_iter_cb *cb, void *opaque); + +/* Compact eager output metadata. */ +int +fsm_eager_output_compact(struct fsm *fsm, fsm_state_t *mapping, size_t mapping_count); + +#endif diff --git a/src/libfsm/edge.c b/src/libfsm/edge.c index d48ca141c..379a19752 100644 --- a/src/libfsm/edge.c +++ b/src/libfsm/edge.c @@ -24,7 +24,7 @@ fsm_addedge_epsilon(struct fsm *fsm, assert(from < fsm->statecount); assert(to < fsm->statecount); - if (!state_set_add(&fsm->states[from].epsilons, fsm->opt->alloc, to)) { + if (!state_set_add(&fsm->states[from].epsilons, fsm->alloc, to)) { return 0; } @@ -60,7 +60,7 @@ fsm_addedge_literal(struct fsm *fsm, assert(from < fsm->statecount); assert(to < fsm->statecount); - if (!edge_set_add(&fsm->states[from].edges, fsm->opt->alloc, (unsigned char)c, to)) { + if (!edge_set_add(&fsm->states[from].edges, fsm->alloc, (unsigned char)c, to)) { return 0; } diff --git a/src/libfsm/endids.c b/src/libfsm/endids.c index 444ccbc2e..5956e30fc 100644 --- a/src/libfsm/endids.c +++ b/src/libfsm/endids.c @@ -4,10 +4,49 @@ * See LICENCE for the full copyright terms. */ +#include +#include + #include +#include +#include #include -#include "endids_internal.h" +#include +#include +#include +#include + +#include +#include + +#include +#include +#include + +#include "internal.h" +#include "endids.h" + +#define BUCKET_NO_STATE ((fsm_state_t)-1) +#define DEF_BUCKET_COUNT 4 +#define DEF_BUCKET_ID_COUNT 16 + +struct endid_info { + /* Add-only hash table, with a state ID and an associated + * non-empty ordered array of unique end IDs. The state is the + * key. Grows when the buckets are more than half full. */ + unsigned bucket_count; + unsigned buckets_used; + + struct endid_info_bucket { + fsm_state_t state; + struct end_info_ids { + unsigned count; + unsigned ceil; + fsm_end_id_t ids[1]; + } *ids; + } *buckets; +}; #define LOG_ENDIDS 0 @@ -51,9 +90,16 @@ dump_buckets(const char *tag, const struct endid_info *ei) size_t j; for (j = 0; j < ei->bucket_count; j++) { struct endid_info_bucket *b = &ei->buckets[j]; - fprintf(stderr, "%s[%lu/%u]: %d (on %p)\n", + fprintf(stderr, "%s[%lu/%u]: %d (on %p)", tag, j, ei->bucket_count, b->state, (void *)ei->buckets); + if (b->state != BUCKET_NO_STATE) { + fprintf(stderr, ":"); + for (size_t i = 0; i < b->ids->count; i++) { + fprintf(stderr, " %u", b->ids->ids[i]); + } + } + fprintf(stderr, "\n"); } #else (void)tag; @@ -69,13 +115,11 @@ fsm_setendid(struct fsm *fsm, fsm_end_id_t id) /* for every end state */ for (i = 0; i < fsm->statecount; i++) { if (fsm_isend(fsm, i)) { - enum fsm_endid_set_res sres; #if LOG_ENDIDS > 3 fprintf(stderr, "fsm_setendid: setting id %u on state %d\n", id, i); #endif - sres = fsm_endid_set(fsm, i, id); - if (sres == FSM_ENDID_SET_ERROR_ALLOC_FAIL) { + if (fsm_endid_set(fsm, i, id) < 0) { return 0; } } @@ -84,13 +128,20 @@ fsm_setendid(struct fsm *fsm, fsm_end_id_t id) return 1; } +int +fsm_setendid_state(struct fsm *fsm, fsm_state_t s, fsm_end_id_t id) +{ + enum fsm_endid_set_res sres; + sres = fsm_endid_set(fsm, s, id); + return sres != FSM_ENDID_SET_ERROR_ALLOC_FAIL; +} + enum fsm_getendids_res fsm_getendids(const struct fsm *fsm, fsm_state_t end_state, - size_t id_buf_count, fsm_end_id_t *id_buf, - size_t *ids_written) + size_t id_buf_count, fsm_end_id_t *id_buf) { return fsm_endid_get(fsm, end_state, - id_buf_count, id_buf, ids_written); + id_buf_count, id_buf); } size_t @@ -104,15 +155,15 @@ fsm_endid_init(struct fsm *fsm) { struct endid_info_bucket *buckets = NULL; size_t i; - struct endid_info *res = f_calloc(fsm->opt->alloc, 1, sizeof(*res)); + struct endid_info *res = f_malloc(fsm->alloc, sizeof(*res)); if (res == NULL) { return 0; } - buckets = f_malloc(fsm->opt->alloc, + buckets = f_malloc(fsm->alloc, DEF_BUCKET_COUNT * sizeof(buckets[0])); if (buckets == NULL) { - f_free(fsm->opt->alloc, res); + f_free(fsm->alloc, res); return 0; } @@ -148,10 +199,10 @@ fsm_endid_free(struct fsm *fsm) if (b->state == BUCKET_NO_STATE) { continue; } - f_free(fsm->opt->alloc, fsm->endid_info->buckets[i].ids); + f_free(fsm->alloc, fsm->endid_info->buckets[i].ids); } - f_free(fsm->opt->alloc, fsm->endid_info->buckets); - f_free(fsm->opt->alloc, fsm->endid_info); + f_free(fsm->alloc, fsm->endid_info->buckets); + f_free(fsm->alloc, fsm->endid_info); } static int @@ -245,7 +296,7 @@ endid_find_bucket(struct fsm *fsm, fsm_state_t state) else if (b->state == BUCKET_NO_STATE) { /* empty */ - LOG_2("fsm_endid_set: setting empty bucket %lu\n", b_i); + LOG_2("fsm_endid_set: found empty bucket %lu\n", b_i); if (ei->buckets_used == ei->bucket_count / 2) { assert(!has_grown); /* should only happen once */ @@ -253,7 +304,7 @@ endid_find_bucket(struct fsm *fsm, fsm_state_t state) LOG_2(" -- growing buckets [%u/%u used], then rehashing\n", ei->buckets_used, ei->bucket_count); - if (!grow_endid_buckets(fsm->opt->alloc, ei)) { + if (!grow_endid_buckets(fsm->alloc, ei)) { return NULL; } has_grown = 1; @@ -381,7 +432,7 @@ allocate_ids(const struct fsm *fsm, struct end_info_ids *prev, size_t n) assert(n > 0); id_alloc_size = sizeof(*ids) + (n - 1) * sizeof(ids->ids[0]); - return f_realloc(fsm->opt->alloc, prev, id_alloc_size); + return f_realloc(fsm->alloc, prev, id_alloc_size); } enum fsm_endid_set_res @@ -495,7 +546,7 @@ cmp_endids(const void *pa, const void *pb) return 0; } -enum fsm_endid_set_res +int fsm_endid_set_bulk(struct fsm *fsm, fsm_state_t state, size_t num_ids, const fsm_end_id_t *ids, enum fsm_endid_bulk_op op) { @@ -509,7 +560,7 @@ fsm_endid_set_bulk(struct fsm *fsm, b = endid_find_bucket(fsm, state); if (b == NULL) { - return FSM_ENDID_SET_ERROR_ALLOC_FAIL; + return 0; } if (b->state == state) { @@ -532,12 +583,12 @@ fsm_endid_set_bulk(struct fsm *fsm, if (new_ceil < total_count) { /* num_ids is too large? */ - return FSM_ENDID_SET_ERROR_ALLOC_FAIL; + return 0; } new_ids = allocate_ids(fsm, b->ids, new_ceil); if (new_ids == NULL) { - return FSM_ENDID_SET_ERROR_ALLOC_FAIL; + return 0; } b->ids = new_ids; @@ -567,12 +618,12 @@ fsm_endid_set_bulk(struct fsm *fsm, if (n < num_ids) { /* num_ids is too large? */ - return FSM_ENDID_SET_ERROR_ALLOC_FAIL; + return 0; } new_ids = allocate_ids(fsm, NULL, n); if (new_ids == NULL) { - return FSM_ENDID_SET_ERROR_ALLOC_FAIL; + return 0; } memcpy(&new_ids->ids[0], &ids[0], num_ids * sizeof ids[0]); @@ -609,7 +660,7 @@ fsm_endid_set_bulk(struct fsm *fsm, b->ids->count = j; } - return FSM_ENDID_SET_ADDED; + return 1; } int @@ -700,24 +751,19 @@ fsm_endid_count(const struct fsm *fsm, enum fsm_getendids_res fsm_endid_get(const struct fsm *fsm, fsm_state_t end_state, - size_t id_buf_count, fsm_end_id_t *id_buf, - size_t *ids_written) + size_t id_buf_count, fsm_end_id_t *id_buf) { size_t i; - size_t written = 0; const struct endid_info *ei = NULL; uint64_t hash = hash_id(end_state); uint64_t mask; - (void)written; - assert(fsm != NULL); ei = fsm->endid_info; assert(ei != NULL); assert(id_buf != NULL); - assert(ids_written != NULL); mask = ei->bucket_count - 1; /* bucket count is a power of 2 */ @@ -738,7 +784,6 @@ fsm_endid_get(const struct fsm *fsm, fsm_state_t end_state, #if LOG_ENDIDS > 2 fprintf(stderr, "fsm_endid_get: not found\n"); #endif - *ids_written = 0; /* not found */ return FSM_GETENDIDS_NOT_FOUND; } else if (b->state == end_state) { size_t id_i; @@ -755,8 +800,13 @@ fsm_endid_get(const struct fsm *fsm, fsm_state_t end_state, id_buf[id_i] = b->ids->ids[id_i]; } - /* todo: could sort them here, if it matters. */ - *ids_written = b->ids->count; + #if EXPENSIVE_CHECKS + /* confirm they are sorted and unique */ + for (id_i = 1; id_i < b->ids->count; id_i++) { + assert(id_buf[id_i] > id_buf[id_i - 1]); + } + #endif + return FSM_GETENDIDS_FOUND; } else { /* collision */ #if LOG_ENDIDS > 4 @@ -771,24 +821,27 @@ fsm_endid_get(const struct fsm *fsm, fsm_state_t end_state, } struct carry_env { +#ifndef NDEBUG char tag; +#endif struct fsm *dst; fsm_state_t dst_state; - int ok; + bool ok; }; static int -carry_iter_cb(fsm_state_t state, fsm_end_id_t id, void *opaque) +carry_iter_cb(const struct fsm *fsm, fsm_state_t state, + size_t nth, fsm_end_id_t id, void *opaque) { - enum fsm_endid_set_res sres; struct carry_env *env = opaque; assert(env->tag == 'C'); + (void)fsm; (void)state; + (void)nth; - sres = fsm_endid_set(env->dst, env->dst_state, id); - if (sres == FSM_ENDID_SET_ERROR_ALLOC_FAIL) { - env->ok = 0; + if (fsm_endid_set(env->dst, env->dst_state, id) < 0) { + env->ok = false; return 0; } return 1; @@ -816,10 +869,12 @@ fsm_endid_carry(const struct fsm *src_fsm, const struct state_set *src_set, for (state_set_reset(src_set, &it); state_set_next(&it, &s); ) { struct carry_env env; +#ifndef NDEBUG env.tag = 'C'; +#endif env.dst = dst_fsm; env.dst_state = dst_state; - env.ok = 1; + env.ok = true; if (!fsm_isend(src_fsm, s)) { continue; @@ -839,6 +894,72 @@ fsm_endid_carry(const struct fsm *src_fsm, const struct state_set *src_set, return 1; } +/* Make a new hash table, copying over converted entries and/or discarding. */ +int +fsm_endid_compact(struct fsm *fsm, + const fsm_state_t *mapping, size_t mapping_count) +{ + struct endid_info *info = fsm->endid_info; + const size_t ocount = info->bucket_count; + const size_t ncount = ocount; /* does not need to grow */ + struct endid_info_bucket *obuckets = info->buckets; + struct endid_info_bucket *nbuckets = f_malloc(fsm->alloc, + ncount * sizeof(nbuckets[0])); + const size_t nmask = ncount - 1; + size_t ob_i, nb_i; + size_t moved = 0; + +#if LOG_ENDIDS > 3 + fprintf(stderr, "fsm_endid_compact: rehashing mapped entries\n"); +#endif + + if (nbuckets == NULL) { + return 0; + } + + for (nb_i = 0; nb_i < ncount; nb_i++) { /* clear table */ + nbuckets[nb_i].state = BUCKET_NO_STATE; + } + + for (ob_i = 0; ob_i < ocount; ob_i++) { + const struct endid_info_bucket *ob = &obuckets[ob_i]; + uint64_t hash; + fsm_state_t nstate; + + if (ob->state == BUCKET_NO_STATE) { + continue; + } + + assert(ob->state < mapping_count); + nstate = mapping[ob->state]; + + if (nstate == FSM_STATE_REMAP_NO_STATE) { + info->buckets_used--; /* discarded */ + continue; + } + + hash = hash_id(nstate); + for (nb_i = 0; nb_i < ncount; nb_i++) { + struct endid_info_bucket *nb = &nbuckets[(hash + nb_i) & nmask]; + if (nb->state == BUCKET_NO_STATE) { + nb->state = nstate; + nb->ids = ob->ids; + moved++; + break; + } else { + continue; /* collision */ + } + } + } + + assert(moved == info->buckets_used); + + f_free(fsm->alloc, info->buckets); + info->bucket_count = ncount; + info->buckets = nbuckets; + return 1; +} + void fsm_endid_iter(const struct fsm *fsm, fsm_endid_iter_cb *cb, void *opaque) @@ -867,7 +988,7 @@ fsm_endid_iter(const struct fsm *fsm, count = b->ids->count; for (id_i = 0; id_i < count; id_i++) { - if (!cb(b->state, b->ids->ids[id_i], opaque)) { + if (!cb(fsm, b->state, id_i, b->ids->ids[id_i], opaque)) { break; } @@ -894,6 +1015,10 @@ fsm_endid_iter_bulk(const struct fsm *fsm, return 1; } +#ifndef NDEBUG + const fsm_state_t state_count = fsm_countstates(fsm); +#endif + bucket_count = ei->bucket_count; for (b_i = 0; b_i < bucket_count; b_i++) { @@ -905,6 +1030,7 @@ fsm_endid_iter_bulk(const struct fsm *fsm, count = b->ids->count; + assert(b->state < state_count); if (!cb(b->state, &b->ids->ids[0], count, opaque)) { return 0; } @@ -945,8 +1071,8 @@ fsm_endid_iter_state(const struct fsm *fsm, fsm_state_t state, assert((mask & bucket_count) == 0); /* power of 2 */ #if LOG_ENDIDS > 2 - fprintf(stderr, "fsm_endid_iter_state: state %d -> hash %" PRIx64 "\n", - state, hash); + fprintf(stderr, "fsm_endid_iter_state: state %d -> hash %" PRIx64 ", masked %" PRIx64"\n", + state, hash, hash & mask); #endif #if LOG_ENDIDS > 3 @@ -969,7 +1095,7 @@ fsm_endid_iter_state(const struct fsm *fsm, fsm_state_t state, fprintf(stderr, "fsm_endid_iter_state[%d], ids[%ld] -> %d\n", b->state, id_i, b->ids->ids[id_i]); #endif - if (!cb(b->state, b->ids->ids[id_i], opaque)) { + if (!cb(fsm, b->state, id_i, b->ids->ids[id_i], opaque)) { return; } id_i++; @@ -991,10 +1117,13 @@ struct dump_env { }; static int -dump_cb(fsm_state_t state, const fsm_end_id_t id, void *opaque) +dump_cb(const struct fsm *fsm, fsm_state_t state, + size_t nth, const fsm_end_id_t id, void *opaque) { struct dump_env *env = opaque; - fprintf(env->f, "state[%u]: %u\n", state, id); + fprintf(env->f, "endids: state[%u]: %u\n", state, id); + (void)fsm; + (void)nth; return 1; } @@ -1039,4 +1168,3 @@ fsm_increndids(struct fsm * fsm, int delta) fsm_mapendids(fsm, incr_remap, &delta); } - diff --git a/src/libfsm/endids.h b/src/libfsm/endids.h index 6c46567b3..51bbaf956 100644 --- a/src/libfsm/endids.h +++ b/src/libfsm/endids.h @@ -1,20 +1,12 @@ #ifndef ENDIDS_H #define ENDIDS_H -#include -#include - int fsm_endid_init(struct fsm *fsm); void fsm_endid_free(struct fsm *fsm); -enum fsm_endid_set_res { - FSM_ENDID_SET_ADDED, - FSM_ENDID_SET_ALREADY_PRESENT, - FSM_ENDID_SET_ERROR_ALLOC_FAIL = -1 -}; enum fsm_endid_set_res fsm_endid_set(struct fsm *fsm, fsm_state_t state, fsm_end_id_t id); @@ -34,8 +26,7 @@ fsm_endid_set(struct fsm *fsm, * * The caller maintains ownership of ids, and must free it if needed. * - * Only FSM_ENDID_SET_ADDED and FSM_ENDID_SET_ERROR_ALLOC_FAIL - * are returned. + * Returns 1 on success, 0 on failure. */ enum fsm_endid_bulk_op { FSM_ENDID_BULK_REPLACE = 0, @@ -49,19 +40,31 @@ size_t fsm_endid_count(const struct fsm *fsm, fsm_state_t state); +/* Get the end IDs associated with an end state, if any. + * If id_buf has enough cells to store all the end IDs (according + * to id_buf_count) then they are written into id_buf[] and + * *ids_written is set to the number of IDs. The end IDs in the + * buffer may appear in any order, but should not have duplicates. + * + * Returns 0 if there is not enough space in id_buf for the + * end IDs, or 1 if zero or more end IDs were returned. */ enum fsm_getendids_res fsm_endid_get(const struct fsm *fsm, fsm_state_t end_state, - size_t id_buf_count, fsm_end_id_t *id_buf, - size_t *ids_written); + size_t id_buf_count, fsm_end_id_t *id_buf); int fsm_endid_carry(const struct fsm *src_fsm, const struct state_set *src_set, struct fsm *dst_fsm, fsm_state_t dst_state); +int +fsm_endid_compact(struct fsm *fsm, + const fsm_state_t *mapping, size_t mapping_count); + /* Callback when iterating over the endids. * Return 0 to halt, or non-zero to continue. */ typedef int -fsm_endid_iter_cb(fsm_state_t state, const fsm_end_id_t id, void *opaque); +fsm_endid_iter_cb(const struct fsm *fsm, fsm_state_t state, + size_t nth, const fsm_end_id_t id, void *opaque); void fsm_endid_iter(const struct fsm *fsm, diff --git a/src/libfsm/endids_internal.h b/src/libfsm/endids_internal.h deleted file mode 100644 index 27450af3b..000000000 --- a/src/libfsm/endids_internal.h +++ /dev/null @@ -1,43 +0,0 @@ -#ifndef ENDIDS_INTERNAL_H -#define ENDIDS_INTERNAL_H - -#include -#include - -#include -#include -#include -#include - -#include -#include - -#include -#include -#include - -#include "internal.h" -#include "endids.h" - -#define BUCKET_NO_STATE ((fsm_state_t)-1) -#define DEF_BUCKET_COUNT 4 -#define DEF_BUCKET_ID_COUNT 16 - -struct endid_info { - /* Add-only hash table, with a state ID and an associated - * non-empty ordered array of unique end IDs. The state is the - * key. Grows when the buckets are more than half full. */ - unsigned bucket_count; - unsigned buckets_used; - - struct endid_info_bucket { - fsm_state_t state; - struct end_info_ids { - unsigned count; - unsigned ceil; - fsm_end_id_t ids[1]; - } *ids; - } *buckets; -}; - -#endif diff --git a/src/libfsm/epsilons.c b/src/libfsm/epsilons.c index e87d9d974..aadc6ec84 100644 --- a/src/libfsm/epsilons.c +++ b/src/libfsm/epsilons.c @@ -6,67 +6,150 @@ #include #include +#include +#include #include +#include #include #include +#include #include +#include #include #include #include +#include #include "internal.h" #include "capture.h" #include "endids.h" +#include "eager_output.h" #define DUMP_EPSILON_CLOSURES 0 -#define DEF_PENDING_CAPTURE_ACTIONS_CEIL 2 #define LOG_RM_EPSILONS_CAPTURES 0 -#define DEF_CARRY_ENDIDS_COUNT 2 +#define LOG_COPYING 0 +#define LOG_RESULT 0 -struct remap_env { - char tag; +/* #define DEF_CARRY_ENDIDS_COUNT 2 */ +/* #define DEF_CARRY_CAPTUREIDS_COUNT 2 */ + +#if LOG_RESULT +#include +#endif + +#define DEF_END_METADATA_ENDIDS_CEIL 4 +#define DEF_END_METADATA_CAPTUREIDS_CEIL 4 +#define DEF_END_METADATA_PROGRAMIDS_CEIL 4 +struct carry_end_metadata_env { + struct fsm *fsm; const struct fsm_alloc *alloc; - struct state_set **rmap; - int ok; - size_t count; - size_t ceil; - struct remap_action { - fsm_state_t state; - enum capture_action_type type; - unsigned capture_id; - fsm_state_t to; - } *actions; + struct { + size_t ceil; + fsm_end_id_t *ids; + } end; + struct { + int ok; + size_t count; + size_t ceil; + unsigned *ids; + } capture; + struct { + int ok; + size_t count; + size_t ceil; + uint32_t *ids; + } program; }; static int -remap_capture_actions(struct fsm *nfa, struct state_set **eclosures); +carry_end_metadata(struct carry_end_metadata_env *env, + fsm_state_t end_state, fsm_state_t dst_state); -static int -remap_capture_action_cb(fsm_state_t state, - enum capture_action_type type, unsigned capture_id, fsm_state_t to, - void *opaque); +static void +mark_states_reachable_by_label(const struct fsm *nfa, uint64_t *reachable_by_label); + +struct eager_output_buf { +#define DEF_EAGER_OUTPUT_BUF_CEIL 8 + bool ok; + const struct fsm_alloc *alloc; + size_t ceil; + size_t used; + fsm_output_id_t *ids; +}; + +static bool +append_eager_output_id(struct eager_output_buf *buf, fsm_output_id_t id) +{ + if (buf->used == buf->ceil) { + const size_t nceil = buf->ceil == 0 ? DEF_EAGER_OUTPUT_BUF_CEIL : 2*buf->ceil; + fsm_output_id_t *nids = f_realloc(buf->alloc, buf->ids, nceil * sizeof(nids[0])); + if (nids == NULL) { + buf->ok = false; + return false; + } + buf->ids = nids; + buf->ceil = nceil; + } + + for (size_t i = 0; i < buf->used; i++) { + /* avoid duplicates */ + if (buf->ids[i] == id) { return true; } + } + + buf->ids[buf->used++] = id; + return true; +} static int -carry_endids(struct fsm *fsm, struct state_set *states, - fsm_state_t s); +collect_eager_output_ids_cb(fsm_state_t state, fsm_output_id_t id, void *opaque) +{ + (void)state; + struct eager_output_buf *buf = opaque; + return append_eager_output_id(buf, id) ? 1 : 0; +} int fsm_remove_epsilons(struct fsm *nfa) { +#if LOG_RESULT + fprintf(stderr, "==== before\n"); + fsm_print_fsm(stderr, nfa); + fsm_capture_dump(stderr, "#### before_remove_epsilons", nfa); + fprintf(stderr, "====\n"); +#endif + const size_t state_count = fsm_countstates(nfa); int res = 0; struct state_set **eclosures = NULL; - fsm_state_t s; + fsm_state_t s, start_id; + const struct fsm_alloc *alloc = nfa->alloc; INIT_TIMERS(); + struct carry_end_metadata_env em_env = { 0 }; + em_env.fsm = nfa; + em_env.alloc = alloc; + + struct eager_output_buf eager_output_buf = { + .ok = true, + .alloc = nfa->alloc, + }; + uint64_t *reachable_by_label = NULL; + assert(nfa != NULL); + if (!fsm_getstart(nfa, &start_id)) { + goto cleanup; + } + + /* TODO: This could successfully exit early if none of the + * states have epsilon edges. */ + TIME(&pre); - eclosures = epsilon_closure(nfa); + eclosures = fsm_epsilon_closure(nfa); TIME(&post); DIFF_MSEC("epsilon_closure", pre, post, NULL); @@ -90,6 +173,17 @@ fsm_remove_epsilons(struct fsm *nfa) } #endif + const size_t state_words = u64bitset_words(state_count); + reachable_by_label = f_calloc(nfa->alloc, state_words, sizeof(reachable_by_label[0])); + if (reachable_by_label == NULL) { goto cleanup; } + + mark_states_reachable_by_label(nfa, reachable_by_label); + + fsm_state_t start; + if (!fsm_getstart(nfa, &start)) { + goto cleanup; /* no start state */ + } + for (s = 0; s < state_count; s++) { struct state_iter si; fsm_state_t es_id; @@ -97,6 +191,12 @@ fsm_remove_epsilons(struct fsm *nfa) struct edge_group_iter egi; struct edge_group_iter_info info; + /* If the state isn't reachable by a label and isn't the start state, + * skip processing -- it will soon become garbage. */ + if (!u64bitset_get(reachable_by_label, s) && s != start) { + continue; + } + /* Process the epsilon closure. */ state_set_reset(eclosures[s], &si); while (state_set_next(&si, &es_id)) { @@ -107,24 +207,29 @@ fsm_remove_epsilons(struct fsm *nfa) * end states. * * Similarly, any end state metadata on states - * in its epsilon-closure is copied to it. - * - * Capture actions are copied in a later pass. */ + * in its epsilon-closure is copied to it. */ if (fsm_isend(nfa, es_id)) { #if LOG_COPYING fprintf(stderr, "remove_epsilons: setting end on %d (due to %d)\n", s, es_id); #endif fsm_setend(nfa, s, 1); - /* - * Carry through end IDs, if present. This isn't anything to do - * with the NFA conversion; it's meaningful only to the caller. - */ - if (!carry_endids(nfa, eclosures[s], s)) { + if (!carry_end_metadata(&em_env, es_id, s)) { goto cleanup; } } + /* Collect every eager output ID from any state + * in the current state's epsilon closure to the + * current state. These will be added at the end. */ + { + const size_t count = fsm_eager_output_count(nfa, es_id); + if (count > 0) { + fsm_eager_output_iter_state(nfa, es_id, collect_eager_output_ids_cb, &eager_output_buf); + if (!eager_output_buf.ok) { goto cleanup; } + } + } + /* For every state in this state's transitive * epsilon closure, add all of their sets of * labeled edges. */ @@ -134,12 +239,19 @@ fsm_remove_epsilons(struct fsm *nfa) fprintf(stderr, "%s: bulk-copying edges leading to state %d onto state %d (from state %d)\n", __func__, info.to, s, es_id); #endif - if (!edge_set_add_bulk(&nfa->states[s].edges, nfa->opt->alloc, + if (!edge_set_add_bulk(&nfa->states[s].edges, nfa->alloc, info.symbols, info.to)) { goto cleanup; } } } + + for (size_t i = 0; i < eager_output_buf.used; i++) { + if (!fsm_eager_output_set(nfa, s, eager_output_buf.ids[i])) { + goto cleanup; + } + } + eager_output_buf.used = 0; /* clear */ } /* Remove the epsilon-edge state sets from everything. @@ -150,272 +262,188 @@ fsm_remove_epsilons(struct fsm *nfa) state->epsilons = NULL; } -#if LOG_RESULT - fprintf(stderr, "=== %s: about to update capture actions\n", __func__); - fsm_print_fsm(stderr, nfa); -#endif - - if (!remap_capture_actions(nfa, eclosures)) { - goto cleanup; - } + fsm_capture_integrity_check(nfa); #if LOG_RESULT - fsm_print_fsm(stderr, nfa); + fsm_dump(stderr, nfa); fsm_capture_dump(stderr, "#### post_remove_epsilons", nfa); #endif res = 1; cleanup: if (eclosures != NULL) { - closure_free(eclosures, state_count); + fsm_closure_free(nfa, eclosures, state_count); } - - return res; -} - -static int -remap_capture_actions(struct fsm *nfa, struct state_set **eclosures) -{ - int res = 0; - fsm_state_t s, i; - struct state_set **rmap; - struct state_iter si; - fsm_state_t si_s; - struct remap_env env = { 'R', NULL, NULL, 1, 0, 0, NULL }; - env.alloc = nfa->opt->alloc; - - /* build a reverse mapping */ - rmap = f_calloc(nfa->opt->alloc, nfa->statecount, sizeof(rmap[0])); - if (rmap == NULL) { - goto cleanup; - } - - for (s = 0; s < nfa->statecount; s++) { - if (eclosures[s] == NULL) { continue; } - for (state_set_reset(eclosures[s], &si); state_set_next(&si, &si_s); ) { - if (si_s == s) { - continue; /* ignore identical states */ - } -#if LOG_RM_EPSILONS_CAPTURES - fprintf(stderr, "remap_capture_actions: %u <- %u\n", - s, si_s); -#endif - if (!state_set_add(&rmap[si_s], nfa->opt->alloc, s)) { - goto cleanup; - } - } + if (em_env.end.ids != NULL) { + f_free(alloc, em_env.end.ids); } - env.rmap = rmap; - - /* Iterate over the current set of actions with the reverse - * mapping (containing only states which will be skipped, - * collecting info about every new capture action that will need - * to be added. - * - * It can't be added during the iteration, because that would - * modify the hash table as it's being iterated over. */ - fsm_capture_action_iter(nfa, remap_capture_action_cb, &env); - - /* Now that we're done iterating, add those actions. */ - for (i = 0; i < env.count; i++) { - const struct remap_action *a = &env.actions[i]; - if (!fsm_capture_add_action(nfa, a->state, a->type, - a->capture_id, a->to)) { - goto cleanup; - } + if (em_env.program.ids != NULL) { + f_free(alloc, em_env.program.ids); } - - res = 1; - -cleanup: - if (env.actions != NULL) { - f_free(nfa->opt->alloc, env.actions); + if (em_env.capture.ids != NULL) { + f_free(alloc, em_env.capture.ids); } + f_free(nfa->alloc, reachable_by_label); + f_free(nfa->alloc, eager_output_buf.ids); - if (rmap != NULL) { - for (i = 0; i < nfa->statecount; i++) { - state_set_free(rmap[i]); - } - f_free(nfa->opt->alloc, rmap); - } return res; - } -static int -add_pending_capture_action(struct remap_env *env, - fsm_state_t state, enum capture_action_type type, - unsigned capture_id, fsm_state_t to) +/* For every state, mark every state reached by a labeled edge as + * reachable. This doesn't check that the FROM state is reachable from + * the start state (trim will do that soon enough), it's just used to + * check which states will become unreachable once epsilon edges are + * removed. We don't need to add eager outputs for them, because they + * will soon be disconnected from the epsilon-free NFA. */ +static void +mark_states_reachable_by_label(const struct fsm *nfa, uint64_t *reachable_by_label) { - struct remap_action *a; - if (env->count == env->ceil) { - struct remap_action *nactions; - const size_t nceil = (env->ceil == 0 - ? DEF_PENDING_CAPTURE_ACTIONS_CEIL : 2*env->ceil); - assert(nceil > 0); - nactions = f_realloc(env->alloc, - env->actions, - nceil * sizeof(nactions[0])); - if (nactions == NULL) { - return 0; - } - - env->ceil = nceil; - env->actions = nactions; + fsm_state_t start; + if (!fsm_getstart(nfa, &start)) { + return; /* nothing reachable */ } + u64bitset_set(reachable_by_label, start); - a = &env->actions[env->count]; -#if LOG_RM_EPSILONS_CAPTURES - fprintf(stderr, "add_pending_capture_action: state %d, type %s, capture_id %u, to %d\n", - state, fsm_capture_action_type_name[type], capture_id, to); -#endif - - a->state = state; - a->type = type; - a->capture_id = capture_id; - a->to = to; - env->count++; - return 1; -} - -static int -remap_capture_action_cb(fsm_state_t state, - enum capture_action_type type, unsigned capture_id, fsm_state_t to, - void *opaque) -{ - struct state_iter si; - fsm_state_t si_s; - struct remap_env *env = opaque; - assert(env->tag == 'R'); - -#if LOG_RM_EPSILONS_CAPTURES - fprintf(stderr, "remap_capture_action_cb: state %d, type %s, capture_id %u, to %d\n", - state, fsm_capture_action_type_name[type], capture_id, to); -#endif - - for (state_set_reset(env->rmap[state], &si); state_set_next(&si, &si_s); ) { - struct state_iter si_to; - fsm_state_t si_tos; - -#if LOG_RM_EPSILONS_CAPTURES - fprintf(stderr, " -- rcac: state %d -> %d\n", state, si_s); -#endif - - if (!add_pending_capture_action(env, si_s, type, capture_id, to)) { - goto fail; - } + const fsm_state_t state_count = fsm_countstates(nfa); - if (to == CAPTURE_NO_STATE) { - continue; - } + for (size_t s_i = 0; s_i < state_count; s_i++) { + struct edge_group_iter egi; + struct edge_group_iter_info info; - for (state_set_reset(env->rmap[to], &si_to); state_set_next(&si, &si_tos); ) { -#if LOG_RM_EPSILONS_CAPTURES - fprintf(stderr, " -- rcac: to %d -> %d\n", to, si_tos); -#endif + struct fsm_state *s = &nfa->states[s_i]; - if (!add_pending_capture_action(env, si_tos, type, capture_id, to)) { - goto fail; - } + /* Clear the visited flag, it will be used to avoid cycles. */ + assert(s->visited == 0); /* stale */ + s->visited = 0; + edge_set_group_iter_reset(s->edges, EDGE_GROUP_ITER_ALL, &egi); + while (edge_set_group_iter_next(&egi, &info)) { + u64bitset_set(reachable_by_label, info.to); } } - - return 1; - -fail: - env->ok = 0; - return 0; } -struct collect_env { - char tag; - const struct fsm_alloc *alloc; - size_t count; - size_t ceil; - fsm_end_id_t *ids; - int ok; -}; - static int -collect_cb(fsm_state_t state, fsm_end_id_t id, void *opaque) +collect_captureid_cb(fsm_state_t state, unsigned id, void *opaque) { - struct collect_env *env = opaque; - assert(env->tag == 'E'); - + struct carry_end_metadata_env *env = opaque; (void)state; - if (env->count == env->ceil) { - const size_t nceil = 2 * env->ceil; - fsm_end_id_t *nids; - assert(nceil > env->ceil); - nids = f_realloc(env->alloc, env->ids, - nceil * sizeof(*env->ids)); + if (env->capture.count == env->capture.ceil) { + const size_t nceil = (env->capture.ceil == 0) + ? DEF_END_METADATA_CAPTUREIDS_CEIL + : 2 * env->capture.ceil; + unsigned *nids; + assert(nceil > env->capture.ceil); + nids = f_realloc(env->alloc, env->capture.ids, + nceil * sizeof(env->capture.ids[0])); if (nids == NULL) { - env->ok = 0; + env->capture.ok = 0; return 0; - } - env->ceil = nceil; - env->ids = nids; - } + } + env->capture.ceil = nceil; + env->capture.ids = nids; + } - env->ids[env->count] = id; - env->count++; + env->capture.ids[env->capture.count] = id; + env->capture.count++; + return 1; +} + +static int +collect_progid_cb(fsm_state_t state, unsigned id, void *opaque) +{ + struct carry_end_metadata_env *env = opaque; + uint32_t prog_id = (uint32_t)id; + (void)state; + + if (env->program.count == env->program.ceil) { + const size_t nceil = (env->program.ceil == 0) + ? DEF_END_METADATA_PROGRAMIDS_CEIL + : 2 * env->program.ceil; + unsigned *nids; + assert(nceil > env->program.ceil); + nids = f_realloc(env->alloc, env->program.ids, + nceil * sizeof(env->program.ids[0])); + if (nids == NULL) { + env->program.ok = 0; + return 0; + } + env->program.ceil = nceil; + env->program.ids = nids; + } + env->program.ids[env->program.count] = prog_id; + env->program.count++; return 1; } -/* fsm_remove_epsilons can't use fsm_endid_carry directly, because the src - * and dst FSMs are the same -- that would lead to adding entries to a - * hash table, possibly causing it to resize, while iterating over it. - * - * Instead, collect entries that need to be added (if not already - * present), and then add them in a second pass. */ +/* Because we're modifying the FSM in place, we can't iterate and add + * new entries -- it could lead to the underlying hash table resizing. + * Instead, collect, then add in a second pass. */ static int -carry_endids(struct fsm *fsm, struct state_set *states, - fsm_state_t dst_state) +carry_end_metadata(struct carry_end_metadata_env *env, + fsm_state_t end_state, fsm_state_t dst_state) { - struct state_iter it; - fsm_state_t s; size_t i; + const size_t id_count = fsm_endid_count(env->fsm, end_state); + if (id_count > 0) { /* copy end IDs */ + enum fsm_getendids_res id_res; + if (id_count > env->end.ceil) { /* grow buffer */ + size_t nceil = (env->end.ceil == 0) + ? DEF_END_METADATA_ENDIDS_CEIL + : 2*env->end.ceil; + while (nceil < id_count) { + nceil *= 2; + } + assert(nceil > 0); + fsm_end_id_t *nids = f_realloc(env->alloc, + env->end.ids, nceil * sizeof(env->end.ids[0])); + if (nids == NULL) { + return 0; + } + env->end.ids = nids; + env->end.ceil = nceil; + } - struct collect_env env; - env.tag = 'E'; /* for fsm_remove_epsilons */ - env.alloc = fsm->opt->alloc; - env.count = 0; - env.ceil = DEF_CARRY_ENDIDS_COUNT; - env.ids = f_malloc(fsm->opt->alloc, - env.ceil * sizeof(*env.ids)); - if (env.ids == NULL) { - return 0; - } - env.ok = 1; + id_res = fsm_endid_get(env->fsm, end_state, + id_count, env->end.ids); + assert(id_res == FSM_GETENDIDS_FOUND); - /* collect from states */ - for (state_set_reset(states, &it); state_set_next(&it, &s); ) { - if (!fsm_isend(fsm, s)) { - continue; - } + for (i = 0; i < id_count; i++) { +#if LOG_COPYING + fprintf(stderr, "carry_end_metadata: setting end ID %u on %d (due to %d)\n", + env->end.ids[i], dst_state, end_state); +#endif + if (!fsm_setendid_state(env->fsm, dst_state, env->end.ids[i])) { + return 0; + } + } + } - fsm_endid_iter_state(fsm, s, collect_cb, &env); - if (!env.ok) { - goto cleanup; + env->capture.ok = 1; + env->capture.count = 0; + fsm_capture_iter_active_for_end_state(env->fsm, end_state, + collect_captureid_cb, env); + if (!env->capture.ok) { + return 0; + } + for (i = 0; i < env->capture.count; i++) { + if (!fsm_capture_set_active_for_end(env->fsm, + env->capture.ids[i], dst_state)) { + return 0; } } - /* add them */ - for (i = 0; i < env.count; i++) { - enum fsm_endid_set_res sres; - sres = fsm_endid_set(fsm, dst_state, env.ids[i]); - if (sres == FSM_ENDID_SET_ERROR_ALLOC_FAIL) { - env.ok = 0; - goto cleanup; + env->program.count = 0; + fsm_capture_iter_program_ids_for_end_state(env->fsm, end_state, + collect_progid_cb, env); + for (i = 0; i < env->program.count; i++) { + if (!fsm_capture_associate_program_with_end_state(env->fsm, + env->program.ids[i], dst_state)) { + return 0; } } -cleanup: - f_free(fsm->opt->alloc, env.ids); - - return env.ok; + return 1; } - diff --git a/src/libfsm/equal.c b/src/libfsm/equal.c index f683a95e2..02489b0b1 100644 --- a/src/libfsm/equal.c +++ b/src/libfsm/equal.c @@ -55,7 +55,7 @@ subtract(const struct fsm *a, const struct fsm *b) assert(a != NULL); assert(b != NULL); - assert(a->opt == b->opt); + assert(a->alloc == b->alloc); if (!ensure_dfa(a, &pa)) { return NULL; @@ -121,7 +121,7 @@ fsm_equal(const struct fsm *a, const struct fsm *b) assert(a != NULL); assert(b != NULL); - if (a->opt != b->opt) { + if (a->alloc != b->alloc) { errno = EINVAL; return -1; } diff --git a/src/libfsm/example.c b/src/libfsm/example.c index ca9ce9571..9b30ad91f 100644 --- a/src/libfsm/example.c +++ b/src/libfsm/example.c @@ -92,7 +92,7 @@ fsm_example(const struct fsm *fsm, fsm_state_t goal, bufsz--; } - path_free(fsm->opt->alloc, path); + path_free(fsm->alloc, path); return n; } diff --git a/src/libfsm/exec.c b/src/libfsm/exec.c index 9f7b21802..a5f4d68bd 100644 --- a/src/libfsm/exec.c +++ b/src/libfsm/exec.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -20,12 +21,14 @@ #include "internal.h" #include "capture.h" +#include "eager_output.h" #define LOG_EXEC 0 +#define LOG_EAGER 0 + static int transition(const struct fsm *fsm, fsm_state_t state, int c, - size_t offset, struct fsm_capture *captures, fsm_state_t *next) { assert(state < fsm->statecount); @@ -35,31 +38,121 @@ transition(const struct fsm *fsm, fsm_state_t state, int c, return 0; } - if (captures != NULL && fsm_capture_has_capture_actions(fsm, state)) { - fsm_capture_update_captures(fsm, state, *next, - offset, captures); - } + return 1; +} + +struct check_eager_outputs_for_state_env { + const struct fsm *fsm; + fsm_eager_output_cb *cb; + void *opaque; +}; + +static int +match_eager_outputs_for_state_cb(fsm_state_t state, fsm_end_id_t id, void *opaque) +{ + (void)state; + struct check_eager_outputs_for_state_env *env = opaque; +#if LOG_EAGER + fprintf(stderr, "%s: state %d, id %d\n", __func__, state, id); +#endif + env->cb(id, env->opaque); + return 1; +} +static int +match_eager_outputs_for_state(const struct fsm *fsm, fsm_state_t state) +{ + fsm_eager_output_cb *cb = NULL; + void *opaque = NULL; + fsm_eager_output_get_cb(fsm, &cb, &opaque); + if (cb == NULL) { return 1; } /* nothing to do */ + + struct check_eager_outputs_for_state_env env = { + .fsm = fsm, + .cb = cb, + .opaque = opaque, + }; + fsm_eager_output_iter_state(fsm, + state, match_eager_outputs_for_state_cb, &env); return 1; } int fsm_exec(const struct fsm *fsm, - int (*fsm_getc)(void *opaque), void *opaque, - fsm_state_t *end, struct fsm_capture *captures) + int (*fsm_getc)(void *opaque), void *opaque, fsm_state_t *end) { fsm_state_t state; int c; size_t offset = 0; - unsigned i; - size_t capture_count; assert(fsm != NULL); assert(fsm_getc != NULL); assert(end != NULL); - capture_count = fsm_countcaptures(fsm); + /* TODO: check prerequisites; that it has literal edges, DFA, etc */ + + /* TODO: pass struct of callbacks to call during each event; transitions etc */ + + if (!fsm_all(fsm, fsm_isdfa)) { + errno = EINVAL; + return -1; + } + + if (!fsm_getstart(fsm, &state)) { + errno = EINVAL; + return -1; + } + const fsm_state_t start = state; + +#if LOG_EXEC + fprintf(stderr, "fsm_exec: starting at %d\n", state); +#endif + + if (fsm->states[start].has_eager_outputs) { + if (!match_eager_outputs_for_state(fsm, start)) { + return 0; + } + } + + while (c = fsm_getc(opaque), c != EOF) { + if (!transition(fsm, state, c, &state)) { +#if LOG_EXEC + fprintf(stderr, "fsm_exec: edge not found\n"); +#endif + return 0; + } + + if (fsm->states[state].has_eager_outputs) { + if (!match_eager_outputs_for_state(fsm, state)) { + return 0; + } + } + +#if LOG_EXEC + fprintf(stderr, "fsm_exec: @ %zu, input '%c', new state %u\n", + offset, c, state); +#endif + offset++; + } + + if (!fsm_isend(fsm, state)) { + return 0; + } + + *end = state; + return 1; +} + +int +fsm_exec_with_captures(const struct fsm *fsm, const unsigned char *input, + size_t input_length, fsm_state_t *end, + struct fsm_capture *captures, size_t capture_buf_length) +{ + fsm_state_t state; + size_t offset = 0; + assert(fsm != NULL); + assert(end != NULL); /* TODO: check prerequisites; that it has literal edges, DFA, etc */ /* TODO: pass struct of callbacks to call during each event; transitions etc */ @@ -74,17 +167,26 @@ fsm_exec(const struct fsm *fsm, return -1; } - for (i = 0; i < capture_count; i++) { - captures[i].pos[0] = FSM_CAPTURE_NO_POS; - captures[i].pos[1] = FSM_CAPTURE_NO_POS; + if (captures != NULL) { + const size_t capture_ceil = fsm_capture_ceiling(fsm); + if (capture_buf_length < capture_ceil) { + errno = EINVAL; + return -1; + } + + for (size_t i = 0; i < capture_ceil; i++) { + captures[i].pos[0] = FSM_CAPTURE_NO_POS; + captures[i].pos[1] = FSM_CAPTURE_NO_POS; + } } #if LOG_EXEC fprintf(stderr, "fsm_exec: starting at %d\n", state); #endif - while (c = fsm_getc(opaque), c != EOF) { - if (!transition(fsm, state, c, offset, captures, &state)) { + while (offset < input_length) { + const unsigned char c = input[offset]; + if (!transition(fsm, state, c, &state)) { #if LOG_EXEC fprintf(stderr, "fsm_exec: edge not found\n"); #endif @@ -102,15 +204,15 @@ fsm_exec(const struct fsm *fsm, return 0; } - /* Check for capture actions on end state */ - if (captures != NULL && fsm_capture_has_capture_actions(fsm, state)) { - fsm_capture_update_captures(fsm, state, NEXT_STATE_END, - offset, captures); + /* Resolve captures associated with the end state. */ + if (captures != NULL) { + if (!fsm_capture_resolve_during_exec(fsm, state, + input, offset, captures, capture_buf_length)) { + assert(errno != 0); + return -1; + } } - fsm_capture_finalize_captures(fsm, capture_count, captures); - *end = state; return 1; } - diff --git a/src/libfsm/fsm.c b/src/libfsm/fsm.c index 6a7bb790f..866650a9c 100644 --- a/src/libfsm/fsm.c +++ b/src/libfsm/fsm.c @@ -6,12 +6,12 @@ #include #include +#include #include #include #include #include -#include #include #include @@ -21,6 +21,10 @@ #include "internal.h" #include "capture.h" #include "endids.h" +#include "eager_output.h" + +/* guess for default state allocation */ +#define FSM_DEFAULT_STATEALLOC 128 void free_contents(struct fsm *fsm) @@ -31,64 +35,88 @@ free_contents(struct fsm *fsm) for (i = 0; i < fsm->statecount; i++) { state_set_free(fsm->states[i].epsilons); - edge_set_free(fsm->opt->alloc, fsm->states[i].edges); + edge_set_free(fsm->alloc, fsm->states[i].edges); } fsm_capture_free(fsm); fsm_endid_free(fsm); + fsm_eager_output_free(fsm); - f_free(fsm->opt->alloc, fsm->states); + if (fsm->linkage_info != NULL) { + state_set_free(fsm->linkage_info->anchored_starts); + state_set_free(fsm->linkage_info->anchored_ends); + f_free(fsm->alloc, fsm->linkage_info); + } + f_free(fsm->alloc, fsm->states); } struct fsm * -fsm_new(const struct fsm_options *opt) +fsm_new_statealloc(const struct fsm_alloc *alloc, size_t statealloc) { - static const struct fsm_options defaults; - struct fsm *new, f; + struct fsm *new; - if (opt == NULL) { - opt = &defaults; + if (alloc != NULL) { + assert(alloc->free != NULL); + assert(alloc->calloc != NULL); + assert(alloc->malloc != NULL); + assert(alloc->realloc != NULL); } - f.opt = opt; + if (statealloc == 0) { + return fsm_new(alloc); + } - new = f_malloc(f.opt->alloc, sizeof *new); + new = f_malloc(alloc, sizeof *new); if (new == NULL) { return NULL; } - new->statealloc = 128; /* guess */ - new->statecount = 0; - new->endcount = 0; + new->alloc = alloc; + new->statealloc = statealloc; + new->statecount = 0; + new->endcount = 0; new->capture_info = NULL; - new->endid_info = NULL; + new->endid_info = NULL; + new->linkage_info = NULL; - new->states = f_malloc(f.opt->alloc, new->statealloc * sizeof *new->states); + new->states = f_malloc(new->alloc, new->statealloc * sizeof *new->states); if (new->states == NULL) { - f_free(f.opt->alloc, new); + f_free(new->alloc, new); return NULL; } fsm_clearstart(new); - new->opt = opt; - if (!fsm_capture_init(new)) { - f_free(f.opt->alloc, new->states); - f_free(f.opt->alloc, new); + f_free(new->alloc, new->states); + f_free(new->alloc, new); return NULL; } if (!fsm_endid_init(new)) { - f_free(f.opt->alloc, new->states); - f_free(f.opt->alloc, new); + f_free(new->alloc, new->states); + f_free(new->alloc, new); + fsm_capture_free(new); + return NULL; + } + + if (!fsm_eager_output_init(new)) { + f_free(new->alloc, new->states); + f_free(new->alloc, new); fsm_capture_free(new); + fsm_endid_free(new); return NULL; } return new; } +struct fsm * +fsm_new(const struct fsm_alloc *alloc) +{ + return fsm_new_statealloc(alloc, FSM_DEFAULT_STATEALLOC); +} + void fsm_free(struct fsm *fsm) { @@ -96,19 +124,7 @@ fsm_free(struct fsm *fsm) free_contents(fsm); - f_free(fsm->opt->alloc, fsm); -} - -const struct fsm_options * -fsm_getoptions(const struct fsm *fsm) -{ - return fsm->opt; -} - -void -fsm_setoptions(struct fsm *fsm, const struct fsm_options *opts) -{ - fsm->opt = opts; + f_free(fsm->alloc, fsm); } void @@ -117,7 +133,7 @@ fsm_move(struct fsm *dst, struct fsm *src) assert(src != NULL); assert(dst != NULL); - if (dst->opt != src->opt) { + if (dst->alloc != src->alloc) { errno = EINVAL; return; /* XXX */ } @@ -133,8 +149,10 @@ fsm_move(struct fsm *dst, struct fsm *src) dst->capture_info = src->capture_info; dst->endid_info = src->endid_info; + dst->eager_output_info = src->eager_output_info; + dst->linkage_info = src->linkage_info; - f_free(src->opt->alloc, src); + f_free(src->alloc, src); } unsigned int diff --git a/src/libfsm/gen.c b/src/libfsm/gen.c index 89597f7ae..8b8551489 100644 --- a/src/libfsm/gen.c +++ b/src/libfsm/gen.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -74,22 +75,25 @@ struct gen_ctx { const struct fsm *fsm; size_t max_length; fsm_generate_matches_cb *cb; - void *opaque; + + bool done; + bool randomized; size_t buf_ceil; size_t buf_used; char *buf; + void *opaque; + + unsigned depth; + unsigned steps; + /* This is used to avoid useless cycles -- if a state * was reached since the same match_count, then don't * explore it again. */ unsigned match_count; unsigned *state_counts; - unsigned depth; - unsigned steps; - bool done; - /* Shortest end distance for a state: sed[s_id] */ unsigned *sed; @@ -103,7 +107,7 @@ struct gen_ctx { static bool gen_init_outer(struct fsm *fsm, size_t max_length, fsm_generate_matches_cb *cb, void *opaque, - bool randomized, unsigned seed); + bool randomized); static bool gen_init(struct gen_ctx *ctx, struct fsm *fsm); @@ -136,12 +140,21 @@ static bool grow_stack(struct gen_ctx *ctx); int -fsm_generate_matches(struct fsm *fsm, size_t max_length, +fsm_generate_matches(struct fsm *fsm, size_t max_length, int randomized, fsm_generate_matches_cb *cb, void *opaque) { + if (max_length == 0) { + errno = EINVAL; + return 0; + } + + if (!fsm_has(fsm, fsm_isend)) { + return 1; /* no end state -> nothing to do */ + } + INIT_TIMERS(); TIME(&pre); - int res = gen_init_outer(fsm, max_length, cb, opaque, false, 0); + int res = gen_init_outer(fsm, max_length, cb, opaque, randomized != 0); TIME(&post); DIFF_MSEC("fsm_generate_matches", pre, post, NULL); @@ -191,18 +204,15 @@ fsm_generate_cb_printf(const struct fsm *fsm, static bool gen_init_outer(struct fsm *fsm, size_t max_length, fsm_generate_matches_cb *cb, void *opaque, - bool randomized, unsigned seed) + bool randomized) { - int res = 0; + int res = false; if (fsm == NULL || cb == NULL || max_length == 0) { return false; } assert(fsm_all(fsm, fsm_isdfa)); /* DFA-only */ - assert(!randomized); /* not yet supported */ - (void)seed; - #if LOG_GEN > 1 fprintf(stderr, "%s: %u states\n", __func__, fsm_countstates(fsm)); #endif @@ -211,22 +221,23 @@ gen_init_outer(struct fsm *fsm, size_t max_length, } struct gen_ctx ctx = { - .alloc = fsm->opt->alloc, + .alloc = fsm->alloc, .fsm = fsm, .max_length = max_length, .cb = cb, .opaque = opaque, + .randomized = randomized, }; if (!gen_init(&ctx, fsm)) { goto cleanup; } - res = 1; + res = true; while (!ctx.done) { if (!gen_iter(&ctx)) { - res = 0; + res = false; break; } } @@ -516,6 +527,55 @@ first_symbol(const uint64_t *symbols) return 0; } +static unsigned char +random_symbol(const uint64_t *symbols) +{ + bool has_zero = false; + unsigned i = 0; + + /* printable and non-printable character choices */ + size_t choice_count = 0; + unsigned char choices[256]; + size_t np_choice_count = 0; + unsigned char np_choices[256]; + + while (i < 256) { + const uint64_t w = symbols[i/64]; + if ((i & 63) == 0 && w == 0) { + i += 64; + continue; + } + if (w & (1ULL << (i & 63))) { + if (i == 0) { + has_zero = true; + } else if (isprint(i)) { + choices[choice_count++] = (unsigned char)i; + } else { + np_choices[np_choice_count++] = (unsigned char)i; + } + } + i++; + } + + if (choice_count > 0) { + const size_t c = rand() % choice_count; + return choices[c]; + } + + if (np_choice_count > 0) { + const size_t c = rand() % np_choice_count; + return np_choices[c]; + } + + /* Prefer anything besides 0x00 if present, since that will truncate the string. */ + if (has_zero) { + return 0; + } + + assert(!"empty set"); + return 0; +} + #if DUMP_EDGES static void dump_edges(fsm_state_t state, struct edge_set *edges) @@ -530,6 +590,7 @@ dump_edges(fsm_state_t state, struct edge_set *edges) size_t i = 0; while (edge_set_group_iter_next(&ei, &eg)) { const unsigned char symbol = first_symbol(eg.symbols); + const unsigned char symbol = random_symbol(eg.symbols); fprintf(stderr, "%s: %d -- %zu/%zu -- 0x%02x (%c) -> %d\n", __func__, state, i, count, symbol, isprint(symbol) ? symbol : '.', eg.to); @@ -562,8 +623,8 @@ sfs_step_edges(struct gen_ctx *ctx, struct gen_stack_frame *sf) sf->u.step_edges.initialized = true; } - if (ctx->buf_used + ctx->sed[sf->s_id] >= ctx->max_length) { - LOG(2, "PRUNING due to max length: used:%zu + sed[%d]:%u >= max_length:%zu\n", + if (ctx->buf_used + ctx->sed[sf->s_id] > ctx->max_length) { + LOG(2, "PRUNING due to max length: used:%zu + sed[%d]:%u > max_length:%zu\n", ctx->buf_used, sf->s_id, ctx->sed[sf->s_id], ctx->max_length); sf->t = GEN_SFS_LEAVING_STATE; return true; @@ -577,7 +638,9 @@ sfs_step_edges(struct gen_ctx *ctx, struct gen_stack_frame *sf) struct edge_group_iter_info eg; if (iter_next_transition(ctx, sf, &eg)) { - const unsigned char symbol = first_symbol(eg.symbols); + const unsigned char symbol = ctx->randomized + ? random_symbol(eg.symbols) + : first_symbol(eg.symbols); const fsm_state_t state = eg.to; LOG(2, "sfs_step_edges: got edge 0x%x ('%c')\n", diff --git a/src/libfsm/internal.h b/src/libfsm/internal.h index 6e77510a7..c95da7ec4 100644 --- a/src/libfsm/internal.h +++ b/src/libfsm/internal.h @@ -11,7 +11,6 @@ #include #include -#include #include @@ -19,6 +18,7 @@ struct bm; struct edge_set; struct state_set; struct state_array; +struct linkage_info; /* * The alphabet (Sigma) for libfsm's FSM is arbitrary octets. @@ -45,37 +45,61 @@ struct state_array; #define FSM_CAPTURE_MAX INT_MAX struct fsm_edge { - fsm_state_t state; /* destination */ + fsm_state_t state:24; /* destination. :24 for packing */ unsigned char symbol; }; struct fsm_state { - unsigned int end:1; + struct edge_set *edges; + struct state_set *epsilons; - /* If 0, then this state has no need for checking - * the fsm->capture_info struct. */ - unsigned int has_capture_actions:1; + unsigned int end:1; /* meaningful within one particular transformation only */ unsigned int visited:1; - struct edge_set *edges; - struct state_set *epsilons; + /* If 0, then this state has no need for checking + * the fsm->eager_output_info struct. */ + unsigned int has_eager_outputs:1; }; struct fsm { struct fsm_state *states; /* array */ + const struct fsm_alloc *alloc; size_t statealloc; /* number of elements allocated */ size_t statecount; /* number of elements populated */ - size_t endcount; + size_t endcount:31; /* :31 for packing */ - fsm_state_t start; unsigned int hasstart:1; + fsm_state_t start; struct fsm_capture_info *capture_info; struct endid_info *endid_info; - const struct fsm_options *opt; + struct eager_output_info *eager_output_info; + struct linkage_info *linkage_info; +}; + +#define LINKAGE_NO_STATE ((fsm_state_t)-1) + +/* Internal structure for storing structural info about an NFA. + * This is currently only used by fsm_union_repeated_pattern_group, + * which needs to identify a couple components of the NFA in order + * to link groups of repeated pattern together correctly. */ +struct linkage_info { + /* The states with a /./ self edge representing the unanchored + * start and end, or LINKAGE_NO_STATE. There can be at most one + * of each. */ + fsm_state_t unanchored_start_loop; + fsm_state_t unanchored_end_loop; + + /* The inner end state immediately preceding the unanchored end loop. */ + fsm_state_t end_any_inner; + + /* States that link to paths only reachable from the beginning of input. */ + struct state_set *anchored_starts; + /* States leading to an anchored end. */ + struct state_set *anchored_ends; }; struct fsm * @@ -90,10 +114,10 @@ state_hasnondeterminism(const struct fsm *fsm, fsm_state_t state, struct bm *bm) * for states, with wrapper to populate malloced array of user-facing structs. */ struct state_set ** -epsilon_closure(struct fsm *fsm); +fsm_epsilon_closure(const struct fsm *fsm); void -closure_free(struct state_set **closures, size_t n); +fsm_closure_free(const struct fsm *fsm, struct state_set **closures, size_t n); /* * Internal free function that invokes free(3) by default, or a user-provided diff --git a/src/libfsm/lexer.c b/src/libfsm/lexer.c index 7213bb142..3bf26b3b6 100644 --- a/src/libfsm/lexer.c +++ b/src/libfsm/lexer.c @@ -12,7 +12,29 @@ static enum lx_token z0(struct lx *lx); static enum lx_token z1(struct lx *lx); static enum lx_token z2(struct lx *lx); static enum lx_token z3(struct lx *lx); +static enum lx_token z4(struct lx *lx); +static enum lx_token z5(struct lx *lx); +static int +lx_advance_end(struct lx *lx, int c) +{ + lx->end.byte++; + lx->end.col++; + if (c == '\n') { + lx->end.line++; + lx->end.saved_col = lx->end.col - 1; + lx->end.col = 1; + } + if (lx->push != NULL) { + if (-1 == lx->push(lx->buf_opaque, (char)c)) { + return 0; + } + } + return 1; +} + +/* This wrapper manages one character of lookahead/pushback + * and the line, column, and byte offsets. */ #if __STDC_VERSION__ >= 199901L inline #endif @@ -33,18 +55,19 @@ lx_getc(struct lx *lx) } } - lx->end.byte++; - lx->end.col++; - - if (c == '\n') { - lx->end.line++; - lx->end.saved_col = lx->end.col - 1; - lx->end.col = 1; - } + if (!lx_advance_end(lx, c)) { return EOF; } return c; } +/* This wrapper adapts calling lx_getc to the interface + * in libfsm's generated code. */ +static int +fsm_getc(void *getc_opaque) +{ + return lx_getc((struct lx *)getc_opaque); +} + #if __STDC_VERSION__ >= 199901L inline #endif @@ -53,10 +76,7 @@ lx_ungetc(struct lx *lx, int c) { assert(lx != NULL); assert(lx->c == EOF); - lx->c = c; - - lx->end.byte--; lx->end.col--; @@ -66,13 +86,20 @@ lx_ungetc(struct lx *lx, int c) } } +/* Get a character from fgetc and push it to the buffer */ int lx_fgetc(struct lx *lx) { assert(lx != NULL); assert(lx->getc_opaque != NULL); - return fgetc(lx->getc_opaque); + const int c = fgetc(lx->getc_opaque); + if (c == EOF) { + lx->c = EOF; + return EOF; + } else { + return c; + } } int @@ -117,6 +144,17 @@ lx_dynpush(void *buf_opaque, char c) return 0; } +static void +lx_dynpop(void *buf_opaque) +{ + struct lx_dynbuf *t = buf_opaque; + + assert(t != NULL); + + assert(t->p != t->a); + t->p--; +} + int lx_dynclear(void *buf_opaque) { @@ -156,110 +194,129 @@ lx_dynfree(void *buf_opaque) static enum lx_token z0(struct lx *lx) { + int has_consumed_input = 0; int c; - enum { - S0, S1, S2, NONE - } state; - assert(lx != NULL); if (lx->clear != NULL) { lx->clear(lx->buf_opaque); } - state = NONE; - lx->start = lx->end; - while (c = lx_getc(lx), c != EOF) { - if (state == NONE) { - state = S0; - } + void *getc_opaque = (void *)lx; + enum { + S0, S1, S2 + } state; + + state = S0; + while (c = fsm_getc(getc_opaque), c != EOF) { + has_consumed_input = 1; switch (state) { - case S0: /* start */ + case S0: /* e.g. "" */ switch ((unsigned char) c) { - case '\'': state = S2; break; + case '\n': state = S2; break; default: state = S1; break; } break; - case S1: /* e.g. "a" */ - lx_ungetc(lx, c); return TOK_CHAR; + case S1: /* e.g. "" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return lx->z(lx); - case S2: /* e.g. "'" */ - lx_ungetc(lx, c); return lx->z = z3, TOK_LABEL; + case S2: /* e.g. "" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return lx->z = z1, lx->z(lx); default: ; /* unreached */ } + } - if (lx->push != NULL) { - if (-1 == lx->push(lx->buf_opaque, c)) { - return TOK_ERROR; + /* end states */ + switch (state) { + case S1: return TOK_UNKNOWN; + case S2: return lx->z = z1, lx->z(lx); + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; + } + + switch (state) { + case S0: + case S1: + case S2: + break; + + default: + if (lx->push != NULL) { + if (-1 == lx->push(lx->buf_opaque, (char)c)) { + return TOK_ERROR; + } } + break; + } - } lx->lgetc = NULL; - switch (state) { - case NONE: return TOK_EOF; - case S1: return TOK_CHAR; - case S2: return TOK_LABEL; - default: errno = EINVAL; return TOK_ERROR; - } + if (!has_consumed_input) { + return TOK_EOF; + } + return TOK_ERROR; } static enum lx_token z1(struct lx *lx) { + int has_consumed_input = 0; int c; - enum { - S0, S1, S2, S3, S4, S5, S6, S7, NONE - } state; - assert(lx != NULL); if (lx->clear != NULL) { lx->clear(lx->buf_opaque); } - state = NONE; - lx->start = lx->end; - while (c = lx_getc(lx), c != EOF) { - if (state == NONE) { - state = S0; - } + void *getc_opaque = (void *)lx; + enum { + S0, S1, S2, S3, S4, S5 + } state; + + state = S0; + while (c = fsm_getc(getc_opaque), c != EOF) { + has_consumed_input = 1; switch (state) { - case S0: /* start */ + case S0: /* e.g. "" */ switch ((unsigned char) c) { - case '"': state = S2; break; - case '\\': state = S3; break; - default: state = S1; break; + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': state = S1; break; + case ',': state = S2; break; + case '#': state = S3; break; + case '\t': + case '\n': + case '\r': + case ' ': state = S4; break; + case ']': state = S5; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S1: /* e.g. "a" */ - lx_ungetc(lx, c); return TOK_CHAR; - - case S2: /* e.g. "\"" */ - lx_ungetc(lx, c); return lx->z = z3, TOK_LABEL; - - case S3: /* e.g. "\\" */ + case S1: /* e.g. "0" */ switch ((unsigned char) c) { - case '"': - case '\\': - case 'f': - case 'n': - case 'r': - case 't': - case 'v': state = S4; break; case '0': case '1': case '2': @@ -267,17 +324,172 @@ z1(struct lx *lx) case '4': case '5': case '6': - case '7': state = S5; break; - case 'x': state = S6; break; - default: lx_ungetc(lx, c); return TOK_CHAR; + case '7': + case '8': + case '9': break; + default: lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_ENDID; } break; - case S4: /* e.g. "\\f" */ - lx_ungetc(lx, c); return TOK_ESC; + case S2: /* e.g. "," */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_COMMA; - case S5: /* e.g. "\\0" */ + case S3: /* e.g. "#" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return lx->z = z0, lx->z(lx); + + case S4: /* e.g. "\\x09" */ + switch ((unsigned char) c) { + case '\t': + case '\n': + case '\r': + case ' ': break; + default: lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return lx->z(lx); + } + break; + + case S5: /* e.g. "]" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return lx->z = z5, TOK_CLOSEENDIDS; + + default: + ; /* unreached */ + } + } + + /* end states */ + switch (state) { + case S1: return TOK_ENDID; + case S2: return TOK_COMMA; + case S3: return lx->z = z0, lx->z(lx); + case S4: return TOK_EOF; + case S5: return lx->z = z5, TOK_CLOSEENDIDS; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; + } + + switch (state) { + case S3: + case S4: + break; + + default: + if (lx->push != NULL) { + if (-1 == lx->push(lx->buf_opaque, (char)c)) { + return TOK_ERROR; + } + } + break; + + } + + lx->lgetc = NULL; + + if (!has_consumed_input) { + return TOK_EOF; + } + return TOK_ERROR; +} + +static enum lx_token +z2(struct lx *lx) +{ + int has_consumed_input = 0; + int c; + + assert(lx != NULL); + + if (lx->clear != NULL) { + lx->clear(lx->buf_opaque); + } + + lx->start = lx->end; + + void *getc_opaque = (void *)lx; + enum { + S0, S1, S2 + } state; + + state = S0; + + while (c = fsm_getc(getc_opaque), c != EOF) { + has_consumed_input = 1; + switch (state) { + case S0: /* e.g. "" */ + switch ((unsigned char) c) { + case '\'': state = S2; break; + default: state = S1; break; + } + break; + + case S1: /* e.g. "" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_CHAR; + + case S2: /* e.g. "'" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return lx->z = z5, TOK_LABEL; + + default: + ; /* unreached */ + } + } + + /* end states */ + switch (state) { + case S1: return TOK_CHAR; + case S2: return lx->z = z5, TOK_LABEL; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; + } + + if (lx->push != NULL) { + if (-1 == lx->push(lx->buf_opaque, (char)c)) { + return TOK_ERROR; + } + } + + lx->lgetc = NULL; + + if (!has_consumed_input) { + return TOK_EOF; + } + return TOK_ERROR; +} + +static enum lx_token +z3(struct lx *lx) +{ + int has_consumed_input = 0; + int c; + + assert(lx != NULL); + + if (lx->clear != NULL) { + lx->clear(lx->buf_opaque); + } + + lx->start = lx->end; + + void *getc_opaque = (void *)lx; + enum { + S0, S1, S2, S3, S4, S5, S6, S7 + } state; + + state = S0; + + while (c = fsm_getc(getc_opaque), c != EOF) { + has_consumed_input = 1; + switch (state) { + case S0: /* e.g. "" */ switch ((unsigned char) c) { + case '\\': state = S1; break; + case '"': state = S3; break; + default: state = S2; break; + } + break; + + case S1: /* e.g. "\\" */ + switch ((unsigned char) c) { + case 'x': state = S4; break; case '0': case '1': case '2': @@ -285,12 +497,25 @@ z1(struct lx *lx) case '4': case '5': case '6': - case '7': break; - default: lx_ungetc(lx, c); return TOK_OCT; + case '7': state = S5; break; + case '"': + case '\\': + case 'f': + case 'n': + case 'r': + case 't': + case 'v': state = S6; break; + default: lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_CHAR; } break; - case S6: /* e.g. "\\x" */ + case S2: /* e.g. "\\x00" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_CHAR; + + case S3: /* e.g. "\"" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return lx->z = z5, TOK_LABEL; + + case S4: /* e.g. "\\x" */ switch ((unsigned char) c) { case '0': case '1': @@ -314,11 +539,30 @@ z1(struct lx *lx) case 'd': case 'e': case 'f': state = S7; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; + } + break; + + case S5: /* e.g. "\\0" */ + switch ((unsigned char) c) { + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': break; + default: lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_OCT; } break; - case S7: /* e.g. "\\xa" */ + case S6: /* e.g. "\\\"" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_ESC; + + case S7: /* e.g. "\\x0" */ switch ((unsigned char) c) { case '0': case '1': @@ -342,76 +586,92 @@ z1(struct lx *lx) case 'd': case 'e': case 'f': break; - default: lx_ungetc(lx, c); return TOK_HEX; + default: lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_HEX; } break; default: ; /* unreached */ } + } + + /* end states */ + switch (state) { + case S1: return TOK_CHAR; + case S2: return TOK_CHAR; + case S3: return lx->z = z5, TOK_LABEL; + case S5: return TOK_OCT; + case S6: return TOK_ESC; + case S7: return TOK_HEX; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; + } if (lx->push != NULL) { - if (-1 == lx->push(lx->buf_opaque, c)) { + if (-1 == lx->push(lx->buf_opaque, (char)c)) { return TOK_ERROR; } } - } lx->lgetc = NULL; - switch (state) { - case NONE: return TOK_EOF; - case S1: return TOK_CHAR; - case S2: return TOK_LABEL; - case S3: return TOK_CHAR; - case S4: return TOK_ESC; - case S5: return TOK_OCT; - case S7: return TOK_HEX; - default: errno = EINVAL; return TOK_ERROR; - } + if (!has_consumed_input) { + return TOK_EOF; + } + return TOK_ERROR; } static enum lx_token -z2(struct lx *lx) +z4(struct lx *lx) { + int has_consumed_input = 0; int c; - enum { - S0, S1, S2, NONE - } state; - assert(lx != NULL); if (lx->clear != NULL) { lx->clear(lx->buf_opaque); } - state = NONE; - lx->start = lx->end; - while (c = lx_getc(lx), c != EOF) { - if (state == NONE) { - state = S0; - } + void *getc_opaque = (void *)lx; + enum { + S0, S1, S2 + } state; + + state = S0; + while (c = fsm_getc(getc_opaque), c != EOF) { + has_consumed_input = 1; switch (state) { - case S0: /* start */ + case S0: /* e.g. "" */ switch ((unsigned char) c) { case '\n': state = S2; break; default: state = S1; break; } break; - case S1: /* e.g. "a" */ - lx_ungetc(lx, c); return lx->z(lx); + case S1: /* e.g. "" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return lx->z(lx); case S2: /* e.g. "" */ - lx_ungetc(lx, c); return lx->z = z3, lx->z(lx); + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return lx->z = z5, lx->z(lx); default: ; /* unreached */ } + } + + /* end states */ + switch (state) { + case S1: return TOK_UNKNOWN; + case S2: return lx->z = z5, lx->z(lx); + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; + } switch (state) { case S0: @@ -421,63 +681,58 @@ z2(struct lx *lx) default: if (lx->push != NULL) { - if (-1 == lx->push(lx->buf_opaque, c)) { + if (-1 == lx->push(lx->buf_opaque, (char)c)) { return TOK_ERROR; } } break; } - } lx->lgetc = NULL; - switch (state) { - case NONE: return TOK_EOF; - case S1: return TOK_EOF; - case S2: return TOK_EOF; - default: errno = EINVAL; return TOK_ERROR; - } + if (!has_consumed_input) { + return TOK_EOF; + } + return TOK_ERROR; } static enum lx_token -z3(struct lx *lx) +z5(struct lx *lx) { + int has_consumed_input = 0; int c; - enum { - S0, S1, S2, S3, S4, S5, S6, S7, S8, S9, - S10, S11, S12, S13, S14, S15, S16, S17, S18, S19, - S20, NONE - } state; - assert(lx != NULL); if (lx->clear != NULL) { lx->clear(lx->buf_opaque); } - state = NONE; - lx->start = lx->end; - while (c = lx_getc(lx), c != EOF) { - if (state == NONE) { - state = S0; - } + void *getc_opaque = (void *)lx; + enum { + S0, S1, S2, S3, S4, S5, S6, S7, S8, S9, + S10, S11, S12, S13, S14, S15, S16, S17, S18, S19, + S20, S21, S22 + } state; + state = S0; + + while (c = fsm_getc(getc_opaque), c != EOF) { + has_consumed_input = 1; switch (state) { - case S0: /* start */ + case S0: /* e.g. "" */ switch ((unsigned char) c) { - case '\t': - case '\n': - case '\r': - case ' ': state = S1; break; - case '"': state = S2; break; - case '#': state = S3; break; - case '\'': state = S4; break; - case ',': state = S5; break; - case '-': state = S6; break; + case ',': state = S1; break; + case ';': state = S2; break; + case '?': state = S3; break; + case '-': state = S4; break; + case '[': state = S5; break; + case '=': state = S6; break; + case 'e': state = S7; break; + case 's': state = S8; break; case '0': case '1': case '2': @@ -538,45 +793,45 @@ z3(struct lx *lx) case 'w': case 'x': case 'y': - case 'z': state = S7; break; - case ';': state = S8; break; - case '?': state = S9; break; - case 'e': state = S10; break; - case 's': state = S11; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; - } - break; - - case S1: /* e.g. "\\x09" */ - switch ((unsigned char) c) { + case 'z': state = S9; break; + case '\'': state = S10; break; + case '"': state = S11; break; + case '#': state = S12; break; case '\t': case '\n': case '\r': - case ' ': break; - default: lx_ungetc(lx, c); return lx->z(lx); + case ' ': state = S13; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S2: /* e.g. "\"" */ - lx_ungetc(lx, c); return lx->z = z1, lx->z(lx); - - case S3: /* e.g. "#" */ - lx_ungetc(lx, c); return lx->z = z2, lx->z(lx); + case S1: /* e.g. "," */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_COMMA; - case S4: /* e.g. "'" */ - lx_ungetc(lx, c); return lx->z = z0, lx->z(lx); + case S2: /* e.g. ";" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_SEP; - case S5: /* e.g. "," */ - lx_ungetc(lx, c); return TOK_COMMA; + case S3: /* e.g. "\077" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_ANY; - case S6: /* e.g. "-" */ + case S4: /* e.g. "-" */ switch ((unsigned char) c) { - case '>': state = S20; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case '>': state = S22; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S7: /* e.g. "a" */ + case S5: /* e.g. "[" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return lx->z = z1, TOK_OPENENDIDS; + + case S6: /* e.g. "=" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_EQUALS; + + case S7: /* e.g. "e" */ switch ((unsigned char) c) { case '0': case '1': @@ -628,7 +883,6 @@ z3(struct lx *lx) case 'k': case 'l': case 'm': - case 'n': case 'o': case 'p': case 'q': @@ -640,18 +894,13 @@ z3(struct lx *lx) case 'w': case 'x': case 'y': - case 'z': break; - default: lx_ungetc(lx, c); return TOK_IDENT; + case 'z': state = S9; break; + case 'n': state = S19; break; + default: lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_IDENT; } break; - case S8: /* e.g. ";" */ - lx_ungetc(lx, c); return TOK_SEP; - - case S9: /* e.g. "?" */ - lx_ungetc(lx, c); return TOK_ANY; - - case S10: /* e.g. "e" */ + case S8: /* e.g. "s" */ switch ((unsigned char) c) { case '0': case '1': @@ -703,24 +952,24 @@ z3(struct lx *lx) case 'k': case 'l': case 'm': + case 'n': case 'o': case 'p': case 'q': case 'r': case 's': - case 't': case 'u': case 'v': case 'w': case 'x': case 'y': - case 'z': state = S7; break; - case 'n': state = S17; break; - default: lx_ungetc(lx, c); return TOK_IDENT; + case 'z': state = S9; break; + case 't': state = S14; break; + default: lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_IDENT; } break; - case S11: /* e.g. "s" */ + case S9: /* e.g. "0" */ switch ((unsigned char) c) { case '0': case '1': @@ -778,18 +1027,37 @@ z3(struct lx *lx) case 'q': case 'r': case 's': + case 't': case 'u': case 'v': case 'w': case 'x': case 'y': - case 'z': state = S7; break; - case 't': state = S12; break; - default: lx_ungetc(lx, c); return TOK_IDENT; + case 'z': break; + default: lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_IDENT; } break; - case S12: /* e.g. "st" */ + case S10: /* e.g. "'" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return lx->z = z2, lx->z(lx); + + case S11: /* e.g. "\"" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return lx->z = z3, lx->z(lx); + + case S12: /* e.g. "#" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return lx->z = z4, lx->z(lx); + + case S13: /* e.g. "\\x09" */ + switch ((unsigned char) c) { + case '\t': + case '\n': + case '\r': + case ' ': break; + default: lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return lx->z(lx); + } + break; + + case S14: /* e.g. "st" */ switch ((unsigned char) c) { case '0': case '1': @@ -852,13 +1120,13 @@ z3(struct lx *lx) case 'w': case 'x': case 'y': - case 'z': state = S7; break; - case 'a': state = S13; break; - default: lx_ungetc(lx, c); return TOK_IDENT; + case 'z': state = S9; break; + case 'a': state = S15; break; + default: lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_IDENT; } break; - case S13: /* e.g. "sta" */ + case S15: /* e.g. "sta" */ switch ((unsigned char) c) { case '0': case '1': @@ -921,13 +1189,13 @@ z3(struct lx *lx) case 'w': case 'x': case 'y': - case 'z': state = S7; break; - case 'r': state = S14; break; - default: lx_ungetc(lx, c); return TOK_IDENT; + case 'z': state = S9; break; + case 'r': state = S16; break; + default: lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_IDENT; } break; - case S14: /* e.g. "star" */ + case S16: /* e.g. "star" */ switch ((unsigned char) c) { case '0': case '1': @@ -990,13 +1258,13 @@ z3(struct lx *lx) case 'w': case 'x': case 'y': - case 'z': state = S7; break; - case 't': state = S15; break; - default: lx_ungetc(lx, c); return TOK_IDENT; + case 'z': state = S9; break; + case 't': state = S17; break; + default: lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_IDENT; } break; - case S15: /* e.g. "start" */ + case S17: /* e.g. "start" */ switch ((unsigned char) c) { case '0': case '1': @@ -1060,16 +1328,16 @@ z3(struct lx *lx) case 'w': case 'x': case 'y': - case 'z': state = S7; break; - case ':': state = S16; break; - default: lx_ungetc(lx, c); return TOK_IDENT; + case 'z': state = S9; break; + case ':': state = S18; break; + default: lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_IDENT; } break; - case S16: /* e.g. "start:" */ - lx_ungetc(lx, c); return TOK_START; + case S18: /* e.g. "start:" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_START; - case S17: /* e.g. "en" */ + case S19: /* e.g. "en" */ switch ((unsigned char) c) { case '0': case '1': @@ -1132,13 +1400,13 @@ z3(struct lx *lx) case 'w': case 'x': case 'y': - case 'z': state = S7; break; - case 'd': state = S18; break; - default: lx_ungetc(lx, c); return TOK_IDENT; + case 'z': state = S9; break; + case 'd': state = S20; break; + default: lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_IDENT; } break; - case S18: /* e.g. "end" */ + case S20: /* e.g. "end" */ switch ((unsigned char) c) { case '0': case '1': @@ -1202,75 +1470,88 @@ z3(struct lx *lx) case 'w': case 'x': case 'y': - case 'z': state = S7; break; - case ':': state = S19; break; - default: lx_ungetc(lx, c); return TOK_IDENT; + case 'z': state = S9; break; + case ':': state = S21; break; + default: lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_IDENT; } break; - case S19: /* e.g. "end:" */ - lx_ungetc(lx, c); return TOK_END; + case S21: /* e.g. "end:" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_END; - case S20: /* e.g. "->" */ - lx_ungetc(lx, c); return TOK_TO; + case S22: /* e.g. "->" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_TO; default: ; /* unreached */ } + } + + /* end states */ + switch (state) { + case S1: return TOK_COMMA; + case S2: return TOK_SEP; + case S3: return TOK_ANY; + case S5: return lx->z = z1, TOK_OPENENDIDS; + case S6: return TOK_EQUALS; + case S7: return TOK_IDENT; + case S8: return TOK_IDENT; + case S9: return TOK_IDENT; + case S10: return lx->z = z2, lx->z(lx); + case S11: return lx->z = z3, lx->z(lx); + case S12: return lx->z = z4, lx->z(lx); + case S13: return TOK_EOF; + case S14: return TOK_IDENT; + case S15: return TOK_IDENT; + case S16: return TOK_IDENT; + case S17: return TOK_IDENT; + case S18: return TOK_START; + case S19: return TOK_IDENT; + case S20: return TOK_IDENT; + case S21: return TOK_END; + case S22: return TOK_TO; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; + } switch (state) { - case S1: - case S2: - case S3: - case S4: + case S10: + case S11: + case S12: + case S13: break; default: if (lx->push != NULL) { - if (-1 == lx->push(lx->buf_opaque, c)) { + if (-1 == lx->push(lx->buf_opaque, (char)c)) { return TOK_ERROR; } } break; } - } lx->lgetc = NULL; - switch (state) { - case NONE: return TOK_EOF; - case S1: return TOK_EOF; - case S2: return TOK_EOF; - case S3: return TOK_EOF; - case S4: return TOK_EOF; - case S5: return TOK_COMMA; - case S7: return TOK_IDENT; - case S8: return TOK_SEP; - case S9: return TOK_ANY; - case S10: return TOK_IDENT; - case S11: return TOK_IDENT; - case S12: return TOK_IDENT; - case S13: return TOK_IDENT; - case S14: return TOK_IDENT; - case S15: return TOK_IDENT; - case S16: return TOK_START; - case S17: return TOK_IDENT; - case S18: return TOK_IDENT; - case S19: return TOK_END; - case S20: return TOK_TO; - default: errno = EINVAL; return TOK_ERROR; - } + if (!has_consumed_input) { + return TOK_EOF; + } + return TOK_ERROR; } const char * lx_name(enum lx_token t) { switch (t) { - case TOK_COMMA: return "COMMA"; case TOK_SEP: return "SEP"; case TOK_ANY: return "ANY"; case TOK_TO: return "TO"; + case TOK_ENDID: return "ENDID"; + case TOK_COMMA: return "COMMA"; + case TOK_CLOSEENDIDS: return "CLOSEENDIDS"; + case TOK_OPENENDIDS: return "OPENENDIDS"; + case TOK_EQUALS: return "EQUALS"; case TOK_IDENT: return "IDENT"; case TOK_END: return "END"; case TOK_START: return "START"; @@ -1293,10 +1574,14 @@ lx_example(enum lx_token (*z)(struct lx *), enum lx_token t) if (z == z0) { switch (t) { - case TOK_COMMA: return ""; case TOK_SEP: return ""; case TOK_ANY: return ""; case TOK_TO: return ""; + case TOK_ENDID: return ""; + case TOK_COMMA: return ""; + case TOK_CLOSEENDIDS: return ""; + case TOK_OPENENDIDS: return ""; + case TOK_EQUALS: return ""; case TOK_IDENT: return ""; case TOK_END: return ""; case TOK_START: return ""; @@ -1310,10 +1595,14 @@ lx_example(enum lx_token (*z)(struct lx *), enum lx_token t) } else if (z == z1) { switch (t) { - case TOK_COMMA: return ""; case TOK_SEP: return ""; case TOK_ANY: return ""; case TOK_TO: return ""; + case TOK_ENDID: return ""; + case TOK_COMMA: return ""; + case TOK_CLOSEENDIDS: return ""; + case TOK_OPENENDIDS: return ""; + case TOK_EQUALS: return ""; case TOK_IDENT: return ""; case TOK_END: return ""; case TOK_START: return ""; @@ -1327,10 +1616,14 @@ lx_example(enum lx_token (*z)(struct lx *), enum lx_token t) } else if (z == z2) { switch (t) { - case TOK_COMMA: return ""; case TOK_SEP: return ""; case TOK_ANY: return ""; case TOK_TO: return ""; + case TOK_ENDID: return ""; + case TOK_COMMA: return ""; + case TOK_CLOSEENDIDS: return ""; + case TOK_OPENENDIDS: return ""; + case TOK_EQUALS: return ""; case TOK_IDENT: return ""; case TOK_END: return ""; case TOK_START: return ""; @@ -1344,10 +1637,56 @@ lx_example(enum lx_token (*z)(struct lx *), enum lx_token t) } else if (z == z3) { switch (t) { + case TOK_SEP: return ""; + case TOK_ANY: return ""; + case TOK_TO: return ""; + case TOK_ENDID: return ""; case TOK_COMMA: return ""; + case TOK_CLOSEENDIDS: return ""; + case TOK_OPENENDIDS: return ""; + case TOK_EQUALS: return ""; + case TOK_IDENT: return ""; + case TOK_END: return ""; + case TOK_START: return ""; + case TOK_CHAR: return ""; + case TOK_HEX: return ""; + case TOK_OCT: return ""; + case TOK_ESC: return ""; + case TOK_LABEL: return ""; + default: goto error; + } + } else + if (z == z4) { + switch (t) { case TOK_SEP: return ""; case TOK_ANY: return ""; case TOK_TO: return ""; + case TOK_ENDID: return ""; + case TOK_COMMA: return ""; + case TOK_CLOSEENDIDS: return ""; + case TOK_OPENENDIDS: return ""; + case TOK_EQUALS: return ""; + case TOK_IDENT: return ""; + case TOK_END: return ""; + case TOK_START: return ""; + case TOK_CHAR: return ""; + case TOK_HEX: return ""; + case TOK_OCT: return ""; + case TOK_ESC: return ""; + case TOK_LABEL: return ""; + default: goto error; + } + } else + if (z == z5) { + switch (t) { + case TOK_SEP: return ""; + case TOK_ANY: return ""; + case TOK_TO: return ""; + case TOK_ENDID: return ""; + case TOK_COMMA: return ""; + case TOK_CLOSEENDIDS: return ""; + case TOK_OPENENDIDS: return ""; + case TOK_EQUALS: return ""; case TOK_IDENT: return ""; case TOK_END: return ""; case TOK_START: return ""; @@ -1376,11 +1715,12 @@ lx_init(struct lx *lx) *lx = lx_default; lx->c = EOF; - lx->z = z3; + lx->z = z5; lx->end.byte = 0; lx->end.line = 1; lx->end.col = 1; + (void)lx_dynpop; } enum lx_token diff --git a/src/libfsm/lexer.h b/src/libfsm/lexer.h index 7848879fc..68e55b612 100644 --- a/src/libfsm/lexer.h +++ b/src/libfsm/lexer.h @@ -4,10 +4,14 @@ #define LX_H enum lx_token { - TOK_COMMA, TOK_SEP, TOK_ANY, TOK_TO, + TOK_ENDID, + TOK_COMMA, + TOK_CLOSEENDIDS, + TOK_OPENENDIDS, + TOK_EQUALS, TOK_IDENT, TOK_END, TOK_START, diff --git a/src/libfsm/lexer.lx b/src/libfsm/lexer.lx index 7aa8cf5bf..0288e1877 100644 --- a/src/libfsm/lexer.lx +++ b/src/libfsm/lexer.lx @@ -31,6 +31,17 @@ /[a-z0-9_]+/i -> $ident; +'=' -> $equals; +'[' -> $openendids .. ']' -> $closeendids { + /[\r\n\t ]+/; + "#" .. "\n"; + + ',' -> $comma; + + # using a lexical zone because $endid overlaps $ident + /[0-9]+/ -> $endid; +} + '->' -> $to; '?' -> $any; ';' -> $sep; diff --git a/src/libfsm/libfsm.syms b/src/libfsm/libfsm.syms index 415bffbea..9a4144e46 100644 --- a/src/libfsm/libfsm.syms +++ b/src/libfsm/libfsm.syms @@ -2,7 +2,9 @@ fsm_complement fsm_union fsm_union_array +fsm_union_repeated_pattern_group fsm_intersect +fsm_intersect_charset # fsm_cost_legible @@ -15,6 +17,7 @@ fsm_reachableall fsm_reachableany fsm_walk_edges fsm_walk_states +fsm_detect_required_characters # fsm_epsilonsonly @@ -34,23 +37,25 @@ fsm_generate_cb_printf_escaped fsm_parse # +fsm_dump +fsm_print +fsm_print_amd64_att +fsm_print_amd64_go +fsm_print_amd64_nasm fsm_print_api fsm_print_awk fsm_print_c fsm_print_dot fsm_print_fsm +fsm_print_go fsm_print_ir fsm_print_irjson fsm_print_json -fsm_print_vmc -fsm_print_vmdot -fsm_print_vmasm -fsm_print_vmasm_amd64_att -fsm_print_vmasm_amd64_nasm -fsm_print_vmasm_amd64_go +fsm_print_llvm fsm_print_rust fsm_print_sh -fsm_print_go +fsm_print_vmc +fsm_print_vmdot fsm_print_vmops_c fsm_print_vmops_h fsm_print_vmops_main @@ -59,14 +64,16 @@ fsm_print_vmops_main fsm_clone fsm_free fsm_new -fsm_setoptions -fsm_getoptions +fsm_new_statealloc fsm_move fsm_merge fsm_addstate fsm_addstate_bulk fsm_removestate fsm_shuffle +fsm_vacuum + +fsm_new_statealloc fsm_addedge_any fsm_addedge_epsilon @@ -84,18 +91,30 @@ fsm_setstart fsm_getstart fsm_setend -fsm_getendidcount -fsm_getendids +fsm_endid_count +fsm_endid_get +fsm_endid_set fsm_setendid fsm_mapendids fsm_increndids +fsm_endid_dump + +fsm_eager_output_set +fsm_eager_output_set_on_ends +fsm_eager_output_count +fsm_eager_output_get_cb +fsm_eager_output_set_cb +fsm_eager_output_dump +fsm_eager_output_get + fsm_countedges fsm_countstates fsm_trim fsm_reverse fsm_determinise +fsm_determinise_with_config fsm_remove_epsilons fsm_complete fsm_minimise @@ -108,6 +127,7 @@ fsm_shortest fsm_example fsm_exec +fsm_exec_with_captures # exec fsm_fgetc @@ -117,8 +137,8 @@ fsm_print_cfrag # XXX: workaround for lx make_ir # XXX: workaround for lx free_ir # XXX: workaround for lx -epsilon_closure # XXX: workaround for fsm -closure_free # XXX: workaround for fsm +fsm_epsilon_closure # XXX: workaround for fsm +fsm_closure_free # XXX: workaround for fsm fsm_mergeab @@ -129,11 +149,19 @@ fsm_vm_match_buffer fsm_vm_match_file # -fsm_countcaptures -fsm_capture_has_capture_actions +fsm_capture_ceiling fsm_capture_set_path fsm_capture_rebase_capture_id -fsm_capture_alloc +fsm_capture_alloc_capture_buffer +fsm_capture_free_capture_buffer fsm_capture_dump +fsm_capture_set_active_for_end +fsm_capture_add_program +fsm_capture_associate_program_with_end_state + +# capture_vm* +fsm_capvm_program_free +fsm_capvm_program_dump +fsm_capvm_program_exec fsm_minimise_test_oracle diff --git a/src/libfsm/merge.c b/src/libfsm/merge.c index fd4d35ee5..5cfbf5f93 100644 --- a/src/libfsm/merge.c +++ b/src/libfsm/merge.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -17,25 +18,40 @@ #include #include #include +#include #include "capture.h" +#include "capture_vm.h" #include "internal.h" #include "endids.h" +#include "eager_output.h" #define LOG_MERGE_ENDIDS 0 +#define LOG_COPY_CAPTURE_PROGRAMS 0 + struct copy_capture_env { +#ifndef NDEBUG char tag; +#endif + bool ok; struct fsm *dst; - int ok; }; static int -copy_capture_actions(struct fsm *dst, struct fsm *src); +copy_end_metadata(struct fsm *dst, struct fsm *src, + fsm_state_t base_src, unsigned capture_base_src); static int copy_end_ids(struct fsm *dst, struct fsm *src, fsm_state_t base_src); +static int +copy_active_capture_ids(struct fsm *dst, struct fsm *src, + fsm_state_t base_src, unsigned capture_base_src); + +static int +copy_eager_output_ids(struct fsm *dst, struct fsm *src, fsm_state_t base_src); + static struct fsm * merge(struct fsm *dst, struct fsm *src, fsm_state_t *base_dst, fsm_state_t *base_src, @@ -53,7 +69,7 @@ merge(struct fsm *dst, struct fsm *src, /* TODO: round up to next power of two here? * or let realloc do that internally */ - tmp = f_realloc(dst->opt->alloc, dst->states, newalloc * sizeof *dst->states); + tmp = f_realloc(dst->alloc, dst->states, newalloc * sizeof *dst->states); if (tmp == NULL) { return NULL; } @@ -72,17 +88,12 @@ merge(struct fsm *dst, struct fsm *src, *base_dst = 0; *base_src = dst->statecount; *capture_base_dst = 0; - *capture_base_src = fsm_countcaptures(dst); + *capture_base_src = fsm_capture_ceiling(dst); for (i = 0; i < src->statecount; i++) { state_set_rebase(&src->states[i].epsilons, *base_src); edge_set_rebase(&src->states[i].edges, *base_src); } - - /* FIXME: instead of rebasing these here, they could - * also be updated in copy_capture_actions below. */ - fsm_capture_rebase_capture_id(src, *capture_base_src); - fsm_capture_rebase_capture_action_states(src, *base_src); } memcpy(dst->states + dst->statecount, src->states, @@ -90,27 +101,20 @@ merge(struct fsm *dst, struct fsm *src, dst->statecount += src->statecount; dst->endcount += src->endcount; - /* We need to explicitly copy over the capture actions and end - * ID info here because they're stored on the FSMs as a whole, - * rather than individual states; `memcpy`ing the states alone - * won't transfer them. - * - * They're stored separately because they are likely to only - * be on a small portion of the states, and adding two extra - * NULL pointers to `struct fsm_state` increases memory usage - * significantly. */ - - if (!copy_capture_actions(dst, src)) { + /* We need to explicitly copy over end metadata here. They're + * stored separately because they are likely to only be on a + * small portion of the states. */ + if (!copy_end_metadata(dst, src, *base_src, *capture_base_src)) { /* non-recoverable -- destructive operation */ return NULL; } - if (!copy_end_ids(dst, src, *base_src)) { + if (!copy_eager_output_ids(dst, src, *base_src)) { /* non-recoverable -- destructive operation */ return NULL; } - f_free(src->opt->alloc, src->states); + f_free(src->alloc, src->states); src->states = NULL; src->statealloc = 0; src->statecount = 0; @@ -123,16 +127,91 @@ merge(struct fsm *dst, struct fsm *src, return dst; } +struct copy_capture_programs_env { + const struct fsm_alloc *alloc; + const struct fsm *src; + struct fsm *dst; + int ok; + fsm_state_t state_base_src; + unsigned capture_base_src; + +#define DEF_MAPPING_CEIL 1 + size_t mapping_used; + size_t mapping_ceil; + /* TODO: could cache last_map to check first if this becomes expensive */ + struct prog_mapping { + unsigned src_prog_id; + unsigned dst_prog_id; + } *mappings; +}; + static int -copy_capture_cb(fsm_state_t state, - enum capture_action_type type, unsigned capture_id, fsm_state_t to, +copy_capture_programs_cb(fsm_state_t src_state, unsigned src_prog_id, void *opaque) { - struct copy_capture_env *env = opaque; - assert(env->tag == 'C'); + struct copy_capture_programs_env *env = opaque; + + const fsm_state_t dst_state = src_state + env->state_base_src; + assert(dst_state < fsm_countstates(env->dst)); + +#if LOG_COPY_CAPTURE_PROGRAMS + fprintf(stderr, "%s: src %p, dst %p, src_prog_id %u, src_state %d, dst_state %d, capture_base_src %u\n", + __func__, (void *)env->src, (void *)env->dst, + src_prog_id, src_state, dst_state, env->capture_base_src); +#endif + int found = 0; + uint32_t dst_prog_id; + + for (size_t i = 0; i < env->mapping_used; i++) { + const struct prog_mapping *m = &env->mappings[i]; + if (m->src_prog_id == src_prog_id) { + dst_prog_id = m->dst_prog_id; + found = 1; + } + } + + if (!found) { + if (env->mapping_used == env->mapping_ceil) { /* grow */ + const size_t nceil = 2*env->mapping_ceil; + struct prog_mapping *nmappings = f_realloc(env->alloc, + env->mappings, nceil * sizeof(nmappings[0])); + if (nmappings == NULL) { + env->ok = 0; + return 0; + } + + env->mapping_ceil = nceil; + env->mappings = nmappings; + } + + const struct capvm_program *p = fsm_capture_get_program_by_id(env->src, + src_prog_id); + assert(p != NULL); + + struct capvm_program *cp = capvm_program_copy(env->alloc, p); + if (cp == NULL) { + env->ok = 0; + return 0; + } + capvm_program_rebase(cp, env->capture_base_src); + + /* add program, if not present */ + if (!fsm_capture_add_program(env->dst, + cp, &dst_prog_id)) { + f_free(env->alloc, cp); + env->ok = 0; + return 0; + } + + struct prog_mapping *m = &env->mappings[env->mapping_used]; + m->src_prog_id = src_prog_id; + m->dst_prog_id = dst_prog_id; + env->mapping_used++; + } - if (!fsm_capture_add_action(env->dst, state, type, - capture_id, to)) { + /* associate with end states */ + if (!fsm_capture_associate_program_with_end_state(env->dst, + dst_prog_id, dst_state)) { env->ok = 0; return 0; } @@ -141,18 +220,55 @@ copy_capture_cb(fsm_state_t state, } static int -copy_capture_actions(struct fsm *dst, struct fsm *src) +copy_capture_programs(struct fsm *dst, const struct fsm *src, + fsm_state_t state_base_src, unsigned capture_base_src) { - struct copy_capture_env env; - env.tag = 'C'; - env.dst = dst; - env.ok = 1; + const struct fsm_alloc *alloc = src->alloc; + struct prog_mapping *mappings = f_malloc(alloc, + DEF_MAPPING_CEIL * sizeof(mappings[0])); + if (mappings == NULL) { + return 0; + } - fsm_capture_action_iter(src, copy_capture_cb, &env); + struct copy_capture_programs_env env = { + .alloc = alloc, + .src = src, + .dst = dst, + .ok = 1, + .state_base_src = state_base_src, + .capture_base_src = capture_base_src, + .mapping_ceil = DEF_MAPPING_CEIL, + .mappings = mappings, + }; + fsm_capture_iter_program_ids_for_all_end_states(src, + copy_capture_programs_cb, &env); + + f_free(alloc, env.mappings); return env.ok; } +static int +copy_end_metadata(struct fsm *dst, struct fsm *src, + fsm_state_t base_src, unsigned capture_base_src) +{ + /* TODO: inline */ + + if (!copy_end_ids(dst, src, base_src)) { + return 0; + } + + if (!copy_active_capture_ids(dst, src, base_src, capture_base_src)) { + return 0; + } + + if (!copy_capture_programs(dst, src, base_src, capture_base_src)) { + return 0; + } + + return 1; +} + struct copy_end_ids_env { char tag; struct fsm *dst; @@ -163,7 +279,6 @@ struct copy_end_ids_env { static int copy_end_ids_cb(fsm_state_t state, const fsm_end_id_t *ids, size_t num_ids, void *opaque) { - enum fsm_endid_set_res sres; struct copy_end_ids_env *env = opaque; assert(env->tag == 'M'); @@ -172,26 +287,95 @@ copy_end_ids_cb(fsm_state_t state, const fsm_end_id_t *ids, size_t num_ids, void state + env->base_src, id); #endif - sres = fsm_endid_set_bulk(env->dst, state + env->base_src, num_ids, ids, FSM_ENDID_BULK_REPLACE); - if (sres == FSM_ENDID_SET_ERROR_ALLOC_FAIL) { - return 0; - } - - return 1; + return fsm_endid_set_bulk(env->dst, state + env->base_src, + num_ids, ids, FSM_ENDID_BULK_REPLACE); } static int copy_end_ids(struct fsm *dst, struct fsm *src, fsm_state_t base_src) { struct copy_end_ids_env env; +#ifndef NDEBUG env.tag = 'M'; /* for Merge */ +#endif env.dst = dst; - env.src = src; env.base_src = base_src; return fsm_endid_iter_bulk(src, copy_end_ids_cb, &env); } +struct copy_active_capture_ids_env { + char tag; + struct fsm *dst; + fsm_state_t base_src; + unsigned capture_base_src; + int ok; +}; + +static int +copy_active_capture_ids_cb(fsm_state_t state, unsigned capture_id, void *opaque) +{ + struct copy_active_capture_ids_env *env = opaque; + assert(env->tag == 'A'); + + if (!fsm_capture_set_active_for_end(env->dst, + capture_id + env->capture_base_src, + state + env->base_src)) { + env->ok = 0; + return 0; + } + return 1; +} + +static int +copy_active_capture_ids(struct fsm *dst, struct fsm *src, + fsm_state_t base_src, unsigned capture_base_src) +{ + struct copy_active_capture_ids_env env; + env.tag = 'A'; + env.dst = dst; + env.base_src = base_src; + env.capture_base_src = capture_base_src; + env.ok = 1; + + fsm_capture_iter_active_for_all_end_states(src, + copy_active_capture_ids_cb, &env); + return env.ok; +} + +struct copy_eager_output_ids_env { + bool ok; + struct fsm *dst; + struct fsm *src; + fsm_state_t base_src; +}; + +static int +copy_eager_output_ids_cb(fsm_state_t state, fsm_output_id_t id, void *opaque) +{ + struct copy_eager_output_ids_env *env = opaque; + if (!fsm_eager_output_set(env->dst, state + env->base_src, id)) { + env->ok = false; + return 0; + } + + return 1; + +} + +static int +copy_eager_output_ids(struct fsm *dst, struct fsm *src, fsm_state_t base_src) +{ + struct copy_eager_output_ids_env env = { + .ok = true, + .dst = dst, + .src = src, + .base_src = base_src, + }; + fsm_eager_output_iter_all(src, copy_eager_output_ids_cb, &env); + return env.ok; +} + struct fsm * fsm_mergeab(struct fsm *a, struct fsm *b, fsm_state_t *base_b) @@ -204,7 +388,7 @@ fsm_mergeab(struct fsm *a, struct fsm *b, assert(b != NULL); assert(base_b != NULL); - if (a->opt != b->opt) { + if (a->alloc != b->alloc) { errno = EINVAL; return NULL; } @@ -232,11 +416,6 @@ fsm_merge(struct fsm *a, struct fsm *b, assert(b != NULL); assert(combine_info != NULL); - if (a->opt != b->opt) { - errno = EINVAL; - return NULL; - } - /* * We merge the smaller FSM into the larger FSM. * The number of allocate states is considered first, because realloc diff --git a/src/libfsm/mergestates.c b/src/libfsm/mergestates.c index d525cd7ed..77d5e2f6c 100644 --- a/src/libfsm/mergestates.c +++ b/src/libfsm/mergestates.c @@ -28,10 +28,10 @@ fsm_mergestates(struct fsm *fsm, fsm_state_t a, fsm_state_t b, assert(b < fsm->statecount); /* edges from b */ - if (!state_set_copy(&fsm->states[a].epsilons, fsm->opt->alloc, fsm->states[b].epsilons)) { + if (!state_set_copy(&fsm->states[a].epsilons, fsm->alloc, fsm->states[b].epsilons)) { return 0; } - if (!edge_set_copy(&fsm->states[a].edges, fsm->opt->alloc, fsm->states[b].edges)) { + if (!edge_set_copy(&fsm->states[a].edges, fsm->alloc, fsm->states[b].edges)) { return 0; } diff --git a/src/libfsm/minimise.c b/src/libfsm/minimise.c index 60cec48c5..462967aec 100644 --- a/src/libfsm/minimise.c +++ b/src/libfsm/minimise.c @@ -17,17 +17,17 @@ #include #include -#if EXPENSIVE_CHECKS -#include -#endif - #include +#include #include #include #include +#include #include "internal.h" #include "capture.h" +#include "eager_output.h" +#include "endids.h" #define LOG_MAPPINGS 0 #define LOG_STEPS 0 @@ -38,6 +38,63 @@ #include "minimise_internal.h" #include "minimise_test_oracle.h" +static int +label_sets_match(const uint64_t a[256/64], const uint64_t b[256/64]); + +static int +split_ecs_by_end_metadata(struct min_env *env, const struct fsm *fsm); + +#if EXPENSIVE_CHECKS +#include + +static void +check_done_ec_offset(const struct min_env *env); + +static int +all_end_states_are_currently_together(const struct min_env *env); +#endif + +#define DEF_CAPTURE_ID_CEIL 4 +struct end_metadata { + struct end_metadata_end { + unsigned count; + fsm_end_id_t *ids; + } end; + + struct end_metadata_eager_outputs { + unsigned count; + fsm_output_id_t *ids; + } eager_outputs; + + struct end_metadata_capture { + unsigned count; + unsigned ceil; + unsigned *ids; + } capture; + + struct end_metadata_program { + unsigned count; + unsigned ceil; + unsigned *ids; + } program; +}; + +static int +collect_capture_ids(const struct fsm *fsm, fsm_state_t s, + struct end_metadata_capture *c); + +static int +collect_capture_program_ids(const struct fsm *fsm, fsm_state_t s, + struct end_metadata_program *p); + +static int +collect_end_ids(const struct fsm *fsm, fsm_state_t s, + struct end_metadata_end *e); + +static int +collect_eager_output_ids(const struct fsm *fsm, fsm_state_t s, + struct end_metadata_eager_outputs *e); + int fsm_minimise(struct fsm *fsm) { @@ -55,19 +112,36 @@ fsm_minimise(struct fsm *fsm) assert(fsm != NULL); assert(fsm_all(fsm, fsm_isdfa)); +#if LOG_INIT > 1 + fprintf(stderr, "=== BEFORE TRIM, %d states\n", fsm_countstates(fsm)); + fsm_print_fsm(stderr, fsm); + fsm_capture_dump(stderr, "#### pre_minimise", fsm); + fprintf(stderr, "=== BEFORE TRIM\n"); +#endif + /* The algorithm used below won't remove states without a path * to an end state, because it cannot prove they're * unnecessary, so they must be trimmed away first. */ + TIME(&pre); if (fsm_trim(fsm, FSM_TRIM_START_AND_END_REACHABLE, &shortest_end_distance) < 0) { return 0; } + TIME(&post); + DIFF_MSEC("trim", pre, post, NULL); if (fsm->statecount == 0) { r = 1; goto cleanup; } +#if LOG_INIT > 1 + fprintf(stderr, "=== AFTER TRIM, %d states\n", fsm_countstates(fsm)); + fprintf(stderr, "# pre_minimise\n"); + fsm_print_fsm(stderr, fsm); + fsm_capture_dump(stderr, "#### pre_minimise", fsm); +#endif + TIME(&pre); collect_labels(fsm, labels, &label_count); TIME(&post); @@ -78,7 +152,7 @@ fsm_minimise(struct fsm *fsm) goto cleanup; } - mapping = f_malloc(fsm->opt->alloc, + mapping = f_malloc(fsm->alloc, fsm->statecount * sizeof(mapping[0])); if (mapping == NULL) { goto cleanup; @@ -100,6 +174,10 @@ fsm_minimise(struct fsm *fsm) /* Minimisation should never add states. */ assert(minimised_states <= orig_states); + for (size_t i = 0; i < fsm->statecount; i++) { + assert(mapping[i] < fsm->statecount); + } + /* Use the mapping to consolidate the current states * into a new DFA, combining states that could not be * proven distinguishable. */ @@ -113,6 +191,8 @@ fsm_minimise(struct fsm *fsm) goto cleanup; } + fsm_capture_integrity_check(dst); + #if EXPENSIVE_CHECKS if (!fsm_capture_has_captures(fsm)) { struct fsm *oracle = fsm_minimise_test_oracle(fsm); @@ -122,10 +202,10 @@ fsm_minimise(struct fsm *fsm) fprintf(stderr, "%s: expected minimal DFA with %zu states, got %zu\n", __func__, exp_count, got_count); fprintf(stderr, "== expected:\n"); - fsm_print_fsm(stderr, oracle); + fsm_dump(stderr, oracle); fprintf(stderr, "== got:\n"); - fsm_print_fsm(stderr, dst); + fsm_dump(stderr, dst); assert(!"non-minimal result"); } @@ -135,12 +215,18 @@ fsm_minimise(struct fsm *fsm) fsm_move(fsm, dst); + /* The FSM state count should be settled now, so if the state + * allocation is unnecessarily large move to a smaller one. */ + if (!fsm_vacuum(fsm)) { /* realloc failure */ + r = 0; + } + cleanup: if (mapping != NULL) { - f_free(fsm->opt->alloc, mapping); + f_free(fsm->alloc, mapping); } if (shortest_end_distance != NULL) { - f_free(fsm->opt->alloc, shortest_end_distance); + f_free(fsm->alloc, shortest_end_distance); } return r; @@ -234,14 +320,14 @@ build_minimised_mapping(const struct fsm *fsm, env.dfa_labels = dfa_labels; env.dfa_label_count = dfa_label_count; - env.state_ecs = f_malloc(fsm->opt->alloc, alloc_size); + env.state_ecs = f_malloc(fsm->alloc, alloc_size); if (env.state_ecs == NULL) { goto cleanup; } env.ec_map_count = fsm->statecount + 1; - env.jump = f_malloc(fsm->opt->alloc, alloc_size); + env.jump = f_malloc(fsm->alloc, alloc_size); if (env.jump == NULL) { goto cleanup; } - env.ecs = f_malloc(fsm->opt->alloc, alloc_size); + env.ecs = f_malloc(fsm->alloc, alloc_size); if (env.ecs == NULL) { goto cleanup; } env.ecs[INIT_EC_NOT_FINAL] = NO_ID; @@ -253,6 +339,12 @@ build_minimised_mapping(const struct fsm *fsm, goto cleanup; } + /* This only needs to be run once, but must run before the main + * fixpoint loop below, because it potentially refines ECs. */ + if (!split_ecs_by_end_metadata(&env, fsm)) { + goto cleanup; + } + #if LOG_INIT for (i = 0; i < env.ec_count; i++) { fprintf(stderr, "# --ec[%lu]: %d\n", i, env.ecs[i]); @@ -329,7 +421,7 @@ build_minimised_mapping(const struct fsm *fsm, } } -#if EXPENSIVE_INTEGRITY_CHECKS +#if EXPENSIVE_CHECKS check_done_ec_offset(&env); #endif } @@ -365,6 +457,12 @@ build_minimised_mapping(const struct fsm *fsm, } #endif +#if EXPENSIVE_CHECKS + for (i = 0; i < fsm->statecount; i++) { + assert(mapping[i] < fsm->statecount); + } +#endif + #if LOG_STEPS fprintf(stderr, "# done in %lu iteration(s), %lu step(s), %ld -> %ld states, label_count %lu\n", env.iter, env.steps, fsm->statecount, @@ -375,9 +473,9 @@ build_minimised_mapping(const struct fsm *fsm, /* fall through */ cleanup: - f_free(fsm->opt->alloc, env.ecs); - f_free(fsm->opt->alloc, env.state_ecs); - f_free(fsm->opt->alloc, env.jump); + f_free(fsm->alloc, env.ecs); + f_free(fsm->alloc, env.state_ecs); + f_free(fsm->alloc, env.jump); return res; } @@ -403,7 +501,7 @@ dump_ecs(FILE *f, const struct min_env *env) #endif } -#if EXPENSIVE_INTEGRITY_CHECKS +#if EXPENSIVE_CHECKS static void check_descending_EC_counts(const struct min_env *env) { @@ -473,7 +571,7 @@ populate_initial_ecs(struct min_env *env, const struct fsm *fsm, assert(fsm != NULL); assert(shortest_end_distance != NULL); - counts = f_calloc(fsm->opt->alloc, + counts = f_calloc(fsm->alloc, DEF_INITIAL_COUNT_CEIL, sizeof(counts[0])); if (counts == NULL) { goto cleanup; @@ -499,7 +597,7 @@ populate_initial_ecs(struct min_env *env, const struct fsm *fsm, while (sed >= nceil) { nceil *= 2; } - ncounts = f_realloc(fsm->opt->alloc, + ncounts = f_realloc(fsm->alloc, counts, nceil * sizeof(counts[0])); if (ncounts == NULL) { goto cleanup; @@ -528,7 +626,7 @@ populate_initial_ecs(struct min_env *env, const struct fsm *fsm, /* Build a permutation vector of the counts, such * that counts[pv[i..N]] would return the values * in counts[] in ascending order. */ - pv = permutation_vector(fsm->opt->alloc, + pv = permutation_vector(fsm->alloc, sed_limit, count_max, counts); if (pv == NULL) { goto cleanup; @@ -568,7 +666,7 @@ populate_initial_ecs(struct min_env *env, const struct fsm *fsm, * [1]: http://www.sudleyplace.com/APL/Anatomy%20of%20An%20Idiom.pdf * [2]: https://bitbucket.org/ngn/k/src */ - ranking = permutation_vector(fsm->opt->alloc, + ranking = permutation_vector(fsm->alloc, sed_limit, sed_limit, pv); if (ranking == NULL) { goto cleanup; @@ -611,16 +709,16 @@ populate_initial_ecs(struct min_env *env, const struct fsm *fsm, /* The dead state is not a member of any EC. */ env->state_ecs[env->dead_state] = NO_ID; -#if EXPENSIVE_INTEGRITY_CHECKS +#if EXPENSIVE_CHECKS check_descending_EC_counts(env); #endif res = 1; cleanup: - f_free(fsm->opt->alloc, counts); - f_free(fsm->opt->alloc, pv); - f_free(fsm->opt->alloc, ranking); + f_free(fsm->alloc, counts); + f_free(fsm->alloc, pv); + f_free(fsm->alloc, ranking); return res; #else @@ -646,7 +744,501 @@ populate_initial_ecs(struct min_env *env, const struct fsm *fsm, #endif } -#if EXPENSIVE_INTEGRITY_CHECKS +SUPPRESS_EXPECTED_UNSIGNED_INTEGER_OVERFLOW() +static void +incremental_hash_of_ids(uint64_t *accum, fsm_end_id_t id) +{ + (*accum) += hash_id(id); +} + +static int +same_end_metadata(const struct end_metadata *a, const struct end_metadata *b) +{ + if (a->capture.count != b->capture.count) { + return 0; + } + + if (a->program.count != b->program.count) { + return 0; + } + + if (a->end.count != b->end.count) { + return 0; + } + if (a->eager_outputs.count != b->eager_outputs.count) { + return 0; + } + + /* compare -- these must be sorted */ + + for (size_t i = 0; i < a->capture.count; i++) { + if (a->capture.ids[i] != b->capture.ids[i]) { + return 0; + } + } + for (size_t i = 0; i < a->program.count; i++) { + if (a->program.ids[i] != b->program.ids[i]) { + return 0; + } + } + + for (size_t i = 0; i < a->end.count; i++) { + if (a->end.ids[i] != b->end.ids[i]) { + return 0; + } + } + + for (size_t i = 0; i < a->eager_outputs.count; i++) { + if (a->eager_outputs.ids[i] != b->eager_outputs.ids[i]) { + return 0; + } + } + + return 1; +} + +static int +split_ecs_by_end_metadata(struct min_env *env, const struct fsm *fsm) +{ + int res = 0; + + struct end_metadata *end_md; + fsm_state_t *htab = NULL; + + const size_t state_count = fsm_countstates(fsm); + +#if EXPENSIVE_CHECKS + /* Invariant: For each EC, either all or none of the states + * are end states. We only partition the set(s) of end states + * here. */ + assert(all_end_states_are_currently_together(env)); +#endif + + /* Use the hash table to assign to new groups. */ + + end_md = f_calloc(fsm->alloc, + state_count, sizeof(end_md[0])); + if (end_md == NULL) { + goto cleanup; + } + + size_t bucket_count = 1; + while (bucket_count < state_count) { + bucket_count *= 2; /* power of 2 ceiling */ + } + const size_t mask = bucket_count - 1; + + htab = f_malloc(fsm->alloc, + bucket_count * sizeof(htab[0])); + if (htab == NULL) { + goto cleanup; + } + + /* First pass: collect end state metadata */ + for (size_t ec_i = 0; ec_i < env->ec_count; ec_i++) { + fsm_state_t s = MASK_EC_HEAD(env->ecs[ec_i]); +#if LOG_ECS + fprintf(stderr, "## EC %zu\n", ec_i); +#endif + while (s != NO_ID) { + struct end_metadata *e = &end_md[s]; + const bool is_end = fsm_isend(fsm, s); + const bool has_eager_outputs = fsm_eager_output_state_has_eager_output(fsm, s); + + if (!is_end && !has_eager_outputs) { + break; /* skip */ + } + + if (!collect_end_ids(fsm, s, &e->end)) { + goto cleanup; + } + + if (!collect_capture_ids(fsm, s, &e->capture)) { + goto cleanup; + } + + if (!collect_capture_program_ids(fsm, s, &e->program)) { + goto cleanup; + } + + if (!collect_eager_output_ids(fsm, s, &e->eager_outputs)) { + goto cleanup; + } + + s = env->jump[s]; + } + } + +#if LOG_ECS + fprintf(stderr, "==== BEFORE PARTITIONING BY END METADATA\n"); + dump_ecs(stderr, env); + fprintf(stderr, "====\n"); +#endif + + /* Second pass: partition ECs into groups with identical end IDs. + * for each group with different end IDs, unlink them. */ + const size_t max_ec = env->ec_count; + for (size_t ec_i = 0; ec_i < max_ec; ec_i++) { + fsm_state_t s = MASK_EC_HEAD(env->ecs[ec_i]); + fsm_state_t prev = NO_ID; + + for (size_t i = 0; i < bucket_count; i++) { + htab[i] = NO_ID; /* reset hash table */ + } + + while (s != NO_ID) { + const struct end_metadata *s_md = &end_md[s]; + + uint64_t hash = 0; + const fsm_state_t next = env->jump[s]; + + for (size_t pid_i = 0; pid_i < s_md->program.count; pid_i++) { + incremental_hash_of_ids(&hash, s_md->program.ids[pid_i]); + } + + for (size_t eid_i = 0; eid_i < s_md->end.count; eid_i++) { + incremental_hash_of_ids(&hash, s_md->end.ids[eid_i]); + } + + for (size_t eo_i = 0; eo_i < s_md->eager_outputs.count; eo_i++) { + incremental_hash_of_ids(&hash, s_md->eager_outputs.ids[eo_i]); + } + + for (size_t b_i = 0; b_i < bucket_count; b_i++) { + fsm_state_t *b = &htab[(b_i + hash) & mask]; + const fsm_state_t other = *b; + const struct end_metadata *other_md = &end_md[other]; + + if (other == NO_ID) { /* empty hash bucket */ + *b = s; + if (prev == NO_ID) { + /* keep the first state, along with other states + * with matching end IDs, in this EC. no-op. */ +#if LOG_ECS + fprintf(stderr, " -- keeping state s %d in EC %u\n", + s, env->state_ecs[s]); +#endif + prev = s; + } else { /* not first (prev is set), so it landed somewhere else */ + /* unlink and assign new EC */ +#if LOG_ECS + fprintf(stderr, " -- moving state s %d from EC %u to EC %u\n", + s, env->state_ecs[s], env->ec_count); +#endif + env->jump[prev] = env->jump[s]; /* unlink */ + env->ecs[env->ec_count] = s; /* head of new EC */ + env->state_ecs[s] = env->ec_count; + env->jump[s] = NO_ID; + env->ec_count++; + } + break; + } else if (same_end_metadata(s_md, other_md)) { + if (env->state_ecs[other] == ec_i) { + /* keep in the current EC -- no-op */ +#if LOG_ECS + fprintf(stderr, " -- keeping state s %d in EC %u\n", + s, env->state_ecs[s]); +#endif + prev = s; + } else { + /* unlink and link to other state's EC */ +#if LOG_ECS + fprintf(stderr, " -- appending s %d to EC %u, after state %d, before %d\n", + s, env->state_ecs[other], other, env->jump[other]); +#endif + assert(prev != NO_ID); + env->jump[prev] = env->jump[s]; /* unlink */ + env->state_ecs[s] = env->state_ecs[other]; + env->jump[s] = env->jump[other]; + env->jump[other] = s; /* link after other */ + } + break; + } else { + continue; /* collision */ + } + } + + s = next; + } + + /* If this EC only has one entry and it's before the + * done_ec_offset, then set that here so that invariants + * will be restored while sweeping forward after this loop. */ + + if (env->jump[MASK_EC_HEAD(env->ecs[ec_i])] == NO_ID && ec_i < env->done_ec_offset) { + env->done_ec_offset = ec_i; /* will be readjusted later */ + } + +#if LOG_ECS + fprintf(stderr, "==== AFTER PARTITIONING BY END METADATA -- EC %zu\n", ec_i); + dump_ecs(stderr, env); + fprintf(stderr, "==== (done_ec_offset: %d)\n", env->done_ec_offset); +#endif + } + +#if LOG_ECS + fprintf(stderr, "==== AFTER PARTITIONING BY END IDs\n"); + dump_ecs(stderr, env); + fprintf(stderr, "==== (done_ec_offset: %d)\n", env->done_ec_offset); +#endif + + /* Sweep forward and swap ECs as necessary so all single-entry + * ECs are at the end -- they're done. */ + size_t ec_i = env->done_ec_offset; + + while (ec_i < env->ec_count) { + const fsm_state_t head = MASK_EC_HEAD(env->ecs[ec_i]); + if (env->jump[head] == NO_ID) { + /* offset stays where it is */ +#if LOG_ECS + fprintf(stderr, "ec_i: %zu / %u -- branch a\n", ec_i, env->ec_count); +#endif + env->ecs[ec_i] = SET_SMALL_EC_FLAG(head); + } else { + /* this EC has more than one state, but is after + * the done_ec_offset, so swap it with an EC at + * the boundary. */ + const fsm_state_t n_ec_i = env->done_ec_offset; +#if LOG_ECS + fprintf(stderr, "ec_i: %zu / %u -- branch b -- swap %ld and %d\n", + ec_i, env->ec_count, ec_i, n_ec_i); +#endif + + /* swap ec[n_ec_i] and ec[ec_i] */ + const fsm_state_t tmp = env->ecs[ec_i]; + env->ecs[ec_i] = env->ecs[n_ec_i]; + env->ecs[n_ec_i] = tmp; + /* note: this may set the SMALL_EC_FLAG. */ + update_ec_links(env, ec_i); + update_ec_links(env, n_ec_i); + env->done_ec_offset++; + } + ec_i++; + } + +#if LOG_ECS + fprintf(stderr, "==== (done_ec_offset is now: %d, ec_count %u)\n", env->done_ec_offset, env->ec_count); + dump_ecs(stderr, env); +#endif + + /* check that all ECs are before/after done_ec_offset */ + for (size_t ec_i = 0; ec_i < env->ec_count; ec_i++) { + const fsm_state_t s = MASK_EC_HEAD(env->ecs[ec_i]); +#if LOG_ECS + fprintf(stderr, " -- ec_i %zu: s %d\n", ec_i, s); +#endif + if (ec_i < env->done_ec_offset) { + assert(env->jump[s] != NO_ID); + } else { + assert(env->jump[s] == NO_ID); + } + } + + res = 1; + +cleanup: + if (htab != NULL) { + f_free(fsm->alloc, htab); + } + if (end_md != NULL) { + size_t i; + for (i = 0; i < state_count; i++) { + struct end_metadata *e = &end_md[i]; + if (e->end.ids != NULL) { + f_free(fsm->alloc, e->end.ids); + } + if (e->eager_outputs.ids != NULL) { + f_free(fsm->alloc, e->eager_outputs.ids); + } + if (e->capture.ids != NULL) { + f_free(fsm->alloc, e->capture.ids); + } + if (e->program.ids != NULL) { + f_free(fsm->alloc, e->program.ids); + } + } + f_free(fsm->alloc, end_md); + } + + return res; +} + +static int +cmp_unsigned(const void *pa, const void *pb) +{ + const unsigned a = *(unsigned *)pa; + const unsigned b = *(unsigned *)pb; + return a < b ? -1 : a > b ? 1 : 0; +} + +struct collect_capture_env { + int ok; + const struct fsm_alloc *alloc; + struct end_metadata_capture *c; + struct end_metadata_program *p; +}; + +static int +collect_capture_cb(fsm_state_t state, unsigned capture_id, + void *opaque) +{ + struct collect_capture_env *env = opaque; + struct end_metadata_capture *c = env->c; + (void)state; + if (c->count == c->ceil) { + const size_t nceil = (c->count == 0) + ? DEF_CAPTURE_ID_CEIL + : 2*c->ceil; + unsigned *nids = f_realloc(env->alloc, c->ids, nceil * sizeof(nids[0])); + if (nids == NULL) { + env->ok = 0; + return 0; + } + c->ids = nids; + c->ceil = nceil; + } + + c->ids[c->count] = capture_id; + c->count++; + return 1; +} + +static int +collect_capture_ids(const struct fsm *fsm, fsm_state_t s, + struct end_metadata_capture *c) +{ + struct collect_capture_env env = { + .ok = 1, + .alloc = fsm->alloc, + .c = c, + }; + fsm_capture_iter_active_for_end_state(fsm, s, + collect_capture_cb, &env); + + if (env.ok) { + if (c->ids == NULL) { + assert(c->count == 0); + } else { + qsort(c->ids, c->count, sizeof(c->ids[0]), cmp_unsigned); + } + } + + return env.ok; +} + +static int +collect_capture_program_ids_cb(fsm_state_t state, unsigned prog_id, + void *opaque) +{ + struct collect_capture_env *env = opaque; + struct end_metadata_program *p = env->p; + (void)state; + if (p->count == p->ceil) { + const size_t nceil = (p->count == 0) + ? DEF_CAPTURE_ID_CEIL + : 2*p->ceil; + unsigned *nids = f_realloc(env->alloc, p->ids, nceil * sizeof(nids[0])); + if (nids == NULL) { + env->ok = 0; + return 0; + } + p->ids = nids; + p->ceil = nceil; + } + + p->ids[p->count] = prog_id; + p->count++; + return 1; +} + +static int +collect_capture_program_ids(const struct fsm *fsm, fsm_state_t s, + struct end_metadata_program *p) +{ + struct collect_capture_env env = { + .ok = 1, + .alloc = fsm->alloc, + .p = p, + }; + fsm_capture_iter_program_ids_for_end_state(fsm, s, + collect_capture_program_ids_cb, &env); + + if (env.ok) { + if (p->ids == NULL) { + assert(p->count == 0); + } else { + qsort(p->ids, p->count, sizeof(p->ids[0]), cmp_unsigned); + } + } + + return env.ok; +} + +static int +collect_end_ids(const struct fsm *fsm, fsm_state_t s, + struct end_metadata_end *e) +{ + e->count = fsm_endid_count(fsm, s); + if (e->count == 0) { + return 1; + } + + e->ids = f_malloc(fsm->alloc, + e->count * sizeof(e->ids[0])); + if (e->ids == NULL) { + return 0; + } + + int res = fsm_endid_get(fsm, s, e->count, e->ids); + assert(res == 1); + +#if LOG_ECS + fprintf(stderr, "%d:", s); + for (size_t i = 0; i < e->count; i++) { + fprintf(stderr, " %u", e->ids[i]); + } + fprintf(stderr, "\n"); +#endif + + return 1; +} + +static int +collect_cb(fsm_state_t state, fsm_output_id_t id, void *opaque) +{ + (void)state; + struct end_metadata_eager_outputs *e = opaque; + e->ids[e->count++] = id; + return 1; +} + +static int cmp_eager_output_id(const void *pa, const void *pb) +{ + const fsm_output_id_t a = *(fsm_output_id_t *)pa; + const fsm_output_id_t b = *(fsm_output_id_t *)pb; + return a < b ? -1 : a > b ? 1 : 0; +} + +static int +collect_eager_output_ids(const struct fsm *fsm, fsm_state_t state, + struct end_metadata_eager_outputs *e) +{ + size_t count = fsm_eager_output_count(fsm, state); + if (count == 0) { + return 1; /* nothing to do */ + } + + e->ids = f_malloc(fsm->alloc, count * sizeof(e->ids[0])); + if (e->ids == NULL) { return 0; } + + fsm_eager_output_iter_state(fsm, state, collect_cb, e); + + /* sort, to normalize set */ + qsort(e->ids, e->count, sizeof(e->ids[0]), cmp_eager_output_id); + return 1; +} + +#if EXPENSIVE_CHECKS static void check_done_ec_offset(const struct min_env *env) { @@ -661,13 +1253,34 @@ check_done_ec_offset(const struct min_env *env) * worth the added complexity to avoid checking ECs 0 and 1. */ for (i = 0; i < env->ec_count; i++) { const fsm_state_t head = MASK_EC_HEAD(env->ecs[i]); - if (i >= done_ec_offset) { + if (i >= env->done_ec_offset) { assert(head == NO_ID || env->jump[head] == NO_ID); } else if (i >= 2) { assert(env->jump[head] != NO_ID); } } } + +static int +all_end_states_are_currently_together(const struct min_env *env) +{ + /* For each EC, either all or none of the states in it + * are end states. */ + for (size_t i = 0; i < env->ec_count; i++) { + const fsm_state_t head = MASK_EC_HEAD(env->ecs[i]); + const int ec_first_is_end = fsm_isend(env->fsm, head); + + fsm_state_t s = env->jump[head]; + while (s != NO_ID) { + if (fsm_isend(env->fsm, s) != ec_first_is_end) { + return 0; + } + s = env->jump[s]; + } + } + + return 1; +} #endif static int @@ -817,7 +1430,7 @@ try_partition(struct min_env *env, unsigned char label, const unsigned dead_state_ec = env->state_ecs[env->dead_state]; const struct fsm_state *states = env->fsm->states; -#if EXPENSIVE_INTEGRITY_CHECKS +#if EXPENSIVE_CHECKS /* Count states here, to compare against the partitioned * EC' counts later. */ size_t state_count = 0, psrc_count, pdst_count; @@ -857,7 +1470,7 @@ try_partition(struct min_env *env, unsigned char label, first_ec = dead_state_ec; } #if LOG_PARTITIONS > 1 - fprintf(stderr, "# --- try_partition: label '%c' -> EC %d\n", label, first_ec); + fprintf(stderr, "# --- try_partition: label '%c' -> first_ec %d\n", label, first_ec); #endif partition_counts[0] = 1; @@ -897,7 +1510,7 @@ try_partition(struct min_env *env, unsigned char label, partition_counts[0]++; prev = cur; cur = env->jump[cur]; - } else { /* unlink, split */ + } else if (to_ec != first_ec) { /* definitely different destination EC: unlink, split */ fsm_state_t next; #if LOG_PARTITIONS > 1 fprintf(stderr, "# try_partition: unlinking -- label '%c', src %u, dst %u, first_ec %d, cur %u -> to_ec %d\n", label, ec_src, ec_dst, first_ec, cur, to_ec); @@ -912,10 +1525,21 @@ try_partition(struct min_env *env, unsigned char label, env->ecs[ec_dst] = cur; cur = next; partition_counts[1]++; + } else { + /* Restrict the ones that will be marked as checked + * to the common subset before continuing, so that any + * other labels will still be checked in a later pass. */ + for (size_t i = 0; i < 4; i++) { + checked_labels[i] &= cur_label_set[i]; + } + + partition_counts[0]++; + prev = cur; + cur = env->jump[cur]; } } -#if EXPENSIVE_INTEGRITY_CHECKS +#if EXPENSIVE_CHECKS /* Count how many states were split into each EC * and check that the sum matches the original count. */ psrc_count = 0; diff --git a/src/libfsm/minimise_internal.h b/src/libfsm/minimise_internal.h index 6621d5040..f14352803 100644 --- a/src/libfsm/minimise_internal.h +++ b/src/libfsm/minimise_internal.h @@ -3,10 +3,6 @@ #define NO_ID ((fsm_state_t)-1) -/* If set to non-zero, do extra intensive - * integrity checks inside some inner loops. */ -#define EXPENSIVE_INTEGRITY_CHECKS 0 - #define DEF_INITIAL_COUNT_CEIL 8 struct min_env { diff --git a/src/libfsm/minimise_test_oracle.c b/src/libfsm/minimise_test_oracle.c index c7d1473c8..65bd3b43d 100644 --- a/src/libfsm/minimise_test_oracle.c +++ b/src/libfsm/minimise_test_oracle.c @@ -7,6 +7,8 @@ #include #include #include +#include +#include #include #include @@ -107,6 +109,20 @@ fsm_minimise_test_oracle(const struct fsm *fsm) fsm_state_t *tmp_map = NULL; fsm_state_t *mapping = NULL; + /* End metadata grouping: The fixpoint algorithm here isn't aware + * of endids or eager outputs associated with particular states, + * so do a pass grouping them by matching end metadata. + * + * end_md_group_assignments[X] = Y: state X is in end_md group Y + * end_md_group_leaders[X] = Y: see end state Y for end_md group X */ + unsigned *end_md_group_assignments = NULL; + size_t end_md_group_count = 1; /* group 0 is the empty set */ + unsigned *end_md_group_leaders = NULL; + fsm_end_id_t *ids_a = NULL; + fsm_end_id_t *ids_b = NULL; + fsm_output_id_t *eo_ids_a = NULL; + fsm_output_id_t *eo_ids_b = NULL; + table = calloc(row_words * table_states, sizeof(table[0])); if (table == NULL) { goto cleanup; } @@ -116,15 +132,37 @@ fsm_minimise_test_oracle(const struct fsm *fsm) mapping = malloc(state_count * sizeof(mapping[0])); if (mapping == NULL) { goto cleanup; } + end_md_group_assignments = calloc(state_count, sizeof(tmp_map[0])); + if (end_md_group_assignments == NULL) { goto cleanup; } + + end_md_group_leaders = calloc(state_count, sizeof(tmp_map[0])); + if (end_md_group_leaders == NULL) { goto cleanup; } + /* macros for NxN bit table */ #define POS(X,Y) ((X*table_states) + Y) #define MARK(X,Y) u64bitset_set(table, POS(X,Y)) #define CLEAR(X,Y) u64bitset_clear(table, POS(X,Y)) #define CHECK(X,Y) u64bitset_get(table, POS(X,Y)) + size_t max_endid_count = 0; + size_t max_eager_output_count = 0; + /* Mark all pairs of states where one is final and one is not. * This includes the dead state. */ for (size_t i = 0; i < table_states; i++) { + if (i < state_count && fsm_isend(fsm, i)) { + const size_t count = fsm_endid_count(fsm, i); + if (count > max_endid_count) { + max_endid_count = count; + } + } + + /* count eager outputs, not just on end states */ + const size_t eo_count = fsm_eager_output_count(fsm, i); + if (eo_count > max_eager_output_count) { + max_eager_output_count = eo_count; + } + for (size_t j = 0; j < i; j++) { const bool end_i = i == dead_state ? false : fsm_isend(fsm, i); @@ -141,6 +179,94 @@ fsm_minimise_test_oracle(const struct fsm *fsm) } } + ids_a = malloc(max_endid_count * sizeof(ids_a[0])); + if (ids_a == NULL) { goto cleanup; } + ids_b = malloc(max_endid_count * sizeof(ids_b[0])); + if (ids_b == NULL) { goto cleanup; } + + eo_ids_a = malloc(max_eager_output_count * sizeof(eo_ids_a[0])); + if (eo_ids_a == NULL) { goto cleanup; } + eo_ids_b = malloc(max_eager_output_count * sizeof(eo_ids_b[0])); + if (eo_ids_b == NULL) { goto cleanup; } + + /* For every end state, check if it has endids or eager outputs. + * If not, assign it to group 0 (none). Otherwise, check if its IDs match + * any of the other end states. If so, assign it to the same + * group, otherwise assign a new one and mark it as the leader. */ + for (size_t i = 0; i < state_count; i++) { + if (!fsm_isend(fsm, i)) { + end_md_group_assignments[i] = 0; /* none */ + continue; + } + + const size_t endid_count_a = fsm_endid_count(fsm, i); + assert(endid_count_a <= max_endid_count); + + const size_t eager_output_count_a = fsm_eager_output_count(fsm, i); + assert(eager_output_count_a <= max_eager_output_count); + + if (endid_count_a == 0 && eager_output_count_a == 0) { + continue; + } + + int eres = fsm_endid_get(fsm, i, endid_count_a, ids_a); + assert(eres == 1); + + fsm_eager_output_get(fsm, i, max_eager_output_count, eo_ids_a); + + bool found = false; + /* note: skipping eg 0 here since that's the empty set */ + for (size_t eg_i = 1; eg_i < end_md_group_count; eg_i++) { + size_t endid_count_b = fsm_endid_count(fsm, end_md_group_leaders[eg_i]); + if (endid_count_b != endid_count_a) { + continue; + } + + const size_t eager_output_count_b = fsm_eager_output_count(fsm, end_md_group_leaders[eg_i]); + assert(eager_output_count_b <= max_eager_output_count); + if (eager_output_count_b != eager_output_count_a) { + continue; + } + + assert(endid_count_b > 0 || eager_output_count_b > 0); + assert(endid_count_b <= max_endid_count); + eres = fsm_endid_get(fsm, end_md_group_leaders[eg_i], + endid_count_b, ids_b); + assert(eres == 1); + + fsm_eager_output_get(fsm, end_md_group_leaders[eg_i], max_eager_output_count, eo_ids_b); + + if ((0 == memcmp(ids_a, ids_b, endid_count_a * sizeof(ids_a[0]))) && + (0 == memcmp(eo_ids_a, eo_ids_b, eager_output_count_a * sizeof(eo_ids_a[0])))) { + found = true; + end_md_group_assignments[i] = eg_i; + break; + } + } + + if (!found) { + end_md_group_assignments[i] = end_md_group_count; + end_md_group_leaders[end_md_group_count] = i; + end_md_group_count++; + } + } + + /* Any end states that have not been assigned the same endid + * group must be distinguishable. */ + for (size_t i = 0; i < state_count; i++) { + if (fsm_isend(fsm, i)) { + const unsigned i_group = end_md_group_assignments[i]; + for (size_t j = 0; j < i; j++) { + if (fsm_isend(fsm, j)) { + const unsigned j_group = end_md_group_assignments[j]; + if (i_group != j_group) { + MARK(i, j); + } + } + } + } + } + bool changed; do { /* run until fixpoint */ changed = false; @@ -266,6 +392,12 @@ fsm_minimise_test_oracle(const struct fsm *fsm) free(table); free(tmp_map); free(mapping); + free(end_md_group_assignments); + free(end_md_group_leaders); + free(ids_a); + free(ids_b); + free(eo_ids_a); + free(eo_ids_b); return res; @@ -273,6 +405,12 @@ fsm_minimise_test_oracle(const struct fsm *fsm) if (table != NULL) { free(table); } if (tmp_map != NULL) { free(tmp_map); } if (mapping != NULL) { free(mapping); } + if (end_md_group_assignments != NULL) { free(end_md_group_assignments); } + if (end_md_group_leaders != NULL) { free(end_md_group_leaders); } + if (ids_a != NULL) { free(ids_a); } + if (ids_b != NULL) { free(ids_b); } if (res != NULL) { fsm_free(res); } + free(eo_ids_a); + free(eo_ids_b); return NULL; } diff --git a/src/libfsm/mode.c b/src/libfsm/mode.c index 76c60b8ad..87af0bdf9 100644 --- a/src/libfsm/mode.c +++ b/src/libfsm/mode.c @@ -28,6 +28,7 @@ fsm_findmode(const struct fsm *fsm, fsm_state_t state, unsigned int *freq) } mode; mode.freq = 1; + mode.state = (fsm_state_t)-1; edge_set_group_iter_reset(fsm->states[state].edges, EDGE_GROUP_ITER_ALL, &iter); while (edge_set_group_iter_next(&iter, &info)) { @@ -46,6 +47,9 @@ fsm_findmode(const struct fsm *fsm, fsm_state_t state, unsigned int *freq) *freq = mode.freq; } + /* It's not meaningful to call this on a state without edges. */ + assert(mode.state != (fsm_state_t)-1); + assert(mode.freq >= 1); return mode.state; } diff --git a/src/libfsm/parser.act b/src/libfsm/parser.act index 14ad7f169..9d97db001 100644 --- a/src/libfsm/parser.act +++ b/src/libfsm/parser.act @@ -1,5 +1,5 @@ /* - * Copyright 2008-2017 Katherine Flavel + * Copyright 2008-2024 Katherine Flavel * * See LICENCE for the full copyright terms. */ @@ -22,6 +22,7 @@ char -> char; string -> string; state -> state; + endid -> endid; %header% @{ @@ -41,8 +42,9 @@ #include "lexer.h" #include "parser.h" - typedef char * string; - typedef fsm_state_t state; + typedef char * string; + typedef fsm_state_t state; + typedef fsm_end_id_t endid; struct act_statelist { char *id; @@ -155,7 +157,8 @@ typedef struct fsm * fsm; struct fsm * - fsm_parse(FILE *f, const struct fsm_options *opt); + fsm_parse(FILE *f, + const struct fsm_alloc *alloc); @}; @@ -194,12 +197,14 @@ if ((u == ULONG_MAX && errno == ERANGE) || u > UCHAR_MAX) { err(lex_state, "octal escape %s out of range: expected \\0..\\%o inclusive", lex_state->buf.a, UCHAR_MAX); + /* XXX: don't exit in library code */ exit(EXIT_FAILURE); } if ((u == ULONG_MAX && errno != 0) || *e != '\0') { err(lex_state, "%s: %s: expected \\0..\\%o inclusive", lex_state->buf.a, strerror(errno), UCHAR_MAX); + /* XXX: don't exit in library code */ exit(EXIT_FAILURE); } @@ -221,12 +226,14 @@ if ((u == ULONG_MAX && errno == ERANGE) || u > UCHAR_MAX) { err(lex_state, "hex escape %s out of range: expected \\x0..\\x%x inclusive", lex_state->buf.a, UCHAR_MAX); + /* XXX: don't exit in library code */ exit(EXIT_FAILURE); } if ((u == ULONG_MAX && errno != 0) || *e != '\0') { err(lex_state, "%s: %s: expected \\x0..\\x%x inclusive", lex_state->buf.a, strerror(errno), UCHAR_MAX); + /* XXX: don't exit in library code */ exit(EXIT_FAILURE); } @@ -241,11 +248,33 @@ @}; IDENT: () -> (s :string) = @{ + /* XXX: don't exit in library code */ @s = xstrdup(lex_state->buf.a); - if (@s == NULL) { - perror("xstrdup"); + @}; + + ENDID: () -> (id :endid) = @{ + unsigned long u; + char *e; + + errno = 0; + + u = strtoul(lex_state->buf.a, &e, 16); + + if ((u == ULONG_MAX && errno == ERANGE) || u > FSM_END_ID_MAX) { + err(lex_state, "end id %s out of range: expected 0..%u inclusive", + lex_state->buf.a, FSM_END_ID_MAX); + /* XXX: don't exit in library code */ + exit(EXIT_FAILURE); + } + + if ((u == ULONG_MAX && errno != 0) || *e != '\0') { + err(lex_state, "%s: %s: expected 0..%u inclusive", + lex_state->buf.a, strerror(errno), FSM_END_ID_MAX); + /* XXX: don't exit in library code */ exit(EXIT_FAILURE); } + + @id = (fsm_end_id_t) u; @}; %actions% @@ -288,11 +317,8 @@ exit(EXIT_FAILURE); } + /* XXX: don't exit in library code */ new->id = xstrdup(@n); - if (new->id == NULL) { - perror("xstrdup"); - exit(EXIT_FAILURE); - } if (!fsm_addstate(fsm, &@s)) { perror("fsm_addstate"); @@ -314,6 +340,12 @@ fsm_setend(fsm, @s, 1); @}; + : (s :state, id :endid) -> () = @{ + if (fsm_endid_set(fsm, @s, @id) < 0) { + @!; + } + @}; + : (s :string) -> () = @{ free(@s); @}; @@ -382,7 +414,7 @@ %trailer% @{ - struct fsm *fsm_parse(FILE *f, const struct fsm_options *opt) { + struct fsm *fsm_parse(FILE *f, const struct fsm_alloc *alloc) { struct act_state act_state_s; struct act_state *act_state; struct lex_state lex_state_s; @@ -391,7 +423,6 @@ struct lx *lx; assert(f != NULL); - assert(opt != NULL); lex_state = &lex_state_s; @@ -418,14 +449,25 @@ /* This is a workaround for ADVANCE_LEXER assuming a pointer */ act_state = &act_state_s; - /* TODO: pass in fsm_options */ - new = fsm_new(opt); + new = fsm_new(alloc); if (new == NULL) { perror("fsm_new"); return NULL; } - ADVANCE_LEXER; /* XXX: what if the first token is unrecognised? */ + ADVANCE_LEXER; + switch (CURRENT_TERMINAL) { + case TOK_UNKNOWN: + /* fallthrough */ + case TOK_ERROR: + /* XXX: don't exit in library code */ + err(lex_state, "Syntax error"); + exit(EXIT_FAILURE); + + default: + break; + } + p_fsm(new, lex_state, act_state); free(act_state_s.states.buckets); diff --git a/src/libfsm/parser.c b/src/libfsm/parser.c index 81f1cc030..320a35993 100644 --- a/src/libfsm/parser.c +++ b/src/libfsm/parser.c @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 150 "src/libfsm/parser.act" +#line 27 "src/libfsm/parser.act" #include @@ -28,8 +28,9 @@ #include "lexer.h" #include "parser.h" - typedef char * string; - typedef fsm_state_t state; + typedef char * string; + typedef fsm_state_t state; + typedef fsm_end_id_t endid; struct act_statelist { char *id; @@ -135,7 +136,7 @@ htab->longest_chain_length = max_chain_length; } -#line 138 "src/libfsm/parser.c" +#line 140 "src/libfsm/parser.c" #ifndef ERROR_TERMINAL @@ -146,13 +147,16 @@ static void p_label(fsm, lex_state, act_state, char *); static void p_items(fsm, lex_state, act_state); +static void p_xend_C_Cend_Hstates(fsm, lex_state, act_state); static void p_xstart(fsm, lex_state, act_state); static void p_xend(fsm, lex_state, act_state); -static void p_61(fsm, lex_state, act_state); -static void p_63(fsm, lex_state, act_state, string *); +static void p_xend_C_Cend_Hstate(fsm, lex_state, act_state, state *); extern void p_fsm(fsm, lex_state, act_state); -static void p_xend_C_Cend_Hids(fsm, lex_state, act_state); -static void p_xend_C_Cend_Hid(fsm, lex_state, act_state); +static void p_74(fsm, lex_state, act_state); +static void p_76(fsm, lex_state, act_state); +static void p_78(fsm, lex_state, act_state, string *); +static void p_xend_C_Cend_Hids(fsm, lex_state, act_state, state); +static void p_xend_C_Cend_Hid(fsm, lex_state, act_state, state); /* BEGINNING OF STATIC VARIABLES */ @@ -168,21 +172,21 @@ p_label(fsm fsm, lex_state lex_state, act_state act_state, char *ZOc) return; } { - /* BEGINNING OF INLINE: 34 */ + /* BEGINNING OF INLINE: 40 */ { switch (CURRENT_TERMINAL) { case (TOK_CHAR): { /* BEGINNING OF EXTRACT: CHAR */ { -#line 239 "src/libfsm/parser.act" +#line 243 "src/libfsm/parser.act" assert(lex_state->buf.a[0] != '\0'); assert(lex_state->buf.a[1] == '\0'); ZIc = lex_state->buf.a[0]; -#line 185 "src/libfsm/parser.c" +#line 190 "src/libfsm/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; @@ -210,7 +214,7 @@ p_label(fsm fsm, lex_state lex_state, act_state act_state, char *ZOc) default: break; } -#line 213 "src/libfsm/parser.c" +#line 218 "src/libfsm/parser.c" } /* END OF EXTRACT: ESC */ ADVANCE_LEXER; @@ -220,7 +224,7 @@ p_label(fsm fsm, lex_state lex_state, act_state act_state, char *ZOc) { /* BEGINNING OF EXTRACT: HEX */ { -#line 232 "src/libfsm/parser.act" +#line 214 "src/libfsm/parser.act" unsigned long u; char *e; @@ -236,18 +240,20 @@ p_label(fsm fsm, lex_state lex_state, act_state act_state, char *ZOc) if ((u == ULONG_MAX && errno == ERANGE) || u > UCHAR_MAX) { err(lex_state, "hex escape %s out of range: expected \\x0..\\x%x inclusive", lex_state->buf.a, UCHAR_MAX); + /* XXX: don't exit in library code */ exit(EXIT_FAILURE); } if ((u == ULONG_MAX && errno != 0) || *e != '\0') { err(lex_state, "%s: %s: expected \\x0..\\x%x inclusive", lex_state->buf.a, strerror(errno), UCHAR_MAX); + /* XXX: don't exit in library code */ exit(EXIT_FAILURE); } ZIc = (char) (unsigned char) u; -#line 250 "src/libfsm/parser.c" +#line 257 "src/libfsm/parser.c" } /* END OF EXTRACT: HEX */ ADVANCE_LEXER; @@ -257,7 +263,7 @@ p_label(fsm fsm, lex_state lex_state, act_state act_state, char *ZOc) { /* BEGINNING OF EXTRACT: OCT */ { -#line 205 "src/libfsm/parser.act" +#line 185 "src/libfsm/parser.act" unsigned long u; char *e; @@ -273,18 +279,20 @@ p_label(fsm fsm, lex_state lex_state, act_state act_state, char *ZOc) if ((u == ULONG_MAX && errno == ERANGE) || u > UCHAR_MAX) { err(lex_state, "octal escape %s out of range: expected \\0..\\%o inclusive", lex_state->buf.a, UCHAR_MAX); + /* XXX: don't exit in library code */ exit(EXIT_FAILURE); } if ((u == ULONG_MAX && errno != 0) || *e != '\0') { err(lex_state, "%s: %s: expected \\0..\\%o inclusive", lex_state->buf.a, strerror(errno), UCHAR_MAX); + /* XXX: don't exit in library code */ exit(EXIT_FAILURE); } ZIc = (char) (unsigned char) u; -#line 287 "src/libfsm/parser.c" +#line 296 "src/libfsm/parser.c" } /* END OF EXTRACT: OCT */ ADVANCE_LEXER; @@ -294,7 +302,7 @@ p_label(fsm fsm, lex_state lex_state, act_state act_state, char *ZOc) goto ZL1; } } - /* END OF INLINE: 34 */ + /* END OF INLINE: 40 */ switch (CURRENT_TERMINAL) { case (TOK_LABEL): break; @@ -318,27 +326,24 @@ ZL2_items:; switch (CURRENT_TERMINAL) { case (TOK_IDENT): { - /* BEGINNING OF INLINE: 48 */ + /* BEGINNING OF INLINE: 54 */ { { string ZIa; - /* BEGINNING OF INLINE: id */ + /* BEGINNING OF INLINE: ident */ { { switch (CURRENT_TERMINAL) { case (TOK_IDENT): /* BEGINNING OF EXTRACT: IDENT */ { -#line 243 "src/libfsm/parser.act" +#line 250 "src/libfsm/parser.act" + /* XXX: don't exit in library code */ ZIa = xstrdup(lex_state->buf.a); - if (ZIa == NULL) { - perror("xstrdup"); - exit(EXIT_FAILURE); - } -#line 341 "src/libfsm/parser.c" +#line 347 "src/libfsm/parser.c" } /* END OF EXTRACT: IDENT */ break; @@ -348,20 +353,20 @@ ZL2_items:; ADVANCE_LEXER; } } - /* END OF INLINE: id */ - p_63 (fsm, lex_state, act_state, &ZIa); + /* END OF INLINE: ident */ + p_78 (fsm, lex_state, act_state, &ZIa); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; } } } - /* END OF INLINE: 48 */ + /* END OF INLINE: 54 */ /* BEGINNING OF INLINE: items */ goto ZL2_items; /* END OF INLINE: items */ } - /*UNREACHED*/ + /* UNREACHED */ case (ERROR_TERMINAL): return; default: @@ -373,6 +378,92 @@ ZL1:; return; } +static void +p_xend_C_Cend_Hstates(fsm fsm, lex_state lex_state, act_state act_state) +{ + if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { + return; + } +ZL2_xend_C_Cend_Hstates:; + { + state ZIs; + + p_xend_C_Cend_Hstate (fsm, lex_state, act_state, &ZIs); + /* BEGINNING OF INLINE: 71 */ + { + switch (CURRENT_TERMINAL) { + case (TOK_EQUALS): + { + ADVANCE_LEXER; + switch (CURRENT_TERMINAL) { + case (TOK_OPENENDIDS): + break; + default: + goto ZL1; + } + ADVANCE_LEXER; + /* BEGINNING OF INLINE: 72 */ + { + switch (CURRENT_TERMINAL) { + case (TOK_ENDID): + { + p_xend_C_Cend_Hids (fsm, lex_state, act_state, ZIs); + if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { + RESTORE_LEXER; + goto ZL1; + } + } + break; + default: + break; + } + } + /* END OF INLINE: 72 */ + switch (CURRENT_TERMINAL) { + case (TOK_CLOSEENDIDS): + break; + default: + goto ZL1; + } + ADVANCE_LEXER; + } + break; + case (ERROR_TERMINAL): + RESTORE_LEXER; + goto ZL1; + default: + break; + } + } + /* END OF INLINE: 71 */ + /* BEGINNING OF INLINE: 73 */ + { + switch (CURRENT_TERMINAL) { + case (TOK_COMMA): + { + p_74 (fsm, lex_state, act_state); + /* BEGINNING OF INLINE: xend::end-states */ + if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { + RESTORE_LEXER; + goto ZL1; + } else { + goto ZL2_xend_C_Cend_Hstates; + } + /* END OF INLINE: xend::end-states */ + } + /* UNREACHED */ + default: + break; + } + } + /* END OF INLINE: 73 */ + } + return; +ZL1:; + SAVE_LEXER ((ERROR_TERMINAL)); + return; +} + static void p_xstart(fsm fsm, lex_state lex_state, act_state act_state) { @@ -382,7 +473,7 @@ p_xstart(fsm fsm, lex_state lex_state, act_state act_state) string ZIn; state ZIs; - /* BEGINNING OF INLINE: 50 */ + /* BEGINNING OF INLINE: 56 */ { { switch (CURRENT_TERMINAL) { @@ -398,33 +489,30 @@ p_xstart(fsm fsm, lex_state lex_state, act_state act_state) { /* BEGINNING OF ACTION: err-expected-start */ { -#line 371 "src/libfsm/parser.act" +#line 402 "src/libfsm/parser.act" err_expected(lex_state, "'start:'"); -#line 405 "src/libfsm/parser.c" +#line 497 "src/libfsm/parser.c" } /* END OF ACTION: err-expected-start */ } ZL2:; } - /* END OF INLINE: 50 */ - /* BEGINNING OF INLINE: id */ + /* END OF INLINE: 56 */ + /* BEGINNING OF INLINE: ident */ { { switch (CURRENT_TERMINAL) { case (TOK_IDENT): /* BEGINNING OF EXTRACT: IDENT */ { -#line 243 "src/libfsm/parser.act" +#line 250 "src/libfsm/parser.act" + /* XXX: don't exit in library code */ ZIn = xstrdup(lex_state->buf.a); - if (ZIn == NULL) { - perror("xstrdup"); - exit(EXIT_FAILURE); - } -#line 427 "src/libfsm/parser.c" +#line 516 "src/libfsm/parser.c" } /* END OF EXTRACT: IDENT */ break; @@ -434,15 +522,15 @@ p_xstart(fsm fsm, lex_state lex_state, act_state act_state) ADVANCE_LEXER; } } - /* END OF INLINE: id */ - p_61 (fsm, lex_state, act_state); + /* END OF INLINE: ident */ + p_76 (fsm, lex_state, act_state); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; } /* BEGINNING OF ACTION: add-state */ { -#line 254 "src/libfsm/parser.act" +#line 282 "src/libfsm/parser.act" struct act_statelist *p; const unsigned hash = hash_of_id((ZIn)); @@ -481,11 +569,8 @@ p_xstart(fsm fsm, lex_state lex_state, act_state act_state) exit(EXIT_FAILURE); } + /* XXX: don't exit in library code */ new->id = xstrdup((ZIn)); - if (new->id == NULL) { - perror("xstrdup"); - exit(EXIT_FAILURE); - } if (!fsm_addstate(fsm, &(ZIs))) { perror("fsm_addstate"); @@ -498,25 +583,25 @@ p_xstart(fsm fsm, lex_state lex_state, act_state act_state) act_state->states.buckets[bucket_id] = new; } -#line 501 "src/libfsm/parser.c" +#line 587 "src/libfsm/parser.c" } /* END OF ACTION: add-state */ /* BEGINNING OF ACTION: mark-start */ { -#line 309 "src/libfsm/parser.act" +#line 335 "src/libfsm/parser.act" fsm_setstart(fsm, (ZIs)); -#line 510 "src/libfsm/parser.c" +#line 596 "src/libfsm/parser.c" } /* END OF ACTION: mark-start */ /* BEGINNING OF ACTION: free */ { -#line 317 "src/libfsm/parser.act" +#line 349 "src/libfsm/parser.act" free((ZIn)); -#line 519 "src/libfsm/parser.c" +#line 605 "src/libfsm/parser.c" } /* END OF ACTION: free */ } @@ -538,7 +623,7 @@ p_xend(fsm fsm, lex_state lex_state, act_state act_state) switch (CURRENT_TERMINAL) { case (TOK_END): { - /* BEGINNING OF INLINE: 60 */ + /* BEGINNING OF INLINE: 75 */ { { switch (CURRENT_TERMINAL) { @@ -554,19 +639,19 @@ p_xend(fsm fsm, lex_state lex_state, act_state act_state) { /* BEGINNING OF ACTION: err-expected-end */ { -#line 375 "src/libfsm/parser.act" +#line 406 "src/libfsm/parser.act" err_expected(lex_state, "'end:'"); -#line 561 "src/libfsm/parser.c" +#line 647 "src/libfsm/parser.c" } /* END OF ACTION: err-expected-end */ } ZL2:; } - /* END OF INLINE: 60 */ - p_xend_C_Cend_Hids (fsm, lex_state, act_state); - p_61 (fsm, lex_state, act_state); + /* END OF INLINE: 75 */ + p_xend_C_Cend_Hstates (fsm, lex_state, act_state); + p_76 (fsm, lex_state, act_state); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -585,7 +670,213 @@ ZL1:; } static void -p_61(fsm fsm, lex_state lex_state, act_state act_state) +p_xend_C_Cend_Hstate(fsm fsm, lex_state lex_state, act_state act_state, state *ZOs) +{ + state ZIs; + + if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { + return; + } + { + string ZIn; + + /* BEGINNING OF INLINE: ident */ + { + { + switch (CURRENT_TERMINAL) { + case (TOK_IDENT): + /* BEGINNING OF EXTRACT: IDENT */ + { +#line 250 "src/libfsm/parser.act" + + /* XXX: don't exit in library code */ + ZIn = xstrdup(lex_state->buf.a); + +#line 696 "src/libfsm/parser.c" + } + /* END OF EXTRACT: IDENT */ + break; + default: + goto ZL1; + } + ADVANCE_LEXER; + } + } + /* END OF INLINE: ident */ + /* BEGINNING OF ACTION: add-state */ + { +#line 282 "src/libfsm/parser.act" + + struct act_statelist *p; + const unsigned hash = hash_of_id((ZIn)); + const unsigned mask = act_state->states.bucket_count - 1; + size_t chain_length = 0; + unsigned bucket_id = hash & mask; + + assert((ZIn) != NULL); + + if (act_state->states.longest_chain_length > MAX_CHAIN_LENGTH) { + grow_state_hash_table(&act_state->states); + bucket_id = hash & (act_state->states.bucket_count - 1); + } + + for (p = act_state->states.buckets[bucket_id]; p != NULL; p = p->next) { + assert(p->id != NULL); + + chain_length++; + + if (0 == strcmp(p->id, (ZIn))) { + (ZIs) = p->state; + break; + } + } + + if (chain_length > act_state->states.longest_chain_length) { + act_state->states.longest_chain_length = chain_length; + } + + if (p == NULL) { + struct act_statelist *new; + + new = malloc(sizeof *new); + if (new == NULL) { + perror("malloc"); + exit(EXIT_FAILURE); + } + + /* XXX: don't exit in library code */ + new->id = xstrdup((ZIn)); + + if (!fsm_addstate(fsm, &(ZIs))) { + perror("fsm_addstate"); + exit(EXIT_FAILURE); + } + + new->state = (ZIs); + + new->next = act_state->states.buckets[bucket_id]; + act_state->states.buckets[bucket_id] = new; + } + +#line 762 "src/libfsm/parser.c" + } + /* END OF ACTION: add-state */ + /* BEGINNING OF ACTION: mark-end */ + { +#line 339 "src/libfsm/parser.act" + + fsm_setend(fsm, (ZIs), 1); + +#line 771 "src/libfsm/parser.c" + } + /* END OF ACTION: mark-end */ + /* BEGINNING OF ACTION: free */ + { +#line 349 "src/libfsm/parser.act" + + free((ZIn)); + +#line 780 "src/libfsm/parser.c" + } + /* END OF ACTION: free */ + } + goto ZL0; +ZL1:; + SAVE_LEXER ((ERROR_TERMINAL)); + return; +ZL0:; + *ZOs = ZIs; +} + +void +p_fsm(fsm fsm, lex_state lex_state, act_state act_state) +{ + if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { + return; + } + { + p_items (fsm, lex_state, act_state); + p_xstart (fsm, lex_state, act_state); + p_xend (fsm, lex_state, act_state); + switch (CURRENT_TERMINAL) { + case (TOK_EOF): + break; + case (ERROR_TERMINAL): + RESTORE_LEXER; + goto ZL1; + default: + goto ZL1; + } + ADVANCE_LEXER; + /* BEGINNING OF ACTION: free-statelist */ + { +#line 353 "src/libfsm/parser.act" + + struct act_statelist *p; + struct act_statelist *next; + unsigned b_i; /* bucket */ + + for (b_i = 0; b_i < act_state->states.bucket_count; b_i++) { + for (p = act_state->states.buckets[b_i]; p != NULL; p = next) { + next = p->next; + assert(p->id != NULL); + free(p->id); + free(p); + } + } + +#line 829 "src/libfsm/parser.c" + } + /* END OF ACTION: free-statelist */ + } + return; +ZL1:; + { + /* BEGINNING OF ACTION: err-syntax */ + { +#line 410 "src/libfsm/parser.act" + + err(lex_state, "Syntax error"); + exit(EXIT_FAILURE); + +#line 843 "src/libfsm/parser.c" + } + /* END OF ACTION: err-syntax */ + } +} + +static void +p_74(fsm fsm, lex_state lex_state, act_state act_state) +{ + if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { + return; + } + { + switch (CURRENT_TERMINAL) { + case (TOK_COMMA): + break; + default: + goto ZL1; + } + ADVANCE_LEXER; + } + return; +ZL1:; + { + /* BEGINNING OF ACTION: err-expected-comma */ + { +#line 398 "src/libfsm/parser.act" + + err_expected(lex_state, "','"); + +#line 873 "src/libfsm/parser.c" + } + /* END OF ACTION: err-expected-comma */ + } +} + +static void +p_76(fsm fsm, lex_state lex_state, act_state act_state) { if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { return; @@ -604,18 +895,18 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-sep */ { -#line 359 "src/libfsm/parser.act" +#line 390 "src/libfsm/parser.act" err_expected(lex_state, "';'"); -#line 611 "src/libfsm/parser.c" +#line 903 "src/libfsm/parser.c" } /* END OF ACTION: err-expected-sep */ } } static void -p_63(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) +p_78(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) { switch (CURRENT_TERMINAL) { case (TOK_TO): @@ -625,22 +916,19 @@ p_63(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) state ZIy; ADVANCE_LEXER; - /* BEGINNING OF INLINE: id */ + /* BEGINNING OF INLINE: ident */ { { switch (CURRENT_TERMINAL) { case (TOK_IDENT): /* BEGINNING OF EXTRACT: IDENT */ { -#line 243 "src/libfsm/parser.act" +#line 250 "src/libfsm/parser.act" + /* XXX: don't exit in library code */ ZIb = xstrdup(lex_state->buf.a); - if (ZIb == NULL) { - perror("xstrdup"); - exit(EXIT_FAILURE); - } -#line 643 "src/libfsm/parser.c" +#line 932 "src/libfsm/parser.c" } /* END OF EXTRACT: IDENT */ break; @@ -650,10 +938,10 @@ p_63(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) ADVANCE_LEXER; } } - /* END OF INLINE: id */ + /* END OF INLINE: ident */ /* BEGINNING OF ACTION: add-state */ { -#line 254 "src/libfsm/parser.act" +#line 282 "src/libfsm/parser.act" struct act_statelist *p; const unsigned hash = hash_of_id((*ZIa)); @@ -692,11 +980,8 @@ p_63(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) exit(EXIT_FAILURE); } + /* XXX: don't exit in library code */ new->id = xstrdup((*ZIa)); - if (new->id == NULL) { - perror("xstrdup"); - exit(EXIT_FAILURE); - } if (!fsm_addstate(fsm, &(ZIx))) { perror("fsm_addstate"); @@ -709,12 +994,12 @@ p_63(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) act_state->states.buckets[bucket_id] = new; } -#line 712 "src/libfsm/parser.c" +#line 998 "src/libfsm/parser.c" } /* END OF ACTION: add-state */ /* BEGINNING OF ACTION: add-state */ { -#line 254 "src/libfsm/parser.act" +#line 282 "src/libfsm/parser.act" struct act_statelist *p; const unsigned hash = hash_of_id((ZIb)); @@ -753,11 +1038,8 @@ p_63(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) exit(EXIT_FAILURE); } + /* XXX: don't exit in library code */ new->id = xstrdup((ZIb)); - if (new->id == NULL) { - perror("xstrdup"); - exit(EXIT_FAILURE); - } if (!fsm_addstate(fsm, &(ZIy))) { perror("fsm_addstate"); @@ -770,28 +1052,28 @@ p_63(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) act_state->states.buckets[bucket_id] = new; } -#line 773 "src/libfsm/parser.c" +#line 1056 "src/libfsm/parser.c" } /* END OF ACTION: add-state */ /* BEGINNING OF ACTION: free */ { -#line 317 "src/libfsm/parser.act" +#line 349 "src/libfsm/parser.act" free((*ZIa)); -#line 782 "src/libfsm/parser.c" +#line 1065 "src/libfsm/parser.c" } /* END OF ACTION: free */ /* BEGINNING OF ACTION: free */ { -#line 317 "src/libfsm/parser.act" +#line 349 "src/libfsm/parser.act" free((ZIb)); -#line 791 "src/libfsm/parser.c" +#line 1074 "src/libfsm/parser.c" } /* END OF ACTION: free */ - /* BEGINNING OF INLINE: 42 */ + /* BEGINNING OF INLINE: 48 */ { switch (CURRENT_TERMINAL) { case (TOK_ANY): @@ -799,14 +1081,14 @@ p_63(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) ADVANCE_LEXER; /* BEGINNING OF ACTION: add-edge-any */ { -#line 343 "src/libfsm/parser.act" +#line 375 "src/libfsm/parser.act" if (!fsm_addedge_any(fsm, (ZIx), (ZIy))) { perror("fsm_addedge_any"); exit(EXIT_FAILURE); } -#line 809 "src/libfsm/parser.c" +#line 1092 "src/libfsm/parser.c" } /* END OF ACTION: add-edge-any */ } @@ -822,14 +1104,14 @@ p_63(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) } /* BEGINNING OF ACTION: add-edge-literal */ { -#line 336 "src/libfsm/parser.act" +#line 368 "src/libfsm/parser.act" if (!fsm_addedge_literal(fsm, (ZIx), (ZIy), (ZIc))) { perror("fsm_addedge_literal"); exit(EXIT_FAILURE); } -#line 832 "src/libfsm/parser.c" +#line 1115 "src/libfsm/parser.c" } /* END OF ACTION: add-edge-literal */ } @@ -838,14 +1120,14 @@ p_63(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) { /* BEGINNING OF ACTION: add-edge-epsilon */ { -#line 350 "src/libfsm/parser.act" +#line 382 "src/libfsm/parser.act" if (!fsm_addedge_epsilon(fsm, (ZIx), (ZIy))) { perror("fsm_addedge_epsilon"); exit(EXIT_FAILURE); } -#line 848 "src/libfsm/parser.c" +#line 1131 "src/libfsm/parser.c" } /* END OF ACTION: add-edge-epsilon */ } @@ -856,18 +1138,18 @@ p_63(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) { /* BEGINNING OF ACTION: err-expected-trans */ { -#line 363 "src/libfsm/parser.act" +#line 394 "src/libfsm/parser.act" err_expected(lex_state, "transition"); -#line 863 "src/libfsm/parser.c" +#line 1146 "src/libfsm/parser.c" } /* END OF ACTION: err-expected-trans */ } ZL3:; } - /* END OF INLINE: 42 */ - p_61 (fsm, lex_state, act_state); + /* END OF INLINE: 48 */ + p_76 (fsm, lex_state, act_state); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -876,11 +1158,11 @@ p_63(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) break; case (TOK_SEP): { - state ZI45; + state ZI51; /* BEGINNING OF ACTION: add-state */ { -#line 254 "src/libfsm/parser.act" +#line 282 "src/libfsm/parser.act" struct act_statelist *p; const unsigned hash = hash_of_id((*ZIa)); @@ -901,7 +1183,7 @@ p_63(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) chain_length++; if (0 == strcmp(p->id, (*ZIa))) { - (ZI45) = p->state; + (ZI51) = p->state; break; } } @@ -919,36 +1201,33 @@ p_63(fsm fsm, lex_state lex_state, act_state act_state, string *ZIa) exit(EXIT_FAILURE); } + /* XXX: don't exit in library code */ new->id = xstrdup((*ZIa)); - if (new->id == NULL) { - perror("xstrdup"); - exit(EXIT_FAILURE); - } - if (!fsm_addstate(fsm, &(ZI45))) { + if (!fsm_addstate(fsm, &(ZI51))) { perror("fsm_addstate"); exit(EXIT_FAILURE); } - new->state = (ZI45); + new->state = (ZI51); new->next = act_state->states.buckets[bucket_id]; act_state->states.buckets[bucket_id] = new; } -#line 939 "src/libfsm/parser.c" +#line 1219 "src/libfsm/parser.c" } /* END OF ACTION: add-state */ /* BEGINNING OF ACTION: free */ { -#line 317 "src/libfsm/parser.act" +#line 349 "src/libfsm/parser.act" free((*ZIa)); -#line 948 "src/libfsm/parser.c" +#line 1228 "src/libfsm/parser.c" } /* END OF ACTION: free */ - p_61 (fsm, lex_state, act_state); + p_76 (fsm, lex_state, act_state); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -966,109 +1245,31 @@ ZL1:; return; } -void -p_fsm(fsm fsm, lex_state lex_state, act_state act_state) -{ - if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { - return; - } - { - p_items (fsm, lex_state, act_state); - p_xstart (fsm, lex_state, act_state); - p_xend (fsm, lex_state, act_state); - switch (CURRENT_TERMINAL) { - case (TOK_EOF): - break; - case (ERROR_TERMINAL): - RESTORE_LEXER; - goto ZL1; - default: - goto ZL1; - } - ADVANCE_LEXER; - /* BEGINNING OF ACTION: free-statelist */ - { -#line 333 "src/libfsm/parser.act" - - struct act_statelist *p; - struct act_statelist *next; - unsigned b_i; /* bucket */ - - for (b_i = 0; b_i < act_state->states.bucket_count; b_i++) { - for (p = act_state->states.buckets[b_i]; p != NULL; p = next) { - next = p->next; - assert(p->id != NULL); - free(p->id); - free(p); - } - } - -#line 1006 "src/libfsm/parser.c" - } - /* END OF ACTION: free-statelist */ - } - return; -ZL1:; - { - /* BEGINNING OF ACTION: err-syntax */ - { -#line 380 "src/libfsm/parser.act" - - err(lex_state, "Syntax error"); - exit(EXIT_FAILURE); - -#line 1020 "src/libfsm/parser.c" - } - /* END OF ACTION: err-syntax */ - } -} - static void -p_xend_C_Cend_Hids(fsm fsm, lex_state lex_state, act_state act_state) +p_xend_C_Cend_Hids(fsm fsm, lex_state lex_state, act_state act_state, state ZIs) { if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { return; } ZL2_xend_C_Cend_Hids:; { - p_xend_C_Cend_Hid (fsm, lex_state, act_state); - /* BEGINNING OF INLINE: 58 */ + p_xend_C_Cend_Hid (fsm, lex_state, act_state, ZIs); + /* BEGINNING OF INLINE: 65 */ { switch (CURRENT_TERMINAL) { case (TOK_COMMA): { - /* BEGINNING OF INLINE: 59 */ - { - { - switch (CURRENT_TERMINAL) { - case (TOK_COMMA): - break; - default: - goto ZL5; - } - ADVANCE_LEXER; - } - goto ZL4; - ZL5:; - { - /* BEGINNING OF ACTION: err-expected-comma */ - { -#line 367 "src/libfsm/parser.act" - - err_expected(lex_state, "','"); - -#line 1060 "src/libfsm/parser.c" - } - /* END OF ACTION: err-expected-comma */ - } - ZL4:; - } - /* END OF INLINE: 59 */ + p_74 (fsm, lex_state, act_state); /* BEGINNING OF INLINE: xend::end-ids */ - goto ZL2_xend_C_Cend_Hids; + if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { + RESTORE_LEXER; + goto ZL1; + } else { + goto ZL2_xend_C_Cend_Hids; + } /* END OF INLINE: xend::end-ids */ } - /*UNREACHED*/ + /* UNREACHED */ case (ERROR_TERMINAL): RESTORE_LEXER; goto ZL1; @@ -1076,7 +1277,7 @@ ZL2_xend_C_Cend_Hids:; break; } } - /* END OF INLINE: 58 */ + /* END OF INLINE: 65 */ } return; ZL1:; @@ -1085,120 +1286,62 @@ ZL1:; } static void -p_xend_C_Cend_Hid(fsm fsm, lex_state lex_state, act_state act_state) +p_xend_C_Cend_Hid(fsm fsm, lex_state lex_state, act_state act_state, state ZIs) { if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { return; } { - string ZIn; - state ZIs; + endid ZIid; - /* BEGINNING OF INLINE: id */ - { + switch (CURRENT_TERMINAL) { + case (TOK_ENDID): + /* BEGINNING OF EXTRACT: ENDID */ { - switch (CURRENT_TERMINAL) { - case (TOK_IDENT): - /* BEGINNING OF EXTRACT: IDENT */ - { -#line 243 "src/libfsm/parser.act" +#line 255 "src/libfsm/parser.act" - ZIn = xstrdup(lex_state->buf.a); - if (ZIn == NULL) { - perror("xstrdup"); - exit(EXIT_FAILURE); - } - -#line 1112 "src/libfsm/parser.c" - } - /* END OF EXTRACT: IDENT */ - break; - default: - goto ZL1; - } - ADVANCE_LEXER; - } - } - /* END OF INLINE: id */ - /* BEGINNING OF ACTION: add-state */ - { -#line 254 "src/libfsm/parser.act" - - struct act_statelist *p; - const unsigned hash = hash_of_id((ZIn)); - const unsigned mask = act_state->states.bucket_count - 1; - size_t chain_length = 0; - unsigned bucket_id = hash & mask; - - assert((ZIn) != NULL); - - if (act_state->states.longest_chain_length > MAX_CHAIN_LENGTH) { - grow_state_hash_table(&act_state->states); - bucket_id = hash & (act_state->states.bucket_count - 1); - } + unsigned long u; + char *e; - for (p = act_state->states.buckets[bucket_id]; p != NULL; p = p->next) { - assert(p->id != NULL); + errno = 0; - chain_length++; + u = strtoul(lex_state->buf.a, &e, 16); - if (0 == strcmp(p->id, (ZIn))) { - (ZIs) = p->state; - break; - } + if ((u == ULONG_MAX && errno == ERANGE) || u > FSM_END_ID_MAX) { + err(lex_state, "end id %s out of range: expected 0..%u inclusive", + lex_state->buf.a, FSM_END_ID_MAX); + /* XXX: don't exit in library code */ + exit(EXIT_FAILURE); } - if (chain_length > act_state->states.longest_chain_length) { - act_state->states.longest_chain_length = chain_length; + if ((u == ULONG_MAX && errno != 0) || *e != '\0') { + err(lex_state, "%s: %s: expected 0..%u inclusive", + lex_state->buf.a, strerror(errno), FSM_END_ID_MAX); + /* XXX: don't exit in library code */ + exit(EXIT_FAILURE); } - if (p == NULL) { - struct act_statelist *new; - - new = malloc(sizeof *new); - if (new == NULL) { - perror("malloc"); - exit(EXIT_FAILURE); - } - - new->id = xstrdup((ZIn)); - if (new->id == NULL) { - perror("xstrdup"); - exit(EXIT_FAILURE); - } - - if (!fsm_addstate(fsm, &(ZIs))) { - perror("fsm_addstate"); - exit(EXIT_FAILURE); - } - - new->state = (ZIs); - - new->next = act_state->states.buckets[bucket_id]; - act_state->states.buckets[bucket_id] = new; - } + ZIid = (fsm_end_id_t) u; -#line 1181 "src/libfsm/parser.c" +#line 1327 "src/libfsm/parser.c" + } + /* END OF EXTRACT: ENDID */ + break; + default: + goto ZL1; } - /* END OF ACTION: add-state */ - /* BEGINNING OF ACTION: mark-end */ + ADVANCE_LEXER; + /* BEGINNING OF ACTION: mark-end-id */ { -#line 313 "src/libfsm/parser.act" +#line 343 "src/libfsm/parser.act" - fsm_setend(fsm, (ZIs), 1); - -#line 1190 "src/libfsm/parser.c" + if (fsm_endid_set(fsm, (ZIs), (ZIid)) < 0) { + goto ZL1; } - /* END OF ACTION: mark-end */ - /* BEGINNING OF ACTION: free */ - { -#line 317 "src/libfsm/parser.act" - - free((ZIn)); -#line 1199 "src/libfsm/parser.c" +#line 1343 "src/libfsm/parser.c" } - /* END OF ACTION: free */ + /* END OF ACTION: mark-end-id */ } return; ZL1:; @@ -1208,10 +1351,10 @@ ZL1:; /* BEGINNING OF TRAILER */ -#line 436 "src/libfsm/parser.act" +#line 415 "src/libfsm/parser.act" - struct fsm *fsm_parse(FILE *f, const struct fsm_options *opt) { + struct fsm *fsm_parse(FILE *f, const struct fsm_alloc *alloc) { struct act_state act_state_s; struct act_state *act_state; struct lex_state lex_state_s; @@ -1220,7 +1363,6 @@ ZL1:; struct lx *lx; assert(f != NULL); - assert(opt != NULL); lex_state = &lex_state_s; @@ -1247,14 +1389,25 @@ ZL1:; /* This is a workaround for ADVANCE_LEXER assuming a pointer */ act_state = &act_state_s; - /* TODO: pass in fsm_options */ - new = fsm_new(opt); + new = fsm_new(alloc); if (new == NULL) { perror("fsm_new"); return NULL; } - ADVANCE_LEXER; /* XXX: what if the first token is unrecognised? */ + ADVANCE_LEXER; + switch (CURRENT_TERMINAL) { + case TOK_UNKNOWN: + /* fallthrough */ + case TOK_ERROR: + /* XXX: don't exit in library code */ + err(lex_state, "Syntax error"); + exit(EXIT_FAILURE); + + default: + break; + } + p_fsm(new, lex_state, act_state); free(act_state_s.states.buckets); @@ -1263,6 +1416,6 @@ ZL1:; return new; } -#line 1266 "src/libfsm/parser.c" +#line 1420 "src/libfsm/parser.c" /* END OF FILE */ diff --git a/src/libfsm/parser.h b/src/libfsm/parser.h index 181f13d44..32f562c66 100644 --- a/src/libfsm/parser.h +++ b/src/libfsm/parser.h @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 159 "src/libfsm/parser.act" +#line 153 "src/libfsm/parser.act" typedef struct lex_state * lex_state; @@ -17,17 +17,18 @@ typedef struct fsm * fsm; struct fsm * - fsm_parse(FILE *f, const struct fsm_options *opt); + fsm_parse(FILE *f, + const struct fsm_alloc *alloc); -#line 23 "src/libfsm/parser.h" +#line 24 "src/libfsm/parser.h" /* BEGINNING OF FUNCTION DECLARATIONS */ extern void p_fsm(fsm, lex_state, act_state); /* BEGINNING OF TRAILER */ -#line 437 "src/libfsm/parser.act" +#line 479 "src/libfsm/parser.act" -#line 32 "src/libfsm/parser.h" +#line 33 "src/libfsm/parser.h" /* END OF FILE */ diff --git a/src/libfsm/parser.sid b/src/libfsm/parser.sid index d1da43b30..7d22c7420 100644 --- a/src/libfsm/parser.sid +++ b/src/libfsm/parser.sid @@ -1,5 +1,5 @@ /* - * Copyright 2008-2017 Katherine Flavel + * Copyright 2008-2024 Katherine Flavel * * See LICENCE for the full copyright terms. */ @@ -9,10 +9,12 @@ char; string; state; + endid; %terminals% IDENT: () -> (:string); + ENDID: () -> (:endid); ESC: () -> (:char); OCT: () -> (:char); @@ -23,6 +25,10 @@ START; END; + OPENENDIDS; + CLOSEENDIDS; + + EQUALS; TO; ANY; SEP; @@ -36,6 +42,7 @@ : (:string) -> (:state); : (:state) -> (); : (:state) -> (); + : (:state, :endid) -> (); : (:string) -> (); ; @@ -64,14 +71,14 @@ LABEL; }; - id: () -> (n :string) = { + ident: () -> (n :string) = { n = IDENT; }; edge = { - a = id; + a = ident; TO; - b = id; + b = ident; x = (a); y = (b); @@ -106,7 +113,7 @@ * its own output. */ decl = { - a = id; + a = ident; ! = (a); @@ -137,7 +144,7 @@ ; }; - n = id; + n = ident; { SEP; @@ -155,8 +162,30 @@ xend [ - end-id = { - n = id; + end-id: (s :state) -> () = { + id = ENDID; + + (s, id); + }; + + end-ids: (s :state) -> () = { + end-id(s); + + { + { + COMMA; + ## + ; + }; + + end-ids(s); + || + $; + }; + }; + + end-state: () -> (s :state) = { + n = ident; s = (n); (s); @@ -164,8 +193,18 @@ (n); }; - end-ids = { - end-id; + end-states = { + s = end-state; + + { + EQUALS; + + OPENENDIDS; + { end-ids(s); || $; }; + CLOSEENDIDS; + || + $; + }; { { @@ -174,7 +213,7 @@ ; }; - end-ids; + end-states; || $; }; @@ -187,7 +226,7 @@ ; }; - end-ids; + end-states; { SEP; diff --git a/src/libfsm/pred/hasnondeterminism.c b/src/libfsm/pred/hasnondeterminism.c index 5a3801968..39a3b59f2 100644 --- a/src/libfsm/pred/hasnondeterminism.c +++ b/src/libfsm/pred/hasnondeterminism.c @@ -51,7 +51,7 @@ state_epsilon_closure(const struct fsm *fsm, fsm_state_t state, return *closure; } - if (!state_set_add(closure, fsm->opt->alloc, state)) { + if (!state_set_add(closure, fsm->alloc, state)) { return NULL; } diff --git a/src/libfsm/print.c b/src/libfsm/print.c new file mode 100644 index 000000000..bdb092d96 --- /dev/null +++ b/src/libfsm/print.c @@ -0,0 +1,457 @@ +/* + * Copyright 2008-2024 Katherine Flavel + * + * See LICENCE for the full copyright terms. + */ + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "print.h" +#include "internal.h" + +#include "vm/retlist.h" +#include "vm/vm.h" +#include "print/ir.h" + +int +print_hook_args(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + int (*default_args)(FILE *f, const struct fsm_options *opt, + void *lang_opaque, void *hook_opaque), + void *lang_opaque) +{ + assert(f != NULL); + assert(opt != NULL); + assert(hooks != NULL); + + if (default_args == NULL) { + assert(lang_opaque == NULL); + } + + if (hooks->accept != NULL) { + return hooks->args(f, opt, + lang_opaque, hooks->hook_opaque); + } else if (default_args != NULL) { + return default_args(f, opt, + lang_opaque, hooks->hook_opaque); + } + + return 0; +} + +int +print_hook_accept(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + const struct fsm_state_metadata *state_metadata, + int (*default_accept)(FILE *f, const struct fsm_options *opt, + const struct fsm_state_metadata *state_metadata, + void *lang_opaque, void *hook_opaque), + void *lang_opaque) +{ + assert(f != NULL); + assert(opt != NULL); + assert(hooks != NULL); + + if (opt->ambig == AMBIG_ERROR) { + assert(state_metadata->end_id_count <= 1); + } + + if (default_accept == NULL) { + assert(lang_opaque == NULL); + } + + if (hooks->accept != NULL) { + return hooks->accept(f, opt, state_metadata, + lang_opaque, hooks->hook_opaque); + } else if (default_accept != NULL) { + return default_accept(f, opt, state_metadata, + lang_opaque, hooks->hook_opaque); + } + + return 0; +} + +int +print_hook_comment(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + const struct fsm_state_metadata *state_metadata) +{ + assert(f != NULL); + assert(opt != NULL); + assert(hooks != NULL); + + if (opt->ambig == AMBIG_ERROR) { + assert(state_metadata->end_id_count <= 1); + } + + if (opt->comments && hooks->comment != NULL) { + /* this space is a polyglot */ + fprintf(f, " "); + + return hooks->comment(f, opt, state_metadata, + hooks->hook_opaque); + } + + return 0; +} + +int +print_hook_reject(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + const struct fsm_state_metadata *state_metadata, + int (*default_reject)(FILE *f, const struct fsm_options *opt, + void *lang_opaque, void *hook_opaque), + void *lang_opaque) +{ + assert(f != NULL); + assert(opt != NULL); + assert(hooks != NULL); + + if (default_reject == NULL) { + assert(lang_opaque == NULL); + } + + if (hooks->reject != NULL) { + return hooks->reject(f, opt, state_metadata, + lang_opaque, hooks->hook_opaque); + } else if (default_reject != NULL) { + return default_reject(f, opt, + lang_opaque, hooks->hook_opaque); + } + + return 0; +} + +int +print_hook_conflict(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + const struct fsm_state_metadata *state_metadata, + const char *example) +{ + assert(opt != NULL); + assert(hooks != NULL); + + /* f may be NULL for FSM_PRINT_NONE */ + + if (hooks->conflict != NULL) { + return hooks->conflict(f, opt, state_metadata, + example, + hooks->hook_opaque); + } + + return 0; +} + +static int +print_conflicts(FILE *f, const struct fsm *fsm, + const struct fsm_options *opt, + const struct fsm_hooks *hooks) +{ + fsm_state_t s; + int conflicts; + + assert(fsm != NULL); + assert(opt != NULL); + assert(hooks != NULL); + + conflicts = 0; + + for (s = 0; s < fsm->statecount; s++) { + char *example; + fsm_end_id_t *ids; + size_t count; + int res; + + if (!fsm_isend(fsm, s)) { + continue; + } + + count = fsm_endid_count(fsm, s); + if (count <= 1) { + continue; + } + + /* + * It's unfortunate that we call fsm_example() and allocate ids[] here + * as well as during make_ir(), but I can't find a way to do them in + * just one place. That's because we handle conflicts for output formats + * where there isn't an ir, and because the ir only exists for DFA, + * we can't move all output formats to use the ir. + */ + + if (!make_example(fsm, s, &example)) { + goto error; + } + + ids = f_malloc(fsm->alloc, count * sizeof *ids); + if (ids == NULL) { + goto error; + } + + res = fsm_endid_get(fsm, s, count, ids); + assert(res == 1); + + // TODO: de-duplicate by ids[], so we don't call the conflict hook an unneccessary number of times + // TODO: now i think this is the same as calling once per retlist entry + + /* + * The conflict hook is called here (rather in the caller), + * because it can be called multiple times, for different states. + */ + struct fsm_state_metadata state_metadata = { + .end_ids = ids, + .end_id_count = count, + }; + if (-1 == print_hook_conflict(f, opt, hooks, + &state_metadata, + example)) + { + goto error; + } + + if (example != NULL) { + f_free(fsm->alloc, example); + } + + f_free(fsm->alloc, ids); + + conflicts++; + } + + return conflicts; + +error: + + return -1; +} + +int +fsm_print(FILE *f, const struct fsm *fsm, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + enum fsm_print_lang lang) +{ + static const struct fsm_options opt_zero; + static const struct fsm_hooks hooks_zero; + + struct dfavm_assembler_ir a; + struct ir *ir; + int r; + + static const struct dfavm_assembler_ir zero; + static const struct fsm_vm_compile_opts vm_opts = { + FSM_VM_COMPILE_DEFAULT_FLAGS, + FSM_VM_COMPILE_VM_V1, + NULL + }; + + fsm_print_f *print_fsm = NULL; + ir_print_f *print_ir = NULL; + vm_print_f *print_vm = NULL; + + if (lang != FSM_PRINT_NONE) { + assert(f != NULL); + } + + if (opt == NULL) { + opt = &opt_zero; + } + + if (hooks == NULL) { + hooks = &hooks_zero; + } + + ir = NULL; + + /* + * fsm_print() can print an NFA or a DFA. The ir is DFA-only. + * So we can't use the ir for ambiguity checks, because it + * can't be constructed when we have an NFA. + * + * That also means we can't move all output routines to ir-only, + * we need to keep struct fsm around. So we're walking the fsm + * struct to check for ambiguities. + * + * This could be done conditionally, only when print_fsm != NULL + * and then we'd also check for ambiguities for ir and vm output + * at the point where the accept hook is called. But there's no + * reason to do that, it's simpler to detect all the conflicts + * ahead of time regardless. + */ + if (opt->ambig == AMBIG_ERROR) { + int count; + + count = print_conflicts(f, fsm, opt, hooks); + if (count == -1) { + goto error; + } + + if (count > 0) { + goto conflict; + } + } + + switch (lang) { + case FSM_PRINT_NONE: + return 0; + + case FSM_PRINT_AMD64_ATT: print_vm = fsm_print_amd64_att; break; + case FSM_PRINT_AMD64_GO: print_vm = fsm_print_amd64_go; break; + case FSM_PRINT_AMD64_NASM: print_vm = fsm_print_amd64_nasm; break; + + case FSM_PRINT_API: print_fsm = fsm_print_api; break; + case FSM_PRINT_AWK: print_vm = fsm_print_awk; break; + case FSM_PRINT_C: print_ir = fsm_print_c; break; + case FSM_PRINT_DOT: print_fsm = fsm_print_dot; break; + case FSM_PRINT_FSM: print_fsm = fsm_print_fsm; break; + case FSM_PRINT_GO: print_vm = fsm_print_go; break; + case FSM_PRINT_IR: print_ir = fsm_print_ir; break; + case FSM_PRINT_IRJSON: print_ir = fsm_print_irjson; break; + case FSM_PRINT_JSON: print_fsm = fsm_print_json; break; + case FSM_PRINT_LLVM: print_vm = fsm_print_llvm; break; + case FSM_PRINT_RUST: print_vm = fsm_print_rust; break; + case FSM_PRINT_SH: print_vm = fsm_print_sh; break; + case FSM_PRINT_VMC: print_vm = fsm_print_vmc; break; + case FSM_PRINT_VMDOT: print_vm = fsm_print_vmdot; break; + + case FSM_PRINT_VMOPS_C: print_vm = fsm_print_vmops_c; break; + case FSM_PRINT_VMOPS_H: print_vm = fsm_print_vmops_h; break; + case FSM_PRINT_VMOPS_MAIN: print_vm = fsm_print_vmops_main; break; + + default: + errno = ENOTSUP; + return -1; + } + + r = 0; + + if (print_fsm != NULL) { + r = print_fsm(f, opt, hooks, fsm); + goto done; + } + + /* DFA only after this point */ + + ir = make_ir(fsm, opt); + if (ir == NULL) { + return -1; + } + + if (opt->ambig == AMBIG_ERROR) { + size_t i; + + for (i = 0; i < ir->n; i++) { + if (!ir->states[i].isend) { + continue; + } + + assert(ir->states[i].endids.count <= 1); + } + } + + if (print_ir != NULL) { + r = print_ir(f, opt, hooks, ir); + goto done; + } + + /* + * We're building the retlist here based on the ir. + * I think we could build the retlist earlier instead, + * and then point at the struct ret entries from the ir, + * and then dfavm_compile_ir() would pick those up from there. + * But for now this is good. + */ + struct ret_list retlist; + + if (!build_retlist(&retlist, ir)) { + free_ir(fsm, ir); + goto error; + } + + a = zero; + + /* TODO: non-const a */ + if (!dfavm_compile_ir(&a, ir, &retlist, vm_opts)) { + free_retlist(&retlist); + free_ir(fsm, ir); + return -1; + } + + if (print_vm != NULL) { + r = print_vm(f, opt, hooks, &retlist, a.linked); + } + + dfavm_opasm_finalize_op(&a); + + free_retlist(&retlist); + +done: + + if (ir != NULL) { + free_ir(fsm, ir); + } + + if (ferror(f)) { + return -1; + } + + if (r < 0) { + goto error; + } + + return 0; + +conflict: + + /* + * Finding an ambiguity conflict is not an error; + * we successfully identified AMBIG_ERROR. + * + * Success is >= 0 for a print function, we're + * distinguishing this from done: just for the caller. + */ + + return 1; + +error: + + return -1; +} + +void +(fsm_dump)(FILE *f, const struct fsm *fsm, + const char *file, unsigned line, const char *name) +{ + assert(f != NULL); + assert(file != NULL); + assert(name != NULL); + + fprintf(f, "# %s:%d %s\n", file, line, name); + + if (fsm == NULL) { + fprintf(f, "# (null)\n"); + return; + } + + fsm_print(f, fsm, & (const struct fsm_options) { + .consolidate_edges = 1, + .comments = 1, + .group_edges = 1 + }, NULL, FSM_PRINT_FSM); + +} + diff --git a/src/libfsm/print.h b/src/libfsm/print.h new file mode 100644 index 000000000..bfedf0e62 --- /dev/null +++ b/src/libfsm/print.h @@ -0,0 +1,98 @@ +/* + * Copyright 2008-2024 Katherine Flavel + * + * See LICENCE for the full copyright terms. + */ + +#ifndef FSM_INTERNAL_PRINT_H +#define FSM_INTERNAL_PRINT_H + +struct fsm; +struct fsm_options; +struct fsm_hooks; +struct ir; +struct dfavm_op_ir; +struct ret_list; + +#include + +int +print_hook_args(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + int (*default_args)(FILE *f, const struct fsm_options *opt, + void *lang_opaque, void *hook_opaque), + void *lang_opaque); + +int +print_hook_accept(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + const struct fsm_state_metadata *state_metadata, + int (*default_accept)(FILE *f, const struct fsm_options *opt, + const struct fsm_state_metadata *state_metadata, + void *lang_opaque, void *hook_opaque), + void *lang_opaque); + +int +print_hook_comment(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + const struct fsm_state_metadata *state_metadata); + +int +print_hook_reject(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + const struct fsm_state_metadata *state_metadata, + int (*default_reject)(FILE *f, const struct fsm_options *opt, + void *lang_opaque, void *hook_opaque), + void *lang_opaque); + +int +print_hook_conflict(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + const struct fsm_state_metadata *state_metadata, + const char *example); + +typedef int fsm_print_f(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + const struct fsm *fsm); + +typedef int ir_print_f(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + const struct ir *ir); + +typedef int vm_print_f(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + const struct ret_list *retlist, + struct dfavm_op_ir *ops); + +vm_print_f fsm_print_amd64_att; +vm_print_f fsm_print_amd64_go; +vm_print_f fsm_print_amd64_nasm; + +fsm_print_f fsm_print_api; +vm_print_f fsm_print_awk; +ir_print_f fsm_print_c; +fsm_print_f fsm_print_dot; +fsm_print_f fsm_print_fsm; +vm_print_f fsm_print_go; +ir_print_f fsm_print_ir; +ir_print_f fsm_print_irjson; +fsm_print_f fsm_print_json; +vm_print_f fsm_print_llvm; +vm_print_f fsm_print_rust; +vm_print_f fsm_print_sh; +vm_print_f fsm_print_vmc; + +vm_print_f fsm_print_vmdot; +vm_print_f fsm_print_vmops_c; +vm_print_f fsm_print_vmops_h; +vm_print_f fsm_print_vmops_main; + +#endif diff --git a/src/libfsm/print/Makefile b/src/libfsm/print/Makefile index c2911318b..a75482c6d 100644 --- a/src/libfsm/print/Makefile +++ b/src/libfsm/print/Makefile @@ -3,23 +3,27 @@ SRC += src/libfsm/print/api.c SRC += src/libfsm/print/awk.c SRC += src/libfsm/print/c.c -SRC += src/libfsm/print/vmops.c SRC += src/libfsm/print/dot.c SRC += src/libfsm/print/fsm.c +SRC += src/libfsm/print/go.c SRC += src/libfsm/print/ir.c SRC += src/libfsm/print/irdot.c SRC += src/libfsm/print/irjson.c SRC += src/libfsm/print/json.c +SRC += src/libfsm/print/llvm.c SRC += src/libfsm/print/rust.c SRC += src/libfsm/print/sh.c -SRC += src/libfsm/print/go.c +SRC += src/libfsm/print/vmasm.c SRC += src/libfsm/print/vmc.c SRC += src/libfsm/print/vmdot.c -SRC += src/libfsm/print/vmasm.c +SRC += src/libfsm/print/vmops.c .for src in ${SRC:Msrc/libfsm/print/*.c} CFLAGS.${src} += -I src # XXX: for internal.h DFLAGS.${src} += -I src # XXX: for internal.h + +CFLAGS.${src} += -std=c99 +DFLAGS.${src} += -std=c99 .endfor .for src in ${SRC:Msrc/libfsm/print/*.c} diff --git a/src/libfsm/print/api.c b/src/libfsm/print/api.c index 38e5dfc27..8478b5852 100644 --- a/src/libfsm/print/api.c +++ b/src/libfsm/print/api.c @@ -43,7 +43,10 @@ rangeclass(unsigned char x, unsigned char y) } int -fsm_print_api(FILE *f, const struct fsm *fsm_orig) +fsm_print_api(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + const struct fsm *fsm_orig) { struct fsm *fsm; fsm_state_t start, end; @@ -52,7 +55,8 @@ fsm_print_api(FILE *f, const struct fsm *fsm_orig) assert(f != NULL); assert(fsm_orig != NULL); - assert(fsm_orig->opt != NULL); + assert(opt != NULL); + assert(hooks != NULL); if (!fsm_has(fsm_orig, fsm_isend)) { errno = EINVAL; @@ -75,13 +79,13 @@ fsm_print_api(FILE *f, const struct fsm *fsm_orig) /* TODO: leaf callback for opaques */ - if (fsm->opt->fragment) { + if (opt->fragment) { fprintf(f, "\t"); } fprintf(f, "/* Generated by libfsm */\n"); fprintf(f, "\n"); - if (!fsm->opt->fragment) { + if (!opt->fragment) { fprintf(f, "#include LF_HEADER\n"); fprintf(f, "\n"); @@ -90,11 +94,12 @@ fsm_print_api(FILE *f, const struct fsm *fsm_orig) fprintf(f, "\n"); fprintf(f, "#include \n"); + fprintf(f, "#include \n"); fprintf(f, "\n"); fprintf(f, "int\n"); - fprintf(f, "%sfsm(struct fsm *fsm, struct fsm_state *x, struct fsm_state *y)\n", - fsm->opt->prefix != NULL ? fsm->opt->prefix : ""); + fprintf(f, "%sfsm(struct fsm *fsm, fsm_state_t *x, fsm_state_t *y)\n", + opt->prefix != NULL ? opt->prefix : ""); fprintf(f, "{\n"); } @@ -103,16 +108,20 @@ fsm_print_api(FILE *f, const struct fsm *fsm_orig) fprintf(f, "\tsize_t i;\n"); fprintf(f, "\n"); + fprintf(f, "\tassert(x != NULL);\n"); + fprintf(f, "\tassert(y != NULL);\n"); + fprintf(f, "\n"); + fprintf(f, "\tfor (i = 0; i < %zu; i++) {\n", fsm->statecount); fprintf(f, "\t\tif (i == %u) {\n", start); - fprintf(f, "\t\t\ts[%u] = x;\n", start); + fprintf(f, "\t\t\ts[%u] = *x;\n", start); fprintf(f, "\t\t\tcontinue;\n"); fprintf(f, "\t\t}\n"); fprintf(f, "\n"); fprintf(f, "\t\tif (i == %u) {\n", end); - fprintf(f, "\t\t\ts[%u] = y;\n", end); + fprintf(f, "\t\t\ts[%u] = *y;\n", end); fprintf(f, "\t\t\tcontinue;\n"); fprintf(f, "\t\t}\n"); fprintf(f, "\n"); @@ -124,7 +133,37 @@ fsm_print_api(FILE *f, const struct fsm *fsm_orig) fprintf(f, "\t}\n"); fprintf(f, "\n"); - a = f_malloc(fsm->opt->alloc, fsm->statecount * sizeof *a); + /* endleaf() doesn't make sense for this format */ + + { + size_t count; + + count = fsm_endid_count(fsm, end); + if (count > 0) { + fsm_end_id_t *ids; + int res; + + ids = f_malloc(fsm->alloc, count * sizeof *ids); + if (ids == NULL) { + /* XXX */ + goto error; + } + + res = fsm_endid_get(fsm, end, count, ids); + assert(res == 1); + + fprintf(f, "\tif (fsm_isend(fsm, s[%u])) {\n", end); + for (size_t id = 0; id < count; id++) { + fprintf(f, "\t\tif (fsm_endid_set(fsm, s[%u], %zu) < 0) { return 0; }\n", end, id); + } + fprintf(f, "\t}\n"); + fprintf(f, "\n"); + + f_free(fsm->alloc, ids); + } + } + + a = f_malloc(fsm->alloc, fsm->statecount * sizeof *a); if (a == NULL) { /* XXX */ goto error; @@ -173,12 +212,12 @@ fsm_print_api(FILE *f, const struct fsm *fsm_orig) } else if (lo == hi - 1) { fprintf(f, "\tif (!fsm_addedge_literal(fsm, s[%u], s[%u], ", from, to); - c_escputcharlit(f, fsm->opt, lo); + c_escputcharlit(f, opt, lo); fprintf(f, ")) { return 0; }\n"); } else { fprintf(f, "\tfor (i = 0x%02x; i <= 0x%02x; i++) {", (unsigned int) lo, (unsigned int) hi - 1); - if (rangeclass(lo, hi - 1)) { + if (opt->comments && rangeclass(lo, hi - 1)) { fprintf(f, " /* '%c' .. '%c' */", (unsigned char) lo, (unsigned char) hi - 1); } fprintf(f, "\n"); @@ -191,11 +230,11 @@ fsm_print_api(FILE *f, const struct fsm *fsm_orig) } } - f_free(fsm->opt->alloc, a); + f_free(fsm->alloc, a); fprintf(f, "\n"); - if (!fsm->opt->fragment) { + if (!opt->fragment) { fprintf(f, "\n"); fprintf(f, "\treturn 1;\n"); @@ -205,10 +244,6 @@ fsm_print_api(FILE *f, const struct fsm *fsm_orig) fsm_free(fsm); - if (ferror(f)) { - return -1; - } - return 0; error: diff --git a/src/libfsm/print/awk.c b/src/libfsm/print/awk.c index b71ffc307..4d952cbef 100644 --- a/src/libfsm/print/awk.c +++ b/src/libfsm/print/awk.c @@ -24,29 +24,13 @@ #include #include "libfsm/internal.h" +#include "libfsm/print.h" +#include "libfsm/vm/retlist.h" #include "libfsm/vm/vm.h" -#include "ir.h" - #define START UINT32_MAX -static int -leaf(FILE *f, const struct fsm_end_ids *ids, const void *leaf_opaque) -{ - assert(f != NULL); - assert(leaf_opaque == NULL); - - (void) leaf_opaque; - (void) ids; - - /* XXX: this should be FSM_UNKNOWN or something non-EOF, - * maybe user defined */ - fprintf(f, "return -1;"); - - return 0; -} - static const char * cmp_operator(int cmp) { @@ -65,6 +49,81 @@ cmp_operator(int cmp) } } +static int +default_accept(FILE *f, const struct fsm_options *opt, + const struct fsm_state_metadata *state_metadata, + void *lang_opaque, void *hook_opaque) +{ + assert(f != NULL); + assert(opt != NULL); + assert(lang_opaque == NULL); + + (void) lang_opaque; + (void) hook_opaque; + + switch (opt->ambig) { + case AMBIG_NONE: + /* awk doesn't have a boolean type */ + fprintf(f, "return 1;"); + break; + + case AMBIG_ERROR: +// TODO: decide if we deal with this ahead of the call to print or not + if (state_metadata->end_id_count > 1) { + errno = EINVAL; + return -1; + } + + fprintf(f, "return %u;", state_metadata->end_ids[0]); + break; + + case AMBIG_EARLIEST: + /* + * The libfsm api guarentees these ids are unique, + * and only appear once each, and are sorted. + */ + fprintf(f, "return %u;", state_metadata->end_ids[0]); + break; + + case AMBIG_MULTIPLE: + /* awk can't return an array */ + fprintf(f, "return \""); + + for (size_t i = 0; i < state_metadata->end_id_count; i++) { + fprintf(f, "%u", state_metadata->end_ids[i]); + + if (i < state_metadata->end_id_count - 1) { + fprintf(f, ","); + } + } + + fprintf(f, "\""); + break; + + default: + assert(!"unreached"); + abort(); + } + + return 0; +} + +static int +default_reject(FILE *f, const struct fsm_options *opt, + void *lang_opaque, void *hook_opaque) +{ + assert(f != NULL); + assert(opt != NULL); + assert(lang_opaque == NULL); + + (void) lang_opaque; + (void) hook_opaque; + + fprintf(f, "return 0;"); + + return 0; +} + static void print_label(FILE *f, const struct dfavm_op_ir *op) { @@ -88,23 +147,40 @@ print_cond(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt) } static int -print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, - enum dfavm_op_end end_bits, const struct ir *ir) +print_end(FILE *f, const struct dfavm_op_ir *op, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + enum dfavm_op_end end_bits) { - if (end_bits == VM_END_FAIL) { - fprintf(f, "return -1"); - return 0; - } + switch (end_bits) { + case VM_END_FAIL: + return print_hook_reject(f, opt, hooks, NULL, default_reject, NULL); + + case VM_END_SUCC:; + struct fsm_state_metadata state_metadata = { + .end_ids = op->ret->ids, + .end_id_count = op->ret->count, + }; + if (-1 == print_hook_accept(f, opt, hooks, + &state_metadata, + default_accept, + NULL)) + { + return -1; + } - if (opt->endleaf != NULL) { - if (-1 == opt->endleaf(f, op->ir_state->end_ids, opt->endleaf_opaque)) { + if (-1 == print_hook_comment(f, opt, hooks, + &state_metadata)) + { return -1; } - } else { - fprintf(f, "return %td", op->ir_state - ir->states); - } - return 0; + return 0; + + default: + assert(!"unreached"); + abort(); + } } static void @@ -123,37 +199,25 @@ print_branch(FILE *f, const struct dfavm_op_ir *op, const char *prefix) /* TODO: eventually to be non-static */ static int -fsm_print_awkfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, - const char *cp, const char *prefix, - int (*leaf)(FILE *, const struct fsm_end_ids *ids, const void *leaf_opaque), - const void *leaf_opaque) +fsm_print_awkfrag(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + const struct ret_list *retlist, + struct dfavm_op_ir *ops, + const char *cp, const char *prefix) { - static const struct dfavm_assembler_ir zero; - struct dfavm_assembler_ir a; struct dfavm_op_ir *op; - static const struct fsm_vm_compile_opts vm_opts = { FSM_VM_COMPILE_DEFAULT_FLAGS, FSM_VM_COMPILE_VM_V1, NULL }; - assert(f != NULL); - assert(ir != NULL); assert(opt != NULL); + assert(hooks != NULL); + assert(retlist != NULL); assert(cp != NULL); assert(prefix != NULL); - a = zero; - - /* TODO: we don't currently have .opaque information attached to struct dfavm_op_ir. - * We'll need that in order to be able to use the leaf callback here. */ - (void) leaf; - (void) leaf_opaque; - /* TODO: we'll need to heed cp for e.g. lx's codegen */ (void) cp; - if (!dfavm_compile_ir(&a, ir, vm_opts)) { - return -1; - } - /* * We only output labels for ops which are branched to. This gives * gaps in the sequence for ops which don't need a label. @@ -164,8 +228,8 @@ fsm_print_awkfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, l = START; - for (op = a.linked; op != NULL; op = op->next) { - if (op == a.linked || op->num_incoming > 0) { + for (op = ops; op != NULL; op = op->next) { + if (op == ops|| op->num_incoming > 0) { op->index = l++; } } @@ -173,9 +237,9 @@ fsm_print_awkfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, fprintf(f, " switch (l) {\n"); - for (op = a.linked; op != NULL; op = op->next) { - if (op == a.linked || op->num_incoming > 0) { - if (op != a.linked) { + for (op = ops; op != NULL; op = op->next) { + if (op == ops|| op->num_incoming > 0) { + if (op != ops) { fprintf(f, "\n"); } @@ -183,9 +247,9 @@ fsm_print_awkfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, print_label(f, op); fprintf(f, ":"); - if (op->ir_state != NULL && op->ir_state->example != NULL) { + if (op->example != NULL) { fprintf(f, " /* e.g. \""); - escputs(f, opt, c_escputc_str, op->ir_state->example); + escputs(f, opt, c_escputc_str, op->example); fprintf(f, "\" */"); } @@ -197,7 +261,7 @@ fsm_print_awkfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, switch (op->instr) { case VM_OP_STOP: print_cond(f, op, opt); - if (-1 == print_end(f, op, opt, op->u.stop.end_bits, ir)) { + if (-1 == print_end(f, op, opt, hooks, op->u.stop.end_bits)) { return -1; } fprintf(f, ";"); @@ -205,7 +269,7 @@ fsm_print_awkfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, case VM_OP_FETCH: { fprintf(f, "if (s == \"\") "); - if (-1 == print_end(f, op, opt, op->u.fetch.end_bits, ir)) { + if (-1 == print_end(f, op, opt, hooks, op->u.fetch.end_bits)) { return -1; } fprintf(f, "\n"); @@ -236,28 +300,44 @@ fsm_print_awkfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, fprintf(f, " }\n"); - dfavm_opasm_finalize_op(&a); - return 0; } -static int -fsm_print_awk_complete(FILE *f, const struct ir *ir, - const struct fsm_options *opt, const char *prefix, const char *cp) +int +fsm_print_awk(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + const struct ret_list *retlist, + struct dfavm_op_ir *ops) { + const char *prefix; + const char *cp; + assert(f != NULL); - assert(ir != NULL); assert(opt != NULL); + assert(hooks != NULL); + assert(retlist != NULL); + + if (opt->prefix != NULL) { + prefix = opt->prefix; + } else { + prefix = "fsm_"; + } + + if (hooks->cp != NULL) { + cp = hooks->cp; + } else { + cp = "c"; /* XXX */ + } if (opt->fragment) { - if (-1 == fsm_print_awkfrag(f, ir, opt, cp, prefix, - opt->leaf != NULL ? opt->leaf : leaf, opt->leaf_opaque)) - { + if (-1 == fsm_print_awkfrag(f, opt, hooks, retlist, ops, cp, prefix)) { return -1; } } else { fprintf(f, "\n"); + fprintf(f, "# generated\n"); fprintf(f, "function %smain(", prefix); switch (opt->io) { @@ -274,9 +354,7 @@ fsm_print_awk_complete(FILE *f, const struct ir *ir, fprintf(f, ", l, c) {\n"); - if (-1 == fsm_print_awkfrag(f, ir, opt, cp, prefix, - opt->leaf != NULL ? opt->leaf : leaf, opt->leaf_opaque)) - { + if (-1 == fsm_print_awkfrag(f, opt, hooks, retlist, ops, cp, prefix)) { return -1; } @@ -284,46 +362,6 @@ fsm_print_awk_complete(FILE *f, const struct ir *ir, fprintf(f, "\n"); } - if (ferror(f)) { - return -1; - } - return 0; } -int -fsm_print_awk(FILE *f, const struct fsm *fsm) -{ - struct ir *ir; - const char *prefix; - const char *cp; - int r; - - assert(f != NULL); - assert(fsm != NULL); - assert(fsm->opt != NULL); - - ir = make_ir(fsm); - if (ir == NULL) { - return -1; - } - - if (fsm->opt->prefix != NULL) { - prefix = fsm->opt->prefix; - } else { - prefix = "fsm_"; - } - - if (fsm->opt->cp != NULL) { - cp = fsm->opt->cp; - } else { - cp = "c"; /* XXX */ - } - - r = fsm_print_awk_complete(f, ir, fsm->opt, prefix, cp); - - free_ir(fsm, ir); - - return r; -} - diff --git a/src/libfsm/print/c.c b/src/libfsm/print/c.c index df1d26f8c..b10cd9ace 100644 --- a/src/libfsm/print/c.c +++ b/src/libfsm/print/c.c @@ -22,6 +22,7 @@ #include #include "libfsm/internal.h" +#include "libfsm/print.h" #include "ir.h" @@ -52,18 +53,99 @@ ir_hasend(const struct ir *ir) return 0; } +// TODO: centralise vmc/c static int -leaf(FILE *f, const struct fsm_end_ids *ids, const void *leaf_opaque) +print_ids(FILE *f, + enum fsm_ambig ambig, const struct fsm_state_metadata *state_metadata) +{ + switch (ambig) { + case AMBIG_NONE: + // TODO: for C99 we'd return bool + fprintf(f, "return 1;"); + break; + + case AMBIG_ERROR: +// TODO: decide if we deal with this ahead of the call to print or not + if (state_metadata->end_id_count > 1) { + errno = EINVAL; + return -1; + } + + /* fallthrough */ + + case AMBIG_EARLIEST: + /* + * The libfsm api guarentees these ids are unique, + * and only appear once each, and are sorted. + */ + fprintf(f, "{\n"); + fprintf(f, "\t\t*id = %u;\n", state_metadata->end_ids[0]); + fprintf(f, "\t\treturn 1;\n"); + fprintf(f, "\t}"); + break; + + case AMBIG_MULTIPLE: + /* + * Here I would like to emit (static unsigned []) { 1, 2, 3 } + * but specifying a storage duration for compound literals + * is a compiler extension. + * So I'm emitting a static const variable declaration instead. + */ + + fprintf(f, "{\n"); + fprintf(f, "\t\tstatic const unsigned a[] = { "); + for (fsm_end_id_t i = 0; i < state_metadata->end_id_count; i++) { + fprintf(f, "%u", state_metadata->end_ids[i]); + if (i + 1 < state_metadata->end_id_count) { + fprintf(f, ", "); + } + } + fprintf(f, " };\n"); + fprintf(f, "\t\t*ids = a;\n"); + fprintf(f, "\t\t*count = %zu;\n", state_metadata->end_id_count); + fprintf(f, "\t\treturn 1;\n"); + fprintf(f, "\t}"); + break; + + default: + assert(!"unreached"); + abort(); + } + + return 0; +} + +static int +default_accept(FILE *f, const struct fsm_options *opt, + const struct fsm_state_metadata *state_metadata, + void *lang_opaque, void *hook_opaque) +{ + assert(f != NULL); + assert(opt != NULL); + assert(lang_opaque == NULL); + + (void) lang_opaque; + (void) hook_opaque; + + if (-1 == print_ids(f, opt->ambig, state_metadata)) { + return -1; + } + + return 0; +} + +static int +default_reject(FILE *f, const struct fsm_options *opt, + void *lang_opaque, void *hook_opaque) { assert(f != NULL); - assert(leaf_opaque == NULL); + assert(opt != NULL); + assert(lang_opaque == NULL); - (void) ids; - (void) leaf_opaque; + (void) lang_opaque; + (void) hook_opaque; - /* XXX: this should be FSM_UNKNOWN or something non-EOF, - * maybe user defined */ - fprintf(f, "return -1; /* leaf */"); + fprintf(f, "return 0;"); return 0; } @@ -120,36 +202,44 @@ print_groups(FILE *f, const struct fsm_options *opt, print_ranges(f, opt, groups[j].ranges, groups[j].n); - /* TODO: pad S%u out to maximum state width */ if (groups[j].to != csi) { fprintf(f, " state = S%u;", groups[j].to); } fprintf(f, " break;\n"); - - /* TODO: if greedy, and fsm_isend(fsm, state->edges[i].sl->state) then: - fprintf(f, " return %s%s;\n", prefix.tok, state->edges[i].sl->state's token); - */ } } static int -print_singlecase(FILE *f, const struct ir *ir, const struct fsm_options *opt, +print_case(FILE *f, const struct ir *ir, fsm_state_t state_id, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, const char *cp, - struct ir_state *cs, - int (*leaf)(FILE *, const struct fsm_end_ids *ids, const void *leaf_opaque), - const void *leaf_opaque) + struct ir_state *cs) { assert(ir != NULL); assert(opt != NULL); assert(cp != NULL); assert(f != NULL); assert(cs != NULL); - assert(leaf != NULL); + + assert(state_id < ir->n); + const struct fsm_state_metadata state_metadata = { + .end_ids = ir->states[state_id].endids.ids, + .end_id_count = ir->states[state_id].endids.count, + }; + + if (cs->eager_outputs != NULL && opt->fragment) { + /* If .fragment is set and the state has eager outputs, then emit a call to a + * macro (the caller is expected to define). This is a temporary interface. */ + for (size_t i = 0; i < cs->eager_outputs->count; i++) { + fprintf(f, "\t\t\tFSM_SET_EAGER_OUTPUT(%u);\n", cs->eager_outputs->ids[i]); + } + } switch (cs->strategy) { case IR_NONE: fprintf(f, "\t\t\t"); - if (-1 == leaf(f, cs->end_ids, leaf_opaque)) { + if (-1 == print_hook_reject(f, opt, hooks, &state_metadata, default_reject, NULL)) { return -1; } fprintf(f, "\n"); @@ -178,7 +268,7 @@ print_singlecase(FILE *f, const struct ir *ir, const struct fsm_options *opt, print_groups(f, opt, ir_indexof(ir, cs), cs->u.partial.groups, cs->u.partial.n); fprintf(f, "\t\t\tdefault: "); - if (-1 == leaf(f, cs->end_ids, leaf_opaque)) { + if (-1 == print_hook_reject(f, opt, hooks, &state_metadata, default_reject, NULL)) { return -1; } fprintf(f, "\n"); @@ -209,7 +299,7 @@ print_singlecase(FILE *f, const struct ir *ir, const struct fsm_options *opt, print_ranges(f, opt, cs->u.error.error.ranges, cs->u.error.error.n); fprintf(f, " "); - if (-1 == leaf(f, cs->end_ids, leaf_opaque)) { + if (-1 == print_hook_reject(f, opt, hooks, &state_metadata, default_reject, NULL)) { return -1; } fprintf(f, "\n"); @@ -265,7 +355,10 @@ print_stateenum(FILE *f, size_t n) } static int -endstates(FILE *f, const struct fsm_options *opt, const struct ir *ir) +print_endstates(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + const struct ir *ir) { unsigned i; @@ -275,12 +368,18 @@ endstates(FILE *f, const struct fsm_options *opt, const struct ir *ir) /* no end states */ if (!ir_hasend(ir)) { - fprintf(f, "\treturn -1; /* unexpected EOT */\n"); + fprintf(f, "\treturn 0;"); + if (opt->comments) { + fprintf(f, " /* unexpected EOT */"); + } + fprintf(f, "\n"); return 0; } /* usual case */ - fprintf(f, "\t/* end states */\n"); + if (opt->comments) { + fprintf(f, "\t/* end states */\n"); + } fprintf(f, "\tswitch (state) {\n"); for (i = 0; i < ir->n; i++) { if (!ir->states[i].isend) { @@ -288,26 +387,50 @@ endstates(FILE *f, const struct fsm_options *opt, const struct ir *ir) } fprintf(f, "\tcase S%u: ", i); - if (opt->endleaf != NULL) { - if (-1 == opt->endleaf(f, ir->states[i].end_ids, opt->endleaf_opaque)) { - return -1; - } - } else { - fprintf(f, "return %u;", i); + + const struct fsm_state_metadata state_metadata = { + .end_ids = ir->states[i].endids.ids, + .end_id_count = ir->states[i].endids.count, + + .eager_output_count = (ir->states[i].eager_outputs == NULL + ? 0 : ir->states[i].eager_outputs->count), + .eager_output_ids = (ir->states[i].eager_outputs == NULL + ? NULL : ir->states[i].eager_outputs->ids), + }; + + if (-1 == print_hook_accept(f, opt, hooks, + &state_metadata, + default_accept, + NULL)) + { + return -1; + } + + if (-1 == print_hook_comment(f, opt, hooks, + &state_metadata)) + { + return -1; } + fprintf(f, "\n"); } - fprintf(f, "\tdefault: return -1; /* unexpected EOT */\n"); + + /* unexpected EOT */ + fprintf(f, "\tdefault: "); + if (-1 == print_hook_reject(f, opt, hooks, NULL, default_reject, NULL)) { + return -1; + } + fprintf(f, "\n"); fprintf(f, "\t}\n"); return 0; } int -fsm_print_cfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, - const char *cp, - int (*leaf)(FILE *, const struct fsm_end_ids *ids, const void *leaf_opaque), - const void *leaf_opaque) +fsm_print_cfrag(FILE *f, const struct ir *ir, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + const char *cp) { unsigned i; @@ -325,20 +448,24 @@ fsm_print_cfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, fprintf(f, " /* e.g. \""); escputs(f, opt, c_escputc_str, ir->states[i].example); fprintf(f, "\" */"); - } else if (i == ir->start) { + } else if (i == ir->start && opt->comments) { fprintf(f, " /* start */"); } } fprintf(f, "\n"); - if (-1 == print_singlecase(f, ir, opt, cp, &ir->states[i], leaf, leaf_opaque)) { + if (-1 == print_case(f, ir, i, opt, hooks, cp, &ir->states[i])) { return -1; } fprintf(f, "\n"); } fprintf(f, "\t\tdefault:\n"); - fprintf(f, "\t\t\t; /* unreached */\n"); + fprintf(f, "\t\t\t;"); + if (opt->comments) { + fprintf(f, " /* unreached */"); + } + fprintf(f, "\n"); fprintf(f, "\t\t}\n"); if (ferror(f)) { @@ -349,16 +476,19 @@ fsm_print_cfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, } static int -fsm_print_c_body(FILE *f, const struct ir *ir, const struct fsm_options *opt) +fsm_print_c_body(FILE *f, const struct ir *ir, + const struct fsm_options *opt, + const struct fsm_hooks *hooks) { const char *cp; assert(f != NULL); assert(ir != NULL); assert(opt != NULL); + assert(hooks != NULL); - if (opt->cp != NULL) { - cp = opt->cp; + if (hooks->cp != NULL) { + cp = hooks->cp; } else { switch (opt->io) { case FSM_IO_GETC: cp = "c"; break; @@ -377,7 +507,7 @@ fsm_print_c_body(FILE *f, const struct ir *ir, const struct fsm_options *opt) switch (opt->io) { case FSM_IO_GETC: - fprintf(f, "\twhile (c = fsm_getc(opaque), c != EOF) {\n"); + fprintf(f, "\twhile (c = fsm_getc(getc_opaque), c != EOF) {\n"); break; case FSM_IO_STR: @@ -389,9 +519,13 @@ fsm_print_c_body(FILE *f, const struct ir *ir, const struct fsm_options *opt) break; } - if (-1 == fsm_print_cfrag(f, ir, opt, cp, - opt->leaf != NULL ? opt->leaf : leaf, opt->leaf_opaque)) - { + if (hooks->advance != NULL) { + if (-1 == hooks->advance(f, opt, cp, hooks->hook_opaque)) { + return -1; + } + } + + if (-1 == fsm_print_cfrag(f, ir, opt, hooks, cp)) { return -1; } @@ -399,34 +533,94 @@ fsm_print_c_body(FILE *f, const struct ir *ir, const struct fsm_options *opt) fprintf(f, "\n"); /* end states */ - if (-1 == endstates(f, opt, ir)) { + if (-1 == print_endstates(f, opt, hooks, ir)) { return -1; } return 0; } -static int -fsm_print_c_complete(FILE *f, const struct ir *ir, - const struct fsm_options *opt, const char *prefix) +int +fsm_print_c(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + const struct ir *ir) { + const char *prefix; + assert(f != NULL); - assert(ir != NULL); assert(opt != NULL); + assert(hooks != NULL); + assert(ir != NULL); + + if (opt->prefix != NULL) { + prefix = opt->prefix; + } else { + prefix = "fsm_"; + } if (opt->fragment) { - if (-1 == fsm_print_c_body(f, ir, opt)) { + if (-1 == fsm_print_c_body(f, ir, opt, hooks)) { return -1; } } else { fprintf(f, "\n"); fprintf(f, "int\n%smain", prefix); + fprintf(f, "("); + + switch (opt->io) { + case FSM_IO_GETC: + fprintf(f, "int (*fsm_getc)(void *getc_opaque), void *getc_opaque"); + break; + + case FSM_IO_STR: + fprintf(f, "const char *s"); + break; + + case FSM_IO_PAIR: + fprintf(f, "const char *b, const char *e"); + break; + } + + /* + * unsigned rather than fsm_end_id_t here, so the generated code + * doesn't depend on fsm.h + */ + switch (opt->ambig) { + case AMBIG_NONE: + break; + + case AMBIG_ERROR: + case AMBIG_EARLIEST: + fprintf(f, ",\n"); + fprintf(f, "\tunsigned *id"); + break; + + case AMBIG_MULTIPLE: + fprintf(f, ",\n"); + fprintf(f, "\tconst unsigned **ids, size_t *count"); + break; + + default: + assert(!"unreached"); + abort(); + } + + if (hooks->args != NULL) { + fprintf(f, ",\n"); + fprintf(f, "\t"); + + if (-1 == print_hook_args(f, opt, hooks, NULL, NULL)) { + return -1; + } + } + + fprintf(f, ")\n"); + fprintf(f, "{\n"); switch (opt->io) { case FSM_IO_GETC: - fprintf(f, "(int (*fsm_getc)(void *opaque), void *opaque)\n"); - fprintf(f, "{\n"); if (ir->n > 0) { fprintf(f, "\tint c;\n"); fprintf(f, "\n"); @@ -434,8 +628,6 @@ fsm_print_c_complete(FILE *f, const struct ir *ir, break; case FSM_IO_STR: - fprintf(f, "(const char *s)\n"); - fprintf(f, "{\n"); if (ir->n > 0) { fprintf(f, "\tconst char *p;\n"); fprintf(f, "\n"); @@ -443,8 +635,6 @@ fsm_print_c_complete(FILE *f, const struct ir *ir, break; case FSM_IO_PAIR: - fprintf(f, "(const char *b, const char *e)\n"); - fprintf(f, "{\n"); if (ir->n > 0) { fprintf(f, "\tconst char *p;\n"); fprintf(f, "\n"); @@ -453,9 +643,13 @@ fsm_print_c_complete(FILE *f, const struct ir *ir, } if (ir->n == 0) { - fprintf(f, "\treturn -1; /* no matches */\n"); + fprintf(f, "\treturn 0;"); + if (opt->comments) { + fprintf(f, " /* no matches */"); + } + fprintf(f, "\n"); } else { - if (-1 == fsm_print_c_body(f, ir, opt)) { + if (-1 == fsm_print_c_body(f, ir, opt, hooks)) { return -1; } } @@ -464,41 +658,6 @@ fsm_print_c_complete(FILE *f, const struct ir *ir, fprintf(f, "\n"); } - if (ferror(f)) { - return -1; - } - return 0; } -int -fsm_print_c(FILE *f, const struct fsm *fsm) -{ - struct ir *ir; - const char *prefix; - int r; - - assert(f != NULL); - assert(fsm != NULL); - assert(fsm->opt != NULL); - - ir = make_ir(fsm); - if (ir == NULL) { - return -1; - } - - /* henceforth, no function should be passed struct fsm *, only the ir and options */ - - if (fsm->opt->prefix != NULL) { - prefix = fsm->opt->prefix; - } else { - prefix = "fsm_"; - } - - r = fsm_print_c_complete(f, ir, fsm->opt, prefix); - - free_ir(fsm, ir); - - return r; -} - diff --git a/src/libfsm/print/dot.c b/src/libfsm/print/dot.c index f7a351c0d..69cce10aa 100644 --- a/src/libfsm/print/dot.c +++ b/src/libfsm/print/dot.c @@ -11,6 +11,7 @@ #include #include "libfsm/internal.h" /* XXX: up here for bitmap.h */ +#include "libfsm/print.h" #include @@ -25,7 +26,70 @@ #include static int -singlestate(FILE *f, const struct fsm *fsm, fsm_state_t s) +default_accept(FILE *f, const struct fsm_options *opt, + const struct fsm_state_metadata *state_metadata, + void *lang_opaque, void *hook_opaque) +{ + fsm_state_t s; + + assert(f != NULL); + assert(opt != NULL); + assert(lang_opaque != NULL); + + (void) hook_opaque; + + s = * (fsm_state_t *) lang_opaque; + + fprintf(f, "label = <"); + + if (!opt->anonymous_states) { + fprintf(f, "%u", s); + + if (state_metadata->end_id_count > 0) { + fprintf(f, "
"); + } + } + + for (size_t i = 0; i < state_metadata->end_id_count; i++) { + fprintf(f, "#%u", state_metadata->end_ids[i]); + + if (i < state_metadata->end_id_count - 1) { + fprintf(f, ","); + } + } + + fprintf(f, ">"); + + return 0; +} + +static int +default_reject(FILE *f, const struct fsm_options *opt, + void *lang_opaque, void *hook_opaque) +{ + fsm_state_t s; + + assert(f != NULL); + assert(opt != NULL); + assert(lang_opaque != NULL); + + (void) hook_opaque; + + s = * (fsm_state_t *) lang_opaque; + + if (!opt->anonymous_states) { + fprintf(f, "label = <%u>", s); + } + + return 0; +} + +static int +print_state(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + const struct fsm *fsm, + const char *prefix, fsm_state_t s) { struct fsm_edge e; struct edge_iter it; @@ -33,31 +97,21 @@ singlestate(FILE *f, const struct fsm *fsm, fsm_state_t s) struct state_set *unique; assert(f != NULL); + assert(opt != NULL); + assert(hooks != NULL); assert(fsm != NULL); - assert(fsm->opt != NULL); + assert(prefix != NULL); assert(s < fsm->statecount); - if (!fsm->opt->anonymous_states) { - fprintf(f, "\t%sS%-2u [ label = <", - fsm->opt->prefix != NULL ? fsm->opt->prefix : "", - s); - - fprintf(f, "%u", s); - - fprintf(f, "> ];\n"); - } - - if (!fsm->opt->consolidate_edges) { + if (!opt->consolidate_edges) { { struct state_iter jt; fsm_state_t st; for (state_set_reset(fsm->states[s].epsilons, &jt); state_set_next(&jt, &st); ) { fprintf(f, "\t%sS%-2u -> %sS%-2u [ label = <", - fsm->opt->prefix != NULL ? fsm->opt->prefix : "", - s, - fsm->opt->prefix != NULL ? fsm->opt->prefix : "", - st); + prefix, s, + prefix, st); fprintf(f, "ε"); @@ -67,12 +121,10 @@ singlestate(FILE *f, const struct fsm *fsm, fsm_state_t s) for (edge_set_ordered_iter_reset(fsm->states[s].edges, &eoi); edge_set_ordered_iter_next(&eoi, &e); ) { fprintf(f, "\t%sS%-2u -> %sS%-2u [ label = <", - fsm->opt->prefix != NULL ? fsm->opt->prefix : "", - s, - fsm->opt->prefix != NULL ? fsm->opt->prefix : "", - e.state); + prefix, s, + prefix, e.state); - dot_escputc_html(f, fsm->opt, (char) e.symbol); + dot_escputc_html(f, opt, (char) e.symbol); fprintf(f, "> ];\n"); } @@ -83,7 +135,7 @@ singlestate(FILE *f, const struct fsm *fsm, fsm_state_t s) unique = NULL; for (edge_set_reset(fsm->states[s].edges, &it); edge_set_next(&it, &e); ) { - if (!state_set_add(&unique, fsm->opt->alloc, e.state)) { + if (!state_set_add(&unique, fsm->alloc, e.state)) { return -1; } } @@ -119,17 +171,15 @@ singlestate(FILE *f, const struct fsm *fsm, fsm_state_t s) } fprintf(f, "\t%sS%-2u -> %sS%-2u [ ", - fsm->opt->prefix != NULL ? fsm->opt->prefix : "", - s, - fsm->opt->prefix != NULL ? fsm->opt->prefix : "", - e.state); + prefix, s, + prefix, e.state); if (bm_count(&bm) > 4) { fprintf(f, "weight = 3, "); } fprintf(f, "label = <"); - bm_print(f, fsm->opt, &bm, 0, dot_escputc_html); + bm_print(f, opt, &bm, 0, dot_escputc_html); fprintf(f, "> ];\n"); } @@ -142,10 +192,8 @@ singlestate(FILE *f, const struct fsm *fsm, fsm_state_t s) for (state_set_reset(fsm->states[s].epsilons, &jt); state_set_next(&jt, &st); ) { fprintf(f, "\t%sS%-2u -> %sS%-2u [ weight = 1, label = <", - fsm->opt->prefix != NULL ? fsm->opt->prefix : "", - s, - fsm->opt->prefix != NULL ? fsm->opt->prefix : "", - st); + prefix, s, + prefix, st); fprintf(f, "ε"); @@ -161,53 +209,85 @@ singlestate(FILE *f, const struct fsm *fsm, fsm_state_t s) } static int -print_dotfrag(FILE *f, const struct fsm *fsm) +print_dotfrag(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + const struct fsm *fsm, const char *prefix) { - fsm_state_t i; + fsm_state_t s; assert(f != NULL); + assert(opt != NULL); assert(fsm != NULL); - assert(fsm->opt != NULL); - - for (i = 0; i < fsm->statecount; i++) { - if (fsm_isend(fsm, i)) { - enum fsm_getendids_res res; - size_t written; - struct fsm_end_ids *ids = NULL; - const size_t count = fsm_getendidcount(fsm, i); - if (count > 0) { - ids = f_malloc(fsm->opt->alloc, - sizeof(*ids) + ((count - 1) * sizeof(ids->ids))); - assert(ids != NULL); - if (ids == NULL) { + assert(prefix != NULL); + + for (s = 0; s < fsm->statecount; s++) { + if (!fsm_isend(fsm, s)) { + if (!opt->anonymous_states) { + fprintf(f, "\t%sS%-2u [ ", prefix, s); + + if (-1 == print_hook_reject(f, opt, hooks, NULL, default_reject, &s)) { return -1; } - res = fsm_getendids(fsm, i, count, - ids->ids, &written); - if (res == FSM_GETENDIDS_FOUND) { - ids->count = (unsigned)written; - } else { - assert(res == FSM_GETENDIDS_NOT_FOUND); + fprintf(f, " ];\n"); + } + } else { + fsm_end_id_t *ids; + size_t count; + + count = fsm_endid_count(fsm, s); + if (count == 0) { + ids = NULL; + } else { + int res; + + ids = f_malloc(fsm->alloc, count * sizeof *ids); + if (ids == NULL) { + return -1; } + + res = fsm_endid_get(fsm, s, count, ids); + assert(res == 1); } fprintf(f, "\t%sS%-2u [ shape = doublecircle", - fsm->opt->prefix != NULL ? fsm->opt->prefix : "", i); + prefix, s); + + assert(f != NULL); + + fprintf(f, ", "); + + const struct fsm_state_metadata state_metadata = { + .end_ids = ids, + .end_id_count = count, + }; + + if (-1 == print_hook_accept(f, opt, hooks, + &state_metadata, + default_accept, &s)) + { + return -1; + } + + if (opt->comments && hooks->comment != NULL) { + fprintf(f, ","); - if (fsm->opt->endleaf != NULL) { - fprintf(f, ", "); - if (-1 == fsm->opt->endleaf(f, ids, fsm->opt->endleaf_opaque)) { + if (-1 == print_hook_comment(f, opt, hooks, + &state_metadata)) + { return -1; } } fprintf(f, " ];\n"); + + f_free(fsm->alloc, ids); } - /* TODO: show example here, unless !opt->comments */ + /* TODO: comment example per state */ - if (-1 == singlestate(f, fsm, i)) { + if (-1 == print_state(f, opt, hooks, fsm, prefix, s)) { return -1; } } @@ -216,14 +296,26 @@ print_dotfrag(FILE *f, const struct fsm *fsm) } int -fsm_print_dot(FILE *f, const struct fsm *fsm) +fsm_print_dot(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + const struct fsm *fsm) { + const char *prefix; + assert(f != NULL); assert(fsm != NULL); - assert(fsm->opt != NULL); + assert(opt != NULL); + assert(hooks != NULL); - if (fsm->opt->fragment) { - if (-1 == print_dotfrag(f, fsm)) { + if (opt->prefix != NULL) { + prefix = opt->prefix; + } else { + prefix = ""; + } + + if (opt->fragment) { + if (-1 == print_dotfrag(f, opt, hooks, fsm, prefix)) { return -1; } } else { @@ -235,7 +327,7 @@ fsm_print_dot(FILE *f, const struct fsm *fsm) fprintf(f, "\tnode [ shape = circle ];\n"); fprintf(f, "\tedge [ weight = 2 ];\n"); - if (fsm->opt->anonymous_states) { + if (opt->anonymous_states) { fprintf(f, "\tnode [ label = \"\", width = 0.3 ];\n"); } @@ -246,13 +338,13 @@ fsm_print_dot(FILE *f, const struct fsm *fsm) if (fsm_getstart(fsm, &start)) { fprintf(f, "\tstart -> %sS%u;\n", - fsm->opt->prefix != NULL ? fsm->opt->prefix : "", + opt->prefix != NULL ? opt->prefix : "", start); fprintf(f, "\n"); } - if (-1 == print_dotfrag(f, fsm)) { + if (-1 == print_dotfrag(f, opt, hooks, fsm, prefix)) { return -1; } @@ -260,10 +352,6 @@ fsm_print_dot(FILE *f, const struct fsm *fsm) fprintf(f, "\n"); } - if (ferror(f)) { - return -1; - } - return 0; } diff --git a/src/libfsm/print/fsm.c b/src/libfsm/print/fsm.c index ea41ea757..027531514 100644 --- a/src/libfsm/print/fsm.c +++ b/src/libfsm/print/fsm.c @@ -9,6 +9,7 @@ #include #include "libfsm/internal.h" /* XXX: up here for bitmap.h */ +#include "libfsm/print.h" #include @@ -24,6 +25,36 @@ #include #include +static int +default_accept(FILE *f, const struct fsm_options *opt, + const struct fsm_state_metadata *state_metadata, + void *lang_opaque, void *hook_opaque) +{ + assert(f != NULL); + assert(opt != NULL); + assert(lang_opaque == NULL); + + (void) hook_opaque; + + if (state_metadata->end_id_count == 0) { + return 0; + } + + fprintf(f, " = ["); + + for (size_t i = 0; i < state_metadata->end_id_count; i++) { + fprintf(f, "%u", state_metadata->end_ids[i]); + + if (i + 1 < state_metadata->end_id_count) { + fprintf(f, ", "); + } + } + + fprintf(f, "]"); + + return 0; +} + /* TODO: centralise */ static int findany(const struct fsm *fsm, fsm_state_t state, fsm_state_t *a) @@ -88,10 +119,17 @@ findany(const struct fsm *fsm, fsm_state_t state, fsm_state_t *a) } static int -print_state_comments(FILE *f, const struct fsm *fsm, fsm_state_t dst) +print_state_comments(FILE *f, + const struct fsm_options *opt, + const struct fsm *fsm, + fsm_state_t dst) { fsm_state_t start; + assert(f != NULL); + assert(opt != NULL); + assert(fsm != NULL); + if (!fsm_getstart(fsm, &start)) { return 0; } @@ -109,7 +147,7 @@ print_state_comments(FILE *f, const struct fsm *fsm, fsm_state_t dst) if (n > 0) { fprintf(f, " # e.g. \""); - escputs(f, fsm->opt, fsm_escputc, buf); + escputs(f, opt, fsm_escputc, buf); fprintf(f, "%s\"", n >= (int) sizeof buf - 1 ? "..." : ""); } @@ -121,6 +159,9 @@ print_state_comments(FILE *f, const struct fsm *fsm, fsm_state_t dst) static void print_char_range(FILE *f, const struct fsm_options *opt, char lower, char upper) { + assert(f != NULL); + assert(opt != NULL); + if (lower == upper) { fprintf(f, "\""); fsm_escputc(f, opt, (char) lower); @@ -135,8 +176,19 @@ print_char_range(FILE *f, const struct fsm_options *opt, char lower, char upper) } static int -print_state(FILE *f, const struct fsm *fsm, fsm_state_t s) +print_state(FILE *f, const struct fsm_options *opt, const struct fsm_hooks *hooks, + const struct fsm *fsm, + fsm_state_t s) { + assert(f != NULL); + assert(opt != NULL); + + if (!fsm_isend(fsm, s)) { + if (-1 == print_hook_reject(f, opt, hooks, NULL, NULL, NULL)) { + return -1; + } + } + { struct state_iter jt; fsm_state_t st; @@ -156,7 +208,7 @@ print_state(FILE *f, const struct fsm *fsm, fsm_state_t s) } assert(s < fsm->statecount); - if (fsm->opt->group_edges) { + if (opt->group_edges) { struct edge_group_iter egi; struct edge_group_iter_info info; @@ -180,7 +232,7 @@ print_state(FILE *f, const struct fsm *fsm, fsm_state_t s) fprintf(f, ", "); } - print_char_range(f, fsm->opt, (char) lower, (char) i - 1); + print_char_range(f, opt, (char) lower, (char) i - 1); ranges++; } lower = 256; @@ -191,13 +243,13 @@ print_state(FILE *f, const struct fsm *fsm, fsm_state_t s) fprintf(f, ", "); } - print_char_range(f, fsm->opt, (char) lower, (char) 255); + print_char_range(f, opt, (char) lower, (char) 255); } fprintf(f, ";"); - if (fsm->opt->comments) { - if (-1 == print_state_comments(f, fsm, info.to)) { + if (opt->comments) { + if (-1 == print_state_comments(f, opt, fsm, info.to)) { return -1; } } @@ -215,13 +267,13 @@ print_state(FILE *f, const struct fsm *fsm, fsm_state_t s) fprintf(f, "%-2u -> %2u", s, e.state); fprintf(f, " \""); - fsm_escputc(f, fsm->opt, (char) e.symbol); + fsm_escputc(f, opt, (char) e.symbol); putc('\"', f); fprintf(f, ";"); - if (fsm->opt->comments) { - if (-1 == print_state_comments(f, fsm, e.state)) { + if (opt->comments) { + if (-1 == print_state_comments(f, opt, fsm, e.state)) { return -1; } } @@ -234,15 +286,20 @@ print_state(FILE *f, const struct fsm *fsm, fsm_state_t s) } int -fsm_print_fsm(FILE *f, const struct fsm *fsm) +fsm_print_fsm(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + const struct fsm *fsm) { fsm_state_t s, start; size_t end; assert(f != NULL); + assert(opt != NULL); + assert(hooks != NULL); assert(fsm != NULL); - if (!fsm->opt->anonymous_states) { + if (!opt->anonymous_states) { /* * States are output in order here so as to force ordering when * parsing the .fsm format and creating new states. This ensures @@ -257,7 +314,7 @@ fsm_print_fsm(FILE *f, const struct fsm *fsm) } for (s = 0; s < fsm->statecount; s++) { - if (-1 == print_state(f, fsm, s)) { + if (-1 == print_state(f, opt, hooks, fsm, s)) { return -1; } } @@ -278,19 +335,60 @@ fsm_print_fsm(FILE *f, const struct fsm *fsm) fprintf(f, "end: "); for (s = 0; s < fsm->statecount; s++) { - if (fsm_isend(fsm, s)) { - end--; + fsm_end_id_t *ids; + size_t count; - fprintf(f, "%u%s", s, end > 0 ? ", " : ";\n"); + if (!fsm_isend(fsm, s)) { + continue; } - } -done: + end--; + + count = fsm_endid_count(fsm, s); + if (count == 0) { + ids = NULL; + } else { + int res; + + ids = f_malloc(fsm->alloc, count * sizeof *ids); + if (ids == NULL) { + return -1; + } + + res = fsm_endid_get(fsm, s, count, ids); + assert(res == 1); + } + + fprintf(f, "%u", s); - if (ferror(f)) { - return -1; + const struct fsm_state_metadata state_metadata = { + .end_ids = ids, + .end_id_count = count, + }; + + if (-1 == print_hook_accept(f, opt, hooks, + &state_metadata, + default_accept, + NULL)) + { + return -1; + } + + if (-1 == print_hook_comment(f, opt, hooks, + &state_metadata)) + { + return -1; + } + + if (count > 0) { + f_free(fsm->alloc, ids); + } + + fprintf(f, "%s", end > 0 ? ", " : ";\n"); } +done: + return 0; } diff --git a/src/libfsm/print/go.c b/src/libfsm/print/go.c index d0a4edc47..1f6b7a0dc 100644 --- a/src/libfsm/print/go.c +++ b/src/libfsm/print/go.c @@ -23,27 +23,11 @@ #include #include "libfsm/internal.h" +#include "libfsm/print.h" +#include "libfsm/vm/retlist.h" #include "libfsm/vm/vm.h" -#include "ir.h" - -static int -leaf(FILE *f, const struct fsm_end_ids *ids, const void *leaf_opaque) -{ - assert(f != NULL); - assert(leaf_opaque == NULL); - - (void) ids; - (void) leaf_opaque; - - /* XXX: this should be FSM_UNKNOWN or something non-EOF, - * maybe user defined */ - fprintf(f, "return TOK_UNKNOWN;"); - - return 0; -} - static const char * cmp_operator(int cmp) { @@ -62,15 +46,117 @@ cmp_operator(int cmp) } } +static int +print_ids(FILE *f, + enum fsm_ambig ambig, const struct fsm_state_metadata *state_metadata, + size_t i) +{ + switch (ambig) { + case AMBIG_NONE: + break; + + case AMBIG_ERROR: +// TODO: decide if we deal with this ahead of the call to print or not + if (state_metadata->end_id_count > 1) { + errno = EINVAL; + return -1; + } + + fprintf(f, ", %u", state_metadata->end_ids[0]); + break; + + case AMBIG_EARLIEST: + /* + * The libfsm api guarentees these ids are unique, + * and only appear once each, and are sorted. + */ + fprintf(f, ", %u", state_metadata->end_ids[0]); + break; + + case AMBIG_MULTIPLE: + fprintf(f, ", ret%zu", i); + break; + + default: + assert(!"unreached"); + abort(); + } + + return 0; +} + +static int +default_accept(FILE *f, const struct fsm_options *opt, + const struct fsm_state_metadata *state_metadata, + void *lang_opaque, void *hook_opaque) +{ + size_t i; + + assert(f != NULL); + assert(opt != NULL); + assert(lang_opaque != NULL); + + (void) hook_opaque; + + i = * (const size_t *) lang_opaque; + + fprintf(f, "return true"); + + if (-1 == print_ids(f, opt->ambig, state_metadata, i)) { + return -1; + } + + return 0; +} + +static int +default_reject(FILE *f, const struct fsm_options *opt, + void *lang_opaque, void *hook_opaque) +{ + assert(f != NULL); + assert(opt != NULL); + assert(lang_opaque == NULL); + + (void) lang_opaque; + (void) hook_opaque; + + fprintf(f, "{\n\t\treturn false"); + + switch (opt->ambig) { + case AMBIG_NONE: + break; + + case AMBIG_ERROR: + fprintf(f, ", 0"); + break; + + case AMBIG_EARLIEST: + fprintf(f, ", 0"); + break; + + case AMBIG_MULTIPLE: + fprintf(f, ", nil"); + break; + + default: + assert(!"unreached"); + abort(); + } + + fprintf(f, "\n\t}\n"); + + return 0; +} + static void print_label(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt) { fprintf(f, "l%" PRIu32 ":", op->index); - if (op->ir_state->example != NULL) { + if (op->example != NULL) { fprintf(f, " // e.g. \""); /* Go's string escape rules are a superset of C's. */ - escputs(f, opt, c_escputc_str, op->ir_state->example); + escputs(f, opt, c_escputc_str, op->example); fprintf(f, "\""); } } @@ -89,23 +175,52 @@ print_cond(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt) } static int -print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, - enum dfavm_op_end end_bits, const struct ir *ir) +print_end(FILE *f, const struct dfavm_op_ir *op, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + const struct ret_list *retlist, + enum dfavm_op_end end_bits) { - if (end_bits == VM_END_FAIL) { - fprintf(f, "{\n\t\treturn -1\n\t}\n"); - return 0; - } + size_t i; + + switch (end_bits) { + case VM_END_FAIL: + return print_hook_reject(f, opt, hooks, NULL, default_reject, NULL); - if (opt->endleaf != NULL) { - if (-1 == opt->endleaf(f, op->ir_state->end_ids, opt->endleaf_opaque)) { + case VM_END_SUCC: + assert(op->ret >= retlist->a); + + i = op->ret - retlist->a; + + fprintf(f, "{\n"); + fprintf(f, "\t\t"); + + const struct fsm_state_metadata state_metadata = { + .end_ids = op->ret->ids, + .end_id_count = op->ret->count, + }; + + if (-1 == print_hook_accept(f, opt, hooks, + &state_metadata, + default_accept, &i)) + { return -1; } - } else { - fprintf(f, "{\n\t\treturn %td\n\t}\n", op->ir_state - ir->states); - } - return 0; + if (-1 == print_hook_comment(f, opt, hooks, + &state_metadata)) + { + return -1; + } + + fprintf(f, "\n\t}\n"); + + return 0; + + default: + assert(!"unreached"); + abort(); + } } static void @@ -128,38 +243,40 @@ print_fetch(FILE *f, const struct fsm_options *opt) } } +static void +print_ret(FILE *f, const unsigned *ids, size_t count) +{ + size_t i; + + fprintf(f, "[]uint{"); + for (i = 0; i < count; i++) { + fprintf(f, "%u", ids[i]); + if (i + 1 < count) { + fprintf(f, ", "); + } + } + fprintf(f, "}"); +} + /* TODO: eventually to be non-static */ static int -fsm_print_gofrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, - const char *cp, - int (*leaf)(FILE *, const struct fsm_end_ids *ids, const void *leaf_opaque), - const void *leaf_opaque) +fsm_print_gofrag(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + const struct ret_list *retlist, + struct dfavm_op_ir *ops, + const char *cp) { - static const struct dfavm_assembler_ir zero; - struct dfavm_assembler_ir a; struct dfavm_op_ir *op; - static const struct fsm_vm_compile_opts vm_opts = { FSM_VM_COMPILE_DEFAULT_FLAGS, FSM_VM_COMPILE_VM_V1, NULL }; - assert(f != NULL); - assert(ir != NULL); assert(opt != NULL); + assert(retlist != NULL); assert(cp != NULL); - a = zero; - - /* TODO: we don't currently have .opaque information attached to struct dfavm_op_ir. - * We'll need that in order to be able to use the leaf callback here. */ - (void) leaf; - (void) leaf_opaque; - /* TODO: we'll need to heed cp for e.g. lx's codegen */ (void) cp; - if (!dfavm_compile_ir(&a, ir, vm_opts)) { - return -1; - } - /* * We only output labels for ops which are branched to. This gives * gaps in the sequence for ops which don't need a label. @@ -170,7 +287,7 @@ fsm_print_gofrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, l = 0; - for (op = a.linked; op != NULL; op = op->next) { + for (op = ops; op != NULL; op = op->next) { if (op->num_incoming > 0) { op->index = l++; } @@ -180,12 +297,12 @@ fsm_print_gofrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, /* * Only declare variables if we're actually going to use them. */ - if (a.linked->cmp == VM_CMP_ALWAYS && a.linked->instr == VM_OP_STOP) { - assert(a.linked->next == NULL); + if (ops->cmp == VM_CMP_ALWAYS && ops->instr == VM_OP_STOP) { + assert(ops->next == NULL); } else { switch (opt->io) { case FSM_IO_PAIR: - if (ir->n > 0) { + if (ops != NULL) { /* start idx at -1 unsigned so after first increment we're correct at index 0 */ fprintf(f, "\tvar idx = ^uint(0)\n"); fprintf(f, "\n"); @@ -193,7 +310,7 @@ fsm_print_gofrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, break; case FSM_IO_STR: - if (ir->n > 0) { + if (ops != NULL) { /* start idx at -1 unsigned so after first increment we're correct at index 0 */ fprintf(f, "\tvar idx = ^uint(0)\n"); fprintf(f, "\n"); @@ -206,7 +323,7 @@ fsm_print_gofrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, } } - for (op = a.linked; op != NULL; op = op->next) { + for (op = ops; op != NULL; op = op->next) { if (op->num_incoming > 0) { print_label(f, op, opt); fprintf(f, "\n"); @@ -217,12 +334,12 @@ fsm_print_gofrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, switch (op->instr) { case VM_OP_STOP: print_cond(f, op, opt); - print_end(f, op, opt, op->u.stop.end_bits, ir); + print_end(f, op, opt, hooks, retlist, op->u.stop.end_bits); break; case VM_OP_FETCH: print_fetch(f, opt); - print_end(f, op, opt, op->u.fetch.end_bits, ir); + print_end(f, op, opt, hooks, retlist, op->u.fetch.end_bits); break; case VM_OP_BRANCH: @@ -238,43 +355,67 @@ fsm_print_gofrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, fprintf(f, "\n"); } - dfavm_opasm_finalize_op(&a); - return 0; } -static int -fsm_print_go_complete(FILE *f, const struct ir *ir, - const struct fsm_options *opt, const char *prefix, const char *package_prefix) +int +fsm_print_go(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + const struct ret_list *retlist, + struct dfavm_op_ir *ops) { + const char *prefix; + const char *package_prefix; + /* TODO: currently unused, but must be non-NULL */ const char *cp = ""; assert(f != NULL); - assert(ir != NULL); assert(opt != NULL); - assert(prefix != NULL); - assert(package_prefix != NULL); + assert(hooks != NULL); + assert(retlist != NULL); + + if (opt->prefix != NULL) { + prefix = opt->prefix; + } else { + prefix = "fsm_"; + } + + if (opt->package_prefix != NULL) { + package_prefix = opt->package_prefix; + } else { + package_prefix = prefix; + } if (opt->fragment) { - if (-1 == fsm_print_gofrag(f, ir, opt, cp, - opt->leaf != NULL ? opt->leaf : leaf, opt->leaf_opaque)) - { + if (-1 == fsm_print_gofrag(f, opt, hooks, retlist, ops, cp)) { return -1; } } else { fprintf(f, "package %sfsm\n", package_prefix); fprintf(f, "\n"); + fprintf(f, "// generated\n"); + + if (opt->ambig == AMBIG_MULTIPLE) { + for (size_t i = 0; i < retlist->count; i++) { + fprintf(f, "var ret%zu []uint = ", i); + print_ret(f, retlist->a[i].ids, retlist->a[i].count); + fprintf(f, "\n"); + } + fprintf(f, "\n"); + } + fprintf(f, "func %sMatch", prefix); switch (opt->io) { case FSM_IO_PAIR: - fprintf(f, "(data []byte) int {\n"); + fprintf(f, "(data []byte)"); break; case FSM_IO_STR: - fprintf(f, "(data string) int {\n"); + fprintf(f, "(data string)"); break; default: @@ -282,57 +423,44 @@ fsm_print_go_complete(FILE *f, const struct ir *ir, return -1; } - if (-1 == fsm_print_gofrag(f, ir, opt, cp, - opt->leaf != NULL ? opt->leaf : leaf, opt->leaf_opaque)) - { - return -1; - } + if (hooks->args != NULL) { + fprintf(f, ", "); - fprintf(f, "}\n"); - } + if (-1 == print_hook_args(f, opt, hooks, NULL, NULL)) { + return -1; + } + } - if (ferror(f)) { - return -1; - } + fprintf(f, " "); - return 0; -} + switch (opt->ambig) { + case AMBIG_NONE: + fprintf(f, "bool"); + break; -int -fsm_print_go(FILE *f, const struct fsm *fsm) -{ - struct ir *ir; - const char *prefix; - const char *package_prefix; - int r; + case AMBIG_ERROR: + case AMBIG_EARLIEST: + fprintf(f, "(bool, uint)"); + break; - assert(f != NULL); - assert(fsm != NULL); - assert(fsm->opt != NULL); + case AMBIG_MULTIPLE: + fprintf(stdout, "(bool, []uint)"); + break; - ir = make_ir(fsm); - if (ir == NULL) { - return -1; - } + default: + assert(!"unreached"); + abort(); + } - /* henceforth, no function should be passed struct fsm *, only the ir and options */ + fprintf(f, " {\n"); - if (fsm->opt->prefix != NULL) { - prefix = fsm->opt->prefix; - } else { - prefix = "fsm_"; - } + if (-1 == fsm_print_gofrag(f, opt, hooks, retlist, ops, cp)) { + return -1; + } - if (fsm->opt->package_prefix != NULL) { - package_prefix = fsm->opt->package_prefix; - } else { - package_prefix = prefix; + fprintf(f, "}\n"); } - r = fsm_print_go_complete(f, ir, fsm->opt, prefix, package_prefix); - - free_ir(fsm, ir); - - return r; + return 0; } diff --git a/src/libfsm/print/ir.c b/src/libfsm/print/ir.c index 3a7aee222..d8e29b2e7 100644 --- a/src/libfsm/print/ir.c +++ b/src/libfsm/print/ir.c @@ -26,6 +26,7 @@ #include #include "libfsm/internal.h" +#include "libfsm/eager_output.h" #include "ir.h" @@ -127,7 +128,7 @@ make_groups(const struct fsm *fsm, fsm_state_t state, * together by their common destination states. */ - groups = f_malloc(fsm->opt->alloc, sizeof *groups * n); /* worst case */ + groups = f_malloc(fsm->alloc, sizeof *groups * n); /* worst case */ if (groups == NULL) { return NULL; } @@ -141,7 +142,7 @@ make_groups(const struct fsm *fsm, fsm_state_t state, to = ranges[i].to; - p = f_malloc(fsm->opt->alloc, sizeof *p * (n - i)); /* worst case */ + p = f_malloc(fsm->alloc, sizeof *p * (n - i)); /* worst case */ if (p == NULL) { j++; goto error; @@ -163,7 +164,7 @@ make_groups(const struct fsm *fsm, fsm_state_t state, { void *tmp; - tmp = f_realloc(fsm->opt->alloc, p, sizeof *p * k); + tmp = f_realloc(fsm->alloc, p, sizeof *p * k); if (tmp == NULL) { j++; goto error; @@ -182,7 +183,7 @@ make_groups(const struct fsm *fsm, fsm_state_t state, { void *tmp; - tmp = f_realloc(fsm->opt->alloc, groups, sizeof *groups * j); + tmp = f_realloc(fsm->alloc, groups, sizeof *groups * j); if (tmp == NULL) { goto error; } @@ -195,10 +196,10 @@ make_groups(const struct fsm *fsm, fsm_state_t state, error: for (i = 0; i < j; i++) { - f_free(fsm->opt->alloc, (void *) groups[i].ranges); + f_free(fsm->alloc, (void *) groups[i].ranges); } - f_free(fsm->opt->alloc, groups); + f_free(fsm->alloc, groups); return NULL; } @@ -209,10 +210,10 @@ free_groups(const struct fsm *fsm, struct ir_group *groups, size_t n) size_t j; for (j = 0; j < n; j++) { - f_free(fsm->opt->alloc, (void *) groups[j].ranges); + f_free(fsm->alloc, (void *) groups[j].ranges); } - f_free(fsm->opt->alloc, groups); + f_free(fsm->alloc, groups); } static void @@ -275,7 +276,7 @@ make_holes(const struct fsm *fsm, const struct bm *bm, size_t *n) assert(bm != NULL); assert(n != NULL); - ranges = f_malloc(fsm->opt->alloc, sizeof *ranges * FSM_SIGMA_COUNT); /* worst case */ + ranges = f_malloc(fsm->alloc, sizeof *ranges * FSM_SIGMA_COUNT); /* worst case */ if (ranges == NULL) { return NULL; } @@ -302,7 +303,7 @@ make_holes(const struct fsm *fsm, const struct bm *bm, size_t *n) { void *tmp; - tmp = f_realloc(fsm->opt->alloc, ranges, sizeof *ranges * *n); + tmp = f_realloc(fsm->alloc, ranges, sizeof *ranges * *n); if (tmp == NULL) { goto error; } @@ -314,7 +315,7 @@ make_holes(const struct fsm *fsm, const struct bm *bm, size_t *n) error: - f_free(fsm->opt->alloc, ranges); + f_free(fsm->alloc, ranges); return NULL; } @@ -335,7 +336,6 @@ make_state(const struct fsm *fsm, fsm_state_t state, } mode; assert(fsm != NULL); - assert(fsm->opt != NULL); assert(state < fsm->statecount); /* TODO: IR_TABLE */ @@ -351,6 +351,7 @@ make_state(const struct fsm *fsm, fsm_state_t state, if (fsm_iscomplete(fsm, state)) { mode.state = fsm_findmode(fsm, state, &mode.freq); } else { + mode.state = 0; /* XXX: dummy */ mode.freq = 0; } @@ -401,7 +402,7 @@ make_state(const struct fsm *fsm, fsm_state_t state, cs->strategy = IR_ERROR; cs->u.error.mode = groups[max.j].to; - f_free(fsm->opt->alloc, (void *) groups[max.j].ranges); + f_free(fsm->alloc, (void *) groups[max.j].ranges); if (max.j < n) { memmove(groups + max.j, groups + max.j + 1, sizeof *groups * (n - max.j - 1)); n--; @@ -431,14 +432,88 @@ make_state(const struct fsm *fsm, fsm_state_t state, return 0; } +int +make_example(const struct fsm *fsm, fsm_state_t s, char **example) +{ + fsm_state_t start; + size_t count; + char *p; + int n; + + assert(fsm != NULL); + assert(example != NULL); + + if (!fsm_getstart(fsm, &start)) { + *example = NULL; + return 1; + } + + count = fsm_countstates(fsm); + + /* + * Example lengths are approximately proportional to the number of states, + * and shorter where the graph branches often. + */ + p = f_malloc(fsm->alloc, count + 3 + 1); + if (p == NULL) { + return 0; + } + + /* + * fsm_example() doesn't distinguish an unreachable state from n = 0 + */ + if (s == start) { + p[0] = '\0'; + goto done; + } + + n = fsm_example(fsm, s, p, count + 1); + if (-1 == n) { + goto error; + } + + /* + * The goal state is always reachable for a DFA with + * no "stray" states, but the caller may not trim an FSM, + * and need not pass a DFA. + */ + if (n == 0) { + f_free(fsm->alloc, p); + p = NULL; + } else if ((size_t) n < count + 1) { + char *tmp; + + tmp = f_realloc(fsm->alloc, p, strlen(p) + 1); + if (tmp == NULL) { + goto error; + } + + p = tmp; + } else if ((size_t) n > count + 1) { + strcpy(p + count, "..."); + } + +done: + + *example = p; + + return 1; + +error: + + f_free(fsm->alloc, p); + + return 0; +} + struct ir * -make_ir(const struct fsm *fsm) +make_ir(const struct fsm *fsm, const struct fsm_options *opt) { fsm_state_t start, i; struct ir *ir; assert(fsm != NULL); - assert(fsm->opt != NULL); + assert(opt != NULL); if (!fsm_getstart(fsm, &start)) { goto empty; @@ -449,15 +524,15 @@ make_ir(const struct fsm *fsm) return NULL; } - ir = f_malloc(fsm->opt->alloc, sizeof *ir); + ir = f_malloc(fsm->alloc, sizeof *ir); if (ir == NULL) { return NULL; } ir->n = fsm_countstates(fsm); - ir->states = f_malloc(fsm->opt->alloc, sizeof *ir->states * ir->n); + ir->states = f_malloc(fsm->alloc, sizeof *ir->states * ir->n); if (ir->states == NULL) { - f_free(fsm->opt->alloc, ir); + f_free(fsm->alloc, ir); return NULL; } @@ -467,83 +542,61 @@ make_ir(const struct fsm *fsm) assert(i < ir->n); ir->states[i].isend = fsm_isend(fsm, i); - ir->states[i].end_ids = NULL; + ir->states[i].endids.ids = NULL; + ir->states[i].endids.count = 0; + + ir->states[i].eager_outputs = NULL; + if (fsm_isend(fsm, i)) { - enum fsm_getendids_res res; - size_t written; - const size_t count = fsm_getendidcount(fsm, i); - struct fsm_end_ids *ids = f_malloc(fsm->opt->alloc, - sizeof(*ids) + (count == 0 ? 0 : (count - 1) * sizeof(ids->ids[0]))); - if (ids == NULL) { - goto error; - } + fsm_end_id_t *ids; + size_t count; - res = fsm_getendids(fsm, i, count, - ids->ids, &written); - if (res == FSM_GETENDIDS_FOUND) { - ids->count = (unsigned)written; - ir->states[i].end_ids = ids; + count = fsm_endid_count(fsm, i); + if (count == 0) { + ids = NULL; } else { - assert(res == FSM_GETENDIDS_NOT_FOUND); - f_free(fsm->opt->alloc, ids); + int res; + + ids = f_malloc(fsm->alloc, count * sizeof *ids); + if (ids == NULL) { + goto error; + } + + res = fsm_endid_get(fsm, i, count, ids); + assert(res == 1); + } + + ir->states[i].endids.ids = ids; + ir->states[i].endids.count = count; + } + + const size_t eager_output_count = fsm_eager_output_count(fsm, i); + if (eager_output_count > 0) { + struct ir_state_eager_output *outputs = f_malloc(fsm->alloc, + sizeof(*outputs) + eager_output_count * sizeof(outputs->ids[0])); + if (outputs == NULL) { + goto error; } + fsm_eager_output_get(fsm, i, eager_output_count, outputs->ids); + outputs->count = eager_output_count; + + ir->states[i].eager_outputs = outputs; } if (make_state(fsm, i, &ir->states[i]) == -1) { goto error; } - if (!fsm->opt->comments) { + if (!opt->comments) { ir->states[i].example = NULL; } else { - char *p; - int n; - - if (i == start) { - ir->states[i].example = NULL; - continue; - } + char *example; - /* - * Example lengths are approximately proportional - * to the number of states in an fsm, and shorter where - * the graph branches often. - */ - p = f_malloc(fsm->opt->alloc, ir->n + 3 + 1); - if (p == NULL) { + if (!make_example(fsm, i, &example)) { goto error_example; } - n = fsm_example(fsm, i, p, ir->n + 1); - if (-1 == n) { - f_free(fsm->opt->alloc, p); - goto error_example; - } - - /* - * The goal state is always reachable for a DFA with - * no "stray" states, but the caller may not trim an FSM. - */ - if (n == 0) { - f_free(fsm->opt->alloc, p); - p = NULL; - } else if ((size_t) n < ir->n + 1) { - char *tmp; - - n = strlen(p); - - tmp = f_realloc(fsm->opt->alloc, p, n + 1); - if (tmp == NULL) { - f_free(fsm->opt->alloc, p); - goto error_example; - } - - p = tmp; - } else if ((size_t) n > ir->n + 1) { - strcpy(p + ir->n, "..."); - } - - ir->states[i].example = p; + ir->states[i].example = example; } } @@ -560,7 +613,7 @@ make_ir(const struct fsm *fsm) empty: - ir = f_malloc(fsm->opt->alloc, sizeof *ir); + ir = f_malloc(fsm->alloc, sizeof *ir); if (ir == NULL) { return NULL; } @@ -573,7 +626,6 @@ make_ir(const struct fsm *fsm) error_example: - ir->states[i].example = NULL; i++; error: @@ -592,9 +644,9 @@ free_ir(const struct fsm *fsm, struct ir *ir) assert(ir != NULL); for (i = 0; i < ir->n; i++) { - f_free(fsm->opt->alloc, (void *) ir->states[i].example); - - f_free(fsm->opt->alloc, (void *) ir->states[i].end_ids); + f_free(fsm->alloc, (void *) ir->states[i].example); + f_free(fsm->alloc, (void *) ir->states[i].endids.ids); + f_free(fsm->alloc, (void *) ir->states[i].eager_outputs); switch (ir->states[i].strategy) { case IR_TABLE: @@ -623,15 +675,15 @@ free_ir(const struct fsm *fsm, struct ir *ir) break; case IR_ERROR: - f_free(fsm->opt->alloc, (void *) ir->states[i].u.error.error.ranges); + f_free(fsm->alloc, (void *) ir->states[i].u.error.error.ranges); free_groups(fsm, (void *) ir->states[i].u.error.groups, ir->states[i].u.error.n); break; } } - f_free(fsm->opt->alloc, ir->states); + f_free(fsm->alloc, ir->states); - f_free(fsm->opt->alloc, ir); + f_free(fsm->alloc, ir); } diff --git a/src/libfsm/print/ir.h b/src/libfsm/print/ir.h index f9f5c40c3..b4b93a9eb 100644 --- a/src/libfsm/print/ir.h +++ b/src/libfsm/print/ir.h @@ -18,6 +18,8 @@ * we're done with graph algorithmics. */ +struct fsm_options; + enum ir_strategy { IR_NONE = 1 << 0, IR_SAME = 1 << 1, @@ -51,9 +53,18 @@ struct ir_error { struct ir_state { const char *example; - unsigned int isend:1; - struct fsm_end_ids *end_ids; /* NULL -> 0 */ + struct ir_state_endids { + fsm_end_id_t *ids; /* NULL -> 0 */ + size_t count; + } endids; + + struct ir_state_eager_output { + size_t count; + fsm_output_id_t ids[]; + } *eager_outputs; /* NULL -> 0 */ + + unsigned int isend:1; enum ir_strategy strategy; union { @@ -85,11 +96,7 @@ struct ir_state { } error; struct { - /* Note: This is allocated separately, to avoid - * making the union significantly larger. */ - struct ir_state_table { - unsigned to[FSM_SIGMA_COUNT]; - } *table; + int not_yet_implemented; } table; } u; }; @@ -100,9 +107,13 @@ struct ir { struct ir_state *states; /* array */ }; +/* caller frees */ +int +make_example(const struct fsm *fsm, fsm_state_t s, char **example); + /* TODO: can pass in mask of allowed strategies */ struct ir * -make_ir(const struct fsm *fsm); +make_ir(const struct fsm *fsm, const struct fsm_options *opt); void free_ir(const struct fsm *fsm, struct ir *ir); diff --git a/src/libfsm/print/irdot.c b/src/libfsm/print/irdot.c index 206a42e5e..37dc34296 100644 --- a/src/libfsm/print/irdot.c +++ b/src/libfsm/print/irdot.c @@ -22,6 +22,7 @@ #include #include "libfsm/internal.h" +#include "libfsm/print.h" #include "ir.h" @@ -61,7 +62,7 @@ print_endpoint(FILE *f, const struct fsm_options *opt, unsigned char c) } static void -print_state(FILE *f, unsigned to, unsigned self) +print_name(FILE *f, unsigned to, unsigned self) { assert(f != NULL); @@ -144,7 +145,7 @@ print_grouprows(FILE *f, const struct fsm_options *opt, } else { fprintf(f, "", (unsigned) j); - print_state(f, groups[j].to, self); + print_name(f, groups[j].to, self); fprintf(f, ""); } @@ -173,8 +174,10 @@ print_grouplinks(FILE *f, unsigned self, } } -static void -print_cs(FILE *f, const struct fsm_options *opt, +static int +print_state(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, const struct ir *ir, const struct ir_state *cs) { assert(f != NULL); @@ -188,16 +191,49 @@ print_cs(FILE *f, const struct fsm_options *opt, fprintf(f, "\tcs%u [ label =\n", ir_indexof(ir, cs)); fprintf(f, "\t\t<\n"); + fprintf(f, "\t\t \n", ir_indexof(ir, cs), strategy_name(cs->strategy)); + if (cs->isend && cs->endids.count > 0) { + fprintf(f, "\t\t \n"); + } + if (cs->example != NULL) { fprintf(f, "\t\t \n"); } - /* TODO: leaf callback for dot output */ + /* showing hook in addition to existing content */ + if (cs->isend && hooks->accept != NULL) { + fprintf(f, "\t\t \n"); + } switch (cs->strategy) { case IR_NONE: @@ -205,7 +241,7 @@ print_cs(FILE *f, const struct fsm_options *opt, case IR_SAME: fprintf(f, "\t\t \n"); break; @@ -219,14 +255,14 @@ print_cs(FILE *f, const struct fsm_options *opt, case IR_DOMINANT: fprintf(f, "\t\t \n"); print_grouprows(f, opt, ir_indexof(ir, cs), cs->u.dominant.groups, cs->u.dominant.n); break; case IR_ERROR: fprintf(f, "\t\t \n"); print_errorrows(f, opt, &cs->u.error.error); print_grouprows(f, opt, ir_indexof(ir, cs), cs->u.error.groups, cs->u.error.n); @@ -290,22 +326,24 @@ print_cs(FILE *f, const struct fsm_options *opt, default: ; } + + return 0; } int -fsm_print_ir(FILE *f, const struct fsm *fsm) +fsm_print_ir(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + const struct ir *ir) { - struct ir *ir; size_t i; assert(f != NULL); - assert(fsm != NULL); - - ir = make_ir(fsm); - if (ir == NULL) { - return -1; - } + assert(opt != NULL); + assert(hooks != NULL); + assert(ir != NULL); + fprintf(f, "# generated\n"); fprintf(f, "digraph G {\n"); fprintf(f, "\tnode [ shape = box, style = rounded ];\n"); @@ -320,17 +358,13 @@ fsm_print_ir(FILE *f, const struct fsm *fsm) } for (i = 0; i < ir->n; i++) { - print_cs(f, fsm->opt, ir, &ir->states[i]); + if (-1 == print_state(f, opt, hooks, ir, &ir->states[i])) { + return -1; + } } fprintf(f, "}\n"); - free_ir(fsm, ir); - - if (ferror(f)) { - return -1; - } - return 0; } diff --git a/src/libfsm/print/irjson.c b/src/libfsm/print/irjson.c index 173bef0df..a96250e0f 100644 --- a/src/libfsm/print/irjson.c +++ b/src/libfsm/print/irjson.c @@ -22,6 +22,7 @@ #include #include "libfsm/internal.h" +#include "libfsm/print.h" #include "ir.h" @@ -112,17 +113,51 @@ print_groups(FILE *f, const struct fsm_options *opt, fprintf(f, "\t\t\t]\n"); } -static void -print_cs(FILE *f, const struct fsm_options *opt, +static int +print_state(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, const struct ir_state *cs) { assert(f != NULL); assert(opt != NULL); + assert(hooks != NULL); assert(cs != NULL); fprintf(f, "\t\t{\n"); fprintf(f, "\t\t\t\"end\": %s,\n", cs->isend ? "true" : "false"); + if (cs->isend && cs->endids.count > 0) { + fprintf(f, "\t\t\t\"end_id\": ["); + for (size_t i = 0; i < cs->endids.count; i++) { + fprintf(f, "%u", cs->endids.ids[i]); + + if (i < (size_t) cs->endids.count - 1) { + fprintf(f, ", "); + } + } + fprintf(f, "],\n"); + } + + const struct fsm_state_metadata state_metadata = { + .end_ids = cs->endids.ids, + .end_id_count = cs->endids.count, + }; + + /* showing hook in addition to existing content */ + if (cs->isend && hooks->accept != NULL) { + fprintf(f, "\t\t\t\"endleaf\": "); + + if (-1 == print_hook_accept(f, opt, hooks, + &state_metadata, + NULL, NULL)) + { + return -1; + } + + fprintf(f, ",\n"); + } + fprintf(f, "\t\t\t\"strategy\": \"%s\"", strategy_name(cs->strategy)); if (cs->example != NULL || cs->strategy != IR_NONE) { fprintf(f, ","); @@ -139,8 +174,6 @@ print_cs(FILE *f, const struct fsm_options *opt, fprintf(f, "\n"); } - /* TODO: leaf callback for json output */ - switch (cs->strategy) { case IR_NONE: break; @@ -183,21 +216,22 @@ print_cs(FILE *f, const struct fsm_options *opt, } fprintf(f, "\t\t}"); + + return 0; } int -fsm_print_irjson(FILE *f, const struct fsm *fsm) +fsm_print_irjson(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + const struct ir *ir) { - struct ir *ir; size_t i; assert(f != NULL); - assert(fsm != NULL); - - ir = make_ir(fsm); - if (ir == NULL) { - return -1; - } + assert(opt != NULL); + assert(hooks != NULL); + assert(ir != NULL); fprintf(f, "{\n"); @@ -205,7 +239,9 @@ fsm_print_irjson(FILE *f, const struct fsm *fsm) fprintf(f, "\t\"states\": [\n"); for (i = 0; i < ir->n; i++) { - print_cs(f, fsm->opt, &ir->states[i]); + if (-1 == print_state(f, opt, hooks, &ir->states[i])) { + return -1; + } if (i + 1 < ir->n) { fprintf(f, ","); @@ -217,12 +253,6 @@ fsm_print_irjson(FILE *f, const struct fsm *fsm) fprintf(f, "}\n"); - free_ir(fsm, ir); - - if (ferror(f)) { - return -1; - } - return 0; } diff --git a/src/libfsm/print/json.c b/src/libfsm/print/json.c index 5c3235237..b1e9f0ca2 100644 --- a/src/libfsm/print/json.c +++ b/src/libfsm/print/json.c @@ -11,6 +11,7 @@ #include #include "libfsm/internal.h" /* XXX: up here for bitmap.h */ +#include "libfsm/print.h" #include @@ -114,7 +115,8 @@ print_edge_label(FILE *f, int *notfirst, } static int -singlestate(FILE *f, const struct fsm *fsm, fsm_state_t s, int *notfirst) +print_state(FILE *f, const struct fsm_options *opt, const struct fsm *fsm, + fsm_state_t s, int *notfirst) { struct fsm_edge e; struct edge_iter it; @@ -122,12 +124,12 @@ singlestate(FILE *f, const struct fsm *fsm, fsm_state_t s, int *notfirst) struct state_set *unique; assert(f != NULL); + assert(opt != NULL); assert(fsm != NULL); - assert(fsm->opt != NULL); assert(s < fsm->statecount); assert(notfirst != NULL); - if (!fsm->opt->consolidate_edges) { + if (!opt->consolidate_edges) { struct state_iter jt; fsm_state_t st; @@ -136,7 +138,7 @@ singlestate(FILE *f, const struct fsm *fsm, fsm_state_t s, int *notfirst) } for (edge_set_ordered_iter_reset(fsm->states[s].edges, &eoi); edge_set_ordered_iter_next(&eoi, &e); ) { - print_edge_symbol(f, notfirst, fsm->opt, + print_edge_symbol(f, notfirst, opt, s, e.state, e.symbol); } @@ -146,7 +148,7 @@ singlestate(FILE *f, const struct fsm *fsm, fsm_state_t s, int *notfirst) unique = NULL; for (edge_set_reset(fsm->states[s].edges, &it); edge_set_next(&it, &e); ) { - if (!state_set_add(&unique, fsm->opt->alloc, e.state)) { + if (!state_set_add(&unique, fsm->alloc, e.state)) { return -1; } } @@ -181,9 +183,11 @@ singlestate(FILE *f, const struct fsm *fsm, fsm_state_t s, int *notfirst) } } - print_edge_bitmap(f, notfirst, fsm->opt, s, e.state, &bm); + print_edge_bitmap(f, notfirst, opt, s, e.state, &bm); } + state_set_free(unique); + /* * Special edges are not consolidated above */ @@ -200,14 +204,18 @@ singlestate(FILE *f, const struct fsm *fsm, fsm_state_t s, int *notfirst) } int -fsm_print_json(FILE *f, const struct fsm *fsm) +fsm_print_json(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + const struct fsm *fsm) { fsm_state_t start; fsm_state_t i; assert(f != NULL); + assert(opt != NULL); + assert(hooks != NULL); assert(fsm != NULL); - assert(fsm->opt != NULL); fprintf(f, "{\n"); @@ -237,44 +245,60 @@ fsm_print_json(FILE *f, const struct fsm *fsm) fprintf(f, " ],\n"); } - if (fsm->opt->endleaf != NULL) { + /* showing hook in addition to existing content */ + if (hooks->accept != NULL) { int notfirst; notfirst = 0; fprintf(f, " \"endleaf\": [ "); for (i = 0; i < fsm->statecount; i++) { - if (fsm_isend(fsm, i)) { - enum fsm_getendids_res res; - size_t written; - const size_t count = fsm_getendidcount(fsm, i); - struct fsm_end_ids *ids = f_malloc(fsm->opt->alloc, - sizeof(*ids) + ((count - 1) * sizeof(ids->ids[0]))); - assert(ids != NULL); - - res = fsm_getendids(fsm, i, count, - ids->ids, &written); - if (res == FSM_GETENDIDS_FOUND) { - ids->count = (unsigned)written; - } else { - assert(res == FSM_GETENDIDS_NOT_FOUND); - ids->count = 0; - } + fsm_end_id_t *ids; + size_t count; + int res; - if (notfirst) { - fprintf(f, ", "); + if (!fsm_isend(fsm, i)) { + continue; + } + + count = fsm_endid_count(fsm, i); + if (count == 0) { + ids = NULL; + } else { + ids = f_malloc(fsm->alloc, count * sizeof *ids); + if (ids == NULL) { + return -1; } - fprintf(f, "{ %u, ", i); + res = fsm_endid_get(fsm, i, count, ids); + assert(res == 1); + } - fsm->opt->endleaf(f, ids, fsm->opt->endleaf_opaque); + if (notfirst) { + fprintf(f, ", "); + } - fprintf(f, " }"); + fprintf(f, "{ %u, ", i); - f_free(fsm->opt->alloc, ids); + const struct fsm_state_metadata state_metadata = { + .end_ids = ids, + .end_id_count = count, + }; - notfirst = 1; + if (-1 == print_hook_accept(f, opt, hooks, + &state_metadata, + NULL, NULL)) + { + return -1; } + + fprintf(f, " }"); + + if (count > 0) { + f_free(fsm->alloc, ids); + } + + notfirst = 1; } fprintf(f, " ],\n"); } @@ -286,7 +310,7 @@ fsm_print_json(FILE *f, const struct fsm *fsm) fprintf(f, " \"edges\": [\n"); for (i = 0; i < fsm->statecount; i++) { - if (-1 == singlestate(f, fsm, i, ¬first)) { + if (-1 == print_state(f, opt, fsm, i, ¬first)) { return -1; } } @@ -296,10 +320,6 @@ fsm_print_json(FILE *f, const struct fsm *fsm) fprintf(f, "}\n"); fprintf(f, "\n"); - if (ferror(f)) { - return -1; - } - return 0; } diff --git a/src/libfsm/print/llvm.c b/src/libfsm/print/llvm.c new file mode 100644 index 000000000..98e76d46f --- /dev/null +++ b/src/libfsm/print/llvm.c @@ -0,0 +1,780 @@ +/* + * Copyright 2008-2024 Katherine Flavel + * + * See LICENCE for the full copyright terms. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include +#include +#include +#include +#include +#include + +#include "libfsm/internal.h" +#include "libfsm/print.h" + +#include "libfsm/vm/retlist.h" +#include "libfsm/vm/vm.h" + +#define OPAQUE_POINTERS 1 + +#if OPAQUE_POINTERS // llvm >= 15 +static const char *ptr_i8 = "ptr"; +static const char *ptr_i32 = "ptr"; +static const char *ptr_void = "ptr"; +static const char *ptr_rt = "ptr"; +#else +static const char *ptr_i8 = "i8*"; +static const char *ptr_i32 = "i32*"; +static const char *ptr_void = "i8*"; +static const char *ptr_rt = "%rt*"; +#endif + +static const struct dfavm_op_ir fail; // used as a unqiue address only + +/* + * If we had a stack, the current set of live values would be a frame. + * We're a DFA, so we don't have a stack. But I still think of them as a frame. + */ +struct frame { + unsigned r; + unsigned b; + unsigned c; +}; + +static unsigned +use(const unsigned *n) +{ + assert(*n > 0); + return *n - 1; +} + +static unsigned +decl(unsigned *n) +{ + return (*n)++; +} + +static const char * +cmp_operator(int cmp) +{ + switch (cmp) { + case VM_CMP_LT: return "ult"; + case VM_CMP_LE: return "ule"; + case VM_CMP_EQ: return "eq"; + case VM_CMP_GE: return "uge"; + case VM_CMP_GT: return "ugt"; + case VM_CMP_NE: return "ne"; + + case VM_CMP_ALWAYS: + default: + assert("unreached"); + return NULL; + } +} + +static int +default_accept(FILE *f, const struct fsm_options *opt, + const struct fsm_state_metadata *state_metadata, + void *lang_opaque, void *hook_opaque) +{ + assert(f != NULL); + assert(opt != NULL); + assert(lang_opaque == NULL); + + (void) hook_opaque; + (void) lang_opaque; + + switch (opt->ambig) { + case AMBIG_NONE: + fprintf(f, "%%rt true"); + break; + + case AMBIG_ERROR: +// TODO: decide if we deal with this ahead of the call to print or not + if (state_metadata->end_id_count > 1) { + errno = EINVAL; + return -1; + } + + fprintf(f, "%%rt { i1 true, i32 %u }", state_metadata->end_ids[0]); + break; + + case AMBIG_EARLIEST: + /* + * The libfsm api guarentees these ids are unique, + * and only appear once each, and are sorted. + */ + fprintf(f, "%%rt { i1 true, i32 %u }", state_metadata->end_ids[0]); + break; + + case AMBIG_MULTIPLE: + fprintf(f, "internal unnamed_addr constant [%zu x i32] [", state_metadata->end_id_count); + for (size_t j = 0; j < state_metadata->end_id_count; j++) { + fprintf(f, "i32 %u", state_metadata->end_ids[j]); + if (j + 1 < state_metadata->end_id_count) { + fprintf(f, ", "); + } + } + fprintf(f, "]"); + break; + + default: + assert(!"unreached"); + abort(); + } + + return 0; +} + +static int +default_reject(FILE *f, const struct fsm_options *opt, + void *lang_opaque, void *hook_opaque) +{ + assert(f != NULL); + assert(opt != NULL); + assert(lang_opaque == NULL); + + (void) lang_opaque; + (void) hook_opaque; + + switch (opt->ambig) { + case AMBIG_NONE: + fprintf(f, "%%rt false"); + break; + + case AMBIG_ERROR: + case AMBIG_EARLIEST: + fprintf(f, "%%rt { i1 false, i32 poison }"); + break; + + case AMBIG_MULTIPLE: + fprintf(f, "%%rt { %s poison, i64 -1 }", ptr_i32); + break; + + default: + assert(!"unreached"); + abort(); + } + + if (opt->comments) { + fprintf(f, " ; fail"); + } + + return 0; +} + +static int +print_rettype(FILE *f, const char *name, enum fsm_ambig ambig) +{ + fprintf(f, "%s = type ", name); + + switch (ambig) { + case AMBIG_NONE: + fprintf(f, "i1"); + break; + + case AMBIG_ERROR: + case AMBIG_EARLIEST: + // success, id + fprintf(f, "{ i1, i32 }"); + break; + + case AMBIG_MULTIPLE: + // ids, -1/count + fprintf(f, "{ %s, i64 }", ptr_i32); + break; + + default: + assert(!"unreached"); + abort(); + } + + fprintf(f, "\n"); + + return 0; +} + +static void +print_decl(FILE *f, const char *name, unsigned n) +{ + fprintf(f, "\t%%%s%u = ", name, n); +} + +static void +vprint_label(FILE *f, bool decl, const char *fmt, va_list ap) +{ + assert(fmt != NULL); + + if (!decl) { + fprintf(f, "label %%"); + } + + vfprintf(f, fmt, ap); + + if (decl) { + fprintf(f, ":\n"); + } +} + +static void +print_label(FILE *f, bool decl, const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + vprint_label(f, decl, fmt, ap); + va_end(ap); +} + +static void +print_target(FILE *f, const struct dfavm_op_ir *op, + const char *fmt, ...) +{ + va_list ap; + + if (op == &fail) { + print_label(f, false, "fail"); + } else if (op != NULL && op->instr == VM_OP_STOP && op->cmp == VM_CMP_ALWAYS && op->u.stop.end_bits == VM_END_FAIL) { + print_label(f, false, "fail"); + } else if (op != NULL) { + print_label(f, false, "l%" PRIu32, op->index); + } else if (fmt == NULL) { + assert("!unreached"); + abort(); + } else { + va_start(ap, fmt); + vprint_label(f, false, fmt, ap); + va_end(ap); + } +} + +static void +print_cond(FILE *f, const struct fsm_options *opt, struct dfavm_op_ir *op, + struct frame *frame) +{ + assert(frame != NULL); + assert(op->cmp != VM_CMP_ALWAYS); + + print_decl(f, "r", decl(&frame->r)); + fprintf(f, "icmp %s i8 %%c%u, ", + cmp_operator(op->cmp), use(&frame->c)); + llvm_escputcharlit(f, opt, op->cmp_arg); + + if (opt->comments) { + fprintf(f, " ; "); + c_escputcharlit(f, opt, op->cmp_arg); // C escaping for a comment + } + fprintf(f, "\n"); +} + +static void +print_jump(FILE *f, const struct dfavm_op_ir *dest) +{ + fprintf(f, "\tbr "); + print_target(f, dest, NULL); + fprintf(f, "\n"); +} + +static void +print_incr(FILE *f, const char *name, unsigned n) +{ + // TODO: emit incr()? + print_decl(f, "n.new", n); + fprintf(f, "add i32 1, %%%s%u\n", name, n); + + fprintf(f, "\tstore i32 %%%s.new%u, %s %%%s\n", name, n, ptr_i32, name); +} + +static void +print_branch(FILE *f, const struct frame *frame, + const struct dfavm_op_ir *op_true, const struct dfavm_op_ir *op_false) +{ + fprintf(f, "\tbr i1 %%r%u, ", use(&frame->r)); + print_target(f, op_true, "t%u", use(&frame->b)); + fprintf(f, ", "); + print_target(f, op_false, "f%u", use(&frame->b)); + fprintf(f, "\n"); +} + +static void +print_fetch(FILE *f, const struct fsm_options *opt, + struct frame *frame, const struct dfavm_op_ir *next, + enum dfavm_op_end end_bits) +{ + unsigned n = decl(&frame->c); + unsigned b = decl(&frame->b); + + assert(frame != NULL); + assert(next != NULL); + + /* + * Emitting phi here would mean we'd need to keep track of the + * basic blocks that jump to the current block. That's entirely + * possible with our IR, but seems like a lot of work when we + * can just alloca and let llvm do it for us. + */ + + // TODO: consider emitting fetch(%s, %n) + + switch (opt->io) { + case FSM_IO_GETC: { + print_decl(f, "i", n); + fprintf(f, "tail call i32 %%fsm_getc(%s noundef %%getc_opaque) #1\n", + ptr_i8); + + print_decl(f, "r", decl(&frame->r)); + fprintf(f, "icmp eq i32 %%i%u, -1", n); + if (opt->comments) { + fprintf(f, " ; EOF"); + } + fprintf(f, "\n"); + +// XXX: we don't distinguish error from eof +// https://github.com/katef/libfsm/issues/484 + print_branch(f, frame, + end_bits == VM_END_FAIL ? &fail : NULL, + NULL); + print_label(f, true, "f%u", b); + + print_decl(f, "c", n); + fprintf(f, "trunc i32 %%i%u to i8\n", n); + + print_jump(f, next); + break; + } + + case FSM_IO_STR: { + // TODO: we could increment %s instead of maintaining %n here + print_decl(f, "n", n); + fprintf(f, "load i32, %s %%n\n", ptr_i32); + + print_decl(f, "p", n); + fprintf(f, "getelementptr inbounds i8, %s %%s, i32 %%n%u\n", + ptr_i8, n); + + print_decl(f, "c", n); + fprintf(f, "load i8, %s %%p%u\n", + ptr_i8, n); + + print_decl(f, "r", decl(&frame->r)); + fprintf(f, "icmp eq i8 %%c%u, 0", n); + if (opt->comments) { + fprintf(f, " ; EOT"); + } + fprintf(f, "\n"); + + print_branch(f, frame, + end_bits == VM_END_FAIL ? &fail : NULL, + NULL); + print_label(f, true, "f%u", b); + + print_incr(f, "n", n); + + print_jump(f, next); + break; + } + + case FSM_IO_PAIR: { + print_decl(f, "n", n); + fprintf(f, "load i32, %s %%n\n", ptr_i32); + + print_decl(f, "p", n); + fprintf(f, "getelementptr inbounds i8, %s %%b, i32 %%n%u\n", + ptr_i8, n); + + print_decl(f, "r", decl(&frame->r)); + fprintf(f, "icmp eq %s %%p%u, %%e", ptr_i8, n); + if (opt->comments) { + fprintf(f, " ; EOT"); + } + fprintf(f, "\n"); + + print_branch(f, frame, + end_bits == VM_END_FAIL ? &fail : NULL, + NULL); + print_label(f, true, "f%u", b); + + print_decl(f, "c", n); + fprintf(f, "load i8, %s %%p%u\n", + ptr_i8, n); + + print_incr(f, "n", n); + + print_jump(f, next); + break; + } + + default: + fprintf(stderr, "unsupported IO API\n"); + exit(EXIT_FAILURE); + } + + if (end_bits != VM_END_FAIL) { + // TODO: skip t%u: if the next instruction is ret -1, centralise it to fail: + fprintf(f, "t%u:\n", use(&frame->b)); + } +} + +/* TODO: eventually to be non-static */ +static int +fsm_print_llvmfrag(FILE *f, + const struct fsm_options *opt, + const struct ret_list *retlist, + struct dfavm_op_ir *ops, + const char *cp, + const char *prefix) +{ + struct dfavm_op_ir *op; + + assert(f != NULL); + assert(opt != NULL); + assert(retlist != NULL); + assert(cp != NULL); + assert(prefix != NULL); + + /* TODO: we'll need to heed cp for e.g. lx's codegen */ + (void) cp; + + { + uint32_t l; + + l = 0; + + for (op = ops; op != NULL; op = op->next) { + op->index = l++; + } + } + + { + print_jump(f, ops); + + /* + * We're jumping to ret*: labels, and having each jump + * to a single stop: with a phi instruction. + * + * This looks like: + * + * stop: + * %i = phi i64 + * [0, %ret0], + * [1, %ret1], + * [2, %ret2], + * [3, %fail] + * %p = getelementptr inbounds [4 x %rt], [4 x %rt]* @fsm.r, i64 0, i64 %i + * %ret = load %rt, ptr %p + * ret %rt %ret + * fail: + * br label %stop + * ret0: + * br label %stop + * ret1: + * br label %stop + * ret2: + * br label %stop + * + * where @fsm.r is [4 x %rt] and %rt is the return type. + * + * And we jump to stop: via the ret*: labels rather than + * to a phi node directly. This helps for two reasons: + * + * - We don't need to track every location in the DFA + * that can stop. In other words, many basic blocks + * jump to the same ret*: label. + * + * - The number of basic blocks grows quadratically + * (as DFA grow quadratically), but the set of endids + * remains constant. So the phi list is small. + * + * llvm doesn't find this optimisation for us. + */ + print_label(f, true, "stop"); + + fprintf(f, "\t%%i = phi i64\n"); + for (size_t i = 0; i < retlist->count; i++) { + fprintf(f, "\t [%zu, %%ret%zu],\n", i, i); + } + fprintf(f, "\t [%zu, %%fail]\n", retlist->count); + + fprintf(f, "\t%%p = getelementptr inbounds [%zu x %%rt], [%zu x %%rt]* @%sr, i64 0, i64 %%i\n", + retlist->count + 1, retlist->count + 1, prefix); + fprintf(f, "\t%%ret = load %%rt, %s %%p\n", ptr_rt); + fprintf(f, "\tret %%rt %%ret\n"); + + print_label(f, true, "fail"); + fprintf(f, "\tbr "); + print_label(f, false, "stop"); + fprintf(f, "\n"); + + for (size_t i = 0; i < retlist->count; i++) { + print_label(f, true, "ret%zu", i); + fprintf(f, "\tbr "); + print_label(f, false, "stop"); + fprintf(f, "\n"); + } + } + + struct frame frame = { 0, 0, 0 }; + for (op = ops; op != NULL; op = op->next) { + if (op->instr != VM_OP_STOP || op->cmp != VM_CMP_ALWAYS || op->u.stop.end_bits != VM_END_FAIL) { + print_label(f, true, "l%" PRIu32, op->index); + + /* + * We only show examples when there's a label for the block, + * otherwise it's confusing with the conditionally elided + * optimisations per-instruction below, which can result in + * no block code being emitted for a particular vm op. + */ + if (op->example != NULL) { + /* C's escaping seems to be a subset of llvm's, and these are + * for comments anyway. So I'm borrowing this for C here */ + fprintf(f, "\t; e.g. \""); + escputs(f, opt, c_escputc_str, op->example); + fprintf(f, "\""); + + fprintf(f, "\n"); + } + } + + switch (op->instr) { + case VM_OP_STOP: + if (op->cmp != VM_CMP_ALWAYS) { + unsigned b = decl(&frame.b); + +// TODO: our ret%u: label goes in place of t%u here, which means we're replacing the NULL + print_cond(f, opt, op, &frame); + print_branch(f, &frame, + op->u.stop.end_bits == VM_END_FAIL ? &fail : NULL, + op->next); + if (op->u.stop.end_bits != VM_END_FAIL) { + print_label(f, true, "t%u", b); + } + } + + if (op->u.stop.end_bits == VM_END_FAIL) { + /* handled above */ + } else { + assert(retlist->count > 0); + const struct ret *ret = op->ret; + assert(ret != NULL); + assert(ret >= retlist->a && ret <= (retlist->a + retlist->count)); + fprintf(f, "\tbr "); + print_label(f, false, "ret%u", ret - retlist->a); + fprintf(f, "\n"); + } + break; + + case VM_OP_FETCH: { + print_fetch(f, opt, &frame, op->next, op->u.fetch.end_bits); + + if (op->u.fetch.end_bits == VM_END_FAIL) { + /* handled in print_fetch() */ + } else { + assert(retlist->count > 0); + const struct ret *ret = op->ret; + assert(ret != NULL); + assert(ret >= retlist->a && ret <= (retlist->a + retlist->count)); + fprintf(f, "\tbr "); + print_label(f, false, "ret%u", ret - retlist->a); + fprintf(f, "\n"); + } + break; + } + + case VM_OP_BRANCH: { + const struct dfavm_op_ir *dest = op->u.br.dest_arg; + + if (op->cmp == VM_CMP_ALWAYS) { + print_jump(f, dest); + } else { + print_cond(f, opt, op, &frame); + print_branch(f, &frame, + dest, + op->next); + } + break; + } + + default: + assert(!"unreached"); + break; + } + } + + return 0; +} + +int +fsm_print_llvm(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + const struct ret_list *retlist, + struct dfavm_op_ir *ops) +{ + const char *prefix; + const char *cp; + + assert(f != NULL); + assert(opt != NULL); + assert(hooks != NULL); + assert(retlist != NULL); + + if (opt->prefix != NULL) { + prefix = opt->prefix; + } else { + prefix = "fsm."; + } + + if (hooks->cp != NULL) { + cp = hooks->cp; + } else { + cp = "c"; /* XXX */ + } + + if (opt->fragment) { + fsm_print_llvmfrag(f, opt, retlist, ops, cp, prefix); + return 0; + } + + fprintf(f, "; generated\n"); + print_rettype(f, "%rt", opt->ambig); + + /* + * For AMBIG_MULTIPLE we emit a bunch of arrays and then point at them from + * each %rt. So we call the hook for the arrays, because that's where the id + * list is. For other ambig modes, we call the hook for the %rt instead. + */ + if (opt->ambig == AMBIG_MULTIPLE) { + for (size_t i = 0; i < retlist->count; i++) { + fprintf(f, "@%sr%zu = ", prefix, i); + const struct fsm_state_metadata state_metadata = { + .end_ids = retlist->a[i].ids, + .end_id_count = retlist->a[i].count, + }; + if (-1 == print_hook_accept(f, opt, hooks, + &state_metadata, + default_accept, NULL)) + { + return -1; + } + + if (-1 == print_hook_comment(f, opt, hooks, + &state_metadata)) + { + return -1; + } + + fprintf(f, "\n"); + } + } + + fprintf(f, "@%sr = internal unnamed_addr constant [%zu x %%rt] [\n", prefix, retlist->count + 1); + for (size_t i = 0; i < retlist->count; i++) { + fprintf(f, "\t "); + if (opt->ambig == AMBIG_MULTIPLE) { + fprintf(f, "%%rt { %s bitcast ([%zu x i32]* @%sr%zu to %s), i64 %zu }", + ptr_i32, retlist->a[i].count, prefix, i, ptr_i32, retlist->a[i].count); + fprintf(f, ","); + } else { + const struct fsm_state_metadata state_metadata = { + .end_ids = retlist->a[i].ids, + .end_id_count = retlist->a[i].count, + }; + if (-1 == print_hook_accept(f, opt, hooks, + &state_metadata, + default_accept, NULL)) + { + return -1; + } + + fprintf(f, ","); + + if (-1 == print_hook_comment(f, opt, hooks, + &state_metadata)) + { + return -1; + } + } + fprintf(f, "\n"); + } + fprintf(f, "\t "); + if (-1 == print_hook_reject(f, opt, hooks, NULL, default_reject, NULL)) { + return -1; + } + fprintf(f, "\n"); + fprintf(f, "\t]\n"); + + fprintf(f, "define dso_local %%rt @%smain", prefix); + + switch (opt->io) { + case FSM_IO_GETC: +#ifdef OPAQUE_POINTERS + fprintf(f, "(i32 (%s)* nocapture noundef readonly %%fsm_getc, %s noundef %%getc_opaque)", + ptr_i8, ptr_void); +#else + fprintf(f, "(ptr nocapture noundef readonly %%fsm_getc, ptr noundef %%getc_opaque)"); +#endif + break; + + case FSM_IO_STR: + fprintf(f, "(%s nocapture noundef readonly %%s)", + ptr_i8); + break; + + case FSM_IO_PAIR: + fprintf(f, "(%s noundef readonly %%b, %s noundef readnone %%e)", + ptr_i8, ptr_i8); + break; + + default: + fprintf(f, "(%s nocapture noundef readonly %%b, %s nocapture noundef readonly %%e)", + ptr_i8, ptr_i8); + break; + } + + fprintf(f, " local_unnamed_addr"); + fprintf(f, " hot nosync nounwind norecurse willreturn"); + fprintf(f, " #0 {\n"); + + switch (opt->io) { + case FSM_IO_GETC: + break; + + case FSM_IO_STR: + fprintf(f, "\t%%n = alloca i32\n"); + fprintf(f, "\tstore i32 0, %s %%n\n", ptr_i32); + break; + + case FSM_IO_PAIR: + fprintf(f, "\t%%n = alloca i32\n"); + fprintf(f, "\tstore i32 0, %s %%n\n", ptr_i32); + break; + + default: + fprintf(stderr, "unsupported IO API\n"); + exit(EXIT_FAILURE); + } + + fsm_print_llvmfrag(f, opt, retlist, ops, cp, prefix); + + fprintf(f, "}\n"); + fprintf(f, "\n"); + + return 0; +} + diff --git a/src/libfsm/print/rust.c b/src/libfsm/print/rust.c index e249c6aad..c1fcf5bb4 100644 --- a/src/libfsm/print/rust.c +++ b/src/libfsm/print/rust.c @@ -24,29 +24,13 @@ #include #include "libfsm/internal.h" +#include "libfsm/print.h" +#include "libfsm/vm/retlist.h" #include "libfsm/vm/vm.h" -#include "ir.h" - #define START UINT32_MAX -static int -leaf(FILE *f, const struct fsm_end_ids *ids, const void *leaf_opaque) -{ - assert(f != NULL); - assert(leaf_opaque == NULL); - - (void) ids; - (void) leaf_opaque; - - /* XXX: this should be FSM_UNKNOWN or something non-EOF, - * maybe user defined */ - fprintf(f, "return TOK_UNKNOWN;"); - - return 0; -} - static const char * cmp_operator(int cmp) { @@ -66,9 +50,89 @@ cmp_operator(int cmp) } static int -has_op(const struct dfavm_op_ir *op, enum dfavm_op_instr instr) +print_ids(FILE *f, + enum fsm_ambig ambig, const fsm_end_id_t *ids, size_t count, + size_t i) +{ + switch (ambig) { + case AMBIG_NONE: + fprintf(f, "return Some(())"); + break; + + case AMBIG_ERROR: +// TODO: decide if we deal with this ahead of the call to print or not + if (count > 1) { + errno = EINVAL; + return -1; + } + + fprintf(f, "return Some(%u)", ids[0]); + break; + + case AMBIG_EARLIEST: + /* + * The libfsm api guarentees these ids are unique, + * and only appear once each, and are sorted. + */ + fprintf(f, "return Some(%u)", ids[0]); + break; + + case AMBIG_MULTIPLE: + fprintf(f, "return Some(&RET%zu)", i); + break; + + default: + assert(!"unreached"); + abort(); + } + + return 0; +} + +static int +default_accept(FILE *f, const struct fsm_options *opt, + const struct fsm_state_metadata *state_metadata, + void *lang_opaque, void *hook_opaque) { - for ( ; op != NULL; op = op->next) { + size_t i; + + assert(f != NULL); + assert(opt != NULL); + assert(lang_opaque != NULL); + + (void) hook_opaque; + + i = * (const size_t *) lang_opaque; + + if (-1 == print_ids(f, opt->ambig, state_metadata->end_ids, state_metadata->end_id_count, i)) { + return -1; + } + + return 0; +} + +static int +default_reject(FILE *f, const struct fsm_options *opt, + void *lang_opaque, void *hook_opaque) +{ + assert(f != NULL); + assert(opt != NULL); + assert(lang_opaque == NULL); + + (void) lang_opaque; + (void) hook_opaque; + + fprintf(f, "return None"); + + return 0; +} + +static int +has_op(const struct dfavm_op_ir *ops, enum dfavm_op_instr instr) +{ + const struct dfavm_op_ir *op; + + for (op = ops; op != NULL; op = op->next) { if (op->instr == instr) { return 1; } @@ -100,23 +164,36 @@ print_cond(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt) } static int -print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, - enum dfavm_op_end end_bits, const struct ir *ir) +print_end(FILE *f, const struct dfavm_op_ir *op, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + const struct ret_list *retlist, + enum dfavm_op_end end_bits) { - if (end_bits == VM_END_FAIL) { - fprintf(f, "return None"); - return 0; - } + size_t i; - if (opt->endleaf != NULL) { - if (-1 == opt->endleaf(f, op->ir_state->end_ids, opt->endleaf_opaque)) { - return -1; - } - } else { - fprintf(f, "return Some(%td)", op->ir_state - ir->states); - } + switch (end_bits) { + case VM_END_FAIL: + return print_hook_reject(f, opt, hooks, NULL, default_reject, NULL); - return 0; + case VM_END_SUCC: + assert(op->ret >= retlist->a); + + i = op->ret - retlist->a; + + const struct fsm_state_metadata state_metadata = { + .end_ids = op->ret->ids, + .end_id_count = op->ret->count, + }; + + return print_hook_accept(f, opt, hooks, + &state_metadata, default_accept, + &i); + + default: + assert(!"unreached"); + abort(); + } } static void @@ -139,30 +216,50 @@ print_fetch(FILE *f) fprintf(f, "bytes.next()"); } +static void +print_ret(FILE *f, const unsigned *ids, size_t count) +{ + size_t i; + + fprintf(f, "["); + for (i = 0; i < count; i++) { + fprintf(f, "%u", ids[i]); + if (i + 1 < count) { + fprintf(f, ", "); + } + } + fprintf(f, "];"); +} + /* TODO: eventually to be non-static */ static int -fsm_print_rustfrag(FILE *f, const struct dfavm_assembler_ir *a, - const struct ir *ir, const struct fsm_options *opt, - const char *cp, - int (*leaf)(FILE *, const struct fsm_end_ids *ids, const void *leaf_opaque), - const void *leaf_opaque) +fsm_print_rustfrag(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + const struct ret_list *retlist, + struct dfavm_op_ir *ops, + const char *cp) { struct dfavm_op_ir *op; bool fallthrough; assert(f != NULL); - assert(a != NULL); assert(opt != NULL); + assert(retlist != NULL); assert(cp != NULL); - /* TODO: we don't currently have .opaque information attached to struct dfavm_op_ir. - * We'll need that in order to be able to use the leaf callback here. */ - (void) leaf; - (void) leaf_opaque; - /* TODO: we'll need to heed cp for e.g. lx's codegen */ (void) cp; + if (opt->ambig == AMBIG_MULTIPLE) { + for (size_t i = 0; i < retlist->count; i++) { + fprintf(f, " static RET%zu: [u32; %zu] = ", i, retlist->a[i].count); + print_ret(f, retlist->a[i].ids, retlist->a[i].count); + fprintf(f, "\n"); + } + fprintf(f, "\n"); + } + /* * We only output labels for ops which are branched to. This gives * gaps in the sequence for ops which don't need a label. @@ -173,8 +270,8 @@ fsm_print_rustfrag(FILE *f, const struct dfavm_assembler_ir *a, l = START; - for (op = a->linked; op != NULL; op = op->next) { - if (op == a->linked || op->num_incoming > 0) { + for (op = ops; op != NULL; op = op->next) { + if (op == ops || op->num_incoming > 0) { op->index = l++; } } @@ -183,8 +280,8 @@ fsm_print_rustfrag(FILE *f, const struct dfavm_assembler_ir *a, /* * Only declare variables if we're actually going to use them. */ - if (a->linked->cmp == VM_CMP_ALWAYS && a->linked->instr == VM_OP_STOP) { - assert(a->linked->next == NULL); + if (ops->cmp == VM_CMP_ALWAYS && ops->instr == VM_OP_STOP) { + assert(ops->next == NULL); fprintf(f, "\n"); } else { switch (opt->io) { @@ -208,8 +305,8 @@ fsm_print_rustfrag(FILE *f, const struct dfavm_assembler_ir *a, } fprintf(f, " pub enum Label {\n "); - for (op = a->linked; op != NULL; op = op->next) { - if (op == a->linked || op->num_incoming > 0) { + for (op = ops; op != NULL; op = op->next) { + if (op == ops || op->num_incoming > 0) { fprintf(f, " "); print_label(f, op); fprintf(f, ","); @@ -219,7 +316,7 @@ fsm_print_rustfrag(FILE *f, const struct dfavm_assembler_ir *a, fprintf(f, " }\n"); fprintf(f, "\n"); - fprintf(f, " let %sl = Ls;\n", has_op(a->linked, VM_OP_BRANCH) ? "mut " : ""); + fprintf(f, " let %sl = Ls;\n", has_op(ops, VM_OP_BRANCH) ? "mut " : ""); fprintf(f, "\n"); fprintf(f, " loop {\n"); @@ -227,9 +324,9 @@ fsm_print_rustfrag(FILE *f, const struct dfavm_assembler_ir *a, fallthrough = true; - for (op = a->linked; op != NULL; op = op->next) { - if (op == a->linked || op->num_incoming > 0) { - if (op != a->linked) { + for (op = ops; op != NULL; op = op->next) { + if (op == ops || op->num_incoming > 0) { + if (op != ops) { if (fallthrough) { fprintf(f, " "); print_jump(f, op); @@ -243,11 +340,11 @@ fsm_print_rustfrag(FILE *f, const struct dfavm_assembler_ir *a, print_label(f, op); fprintf(f, " => {"); - if (op->ir_state != NULL && op->ir_state->example != NULL) { + if (op->example != NULL) { /* C's escaping seems to be a subset of rust's, and these are * for comments anyway. So I'm borrowing this for C here */ fprintf(f, " // e.g. \""); - escputs(f, opt, c_escputc_str, op->ir_state->example); + escputs(f, opt, c_escputc_str, op->example); fprintf(f, "\""); } @@ -264,14 +361,25 @@ fsm_print_rustfrag(FILE *f, const struct dfavm_assembler_ir *a, if (op->cmp != VM_CMP_ALWAYS) { fprintf(f, "{ "); } - if (-1 == print_end(f, op, opt, op->u.stop.end_bits, ir)) { + if (-1 == print_end(f, op, opt, hooks, retlist, op->u.stop.end_bits)) { return -1; } - fprintf(f, ";"); if (op->cmp != VM_CMP_ALWAYS) { fprintf(f, " }"); } + if (op->u.stop.end_bits == VM_END_SUCC) { + const struct fsm_state_metadata state_metadata = { + .end_ids = op->ret->ids, + .end_id_count = op->ret->count, + }; + if (-1 == print_hook_comment(f, opt, hooks, + &state_metadata)) + { + return -1; + } + } + if (op->cmp == VM_CMP_ALWAYS) { /* the code for fallthrough would be unreachable */ fallthrough = false; @@ -313,7 +421,7 @@ fsm_print_rustfrag(FILE *f, const struct dfavm_assembler_ir *a, fprintf(f, " "); fprintf(f, "None => "); - print_end(f, op, opt, op->u.fetch.end_bits, ir); + print_end(f, op, opt, hooks, retlist, op->u.fetch.end_bits); fprintf(f, ",\n"); fprintf(f, " "); @@ -357,27 +465,37 @@ fsm_print_rustfrag(FILE *f, const struct dfavm_assembler_ir *a, return 0; } -static int -fsm_print_rust_complete(FILE *f, const struct ir *ir, - const struct fsm_options *opt, const char *prefix, const char *cp) +int +fsm_print_rust(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + const struct ret_list *retlist, + struct dfavm_op_ir *ops) { - static const struct dfavm_assembler_ir zero; - static const struct fsm_vm_compile_opts vm_opts = { FSM_VM_COMPILE_DEFAULT_FLAGS, FSM_VM_COMPILE_VM_V1, NULL }; - struct dfavm_assembler_ir a; + const char *prefix; + const char *cp; assert(f != NULL); - assert(ir != NULL); assert(opt != NULL); + assert(hooks != NULL); + assert(retlist != NULL); - a = zero; + if (opt->prefix != NULL) { + prefix = opt->prefix; + } else { + prefix = "fsm_"; + } - if (!dfavm_compile_ir(&a, ir, vm_opts)) { - return -1; + if (hooks->cp != NULL) { + cp = hooks->cp; + } else { + cp = "c"; /* XXX */ } + fprintf(f, "// generated\n"); + if (opt->fragment) { - fsm_print_rustfrag(f, &a, ir, opt, cp, - opt->leaf != NULL ? opt->leaf : leaf, opt->leaf_opaque); + fsm_print_rustfrag(f, opt, hooks, retlist, ops, cp); goto error; } @@ -388,22 +506,19 @@ fsm_print_rust_complete(FILE *f, const struct ir *ir, switch (opt->io) { case FSM_IO_GETC: /* e.g. dbg!(fsm_main("abc".as_bytes().iter().copied())); */ - fprintf(f, "(mut bytes: impl Iterator) -> Option {\n"); - fprintf(f, " use Label::*;\n"); + fprintf(f, "(mut bytes: impl Iterator)"); break; case FSM_IO_STR: /* e.g. dbg!(fsm_main("xabces")); */ - fprintf(f, "(%sinput: &str) -> Option {\n", - has_op(a.linked, VM_OP_FETCH) ? "" : "_"); - fprintf(f, " use Label::*;\n"); + fprintf(f, "(%sinput: &str)", + has_op(ops, VM_OP_FETCH) ? "" : "_"); break; case FSM_IO_PAIR: /* e.g. dbg!(fsm_main("xabces".as_bytes())); */ - fprintf(f, "(%sinput: &[u8]) -> Option {\n", - has_op(a.linked, VM_OP_FETCH) ? "" : "_"); - fprintf(f, " use Label::*;\n"); + fprintf(f, "(%sinput: &[u8])", + has_op(ops, VM_OP_FETCH) ? "" : "_"); break; default: @@ -411,60 +526,39 @@ fsm_print_rust_complete(FILE *f, const struct ir *ir, exit(EXIT_FAILURE); } - fsm_print_rustfrag(f, &a, ir, opt, cp, - opt->leaf != NULL ? opt->leaf : leaf, opt->leaf_opaque); - - fprintf(f, "}\n"); - fprintf(f, "\n"); - - dfavm_opasm_finalize_op(&a); + fprintf(f, " -> "); - if (ferror(f)) { - return -1; - } - - return 0; - -error: - - dfavm_opasm_finalize_op(&a); - - return -1; -} + switch (opt->ambig) { + case AMBIG_NONE: + fprintf(f, "Option<()>"); + break; -int -fsm_print_rust(FILE *f, const struct fsm *fsm) -{ - struct ir *ir; - const char *prefix; - const char *cp; - int r; + case AMBIG_ERROR: + case AMBIG_EARLIEST: + fprintf(f, "Option"); + break; - assert(f != NULL); - assert(fsm != NULL); - assert(fsm->opt != NULL); + case AMBIG_MULTIPLE: + fprintf(f, "Option<&'static [u32]>"); + break; - ir = make_ir(fsm); - if (ir == NULL) { - return -1; + default: + fprintf(stderr, "unsupported ambig mode\n"); + exit(EXIT_FAILURE); } - if (fsm->opt->prefix != NULL) { - prefix = fsm->opt->prefix; - } else { - prefix = "fsm_"; - } + fprintf(f, " {\n"); + fprintf(f, " use Label::*;\n"); - if (fsm->opt->cp != NULL) { - cp = fsm->opt->cp; - } else { - cp = "c"; /* XXX */ - } + fsm_print_rustfrag(f, opt, hooks, retlist, ops, cp); - r = fsm_print_rust_complete(f, ir, fsm->opt, prefix, cp); + fprintf(f, "}\n"); + fprintf(f, "\n"); - free_ir(fsm, ir); + return 0; - return r; +error: + + return -1; } diff --git a/src/libfsm/print/sh.c b/src/libfsm/print/sh.c index 7b6bee8d2..5322e4ffc 100644 --- a/src/libfsm/print/sh.c +++ b/src/libfsm/print/sh.c @@ -24,11 +24,11 @@ #include #include "libfsm/internal.h" +#include "libfsm/print.h" +#include "libfsm/vm/retlist.h" #include "libfsm/vm/vm.h" -#include "ir.h" - static const char * str_operator(int cmp) { @@ -65,6 +65,83 @@ ord_operator(int cmp) } } +static int +print_ids(FILE *f, + enum fsm_ambig ambig, const fsm_end_id_t *ids, size_t count) +{ + switch (ambig) { + case AMBIG_NONE: + break; + + case AMBIG_ERROR: +// TODO: decide if we deal with this ahead of the call to print or not + if (count > 1) { + errno = EINVAL; + return -1; + } + + fprintf(f, " %u", ids[0]); + break; + + case AMBIG_EARLIEST: + /* + * The libfsm api guarentees these ids are unique, + * and only appear once each, and are sorted. + */ + fprintf(f, " %u", ids[0]); + break; + + case AMBIG_MULTIPLE: + for (size_t i = 0; i < count; i++) { + fprintf(f, " %u", ids[i]); + } + break; + + default: + assert(!"unreached"); + abort(); + } + + return 0; +} + +static int +default_accept(FILE *f, const struct fsm_options *opt, + const struct fsm_state_metadata *state_metadata, + void *lang_opaque, void *hook_opaque) +{ + assert(f != NULL); + assert(opt != NULL); + assert(lang_opaque == NULL); + + (void) lang_opaque; + (void) hook_opaque; + + fprintf(f, "matched"); + + if (-1 == print_ids(f, opt->ambig, state_metadata->end_ids, state_metadata->end_id_count)) { + return -1; + } + + return 0; +} + +static int +default_reject(FILE *f, const struct fsm_options *opt, + void *lang_opaque, void *hook_opaque) +{ + assert(f != NULL); + assert(opt != NULL); + assert(lang_opaque == NULL); + + (void) lang_opaque; + (void) hook_opaque; + + fprintf(f, "fail"); + + return 0; +} + static void print_label(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt) { @@ -76,9 +153,9 @@ print_label(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt * arbitrary characters, and nobody can read $'...' style strings. * So I'm just using C-style strings here because it's simple enough. */ - if (op->ir_state->example != NULL) { + if (op->example != NULL) { fprintf(f, " # e.g. \""); - escputs(f, opt, c_escputc_str, op->ir_state->example); + escputs(f, opt, c_escputc_str, op->example); fprintf(f, "\""); } } @@ -118,20 +195,39 @@ print_cond(FILE *f, const struct dfavm_op_ir *op) } static int -print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, - enum dfavm_op_end end_bits, const struct ir *ir) +print_end(FILE *f, const struct dfavm_op_ir *op, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + enum dfavm_op_end end_bits) { - if (end_bits == VM_END_FAIL) { - fprintf(f, "fail"); - return 0; - } + switch (end_bits) { + case VM_END_FAIL: + return print_hook_reject(f, opt, hooks, NULL, default_reject, NULL); + + case VM_END_SUCC:; + const struct fsm_state_metadata state_metadata = { + .end_ids = op->ret->ids, + .end_id_count = op->ret->count, + }; + if (-1 == print_hook_accept(f, opt, hooks, + &state_metadata, + default_accept, + NULL)) + { + return -1; + } - if (opt->endleaf != NULL) { - if (-1 == opt->endleaf(f, op->ir_state->end_ids, opt->endleaf_opaque)) { + if (-1 == print_hook_comment(f, opt, hooks, + &state_metadata)) + { return -1; } - } else { - fprintf(f, "matched %td", op->ir_state - ir->states); + + return 0; + + default: + assert(!"unreached"); + abort(); } return 0; @@ -149,22 +245,22 @@ print_fetch(FILE *f) fprintf(f, "read -rn 1 c || "); } -static int -fsm_print_shfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt) +int +fsm_print_sh(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + const struct ret_list *retlist, + struct dfavm_op_ir *ops) { - static const struct dfavm_assembler_ir zero; - struct dfavm_assembler_ir a; struct dfavm_op_ir *op; - static const struct fsm_vm_compile_opts vm_opts = { FSM_VM_COMPILE_DEFAULT_FLAGS, FSM_VM_COMPILE_VM_V1, NULL }; - assert(f != NULL); - assert(ir != NULL); assert(opt != NULL); + assert(hooks != NULL); + assert(retlist != NULL); - a = zero; - - if (!dfavm_compile_ir(&a, ir, vm_opts)) { + if (opt->io != FSM_IO_STR) { + errno = ENOTSUP; return -1; } @@ -178,7 +274,7 @@ fsm_print_shfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt) l = 0; - for (op = a.linked; op != NULL; op = op->next) { + for (op = ops; op != NULL; op = op->next) { if (op->num_incoming > 0) { op->index = l++; } @@ -195,13 +291,14 @@ fsm_print_shfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt) fprintf(f, "IFS=\n"); fprintf(f, "\n"); + fprintf(f, "# generated\n"); fprintf(f, "while true; do\n"); fprintf(f, "\tcase $l in\n"); fprintf(f, "\tstart)\n"); - for (op = a.linked; op != NULL; op = op->next) { + for (op = ops; op != NULL; op = op->next) { if (op->num_incoming > 0) { - if (op != a.linked) { + if (op != ops) { fprintf(f, "\t\t;&\n"); } @@ -215,14 +312,14 @@ fsm_print_shfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt) switch (op->instr) { case VM_OP_STOP: print_cond(f, op); - if (-1 == print_end(f, op, opt, op->u.stop.end_bits, ir)) { + if (-1 == print_end(f, op, opt, hooks, op->u.stop.end_bits)) { return -1; } break; case VM_OP_FETCH: print_fetch(f); - if (-1 == print_end(f, op, opt, op->u.fetch.end_bits, ir)) { + if (-1 == print_end(f, op, opt, hooks, op->u.fetch.end_bits)) { return -1; } break; @@ -244,39 +341,6 @@ fsm_print_shfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt) fprintf(f, "\tesac\n"); fprintf(f, "done\n"); - dfavm_opasm_finalize_op(&a); - - if (ferror(f)) { - return -1; - } - return 0; } -int -fsm_print_sh(FILE *f, const struct fsm *fsm) -{ - struct ir *ir; - int r; - - assert(f != NULL); - assert(fsm != NULL); - assert(fsm->opt != NULL); - - if (fsm->opt->io != FSM_IO_STR) { - errno = ENOTSUP; - return -1; - } - - ir = make_ir(fsm); - if (ir == NULL) { - return -1; - } - - r = fsm_print_shfrag(f, ir, fsm->opt); - - free_ir(fsm, ir); - - return r; -} - diff --git a/src/libfsm/print/vmasm.c b/src/libfsm/print/vmasm.c index 8e3085b86..bf5551420 100644 --- a/src/libfsm/print/vmasm.c +++ b/src/libfsm/print/vmasm.c @@ -22,11 +22,11 @@ #include #include "libfsm/internal.h" +#include "libfsm/print.h" +#include "libfsm/vm/retlist.h" #include "libfsm/vm/vm.h" -#include "ir.h" - enum asm_dialect { AMD64_ATT, AMD64_NASM, @@ -34,9 +34,73 @@ enum asm_dialect { }; static int -print_asm_amd64(FILE *f, const char *prefix, - const struct ir *ir, const struct fsm_options *opt, - const struct dfavm_assembler_ir *a, enum asm_dialect dialect) +print_end(FILE *f, const struct dfavm_op_ir *op, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + enum asm_dialect dialect, const char *ret_reg, + enum dfavm_op_end end_bits) +{ + bool r; + + switch (end_bits) { + case VM_END_FAIL: + if (-1 == print_hook_reject(f, opt, hooks, NULL, NULL, NULL)) { + return -1; + } + break; + + case VM_END_SUCC:; + const struct fsm_state_metadata state_metadata = { + .end_ids = op->ret->ids, + .end_id_count = op->ret->count, + }; + if (-1 == print_hook_accept(f, opt, hooks, + &state_metadata, + NULL, NULL)) + { + return -1; + } + if (-1 == print_hook_comment(f, opt, hooks, + &state_metadata)) + { + return -1; + } + break; + + default: + assert(!"unreached"); + abort(); + } + + if (opt->ambig != AMBIG_NONE) { + errno = ENOTSUP; + return -1; + } + + r = end_bits == VM_END_SUCC; + + switch (dialect) { + case AMD64_ATT: + fprintf(f, "\tmovl $%d, %%%s\n", r, ret_reg); + break; + case AMD64_NASM: + fprintf(f, "\tMOV %s, %d\n", ret_reg, r); + break; + case AMD64_GO: + fprintf(f, "\tMOVQ $%d, %s\n", r, ret_reg); + break; + } + + return 0; +} + +static int +print_asm_amd64(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + struct dfavm_op_ir *ops, + const char *prefix, + enum asm_dialect dialect) { // const char *sst_reg = NULL; // state struct: not currently used const char *stp_reg = "rdi"; // string pointer @@ -47,7 +111,6 @@ print_asm_amd64(FILE *f, const char *prefix, const char *label_dot = "."; - const struct ir_state *curr_st = NULL; const struct dfavm_op_ir *op; char *comment; @@ -60,9 +123,7 @@ print_asm_amd64(FILE *f, const char *prefix, #endif assert(f != NULL); - assert(ir != NULL); assert(opt != NULL); - assert(a != NULL); switch (dialect) { case AMD64_ATT: comment = "#"; break; @@ -83,12 +144,14 @@ print_asm_amd64(FILE *f, const char *prefix, /* print preamble */ switch (dialect) { case AMD64_ATT: + fprintf(f, "%s generated\n", comment); fprintf(f, ".globl %s%s%s\n", sigil, prefix, "match"); fprintf(f, ".text\n"); fprintf(f, "%s%s%s:\n", sigil, prefix, "match"); break; case AMD64_NASM: + fprintf(f, "%s generated\n", comment); fprintf(f, "section .text\n"); fprintf(f, "global %s%s%s\n", sigil, prefix, "match"); fprintf(f, "%s%s%s:\n", sigil, prefix, "match"); @@ -99,14 +162,19 @@ print_asm_amd64(FILE *f, const char *prefix, fprintf(f, "#include \"textflag.h\"\n"); fprintf(f, "\n"); + fprintf(f, "%s generated\n", comment); switch (opt->io) { case FSM_IO_STR: - fprintf(f, "// func %s%s(data string) int\n", prefix, "Match"); + if (opt->comments) { + fprintf(f, "// func %s%s(data string) int\n", prefix, "Match"); + } fprintf(f, "TEXT ·%s(SB), NOSPLIT, $0-24\n", "Match"); break; case FSM_IO_PAIR: - fprintf(f, "// func %s%s(data []byte) int\n", prefix, "Match"); + if (opt->comments) { + fprintf(f, "// func %s%s(data []byte) int\n", prefix, "Match"); + } fprintf(f, "TEXT ·%s%s(SB), NOSPLIT, $0-32\n", prefix, "Match"); break; @@ -140,54 +208,32 @@ print_asm_amd64(FILE *f, const char *prefix, fprintf(f, "\tADDQ %s, %s\n\n", stp_reg, stn_reg); } - for (op = a->linked; op != NULL; op = op->next) { - if (curr_st != op->ir_state) { - unsigned state = op->ir_state - ir->states; - if (op->num_incoming > 0) { - fprintf(f, "%sstate_%u:\n", label_dot, state); - } else { - fprintf(f, "%s state %u\n", comment, state); - } + for (op = ops; op != NULL; op = op->next) { + if (op->num_incoming > 0) { + fprintf(f, "%sl%u:\n", label_dot, op->index); - curr_st = op->ir_state; + // TODO: example + } else if (opt->comments) { + fprintf(f, "%s l%u\n", comment, op->index); } switch (op->instr) { case VM_OP_STOP: { - const char *jmp_op; - - if (op->u.stop.end_bits == VM_END_SUCC) { - unsigned end_st = op->ir_state - ir->states; - switch (dialect) { - case AMD64_ATT: - fprintf(f, "\tmovl $0x%02x, %%%s\n", end_st, ret_reg); - break; - case AMD64_NASM: - fprintf(f, "\tMOV %s, %02xh\n", ret_reg, end_st); - break; - case AMD64_GO: - fprintf(f, "\tMOVQ $0x%02x, %s\n", end_st, ret_reg); - break; - - } - } else { - switch (dialect) { - case AMD64_ATT: - fprintf(f, "\tmovl $-1, %%%s\n", ret_reg); - break; - case AMD64_NASM: - fprintf(f, "\tMOV %s, -1\n", ret_reg); - break; - case AMD64_GO: - fprintf(f, "\tMOVQ $-1, %s\n", ret_reg); - break; - } + if (-1 == print_end(f, op, opt, hooks, + dialect, ret_reg, + op->u.stop.end_bits)) + { + return -1; } if (op->cmp == VM_CMP_ALWAYS && op->next == NULL) { - fprintf(f, "\t%s elided jmp to %sfinish\n", comment, label_dot); + if (opt->comments) { + fprintf(f, "\t%s elided jmp to %sfinish\n", comment, label_dot); + } } else { + const char *jmp_op; + if (op->cmp != VM_CMP_ALWAYS) { switch (dialect) { case AMD64_ATT: @@ -221,41 +267,13 @@ print_asm_amd64(FILE *f, const char *prefix, case VM_OP_FETCH: { - if (op->u.fetch.end_bits == VM_END_SUCC) { - unsigned end_st = op->ir_state - ir->states; - - switch (dialect) { - case AMD64_ATT: - fprintf(f, "\tmovl $0x%02x,%%%s\n", (unsigned)end_st, ret_reg); - break; - - case AMD64_NASM: - fprintf(f, "\tMOV %s, %02xh\n", ret_reg, end_st); - break; - - case AMD64_GO: - fprintf(f, "\tMOVL $0x%02x, %s\n", end_st, ret_reg); - break; - - } - } else { - - switch (dialect) { - case AMD64_ATT: - fprintf(f, "\tmovl $-1, %%%s\n", ret_reg); - break; - - case AMD64_NASM: - fprintf(f, "\tMOV %s, -1\n", ret_reg); - break; - - case AMD64_GO: - fprintf(f, "\tMOVQ $-1, %s\n", ret_reg); - break; - } + if (-1 == print_end(f, op, opt, hooks, + dialect, ret_reg, + op->u.fetch.end_bits)) + { + return -1; } - switch (dialect) { case AMD64_ATT: fprintf(f, "\tcmpq %%%s,%%%s\n", stp_reg, stn_reg); @@ -285,7 +303,6 @@ print_asm_amd64(FILE *f, const char *prefix, case VM_OP_BRANCH: { const char *jmp_op; - unsigned dest_state = op->u.br.dest_arg->ir_state - ir->states; if (op->cmp != VM_CMP_ALWAYS) { switch (dialect) { @@ -313,7 +330,7 @@ print_asm_amd64(FILE *f, const char *prefix, case VM_CMP_NE: jmp_op = (dialect == AMD64_ATT) ? "jne " : "JNE "; break; } - fprintf(f, "\t%-3s %sstate_%u\n", jmp_op, label_dot, dest_state); + fprintf(f, "\t%-3s %sl%u\n", jmp_op, label_dot, op->u.br.dest_arg->index); } break; @@ -355,95 +372,72 @@ print_asm_amd64(FILE *f, const char *prefix, break; } - if (ferror(f)) { - return -1; - } - return 0; } static int -print_vmasm_encoding(FILE *f, const struct fsm *fsm, enum asm_dialect dialect) +print_vmasm_encoding(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + const struct ret_list *retlist, + struct dfavm_op_ir *ops, + enum asm_dialect dialect) { - struct ir *ir; const char *prefix; - int r; - - static const struct dfavm_assembler_ir zero; - struct dfavm_assembler_ir a; - - static const struct fsm_vm_compile_opts vm_opts = { FSM_VM_COMPILE_DEFAULT_FLAGS, FSM_VM_COMPILE_VM_V1, NULL }; assert(f != NULL); - assert(fsm != NULL); - assert(fsm->opt != NULL); + assert(opt != NULL); + assert(hooks != NULL); + assert(retlist != NULL); if (dialect == AMD64_GO) { - if (fsm->opt->io != FSM_IO_STR && fsm->opt->io != FSM_IO_PAIR) { + if (opt->io != FSM_IO_STR && opt->io != FSM_IO_PAIR) { errno = ENOTSUP; return -1; } } else { - if (fsm->opt->io != FSM_IO_PAIR) { + if (opt->io != FSM_IO_PAIR) { errno = ENOTSUP; return -1; } } - ir = make_ir(fsm); - if (ir == NULL) { - return -1; - } - - assert(f != NULL); - assert(ir != NULL); - - a = zero; - - if (!dfavm_compile_ir(&a, ir, vm_opts)) { - free_ir(fsm, ir); - return -1; - } - - /* henceforth, no function should be passed struct fsm *, only the ir and options */ - - if (fsm->opt->prefix != NULL) { - prefix = fsm->opt->prefix; + if (opt->prefix != NULL) { + prefix = opt->prefix; } else { prefix = "fsm_"; } - r = print_asm_amd64(f, prefix, ir, fsm->opt, &a, dialect); - - dfavm_opasm_finalize_op(&a); - free_ir(fsm, ir); - - return r; -} - -int -fsm_print_vmasm(FILE *f, const struct fsm *fsm) -{ - static const enum asm_dialect default_dialect = AMD64_NASM; - - return print_vmasm_encoding(f, fsm, default_dialect); + return print_asm_amd64(f, opt, hooks, ops, prefix, dialect); } int -fsm_print_vmasm_amd64_att(FILE *f, const struct fsm *fsm) +fsm_print_amd64_att(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + const struct ret_list *retlist, + struct dfavm_op_ir *ops) { - return print_vmasm_encoding(f, fsm, AMD64_ATT); + return print_vmasm_encoding(f, opt, hooks, retlist, ops, AMD64_ATT); } int -fsm_print_vmasm_amd64_nasm(FILE *f, const struct fsm *fsm) +fsm_print_amd64_nasm(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + const struct ret_list *retlist, + struct dfavm_op_ir *ops) { - return print_vmasm_encoding(f, fsm, AMD64_NASM); + return print_vmasm_encoding(f, opt, hooks, retlist, ops, AMD64_NASM); } int -fsm_print_vmasm_amd64_go(FILE *f, const struct fsm *fsm) +fsm_print_amd64_go(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + const struct ret_list *retlist, + struct dfavm_op_ir *ops) { - return print_vmasm_encoding(f, fsm, AMD64_GO); + return print_vmasm_encoding(f, opt, hooks, retlist, ops, AMD64_GO); } diff --git a/src/libfsm/print/vmc.c b/src/libfsm/print/vmc.c index b92d68f6d..b427a9cb8 100644 --- a/src/libfsm/print/vmc.c +++ b/src/libfsm/print/vmc.c @@ -23,27 +23,11 @@ #include #include "libfsm/internal.h" +#include "libfsm/print.h" +#include "libfsm/vm/retlist.h" #include "libfsm/vm/vm.h" -#include "ir.h" - -static int -leaf(FILE *f, const struct fsm_end_ids *ids, const void *leaf_opaque) -{ - assert(f != NULL); - assert(leaf_opaque == NULL); - - (void) ids; - (void) leaf_opaque; - - /* XXX: this should be FSM_UNKNOWN or something non-EOF, - * maybe user defined */ - fprintf(f, "return TOK_UNKNOWN;"); - - return 0; -} - static const char * cmp_operator(int cmp) { @@ -62,14 +46,111 @@ cmp_operator(int cmp) } } +// TODO: centralise vmc/c +static int +print_ids(FILE *f, + enum fsm_ambig ambig, const struct fsm_state_metadata *state_metadata) +{ + switch (ambig) { + case AMBIG_NONE: + // TODO: for C99 we'd return bool + fprintf(f, "return 1;"); + break; + + case AMBIG_ERROR: +// TODO: decide if we deal with this ahead of the call to print or not + if (state_metadata->end_id_count > 1) { + errno = EINVAL; + return -1; + } + + /* fallthrough */ + + case AMBIG_EARLIEST: + /* + * The libfsm api guarentees these ids are unique, + * and only appear once each, and are sorted. + */ + fprintf(f, "{\n"); + fprintf(f, "\t\t*id = %u;\n", state_metadata->end_ids[0]); + fprintf(f, "\t\treturn 1;\n"); + fprintf(f, "\t}"); + break; + + case AMBIG_MULTIPLE: + /* + * Here I would like to emit (static unsigned []) { 1, 2, 3 } + * but specifying a storage duration for compound literals + * is a compiler extension. + * So I'm emitting a static const variable declaration instead. + */ + + fprintf(f, "{\n"); + fprintf(f, "\t\tstatic const unsigned a[] = { "); + for (fsm_end_id_t i = 0; i < state_metadata->end_id_count; i++) { + fprintf(f, "%u", state_metadata->end_ids[i]); + if (i + 1 < state_metadata->end_id_count) { + fprintf(f, ", "); + } + } + fprintf(f, " };\n"); + fprintf(f, "\t\t*ids = a;\n"); + fprintf(f, "\t\t*count = %zu;\n", state_metadata->end_id_count); + fprintf(f, "\t\treturn 1;\n"); + fprintf(f, "\t}"); + break; + + default: + assert(!"unreached"); + abort(); + } + + return 0; +} + +static int +default_accept(FILE *f, const struct fsm_options *opt, + const struct fsm_state_metadata *state_metadata, + void *lang_opaque, void *hook_opaque) +{ + assert(f != NULL); + assert(opt != NULL); + assert(lang_opaque == NULL); + + (void) lang_opaque; + (void) hook_opaque; + + if (-1 == print_ids(f, opt->ambig, state_metadata)) { + return -1; + } + + return 0; +} + +static int +default_reject(FILE *f, const struct fsm_options *opt, + void *lang_opaque, void *hook_opaque) +{ + assert(f != NULL); + assert(opt != NULL); + assert(lang_opaque == NULL); + + (void) lang_opaque; + (void) hook_opaque; + + fprintf(f, "return 0;"); + + return 0; +} + static void print_label(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt) { fprintf(f, "l%" PRIu32 ":", op->index); - if (op->ir_state->example != NULL) { + if (op->example != NULL) { fprintf(f, " /* e.g. \""); - escputs(f, opt, c_escputc_str, op->ir_state->example); + escputs(f, opt, c_escputc_str, op->example); fprintf(f, "\" */"); } } @@ -87,23 +168,40 @@ print_cond(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt) } static int -print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, - enum dfavm_op_end end_bits, const struct ir *ir) +print_end(FILE *f, const struct dfavm_op_ir *op, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + enum dfavm_op_end end_bits) { - if (end_bits == VM_END_FAIL) { - fprintf(f, "return -1;"); - return 0; - } + switch (end_bits) { + case VM_END_FAIL: + return print_hook_reject(f, opt, hooks, NULL, default_reject, NULL); + + case VM_END_SUCC:; + struct fsm_state_metadata state_metadata = { + .end_ids = op->ret->ids, + .end_id_count = op->ret->count, + }; + if (-1 == print_hook_accept(f, opt, hooks, + &state_metadata, + default_accept, + NULL)) + { + return -1; + } - if (opt->endleaf != NULL) { - if (-1 == opt->endleaf(f, op->ir_state->end_ids, opt->endleaf_opaque)) { + if (-1 == print_hook_comment(f, opt, hooks, + &state_metadata)) + { return -1; } - } else { - fprintf(f, "return %td;", op->ir_state - ir->states); - } - return 0; + return 0; + + default: + assert(!"unreached"); + abort(); + } } static void @@ -121,7 +219,7 @@ print_fetch(FILE *f, const struct fsm_options *opt) * Per its API, fsm_getc() is expected to return a positive character * value (as if cast via unsigned char), or EOF. Just like fgetc() does. */ - fprintf(f, "if (c = fsm_getc(opaque), c == EOF) "); + fprintf(f, "if (c = fsm_getc(getc_opaque), c == EOF) "); break; case FSM_IO_STR: @@ -152,10 +250,10 @@ walk_sequence(struct dfavm_op_ir *op, /* * Here we're looking for a sequence of: - * + * * FETCH: (or fail) * STOP: c != 'x' (or fail) - * + * * This catches situations like the "abc" and "xyz" in /^abc[01]xyz/, * but not for runs in unanchored regexes. For those, we'd be better * off adding VM instructions for sets of strings, and producing @@ -277,36 +375,23 @@ walk_sequence(struct dfavm_op_ir *op, /* TODO: eventually to be non-static */ static int -fsm_print_cfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, - const char *cp, - int (*leaf)(FILE *, const struct fsm_end_ids *ids, const void *leaf_opaque), - const void *leaf_opaque) +fsm_print_cfrag(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + const struct ret_list *retlist, + struct dfavm_op_ir *ops, + const char *cp) { - static const struct dfavm_assembler_ir zero; - struct dfavm_assembler_ir a; struct dfavm_op_ir *op; - static const struct fsm_vm_compile_opts vm_opts = { FSM_VM_COMPILE_DEFAULT_FLAGS, FSM_VM_COMPILE_VM_V1, NULL }; - assert(f != NULL); - assert(ir != NULL); assert(opt != NULL); + assert(retlist != NULL); assert(cp != NULL); - a = zero; - - /* TODO: we don't currently have .opaque information attached to struct dfavm_op_ir. - * We'll need that in order to be able to use the leaf callback here. */ - (void) leaf; - (void) leaf_opaque; - /* TODO: we'll need to heed cp for e.g. lx's codegen */ (void) cp; - if (!dfavm_compile_ir(&a, ir, vm_opts)) { - return -1; - } - /* * We only output labels for ops which are branched to. This gives * gaps in the sequence for ops which don't need a label. @@ -317,20 +402,20 @@ fsm_print_cfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, l = 0; - for (op = a.linked; op != NULL; op = op->next) { + for (op = ops; op != NULL; op = op->next) { if (op->num_incoming > 0) { op->index = l++; } } } - assert(a.linked != NULL); + assert(ops != NULL); /* * Only declare variables if we're actually going to use them. */ - if (a.linked->cmp == VM_CMP_ALWAYS && a.linked->instr == VM_OP_STOP) { - assert(a.linked->next == NULL); + if (ops->cmp == VM_CMP_ALWAYS && ops->instr == VM_OP_STOP) { + assert(ops->next == NULL); } else { switch (opt->io) { case FSM_IO_GETC: @@ -353,7 +438,7 @@ fsm_print_cfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, } } - for (op = a.linked; op != NULL; op = op->next) { + for (op = ops; op != NULL; op = op->next) { if (op->num_incoming > 0) { fprintf(f, "\n"); print_label(f, op, opt); @@ -365,7 +450,7 @@ fsm_print_cfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, switch (op->instr) { case VM_OP_STOP: print_cond(f, op, opt); - if (-1 == print_end(f, op, opt, op->u.stop.end_bits, ir)) { + if (-1 == print_end(f, op, opt, hooks, op->u.stop.end_bits)) { return -1; } break; @@ -383,7 +468,7 @@ fsm_print_cfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, fprintf(f, "if (e - p < %zu || 0 != memcmp(p, \"", n); escputbuf(f, opt, c_escputc_str, buf, n); fprintf(f, "\", %zu)) ", n); - if (-1 == print_end(f, NULL, opt, end_bits, ir)) { + if (-1 == print_end(f, NULL, opt, hooks, end_bits)) { return -1; } fprintf(f, "\n"); @@ -396,7 +481,7 @@ fsm_print_cfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, fprintf(f, "if (0 != strncmp(p, \""); escputbuf(f, opt, c_escputc_str, buf, n); fprintf(f, "\", %zu)) ", n); - if (-1 == print_end(f, NULL, opt, end_bits, ir)) { + if (-1 == print_end(f, NULL, opt, hooks, end_bits)) { return -1; } fprintf(f, "\n"); @@ -407,7 +492,7 @@ fsm_print_cfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, op = tail; } else { print_fetch(f, opt); - if (-1 == print_end(f, op, opt, op->u.fetch.end_bits, ir)) { + if (-1 == print_end(f, op, opt, hooks, op->u.fetch.end_bits)) { return -1; } if (opt->io == FSM_IO_PAIR) { @@ -440,93 +525,101 @@ fsm_print_cfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, fprintf(f, "\n"); } - dfavm_opasm_finalize_op(&a); - return 0; } -static int -fsm_print_c_complete(FILE *f, const struct ir *ir, const struct fsm_options *opt, const char *prefix) +int +fsm_print_vmc(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + const struct ret_list *retlist, + struct dfavm_op_ir *ops) { + const char *prefix; + /* TODO: currently unused, but must be non-NULL */ const char *cp = ""; assert(f != NULL); - assert(ir != NULL); assert(opt != NULL); - assert(prefix != NULL); + assert(hooks != NULL); + assert(retlist != NULL); + + if (opt->prefix != NULL) { + prefix = opt->prefix; + } else { + prefix = "fsm_"; + } if (opt->fragment) { - if (-1 == fsm_print_cfrag(f, ir, opt, cp, - opt->leaf != NULL ? opt->leaf : leaf, opt->leaf_opaque)) - { + if (-1 == fsm_print_cfrag(f, opt, hooks, retlist, ops, cp)) { return -1; } } else { fprintf(f, "int\n%smain", prefix); + fprintf(f, "("); switch (opt->io) { case FSM_IO_GETC: - fprintf(f, "(int (*fsm_getc)(void *opaque), void *opaque)\n"); - fprintf(f, "{\n"); + fprintf(f, "int (*fsm_getc)(void *opaque), void *getc_opaque"); break; case FSM_IO_STR: - fprintf(f, "(const char *s)\n"); - fprintf(f, "{\n"); + fprintf(f, "const char *s"); break; case FSM_IO_PAIR: - fprintf(f, "(const char *b, const char *e)\n"); - fprintf(f, "{\n"); + fprintf(f, "const char *b, const char *e"); break; - } - if (-1 == fsm_print_cfrag(f, ir, opt, cp, - opt->leaf != NULL ? opt->leaf : leaf, opt->leaf_opaque)) - { - return -1; + default: + assert(!"unreached"); + abort(); } - fprintf(f, "}\n"); - fprintf(f, "\n"); - } - - if (ferror(f)) { - return -1; - } + /* + * unsigned rather than fsm_end_id_t here, so the generated code + * doesn't depend on fsm.h + */ + switch (opt->ambig) { + case AMBIG_NONE: + break; - return 0; -} + case AMBIG_ERROR: + case AMBIG_EARLIEST: + fprintf(f, ",\n"); + fprintf(f, "\tunsigned *id"); + break; -int -fsm_print_vmc(FILE *f, const struct fsm *fsm) -{ - struct ir *ir; - const char *prefix; - int r; + case AMBIG_MULTIPLE: + fprintf(f, ",\n"); + fprintf(f, "\tconst unsigned **ids, size_t *count"); + break; - assert(f != NULL); - assert(fsm != NULL); - assert(fsm->opt != NULL); + default: + assert(!"unreached"); + abort(); + } - ir = make_ir(fsm); - if (ir == NULL) { - return -1; - } + if (hooks->args != NULL) { + fprintf(f, ",\n"); + fprintf(f, "\t"); - /* henceforth, no function should be passed struct fsm *, only the ir and options */ + if (-1 == print_hook_args(f, opt, hooks, NULL, NULL)) { + return -1; + } + } - if (fsm->opt->prefix != NULL) { - prefix = fsm->opt->prefix; - } else { - prefix = "fsm_"; - } + fprintf(f, ")\n"); + fprintf(f, "{\n"); - r = fsm_print_c_complete(f, ir, fsm->opt, prefix); + if (-1 == fsm_print_cfrag(f, opt, hooks, retlist, ops, cp)) { + return -1; + } - free_ir(fsm, ir); + fprintf(f, "}\n"); + fprintf(f, "\n"); + } - return r; + return 0; } - diff --git a/src/libfsm/print/vmdot.c b/src/libfsm/print/vmdot.c index 101934f17..57344cddb 100644 --- a/src/libfsm/print/vmdot.c +++ b/src/libfsm/print/vmdot.c @@ -23,20 +23,11 @@ #include #include "libfsm/internal.h" +#include "libfsm/print.h" +#include "libfsm/vm/retlist.h" #include "libfsm/vm/vm.h" -#include "ir.h" - -static unsigned int -ir_indexof(const struct ir *ir, const struct ir_state *cs) -{ - assert(ir != NULL); - assert(cs != NULL); - - return cs - &ir->states[0]; -} - static const char * cmp_operator(int cmp) { @@ -55,6 +46,51 @@ cmp_operator(int cmp) } } +static int +default_accept(FILE *f, const struct fsm_options *opt, + const struct fsm_state_metadata *state_metadata, + void *lang_opaque, void *hook_opaque) +{ + assert(f != NULL); + assert(opt != NULL); + assert(lang_opaque == NULL); + + (void) lang_opaque; + (void) hook_opaque; + + fprintf(f, "match"); + + if (state_metadata->end_id_count > 0) { + fprintf(f, " / "); + + for (size_t i = 0; i < state_metadata->end_id_count; i++) { + fprintf(f, "#%u", state_metadata->end_ids[i]); + + if (i < state_metadata->end_id_count - 1) { + fprintf(f, " "); + } + } + } + + return 0; +} + +static int +default_reject(FILE *f, const struct fsm_options *opt, + void *lang_opaque, void *hook_opaque) +{ + assert(f != NULL); + assert(opt != NULL); + assert(lang_opaque == NULL); + + (void) lang_opaque; + (void) hook_opaque; + + fprintf(f, "fail"); + + return 0; +} + static int op_can_fallthrough(const struct dfavm_op_ir *op) { @@ -70,7 +106,7 @@ op_can_fallthrough(const struct dfavm_op_ir *op) } static void -print_cond(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt) +print_cond(FILE *f, const struct fsm_options *opt, const struct dfavm_op_ir *op) { if (op->cmp == VM_CMP_ALWAYS) { fprintf(f, "always"); @@ -83,23 +119,36 @@ print_cond(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt) } static int -print_end(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, - enum dfavm_op_end end_bits, const struct ir *ir) +print_end(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + const struct dfavm_op_ir *op, enum dfavm_op_end end_bits) { - if (end_bits == VM_END_FAIL) { - fprintf(f, "fail"); - return 0; - } - - if (opt->endleaf != NULL) { - if (-1 == opt->endleaf(f, op->ir_state->end_ids, opt->endleaf_opaque)) { + switch (end_bits) { + case VM_END_FAIL: + return print_hook_reject(f, opt, hooks, NULL, default_reject, NULL); + + case VM_END_SUCC:; + struct fsm_state_metadata state_metadata = { + .end_ids = op->ret->ids, + .end_id_count = op->ret->count, + }; + if (-1 == print_hook_accept(f, opt, hooks, + &state_metadata, + default_accept, + NULL)) + { return -1; } - } else { - fprintf(f, "ret %td", op->ir_state - ir->states); - } - return 0; + /* no print_hook_comment() for dot output */ + + return 0; + + default: + assert(!"unreached"); + abort(); + } } static void @@ -109,20 +158,25 @@ print_branch(FILE *f, const struct dfavm_op_ir *op) } static int -fsm_print_nodes(FILE *f, const struct ir *ir, const struct fsm_options *opt, - const struct dfavm_assembler_ir *a) +fsm_print_nodes(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + struct dfavm_op_ir *ops) { - const struct dfavm_op_ir *op; - const struct ir_state *ir_state; + struct dfavm_op_ir *op; + const char *example; unsigned block; + assert(f != NULL); + assert(opt != NULL); + block = 0; - ir_state = NULL; + example = NULL; - for (op = a->linked; op != NULL; op = op->next) { - if (op->num_incoming > 0 || op == a->linked) { - if (op != a->linked) { + for (op = ops; op != NULL; op = op->next) { + if (op->num_incoming > 0 || op == ops) { + if (op != ops) { fprintf(f, "\t\t
S%u%s
end id"); + + for (size_t i = 0; i < cs->endids.count; i++) { + fprintf(f, "#%u", cs->endids.ids[i]); + + if (i < (size_t) cs->endids.count - 1) { + fprintf(f, " "); + } + } + + fprintf(f, "
example"); escputs(f, opt, dot_escputc_html, cs->example); fprintf(f, "
"); + + const struct fsm_state_metadata state_metadata = { + .end_ids = cs->endids.ids, + .end_id_count = cs->endids.count, + }; + + if (-1 == print_hook_accept(f, opt, hooks, + &state_metadata, + NULL, + NULL)) + { + return -1; + } + + fprintf(f, "
to"); - print_state(f, cs->u.same.to, ir_indexof(ir, cs)); + print_name(f, cs->u.same.to, ir_indexof(ir, cs)); fprintf(f, "
mode"); - print_state(f, cs->u.dominant.mode, ir_indexof(ir, cs)); + print_name(f, cs->u.dominant.mode, ir_indexof(ir, cs)); fprintf(f, "
mode"); - print_state(f, cs->u.error.mode, ir_indexof(ir, cs)); + print_name(f, cs->u.error.mode, ir_indexof(ir, cs)); fprintf(f, "
\n"); fprintf(f, "\t> ];\n"); } @@ -133,29 +187,28 @@ fsm_print_nodes(FILE *f, const struct ir *ir, const struct fsm_options *opt, fprintf(f, "\t\t\n"); fprintf(f, "\t\t\n"); - if (op == a->linked) { + if (op == ops) { fprintf(f, "\t\t\t\n"); } else { fprintf(f, "\t\t\t\n", block++); } - fprintf(f, "\t\t\t\n"); fprintf(f, "\t\t\t\n"); fprintf(f, "\t\t\t\n"); fprintf(f, "\t\t\n"); } - if (ir_state != op->ir_state || op->num_incoming > 0) { - if (op->ir_state->example != NULL) { + if (example != op->example) { + if (op->example != NULL) { fprintf(f, "\t\t\n"); - fprintf(f, "\t\t\t\n"); fprintf(f, "\t\t\n"); } - ir_state = op->ir_state; + example = op->example; } fprintf(f, "\t\t\n"); @@ -164,21 +217,13 @@ fsm_print_nodes(FILE *f, const struct ir *ir, const struct fsm_options *opt, fprintf(f, "#%" PRIu32, op->index); fprintf(f, "\n"); - fprintf(f, "\t\t\t\n"); - switch (op->instr) { case VM_OP_STOP: fprintf(f, "\t\t\t\n"); fprintf(f, "\t\t\t\n"); @@ -187,7 +232,7 @@ fsm_print_nodes(FILE *f, const struct ir *ir, const struct fsm_options *opt, case VM_OP_FETCH: fprintf(f, "\t\t\t\n"); fprintf(f, "\t\t\t\n"); @@ -195,7 +240,7 @@ fsm_print_nodes(FILE *f, const struct ir *ir, const struct fsm_options *opt, case VM_OP_BRANCH: fprintf(f, "\t\t\t\n"); fprintf(f, "\t\t\t
L-L%uStateCondOp
"); + fprintf(f, "\t\t\t"); fprintf(f, "e.g. \""); - escputs(f, opt, dot_escputc_html, op->ir_state->example); + escputs(f, opt, dot_escputc_html, op->example); fprintf(f, "\""); fprintf(f, "
"); - if (op->ir_state != NULL) { - fprintf(f, op->ir_state->isend ? "(S%u)" : "S%u", ir_indexof(ir, op->ir_state)); - } else { - fprintf(f, "(none)"); - } - fprintf(f, ""); - print_cond(f, op, opt); + print_cond(f, opt, op); fprintf(f, "", op->index); - if (-1 == print_end(f, op, opt, op->u.stop.end_bits, ir)) { + if (-1 == print_end(f, opt, hooks, op, op->u.stop.end_bits)) { return -1; } fprintf(f, "fetch", op->index); - if (-1 == print_end(f, op, opt, op->u.fetch.end_bits, ir)) { + if (-1 == print_end(f, opt, hooks, op, op->u.fetch.end_bits)) { return -1; } fprintf(f, ""); - print_cond(f, op, opt); + print_cond(f, opt, op); fprintf(f, "", op->index); print_branch(f, op); @@ -217,24 +262,29 @@ fsm_print_nodes(FILE *f, const struct ir *ir, const struct fsm_options *opt, } static void -fsm_print_edges(FILE *f, const struct fsm_options *opt, - const struct dfavm_assembler_ir *a) +fsm_print_edges(FILE *f, const struct fsm_options *opt, const struct dfavm_op_ir *ops) { const struct dfavm_op_ir *op; unsigned long block; int can_fallthrough; + assert(f != NULL); + assert(opt != NULL); + (void) opt; can_fallthrough = 1; - for (op = a->linked; op != NULL; op = op->next) { - if (op->num_incoming > 0 || op == a->linked) { - if (op != a->linked && can_fallthrough) { + for (op = ops; op != NULL; op = op->next) { + if (op->num_incoming > 0 || op == ops) { + if (op != ops && can_fallthrough) { fprintf(f, "\t"); - fprintf(f, "S%lu:s -> S%" PRIu32 ":n [ style = bold ]; /* fallthrough */", + fprintf(f, "S%lu:s -> S%" PRIu32 ":n [ style = bold ];", block, op->index); + if (opt->comments) { + fprintf(f, " /* fallthrough */"); + } fprintf(f, "\n"); } @@ -270,11 +320,14 @@ fsm_print_edges(FILE *f, const struct fsm_options *opt, } else { /* relative branch within the same block, entry on the east */ /* XXX: would like to make these edges shorter, but I don't know how */ - fprintf(f, "S%lu:b%" PRIu32 ":e -> S%lu:b%" PRIu32 ":e [ constraint = false ]; /* relative */", + fprintf(f, "S%lu:b%" PRIu32 ":e -> S%lu:b%" PRIu32 ":e [ constraint = false ];", block, op->index, block, op->u.br.dest_arg->index); + if (opt->comments) { + fprintf(f, " /* relative */"); + } } fprintf(f, "\n"); @@ -283,50 +336,44 @@ fsm_print_edges(FILE *f, const struct fsm_options *opt, /* TODO: eventually to be non-static */ static int -fsm_print_vmdotfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt) +fsm_print_vmdotfrag(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + const struct ret_list *retlist, + struct dfavm_op_ir *ops) { - static const struct dfavm_assembler_ir zero; - struct dfavm_assembler_ir a; - - static const struct fsm_vm_compile_opts vm_opts = { FSM_VM_COMPILE_DEFAULT_FLAGS, FSM_VM_COMPILE_VM_V1, NULL }; - assert(f != NULL); - assert(ir != NULL); assert(opt != NULL); + assert(retlist != NULL); - a = zero; - - if (!dfavm_compile_ir(&a, ir, vm_opts)) { - return -1; - } - - if (-1 == fsm_print_nodes(f, ir, opt, &a)) { + if (-1 == fsm_print_nodes(f, opt, hooks, ops)) { return -1; } fprintf(f, "\n"); - fsm_print_edges(f, opt, &a); - - dfavm_opasm_finalize_op(&a); + fsm_print_edges(f, opt, ops); return 0; } - -static int -fsm_print_vmdot_complete(FILE *f, const struct ir *ir, const struct fsm_options *opt) +int +fsm_print_vmdot(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + const struct ret_list *retlist, + struct dfavm_op_ir *ops) { assert(f != NULL); - assert(ir != NULL); assert(opt != NULL); + assert(hooks != NULL); + assert(retlist != NULL); if (opt->fragment) { - if (-1 == fsm_print_vmdotfrag(f, ir, opt)) { + if (-1 == fsm_print_vmdotfrag(f, opt, hooks, retlist, ops)) { return -1; } } else { - fprintf(f, "\n"); - + fprintf(f, "# generated\n"); fprintf(f, "digraph G {\n"); fprintf(f, "\tnode [ shape = plaintext ];\n"); @@ -339,7 +386,7 @@ fsm_print_vmdot_complete(FILE *f, const struct ir *ir, const struct fsm_options fprintf(f, "\tstart [ shape = none, label = \"\" ];\n"); fprintf(f, "\tstart -> S0:i0:w [ style = bold ];\n"); - if (-1 == fsm_print_vmdotfrag(f, ir, opt)) { + if (-1 == fsm_print_vmdotfrag(f, opt, hooks, retlist, ops)) { return -1; } @@ -347,34 +394,6 @@ fsm_print_vmdot_complete(FILE *f, const struct ir *ir, const struct fsm_options fprintf(f, "\n"); } - if (ferror(f)) { - return -1; - } - return 0; } -int -fsm_print_vmdot(FILE *f, const struct fsm *fsm) -{ - struct ir *ir; - int r; - - assert(f != NULL); - assert(fsm != NULL); - assert(fsm->opt != NULL); - - ir = make_ir(fsm); - if (ir == NULL) { - return -1; - } - - /* henceforth, no function should be passed struct fsm *, only the ir and options */ - - r = fsm_print_vmdot_complete(f, ir, fsm->opt); - - free_ir(fsm, ir); - - return r; -} - diff --git a/src/libfsm/print/vmops.c b/src/libfsm/print/vmops.c index 7804d3cd9..74fc6d350 100644 --- a/src/libfsm/print/vmops.c +++ b/src/libfsm/print/vmops.c @@ -23,33 +23,11 @@ #include #include "libfsm/internal.h" +#include "libfsm/print.h" +#include "libfsm/vm/retlist.h" #include "libfsm/vm/vm.h" -#include "ir.h" - -enum vmops_dialect { - VMOPS_C, - VMOPS_H, - VMOPS_MAIN, -}; - -static int -leaf(FILE *f, const struct fsm_end_ids *ids, const void *leaf_opaque) -{ - assert(f != NULL); - assert(leaf_opaque == NULL); - - (void) ids; - (void) leaf_opaque; - - /* XXX: this should be FSM_UNKNOWN or something non-EOF, - * maybe user defined */ - fprintf(f, "return TOK_UNKNOWN;"); - - return 0; -} - static const char * cmp_operator(int cmp) { @@ -67,15 +45,76 @@ cmp_operator(int cmp) } } +static int +default_accept(FILE *f, const struct fsm_options *opt, + const struct fsm_state_metadata *state_metadata, + void *lang_opaque, void *hook_opaque) +{ + const char *prefix; + size_t i; + + assert(f != NULL); + assert(opt != NULL); + assert(lang_opaque != NULL); + + (void) hook_opaque; + + if (opt->prefix != NULL) { + prefix = opt->prefix; + } else { + prefix = "fsm_"; + } + + (void) state_metadata; + + fprintf(f, "%sactionRET, 1", prefix); + if (opt->ambig != AMBIG_NONE) { + i = * (const size_t *) lang_opaque; + + fprintf(f, ", %zu", i); + } + + return 0; +} + +static int +default_reject(FILE *f, const struct fsm_options *opt, + void *lang_opaque, void *hook_opaque) +{ + const char *prefix; + + assert(f != NULL); + assert(opt != NULL); + assert(lang_opaque == NULL); + + (void) lang_opaque; + (void) hook_opaque; + + if (opt->prefix != NULL) { + prefix = opt->prefix; + } else { + prefix = "fsm_"; + } + + fprintf(f, "%sactionRET, 0", prefix); + if (opt->ambig != AMBIG_NONE) { + fprintf(f, ", 0"); + } + + return 0; +} + static int print_label(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt) { - fprintf(f, "\t\t/* l%" PRIu32 " */\n", op->index); + if (opt->comments) { + fprintf(f, "\t\t/* l%" PRIu32 " */\n", op->index); + } - if (op->ir_state->example != NULL) { + if (op->example != NULL) { fprintf(f, "\t\t/* e.g. \""); /* Go's string escape rules are a superset of C's. */ - if (-1 == escputs(f, opt, c_escputc_str, op->ir_state->example)) { + if (-1 == escputs(f, opt, c_escputc_str, op->example)) { return -1; } fprintf(f, "\" */\n"); @@ -87,7 +126,7 @@ print_label(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt static int print_cond(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, const char *prefix) { - fprintf(f, "\t\t{%s%s, ", prefix, cmp_operator(op->cmp)); + fprintf(f, "%s%s, ", prefix, cmp_operator(op->cmp)); if (-1 == c_escputcharlit(f, opt, op->cmp_arg)) { return -1; } @@ -97,23 +136,53 @@ print_cond(FILE *f, const struct dfavm_op_ir *op, const struct fsm_options *opt, } static int -print_end(FILE *f, const struct dfavm_op_ir *op, const char *prefix, - enum dfavm_op_end end_bits, const struct ir *ir) +print_end(FILE *f, const struct dfavm_op_ir *op, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + const struct ret_list *retlist, + enum dfavm_op_end end_bits) { - if (end_bits == VM_END_FAIL) { - fprintf(f, "%sactionRET, -1},\n", prefix); - return 0; - } + size_t i; + + switch (end_bits) { + case VM_END_FAIL: + if (-1 == print_hook_reject(f, opt, hooks, NULL, default_reject, NULL)) { + return -1; + } + break; + + case VM_END_SUCC:; + struct fsm_state_metadata state_metadata = { + .end_ids = op->ret->ids, + .end_id_count = op->ret->count, + }; + assert(op->ret >= retlist->a); + + i = op->ret - retlist->a; + + if (-1 == print_hook_accept(f, opt, hooks, + &state_metadata, + default_accept, &i)) + { + return -1; + } + break; - fprintf(f, "%sactionRET, %td},\n", prefix, op->ir_state - ir->states); + default: + assert(!"unreached"); + abort(); + } return 0; } static int -print_branch(FILE *f, const struct dfavm_op_ir *op, const char *prefix) +print_branch(FILE *f, const struct fsm_options *opt, const struct dfavm_op_ir *op, const char *prefix) { - fprintf(f, "%sactionGOTO, %" PRIu32 "},\n", prefix, op->u.br.dest_arg->index); + fprintf(f, "%sactionGOTO, %" PRIu32, prefix, op->u.br.dest_arg->index); + if (opt->ambig != AMBIG_NONE) { + fprintf(f, ", 0"); + } return 0; } @@ -122,8 +191,9 @@ static int print_fetch(FILE *f, const struct fsm_options *opt, const char *prefix) { - fprintf(f, "\t\t{%sopEOF, 0, ", prefix); + fprintf(f, "%sopEOF, 0, ", prefix); switch (opt->io) { + case FSM_IO_GETC: case FSM_IO_STR: case FSM_IO_PAIR: break; @@ -134,45 +204,81 @@ print_fetch(FILE *f, const struct fsm_options *opt, const char *prefix) return 0; } -/* TODO: eventually to be non-static */ -static int -fsm_print_vmopsfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, const char *prefix, - int (*leaf)(FILE *, const struct fsm_end_ids *ids, const void *leaf_opaque), - const void *leaf_opaque) +static void +print_ret(FILE *f, const unsigned *ids, size_t count) { - static const struct dfavm_assembler_ir zero; - struct dfavm_assembler_ir a; - struct dfavm_op_ir *op; + size_t i; + + fprintf(f, "{ (const unsigned []) { "); + for (i = 0; i < count; i++) { + fprintf(f, "%u", ids[i]); + if (i + 1 < count) { + fprintf(f, ", "); + } + } + fprintf(f, " }, %zu }", count); +} - static const struct fsm_vm_compile_opts vm_opts = { FSM_VM_COMPILE_DEFAULT_FLAGS, FSM_VM_COMPILE_VM_V1, NULL }; +int +fsm_print_vmops_c(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + const struct ret_list *retlist, + struct dfavm_op_ir *ops) +{ + const struct dfavm_op_ir *op; + const char *prefix; assert(f != NULL); - assert(ir != NULL); assert(opt != NULL); + assert(hooks != NULL); - a = zero; + if (opt->fragment) { + errno = ENOTSUP; + return -1; + } - /* TODO: we don't currently have .opaque information attached to struct dfavm_op_ir. - * We'll need that in order to be able to use the leaf callback here. */ - (void) leaf; - (void) leaf_opaque; + if (opt->prefix != NULL) { + prefix = opt->prefix; + } else { + prefix = "fsm_"; + } - if (!dfavm_compile_ir(&a, ir, vm_opts)) { - return -1; + fprintf(f, "#include \n\n"); + fprintf(f, "#ifndef %sLIBFSM_VMOPS_H\n", prefix); + fprintf(f, "#include \"%svmops.h\"\n", prefix); + fprintf(f, "#endif /* %sLIBFSM_VMOPS_H */\n", prefix); + fprintf(f, "\n"); + + if (opt->ambig != AMBIG_NONE) { + fprintf(f, "struct %sret %sRet[] = {\n", prefix, prefix); + for (size_t i = 0; i < retlist->count; i++) { + fprintf(f, "\t"); + print_ret(f, retlist->a[i].ids, retlist->a[i].count); + fprintf(f, ",\n"); + } + fprintf(f, "};\n"); + fprintf(f, "const size_t %sRet_count = sizeof %sRet / sizeof *%sRet;\n", prefix, prefix, prefix); + fprintf(f, "\n"); } - for (op = a.linked; op != NULL; op = op->next) { + fprintf(f, "struct %sop %sOps[] = {\n", prefix, prefix); + + for (op = ops; op != NULL; op = op->next) { if (op->num_incoming > 0) { if (-1 == print_label(f, op, opt)) { return -1; } } + + fprintf(f, "\t{"); + switch (op->instr) { case VM_OP_STOP: if (-1 == print_cond(f, op, opt, prefix)) { return -1; } - if (-1 == print_end(f, op, prefix, op->u.stop.end_bits, ir)) { + if (-1 == print_end(f, op, opt, hooks, retlist, op->u.stop.end_bits)) { return -1; } break; @@ -181,7 +287,7 @@ fsm_print_vmopsfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, if (-1 == print_fetch(f, opt, prefix)) { return -1; } - if (-1 == print_end(f, op, prefix, op->u.fetch.end_bits, ir)) { + if (-1 == print_end(f, op, opt, hooks, retlist, op->u.fetch.end_bits)) { return -1; } break; @@ -190,7 +296,7 @@ fsm_print_vmopsfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, if (-1 == print_cond(f, op, opt, prefix)) { return -1; } - if (-1 == print_branch(f, op, prefix)) { + if (-1 == print_branch(f, opt, op, prefix)) { return -1; } break; @@ -199,229 +305,276 @@ fsm_print_vmopsfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, assert(!"unreached"); break; } + + fprintf(f, "},"); + + if (op->instr == VM_OP_STOP && op->u.stop.end_bits == VM_END_SUCC) { + struct fsm_state_metadata state_metadata = { + .end_ids = op->ret->ids, + .end_id_count = op->ret->count, + }; + if (-1 == print_hook_comment(f, opt, hooks, + &state_metadata)) + { + return -1; + } + } + + fprintf(f, "\n"); } - dfavm_opasm_finalize_op(&a); + fprintf(f, "\t};\n"); return 0; } int -fsm_print_vmops(FILE *f, const struct fsm *fsm, enum vmops_dialect dialect) +fsm_print_vmops_h(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + const struct ret_list *retlist, + struct dfavm_op_ir *ops) { - struct ir *ir; const char *prefix; assert(f != NULL); - assert(fsm != NULL); - assert(fsm->opt != NULL); + assert(opt != NULL); + assert(hooks != NULL); - ir = make_ir(fsm); - if (ir == NULL) { + (void) retlist; + (void) ops; + + if (opt->fragment) { + errno = ENOTSUP; return -1; } - /* henceforth, no function should be passed struct fsm *, only the ir and options */ - - if (fsm->opt->prefix != NULL) { - prefix = fsm->opt->prefix; + if (opt->prefix != NULL) { + prefix = opt->prefix; } else { prefix = "fsm_"; } - if (fsm->opt->fragment) { - if (dialect == VMOPS_C) { - if (-1 == fsm_print_vmopsfrag(f, ir, fsm->opt, prefix, - fsm->opt->leaf != NULL ? fsm->opt->leaf : leaf, fsm->opt->leaf_opaque)) - { - return -1; - } - } + fprintf(f, "#ifndef %sLIBFSM_VMOPS_H\n", prefix); + fprintf(f, "#define %sLIBFSM_VMOPS_H\n", prefix); + fprintf(f, "#include \n\n"); + fprintf(f, "#include \n\n"); + fprintf(f, "enum %svmOp { %sopEOF, %sopLT, %sopLE, %sopEQ, %sopNE, %sopGE, %sopGT, %sopALWAYS};\n", + prefix, prefix, prefix, prefix, prefix, prefix, prefix, prefix, prefix); + fprintf(f, "enum %sactionOp { %sactionRET, %sactionGOTO };\n", prefix, prefix, prefix); + if (opt->ambig != AMBIG_NONE) { + fprintf(f, "struct %sret { const unsigned *ids; size_t count; };\n\n", prefix); + } + fprintf(f, "struct %sop { enum %svmOp op; unsigned char c; enum %sactionOp action; int32_t arg; int32_t ret; };\n\n", + prefix, prefix, prefix); + if (opt->ambig != AMBIG_NONE) { + fprintf(f, "extern struct %sret %sRet[];\n", prefix, prefix); + fprintf(f, "extern const size_t %sRet_count;\n", prefix); + } + fprintf(f, "#endif /* %sLIBFSM_VMOPS_H */\n", prefix); + + return 0; +} + +int +fsm_print_vmops_main(FILE *f, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + const struct ret_list *retlist, + struct dfavm_op_ir *ops) +{ + const char *prefix; + + assert(f != NULL); + assert(opt != NULL); + assert(hooks != NULL); + + (void) retlist; + (void) ops; + + if (opt->fragment) { + errno = ENOTSUP; + return -1; + } + + if (opt->prefix != NULL) { + prefix = opt->prefix; } else { - switch (dialect) { - case VMOPS_C: - fprintf(f, "#include \n\n"); - fprintf(f, "#ifndef %sLIBFSM_VMOPS_H\n", prefix); - fprintf(f, "#include \"%svmops.h\"\n", prefix); - fprintf(f, "#endif /* %sLIBFSM_VMOPS_H */\n", prefix); - fprintf(f, "struct %sop %sOps[] = {\n", prefix, prefix); - if (-1 == fsm_print_vmopsfrag(f, ir, fsm->opt, prefix, - fsm->opt->leaf != NULL ? fsm->opt->leaf : leaf, fsm->opt->leaf_opaque)) - { - return -1; - } - fprintf(f, "\t};\n"); - break; + prefix = "fsm_"; + } - case VMOPS_H: - fprintf(f, "#ifndef %sLIBFSM_VMOPS_H\n", prefix); - fprintf(f, "#define %sLIBFSM_VMOPS_H\n", prefix); - fprintf(f, "#include \n\n"); - fprintf(f, "enum %svmOp { %sopEOF, %sopLT, %sopLE, %sopEQ, %sopNE, %sopGE, %sopGT, %sopALWAYS};\n", - prefix, prefix, prefix, prefix, prefix, prefix, prefix, prefix, prefix); - fprintf(f, "enum %sactionOp { %sactionRET, %sactionGOTO };\n", prefix, prefix, prefix); - fprintf(f, "struct %sop { enum %svmOp op; unsigned char c; enum %sactionOp action; int32_t arg; };\n\n", - prefix, prefix, prefix); - fprintf(f, "#endif /* %sLIBFSM_VMOPS_H */\n", prefix); - break; + fprintf(f, "#include \n"); + fprintf(f, "#include \n"); + fprintf(f, "#include \n\n"); + fprintf(f, "#ifndef %sLIBFSM_VMOPS_H\n", prefix); + fprintf(f, "#include \"%svmops.h\"\n", prefix); + fprintf(f, "#endif /* %sLIBFSM_VMOPS_H */\n", prefix); + fprintf(f, "extern struct %sop %sOps[];\n", prefix, prefix); + if (opt->ambig != AMBIG_NONE) { + fprintf(f, "extern struct %sret %sRet[];\n", prefix, prefix); + fprintf(f, "extern const size_t %sRet_count;\n", prefix); + } + fprintf(f, "\n"); - case VMOPS_MAIN: - fprintf(f, "#include \n"); - fprintf(f, "#include \n"); - fprintf(f, "#include \n\n"); - fprintf(f, "#ifndef %sLIBFSM_VMOPS_H\n", prefix); - fprintf(f, "#include \"%svmops.h\"\n", prefix); - fprintf(f, "#endif /* %sLIBFSM_VMOPS_H */\n", prefix); - fprintf(f, "extern struct %sop %sOps[];\n", prefix, prefix); - fprintf(f, "\n"); - - switch (fsm->opt->io) { - case FSM_IO_PAIR: - fprintf(f, "int %smatch(const char *b, const char *e)\n", prefix); - break; - - case FSM_IO_STR: - fprintf(f, "int %smatch(const char *s)\n", prefix); - break; - - case FSM_IO_GETC: - errno = ENOTSUP; - return -1; - } - fprintf(f, "{\n"); - fprintf(f, "\tunsigned int i = 0;\n"); - fprintf(f, "\t/* The compiler doesn't know the op stream will have fetch before the first comparison. */\n"); - fprintf(f, "\t/* Initialize to zero to prevent maybe-uninitialized warning. */\n"); - fprintf(f, "\tunsigned char c = 0;\n"); - fprintf(f, "\tint ok;\n"); - fprintf(f, "\tstruct %sop *ops = %sOps;\n", prefix, prefix); - - switch (fsm->opt->io) { - case FSM_IO_PAIR: - fprintf(f, "\tconst char *p = b;\n"); - break; - - case FSM_IO_STR: - fprintf(f, "\tconst char *p = s;\n"); - break; - - case FSM_IO_GETC: - errno = ENOTSUP; - return -1; - } + fprintf(f, "int %smatch(", prefix); + switch (opt->io) { + case FSM_IO_PAIR: + fprintf(f, "const char *b, const char *e"); + break; - fprintf(f, "\n"); - fprintf(f, "\tfor (;;) {\n"); - fprintf(f, "\t\tok = 0;\n"); - fprintf(f, "\t\tswitch (ops[i].op) {\n"); - fprintf(f, "\t\tcase %sopEOF:\n", prefix); - - switch (fsm->opt->io) { - case FSM_IO_PAIR: - fprintf(f, "\t\t\tif (p < e) {\n"); - fprintf(f, "\t\t\t\t/* not at EOF */\n"); - fprintf(f, "\t\t\t\tc = *p++;\n"); - fprintf(f, "\t\t\t\ti++;\n"); - fprintf(f, "\t\t\t\tcontinue;\n"); - fprintf(f, "\t\t\t}\n"); - break; - - case FSM_IO_STR: - fprintf(f, "\t\t\tc = *p++;\n"); - fprintf(f, "\t\t\tif (c != '\\0') {\n"); - fprintf(f, "\t\t\t\t/* not at EOF */\n"); - fprintf(f, "\t\t\t\ti++;\n"); - fprintf(f, "\t\t\t\tcontinue;\n"); - fprintf(f, "\t\t\t}\n"); - break; - - case FSM_IO_GETC: - errno = ENOTSUP; - return -1; - } + case FSM_IO_STR: + fprintf(f, "const char *s"); + break; - fprintf(f, "\t\t\tok = 1;\n"); - fprintf(f, "\t\t\tbreak;\n"); - fprintf(f, "\t\tcase %sopLT: ok = c < ops[i].c; break;\n", prefix); - fprintf(f, "\t\tcase %sopLE: ok = c <= ops[i].c; break;\n", prefix); - fprintf(f, "\t\tcase %sopEQ: ok = c == ops[i].c; break;\n", prefix); - fprintf(f, "\t\tcase %sopNE: ok = c != ops[i].c; break;\n", prefix); - fprintf(f, "\t\tcase %sopGE: ok = c >= ops[i].c; break;\n", prefix); - fprintf(f, "\t\tcase %sopGT: ok = c > ops[i].c; break;\n", prefix); - fprintf(f, "\t\tcase %sopALWAYS: ok = 1; break;\n", prefix); - fprintf(f, "\t\t}\n"); - fprintf(f, "\t\tif (ok) {\n"); - fprintf(f, "\t\t\tif (ops[i].action == %sactionRET) {\n", prefix); - fprintf(f, "\t\t\t\treturn (int) (ops[i].arg);\n"); - fprintf(f, "\t\t\t}\n"); - fprintf(f, "\t\t\ti = ops[i].arg;\n"); - fprintf(f, "\t\t\tcontinue;\n"); - fprintf(f, "\t\t}\n"); - fprintf(f, "\t\ti++;\n"); - fprintf(f, "\t}\n"); - fprintf(f, "}\n"); - fprintf(f, "\n"); - fprintf(f, "#define %sBUFFER_SIZE (1024)\n", prefix); - fprintf(f, "\n"); - fprintf(f, "int main(void)\n"); - fprintf(f, "{\n"); - fprintf(f, "\tchar *buf, *p;\n"); - fprintf(f, "\tint r;\n"); - fprintf(f, "\n"); - fprintf(f, "\tbuf = malloc(%sBUFFER_SIZE);\n", prefix); - fprintf(f, "\tif (!buf) {\n"); - fprintf(f, "\t\tperror(\"malloc\");\n"); - fprintf(f, "\t\texit(1);\n"); - fprintf(f, "\t}\n\n"); - fprintf(f, "\tfor (;;) {\n"); - fprintf(f, "\t\tp = fgets(buf, %sBUFFER_SIZE, stdin);\n", prefix); - fprintf(f, "\t\tif (!p) {\n"); - fprintf(f, "\t\t\tbreak;\n"); - fprintf(f, "\t\t}\n"); - - switch (fsm->opt->io) { - case FSM_IO_PAIR: - fprintf(f, "\t\tr = %smatch(p, p + strlen(p));\n", prefix); - break; - case FSM_IO_STR: - fprintf(f, "\t\tr = %smatch(p);\n", prefix); - break; - case FSM_IO_GETC: - errno = ENOTSUP; - return -1; - } - fprintf(f, "\t\tprintf(\"%%smatch\\n\", (r == -1) ? \"no \" : \"\");\n"); - fprintf(f, "\t}\n"); - fprintf(f, "\treturn 0;\n"); - fprintf(f, "}\n"); - break; - } + case FSM_IO_GETC: + errno = ENOTSUP; + return -1; + } + if (opt->ambig != AMBIG_NONE) { + fprintf(f, ",\n"); + fprintf(f, "\tconst unsigned **ids, size_t *count"); } + fprintf(f, ")\n"); - free_ir(fsm, ir); + fprintf(f, "{\n"); + fprintf(f, "\tunsigned int i = 0;\n"); + if (opt->comments) { + fprintf(f, "\t/* The compiler doesn't know the op stream will have fetch before the first comparison. */\n"); + fprintf(f, "\t/* Initialize to zero to prevent maybe-uninitialized warning. */\n"); + } + fprintf(f, "\tunsigned char c = 0;\n"); + fprintf(f, "\tint ok;\n"); + fprintf(f, "\tstruct %sop *ops = %sOps;\n", prefix, prefix); + + switch (opt->io) { + case FSM_IO_PAIR: + fprintf(f, "\tconst char *p = b;\n"); + break; + + case FSM_IO_STR: + fprintf(f, "\tconst char *p = s;\n"); + break; - if (ferror(f)) { + case FSM_IO_GETC: + errno = ENOTSUP; return -1; } - return 0; -} + fprintf(f, "\n"); + fprintf(f, "\tfor (;;) {\n"); + fprintf(f, "\t\tok = 0;\n"); + fprintf(f, "\t\tswitch (ops[i].op) {\n"); + fprintf(f, "\t\tcase %sopEOF:\n", prefix); -int -fsm_print_vmops_c(FILE *f, const struct fsm *fsm) -{ - return fsm_print_vmops(f, fsm, VMOPS_C); -} + switch (opt->io) { + case FSM_IO_PAIR: + fprintf(f, "\t\t\tif (p < e) {\n"); + if (opt->comments) { + fprintf(f, "\t\t\t\t/* not at EOF */\n"); + } + fprintf(f, "\t\t\t\tc = *p++;\n"); + fprintf(f, "\t\t\t\ti++;\n"); + fprintf(f, "\t\t\t\tcontinue;\n"); + fprintf(f, "\t\t\t}\n"); + break; -int -fsm_print_vmops_h(FILE *f, const struct fsm *fsm) -{ - return fsm_print_vmops(f, fsm, VMOPS_H); -} + case FSM_IO_STR: + fprintf(f, "\t\t\tc = *p++;\n"); + fprintf(f, "\t\t\tif (c != '\\0') {\n"); + if (opt->comments) { + fprintf(f, "\t\t\t\t/* not at EOF */\n"); + } + fprintf(f, "\t\t\t\ti++;\n"); + fprintf(f, "\t\t\t\tcontinue;\n"); + fprintf(f, "\t\t\t}\n"); + break; -int -fsm_print_vmops_main(FILE *f, const struct fsm *fsm) -{ - return fsm_print_vmops(f, fsm, VMOPS_MAIN); + case FSM_IO_GETC: + errno = ENOTSUP; + return -1; + } + + fprintf(f, "\t\t\tok = 1;\n"); + fprintf(f, "\t\t\tbreak;\n"); + fprintf(f, "\t\tcase %sopLT: ok = c < ops[i].c; break;\n", prefix); + fprintf(f, "\t\tcase %sopLE: ok = c <= ops[i].c; break;\n", prefix); + fprintf(f, "\t\tcase %sopEQ: ok = c == ops[i].c; break;\n", prefix); + fprintf(f, "\t\tcase %sopNE: ok = c != ops[i].c; break;\n", prefix); + fprintf(f, "\t\tcase %sopGE: ok = c >= ops[i].c; break;\n", prefix); + fprintf(f, "\t\tcase %sopGT: ok = c > ops[i].c; break;\n", prefix); + fprintf(f, "\t\tcase %sopALWAYS: ok = 1; break;\n", prefix); + fprintf(f, "\t\t}\n"); + fprintf(f, "\t\tif (ok) {\n"); + fprintf(f, "\t\t\tif (ops[i].action == %sactionRET) {\n", prefix); + if (opt->ambig != AMBIG_NONE) { + fprintf(f, "\t\t\t\tif (ops[i].arg) {\n"); + fprintf(f, "\t\t\t\t\t*ids = %sRet[ops[i].ret].ids;\n", prefix); + fprintf(f, "\t\t\t\t\t*count = %sRet[ops[i].ret].count;\n", prefix); + fprintf(f, "\t\t\t\t}\n"); + } + fprintf(f, "\t\t\t\treturn (int) (ops[i].arg);\n"); + fprintf(f, "\t\t\t}\n"); + fprintf(f, "\t\t\ti = ops[i].arg;\n"); + fprintf(f, "\t\t\tcontinue;\n"); + fprintf(f, "\t\t}\n"); + fprintf(f, "\t\ti++;\n"); + fprintf(f, "\t}\n"); + fprintf(f, "}\n"); + fprintf(f, "\n"); + fprintf(f, "#define %sBUFFER_SIZE (1024)\n", prefix); + fprintf(f, "\n"); + fprintf(f, "int main(void)\n"); + fprintf(f, "{\n"); + fprintf(f, "\tchar *buf, *p;\n"); + if (opt->ambig != AMBIG_NONE) { + fprintf(f, "\tconst unsigned *ids;\n"); + fprintf(f, "\tsize_t count, i;\n"); + } + fprintf(f, "\tint r;\n"); + fprintf(f, "\n"); + fprintf(f, "\tbuf = malloc(%sBUFFER_SIZE);\n", prefix); + fprintf(f, "\tif (!buf) {\n"); + fprintf(f, "\t\tperror(\"malloc\");\n"); + fprintf(f, "\t\texit(1);\n"); + fprintf(f, "\t}\n\n"); + fprintf(f, "\tfor (;;) {\n"); + fprintf(f, "\t\tp = fgets(buf, %sBUFFER_SIZE, stdin);\n", prefix); + fprintf(f, "\t\tif (!p) {\n"); + fprintf(f, "\t\t\tbreak;\n"); + fprintf(f, "\t\t}\n"); + + fprintf(f, "\t\tr = %smatch(", prefix); + switch (opt->io) { + case FSM_IO_PAIR: + fprintf(f, "p, p + strlen(p)"); + break; + case FSM_IO_STR: + fprintf(f, "p"); + break; + case FSM_IO_GETC: + // TODO: getc from string buffer + errno = ENOTSUP; + return -1; + } + if (opt->ambig != AMBIG_NONE) { + fprintf(f, ", &ids, &count"); + } + fprintf(f, ");\n"); + + fprintf(f, "\t\tprintf(\"%%smatch\\n\", r == 0 ? \"no \" : \"\");\n"); + if (opt->ambig != AMBIG_NONE) { + fprintf(f, "\t\tif (r) {\n"); + fprintf(f, "\t\t\tprintf(\"ids:\");\n"); + fprintf(f, "\t\t\tfor (i = 0; i < count; i++) {\n"); + fprintf(f, "\t\t\t\tprintf(\" %%u\", ids[i]);\n"); + fprintf(f, "\t\t\t}\n"); + fprintf(f, "\t\t\tprintf(\"\\n\");\n"); + fprintf(f, "\t\t}\n"); + } + fprintf(f, "\t}\n"); + fprintf(f, "\treturn 0;\n"); + fprintf(f, "}\n"); + + return 0; } diff --git a/src/libfsm/reverse.c b/src/libfsm/reverse.c index e896a1bff..c7093738b 100644 --- a/src/libfsm/reverse.c +++ b/src/libfsm/reverse.c @@ -10,7 +10,6 @@ #include #include #include -#include #include #include @@ -29,7 +28,6 @@ fsm_reverse(struct fsm *fsm) struct state_set **epsilons; assert(fsm != NULL); - assert(fsm->opt != NULL); /* * Reversing an FSM means to reverse the language the FSM matches. @@ -54,7 +52,7 @@ fsm_reverse(struct fsm *fsm) if (fsm->endcount == 0 || !fsm_getstart(fsm, &prevstart)) { struct fsm *new; - new = fsm_new(fsm->opt); + new = fsm_new_statealloc(fsm->alloc, 1); if (new == NULL) { return 0; } @@ -92,7 +90,7 @@ fsm_reverse(struct fsm *fsm) continue; } - if (!state_set_add(&endset, fsm->opt->alloc, i)) { + if (!state_set_add(&endset, fsm->alloc, i)) { state_set_free(endset); return 0; } @@ -119,16 +117,16 @@ fsm_reverse(struct fsm *fsm) fsm_setstart(fsm, start); } - edges = f_malloc(fsm->opt->alloc, sizeof *edges * fsm->statecount); + edges = f_malloc(fsm->alloc, sizeof *edges * fsm->statecount); if (edges == NULL) { state_set_free(endset); return 0; } - epsilons = f_malloc(fsm->opt->alloc, sizeof *epsilons * fsm->statecount); + epsilons = f_malloc(fsm->alloc, sizeof *epsilons * fsm->statecount); if (edges == NULL) { state_set_free(endset); - f_free(fsm->opt->alloc, edges); + f_free(fsm->alloc, edges); return 0; } @@ -165,7 +163,7 @@ fsm_reverse(struct fsm *fsm) } for (state_set_reset(fsm->states[i].epsilons, &jt); state_set_next(&jt, &se); ) { - if (!state_set_add(&epsilons[se], fsm->opt->alloc, i)) { + if (!state_set_add(&epsilons[se], fsm->alloc, i)) { goto error1; } } @@ -173,7 +171,7 @@ fsm_reverse(struct fsm *fsm) for (edge_set_reset(fsm->states[i].edges, &it); edge_set_next(&it, &e); ) { assert(e.state < fsm->statecount); - if (!edge_set_add(&edges[e.state], fsm->opt->alloc, e.symbol, i)) { + if (!edge_set_add(&edges[e.state], fsm->alloc, e.symbol, i)) { return 0; } } @@ -209,7 +207,7 @@ fsm_reverse(struct fsm *fsm) continue; } - if (!state_set_add(&epsilons[start], fsm->opt->alloc, s)) { + if (!state_set_add(&epsilons[start], fsm->alloc, s)) { goto error1; } } @@ -226,7 +224,7 @@ fsm_reverse(struct fsm *fsm) continue; } - if (!edge_set_copy(&edges[start], fsm->opt->alloc, edges[s])) { + if (!edge_set_copy(&edges[start], fsm->alloc, edges[s])) { goto error; } } @@ -267,7 +265,7 @@ fsm_reverse(struct fsm *fsm) for (i = 0; i < fsm->statecount; i++) { state_set_free(fsm->states[i].epsilons); - edge_set_free(fsm->opt->alloc, fsm->states[i].edges); + edge_set_free(fsm->alloc, fsm->states[i].edges); fsm->states[i].epsilons = epsilons[i]; fsm->states[i].edges = edges[i]; @@ -276,8 +274,8 @@ fsm_reverse(struct fsm *fsm) state_set_free(endset); - f_free(fsm->opt->alloc, epsilons); - f_free(fsm->opt->alloc, edges); + f_free(fsm->alloc, epsilons); + f_free(fsm->alloc, edges); return 1; @@ -288,7 +286,7 @@ fsm_reverse(struct fsm *fsm) for (i = 0; i < fsm->statecount; i++) { state_set_free(fsm->states[i].epsilons); - edge_set_free(fsm->opt->alloc, fsm->states[i].edges); + edge_set_free(fsm->alloc, fsm->states[i].edges); fsm->states[i].epsilons = epsilons[i]; fsm->states[i].edges = edges[i]; @@ -299,8 +297,8 @@ fsm_reverse(struct fsm *fsm) state_set_free(endset); - f_free(fsm->opt->alloc, epsilons); - f_free(fsm->opt->alloc, edges); + f_free(fsm->alloc, epsilons); + f_free(fsm->alloc, edges); return 0; } diff --git a/src/libfsm/shortest.c b/src/libfsm/shortest.c index e484e9667..45a441a08 100644 --- a/src/libfsm/shortest.c +++ b/src/libfsm/shortest.c @@ -72,7 +72,7 @@ fsm_shortest(const struct fsm *fsm, } for (i = 0; i < fsm->statecount; i++) { - u = priq_push(fsm->opt->alloc, &todo, i, + u = priq_push(fsm->alloc, &todo, i, i == qs ? 0 : FSM_COST_INFINITY); if (u == NULL) { goto error; @@ -149,13 +149,13 @@ fsm_shortest(const struct fsm *fsm, done: for (u = priq_find(done, goal); u != NULL; u = u->prev) { - if (!path_push(fsm->opt->alloc, &path, u->state, u->c)) { + if (!path_push(fsm->alloc, &path, u->state, u->c)) { goto error; } } - priq_free(fsm->opt->alloc, todo); - priq_free(fsm->opt->alloc, done); + priq_free(fsm->alloc, todo); + priq_free(fsm->alloc, done); return path; @@ -171,21 +171,21 @@ fsm_shortest(const struct fsm *fsm, * (otherwise we would have been able to reach it). */ - if (!path_push(fsm->opt->alloc, &path, u->state, u->c)) { + if (!path_push(fsm->alloc, &path, u->state, u->c)) { goto error; } - priq_free(fsm->opt->alloc, todo); - priq_free(fsm->opt->alloc, done); + priq_free(fsm->alloc, todo); + priq_free(fsm->alloc, done); return path; error: - priq_free(fsm->opt->alloc, todo); - priq_free(fsm->opt->alloc, done); + priq_free(fsm->alloc, todo); + priq_free(fsm->alloc, done); - path_free(fsm->opt->alloc, path); + path_free(fsm->alloc, path); return NULL; } diff --git a/src/libfsm/shuffle.c b/src/libfsm/shuffle.c index da8e35bfd..0a7269dab 100644 --- a/src/libfsm/shuffle.c +++ b/src/libfsm/shuffle.c @@ -68,7 +68,7 @@ fsm_shuffle(struct fsm *fsm, unsigned seed) return 1; } - mapping = f_malloc(fsm->opt->alloc, state_count * sizeof(mapping[0])); + mapping = f_malloc(fsm->alloc, state_count * sizeof(mapping[0])); if (mapping == NULL) { goto cleanup; } populate_mapping(mapping, state_count, (uint32_t)seed); @@ -80,6 +80,6 @@ fsm_shuffle(struct fsm *fsm, unsigned seed) res = 1; cleanup: - f_free(fsm->opt->alloc, mapping); + f_free(fsm->alloc, mapping); return res; } diff --git a/src/libfsm/state.c b/src/libfsm/state.c index acf2bff25..1ed9c70f7 100644 --- a/src/libfsm/state.c +++ b/src/libfsm/state.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -17,6 +18,9 @@ #include #include "internal.h" +#include "capture.h" +#include "endids.h" +#include "eager_output.h" int fsm_addstate(struct fsm *fsm, fsm_state_t *state) @@ -33,15 +37,14 @@ fsm_addstate(struct fsm *fsm, fsm_state_t *state) const size_t factor = 2; /* a guess */ const size_t n = fsm->statealloc * factor; struct fsm_state *tmp; - size_t i; - tmp = f_realloc(fsm->opt->alloc, fsm->states, n * sizeof *fsm->states); + tmp = f_realloc(fsm->alloc, fsm->states, n * sizeof *fsm->states); if (tmp == NULL) { return 0; } - for (i = fsm->statealloc; i < n; i++) { - tmp[i].has_capture_actions = 0; + for (size_t i = fsm->statealloc; i < n; i++) { + tmp[i].has_eager_outputs = 0; } fsm->statealloc = n; @@ -61,6 +64,7 @@ fsm_addstate(struct fsm *fsm, fsm_state_t *state) new->visited = 0; new->epsilons = NULL; new->edges = NULL; + new->has_eager_outputs = 0; } fsm->statecount++; @@ -85,6 +89,7 @@ fsm_addstate_bulk(struct fsm *fsm, size_t n) new->visited = 0; new->epsilons = NULL; new->edges = NULL; + new->has_eager_outputs = 0; } fsm->statecount += n; @@ -128,7 +133,7 @@ fsm_removestate(struct fsm *fsm, fsm_state_t state) } state_set_free(fsm->states[state].epsilons); - edge_set_free(fsm->opt->alloc, fsm->states[state].edges); + edge_set_free(fsm->alloc, fsm->states[state].edges); if (fsm_getstart(fsm, &start) && start == state) { fsm_clearstart(fsm); @@ -148,7 +153,7 @@ fsm_removestate(struct fsm *fsm, fsm_state_t state) for (i = 0; i < fsm->statecount - 1; i++) { state_set_replace(&fsm->states[i].epsilons, fsm->statecount - 1, state); - if (!edge_set_replace_state(&fsm->states[i].edges, fsm->opt->alloc, fsm->statecount - 1, state)) { + if (!edge_set_replace_state(&fsm->states[i].edges, fsm->alloc, fsm->statecount - 1, state)) { return 0; } } @@ -177,8 +182,8 @@ fsm_compact_states(struct fsm *fsm, size_t kept, removed_count; const fsm_state_t orig_statecount = fsm->statecount; - fsm_state_t *mapping = f_malloc(fsm->opt->alloc, - fsm->statecount * sizeof(mapping[0])); + fsm_state_t *mapping = f_malloc(fsm->alloc, + orig_statecount * sizeof(mapping[0])); if (mapping == NULL) { return 0; } @@ -223,7 +228,7 @@ fsm_compact_states(struct fsm *fsm, #endif state_set_compact(&s->epsilons, mapping_cb, mapping); if (fsm->states[i].edges != NULL) { - edge_set_compact(&s->edges, fsm->opt->alloc, mapping_cb, mapping); + edge_set_compact(&s->edges, fsm->alloc, mapping_cb, mapping); } } @@ -232,7 +237,7 @@ fsm_compact_states(struct fsm *fsm, assert(dst <= i); if (mapping[i] == FSM_STATE_REMAP_NO_STATE) { /* dead */ state_set_free(fsm->states[i].epsilons); - edge_set_free(fsm->opt->alloc, fsm->states[i].edges); + edge_set_free(fsm->alloc, fsm->states[i].edges); fsm->statecount--; removed_count++; @@ -253,6 +258,23 @@ fsm_compact_states(struct fsm *fsm, } } + /* Remap end metadata */ + if (!fsm_endid_compact(fsm, mapping, orig_statecount)) { + goto error; + } + + if (!fsm_capture_id_compact(fsm, mapping, orig_statecount)) { + goto error; + } + + if (!fsm_capture_program_association_compact(fsm, mapping, orig_statecount)) { + goto error; + } + + if (!fsm_eager_output_compact(fsm, mapping, orig_statecount)) { + goto error;; + } + assert(dst == kept); assert(kept == fsm->statecount); @@ -272,10 +294,15 @@ fsm_compact_states(struct fsm *fsm, /* todo: resize backing array, if significantly smaller? */ } - f_free(fsm->opt->alloc, mapping); + f_free(fsm->alloc, mapping); if (removed != NULL) { *removed = removed_count; } return 1; + +error: + f_free(fsm->alloc, mapping); + + return 0; } diff --git a/src/libfsm/trim.c b/src/libfsm/trim.c index 6a9a25f09..0d8762b3e 100644 --- a/src/libfsm/trim.c +++ b/src/libfsm/trim.c @@ -6,17 +6,21 @@ #include #include +#include #include #include #include +#include #include #include +#include #include #include #include "internal.h" +#include "capture.h" #define DEF_EDGES_CEIL 8 #define DEF_ENDS_CEIL 8 @@ -42,17 +46,18 @@ save_edge(const struct fsm_alloc *alloc, size_t *count, size_t *ceil, struct edge **edges, fsm_state_t from, fsm_state_t to); -static int -cmp_edges_by_to(const void *pa, const void *pb) +static fsm_state_t +get_max_to(const struct edge *edges, size_t edge_count) { - const struct edge *a = (const struct edge *)pa; - const struct edge *b = (const struct edge *)pb; - - return a->to < b->to ? -1 - : a->to > b->to ? 1 - : a->from < b->from ? -1 - : a->from > b->from ? 1 - : 0; + size_t i; + fsm_state_t res = edges[0].to; + for (i = 1; i < edge_count; i++) { + const fsm_state_t to = edges[i].to; + if (to > res) { + res = to; + } + } + return res; } static int @@ -61,7 +66,7 @@ mark_states(struct fsm *fsm, enum fsm_trim_mode mode, { /* Use a queue to walk breath-first over all states reachable * from the start state. Note all end states. Collect all the - * edges, then sort them by the note they lead to, to convert it + * edges, then sort them by the node they lead to, to convert it * to a reverse edge index. Then, enqueue all the end states, * and again use the queue to walk the graph breadth-first, but * this time iterating bottom-up from the end states, and mark @@ -81,14 +86,17 @@ mark_states(struct fsm *fsm, enum fsm_trim_mode mode, fsm_state_t max_end; const size_t state_count = fsm->statecount; + fsm_state_t max_to; + unsigned *pv = NULL; size_t *offsets = NULL; + INIT_TIMERS(); - if (!fsm_getstart(fsm, &start)) { + if (!fsm_getstart(fsm, &start) || state_count == 0) { return 1; /* nothing is reachable */ } - q = queue_new(fsm->opt->alloc, state_count); + q = queue_new(fsm->alloc, state_count); if (q == NULL) { goto cleanup; } @@ -98,13 +106,13 @@ mark_states(struct fsm *fsm, enum fsm_trim_mode mode, } if (mode == FSM_TRIM_START_AND_END_REACHABLE) { - edges = f_malloc(fsm->opt->alloc, + edges = f_malloc(fsm->alloc, edge_ceil * sizeof(edges[0])); if (edges == NULL) { goto cleanup; } - ends = f_malloc(fsm->opt->alloc, + ends = f_malloc(fsm->alloc, end_ceil * sizeof(ends[0])); if (ends == NULL) { goto cleanup; @@ -138,7 +146,7 @@ mark_states(struct fsm *fsm, enum fsm_trim_mode mode, if (ends && fsm_isend(fsm, s_id)) { if (end_count == end_ceil) { - if (!grow_ends(fsm->opt->alloc, + if (!grow_ends(fsm->alloc, &end_ceil, &ends)) { goto cleanup; } @@ -172,7 +180,7 @@ mark_states(struct fsm *fsm, enum fsm_trim_mode mode, if (edges == NULL) { continue; } - if (!save_edge(fsm->opt->alloc, + if (!save_edge(fsm->alloc, &edge_count, &edge_ceil, &edges, s_id, next)) { goto cleanup; @@ -201,7 +209,7 @@ mark_states(struct fsm *fsm, enum fsm_trim_mode mode, if (edges == NULL) { continue; } - if (!save_edge(fsm->opt->alloc, + if (!save_edge(fsm->alloc, &edge_count, &edge_ceil, &edges, s_id, next)) { goto cleanup; @@ -225,7 +233,33 @@ mark_states(struct fsm *fsm, enum fsm_trim_mode mode, } /* Sort edges by state they lead to, inverting the index. */ - qsort(edges, edge_count, sizeof(edges[0]), cmp_edges_by_to); + max_to = edge_count == 0 ? 0 : get_max_to(edges, edge_count); +#if LOG_TRIM + fprintf(stderr, " -- edge count %zu, got max_to %u\n", edge_count, max_to); +#endif + TIME(&pre); + pv = permutation_vector_with_size_and_offset(fsm->alloc, + edge_count, max_to, edges, sizeof(edges[0]), offsetof(struct edge, to)); + TIME(&post); + DIFF_MSEC("trim_pv_so", pre, post, NULL); + + if (EXPENSIVE_CHECKS) { + size_t i; + int ok = 1; +#if LOG_TRIM + fprintf(stderr, "\n#i\tedge\tpv\tsorted, max_to %u\n", max_to); +#endif + for (i = 0; i < edge_count; i++) { +#if LOG_TRIM + fprintf(stderr, "%zu\t%u\t%u\t%u\n", + i, edges[i].to, pv[i], edges[pv[i]].to); +#endif + if (i > 0 && edges[pv[i]].to < edges[pv[i - 1]].to) { + ok = 0; + } + } + assert(ok); + } max_end = 0; @@ -262,7 +296,7 @@ mark_states(struct fsm *fsm, enum fsm_trim_mode mode, } /* The ends are no longer needed. */ - f_free(fsm->opt->alloc, ends); + f_free(fsm->alloc, ends); ends = NULL; } @@ -282,17 +316,16 @@ mark_states(struct fsm *fsm, enum fsm_trim_mode mode, * offsets[i - 1], to represent zero entries. */ { size_t i; - const fsm_state_t max_to = edges[edge_count - 1].to; const size_t offset_count = fsm_countstates(fsm); - offsets = f_calloc(fsm->opt->alloc, + offsets = f_calloc(fsm->alloc, offset_count, sizeof(offsets[0])); if (offsets == NULL) { goto cleanup; } for (i = 0; i < edge_count; i++) { - const fsm_state_t to = edges[i].to; + const fsm_state_t to = edges[pv[i]].to; offsets[to] = i + 1; } @@ -312,8 +345,8 @@ mark_states(struct fsm *fsm, enum fsm_trim_mode mode, if (LOG_TRIM > 1) { size_t i; for (i = 0; i < edge_count; i++) { - fprintf(stderr, "mark_states: edges[%zu]: %d -> %d\n", - i, edges[i].from, edges[i].to); + fprintf(stderr, "mark_states: edges[pv[%zu]]: %d -> %d\n", + i, edges[pv[i]].from, edges[pv[i]].to); } } @@ -331,13 +364,13 @@ mark_states(struct fsm *fsm, enum fsm_trim_mode mode, } for (e_i = base; e_i < limit; e_i++) { - const fsm_state_t from = edges[e_i].from; + const fsm_state_t from = edges[pv[e_i]].from; const unsigned end_distance = (sed == NULL ? 0 : sed[s_id]); assert(from < state_count); if (LOG_TRIM > 0) { - fprintf(stderr, "mark_states: edges[%zu]: from: %d, visited? %d\n", + fprintf(stderr, "mark_states: edges[pv[%zu]]: from: %d, visited? %d\n", e_i, from, fsm->states[from].visited); } @@ -366,10 +399,11 @@ mark_states(struct fsm *fsm, enum fsm_trim_mode mode, res = 1; cleanup: - if (edges != NULL) { f_free(fsm->opt->alloc, edges); } - if (ends != NULL) { f_free(fsm->opt->alloc, ends); } - if (offsets != NULL) { f_free(fsm->opt->alloc, offsets); } + if (edges != NULL) { f_free(fsm->alloc, edges); } + if (ends != NULL) { f_free(fsm->alloc, ends); } + if (offsets != NULL) { f_free(fsm->alloc, offsets); } if (q != NULL) { queue_free(q); } + if (pv != NULL) { f_free(fsm->alloc, pv); } return res; } @@ -457,7 +491,11 @@ integrity_check(const char *descr, const struct fsm *fsm) struct edge_iter edge_iter; struct fsm_edge e; -#ifdef NDEBUG +#if defined(NDEBUG) || !EXPENSIVE_CHECKS + return; +#endif + +#if !EXPENSIVE_CHECKS return; #endif @@ -484,6 +522,8 @@ integrity_check(const char *descr, const struct fsm *fsm) } } + fsm_capture_integrity_check(fsm); + if (LOG_TRIM > 1) { fprintf(stderr, "integrity check: %s...PASS\n", descr); } @@ -504,10 +544,12 @@ fsm_trim(struct fsm *fsm, enum fsm_trim_mode mode, return 1; } + integrity_check("pre", fsm); + if (shortest_end_distance != NULL && mode == FSM_TRIM_START_AND_END_REACHABLE) { size_t s_i; - sed = f_malloc(fsm->opt->alloc, + sed = f_malloc(fsm->alloc, fsm->statecount * sizeof(sed[0])); if (sed == NULL) { goto cleanup; @@ -546,7 +588,7 @@ fsm_trim(struct fsm *fsm, enum fsm_trim_mode mode, if (ret < 0) { if (sed != NULL) { - f_free(fsm->opt->alloc, sed); + f_free(fsm->alloc, sed); } return ret; } @@ -562,10 +604,10 @@ fsm_trim(struct fsm *fsm, enum fsm_trim_mode mode, cleanup: if (marks != NULL) { - f_free(fsm->opt->alloc, marks); + f_free(fsm->alloc, marks); } if (sed != NULL) { - f_free(fsm->opt->alloc, sed); + f_free(fsm->alloc, sed); } return -1; } diff --git a/src/libfsm/union.c b/src/libfsm/union.c index 60e1fbaff..01fb1e889 100644 --- a/src/libfsm/union.c +++ b/src/libfsm/union.c @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -15,10 +16,74 @@ #include #include #include +#include #include "internal.h" -#define LOG_UNION_ARRAY 0 +#include +#include + +#include "eager_output.h" +#include "endids.h" + +#define LOG_UNION_ARRAY (0) +#define LOG_ANALYZE_GROUP_NFA (0) +#define LOG_AFTER_MODIFY_GROUP_NFA (0) +#define LOG_ANALYZE_GROUP_NFA_RESULTS (0 || LOG_ANALYZE_GROUP_NFA > 1) +#define LOG_UNION_REPEATED_PATTERN_GROUP (0) +#define LOG_FSM_UNION_REPEATED_PATTERN_GROUP_OUTPUT (0) + +#define NO_STATE (fsm_state_t)-1 + +/* State/edge info gathered about an NFA. Used by fsm_union_repeated_pattern_group. */ +struct analysis_info { + bool nullable; /* Does the NFA match the empty string? */ + fsm_state_t start; /* start state */ + + /* The states with a /./ self edge representing the unanchored + * start and end, or LINKAGE_NO_STATE. There can be at most one + * of each. Copied from linkage_info. */ + fsm_state_t unanchored_start_loop; + fsm_state_t unanchored_end_loop; + + /* The inner end state immediately preceding the unanchored end loop, + * copied from linkage_info. This is used to identify where the input + * FSMs' original unanchored end loops should be disconnected and + * replaced with a shared end loop. */ + fsm_state_t end_any_inner; + + /* States that link to paths only reachable from the beginning of input. + * Copied from linkage_info. */ + struct state_set *anchored_starts; + /* States leading to an anchored end. Copied from linkage_info. */ + struct state_set *anchored_ends; + + /* States with an outgoing labeled edge to the unanchored end loop. Input + * following those edges has matched, but may still consume trailing input. + * These edges correspond to edges leaving capture group 0 in PCRE. */ + struct state_set *eager_matches; + + /* Edges leading to states that can only match at the start of input. */ + struct edge_set *anchored_firsts; + + /* Edges leading to states that can begin an unanchored match, + * potentially after other combined patterns have matched. */ + struct edge_set *repeatable_firsts; + + /* A new state that may be added while replacing the unanchored_end_loop, + * if present. This state exists to set an eager output ID and have an + * epsilon edge to the combined NFA's global_unanchored_end_loop. + * The old unanchored_end_loop will be disconnected. */ + fsm_state_t eager_match_state; + + /* These states need an epsilon edge added to the eager_matched_state. */ + struct state_set *needs_indirect_epsilon_edge_to_eager_match_state; + + /* States which are reachable from any state besides the start + * state. This can be necessary to correctly identify the + * unanchored start loop. */ + struct state_set *reachable_from_nonstart_state; +}; struct fsm * fsm_union(struct fsm *a, struct fsm *b, @@ -32,13 +97,14 @@ fsm_union(struct fsm *a, struct fsm *b, if (combine_info == NULL) { combine_info = &combine_info_internal; } + memset(combine_info, 0x00, sizeof(*combine_info)); memset(combine_info, 0x00, sizeof(*combine_info)); assert(a != NULL); assert(b != NULL); - if (a->opt != b->opt) { + if (a->alloc != b->alloc) { errno = EINVAL; return NULL; } @@ -101,10 +167,13 @@ fsm_union_array(size_t fsm_count, struct fsm *res = fsms[0]; fsms[0] = NULL; - memset(bases, 0x00, fsm_count * sizeof(bases[0])); + if (bases != NULL) { + memset(bases, 0x00, fsm_count * sizeof(bases[0])); + } for (i = 1; i < fsm_count; i++) { struct fsm_combine_info ci; + struct fsm *combined = fsm_union(res, fsms[i], &ci); fsms[i] = NULL; if (combined == NULL) { @@ -117,9 +186,14 @@ fsm_union_array(size_t fsm_count, return NULL; } + res = combined; + + if (bases == NULL) { + continue; + } + bases[i].state = ci.base_b; bases[i].capture = ci.capture_base_b; - res = combined; /* If the first argument didn't get its states put first * in the union, then shift the bases for everything @@ -134,11 +208,782 @@ fsm_union_array(size_t fsm_count, } #if LOG_UNION_ARRAY - for (i = 0; i < fsm_count; i++) { - fprintf(stderr, "union_array: bases %u: %zu, %zu\n", - i, bases[i].state, bases[i].capture); + if (bases != NULL) { + for (i = 0; i < fsm_count; i++) { + fprintf(stderr, "union_array: bases %u: %zu, %zu\n", + i, bases[i].state, bases[i].capture); + } + } +#endif + + return res; +} + +#if LOG_ANALYZE_GROUP_NFA_RESULTS +static void +dump_state_set(FILE *f, const char *name, const struct state_set *set) +{ + struct state_iter si; + fsm_state_t s; + if (state_set_empty(set)) { return; } + + fprintf(f, " - %s:", name); + state_set_reset(set, &si); + while (state_set_next(&si, &s)) { + fprintf(f, " %d", s); + } + fprintf(f, "\n"); +} + +static void +dump_edge_set(FILE *f, const char *name, fsm_state_t from, const struct edge_set *edges) +{ + struct edge_group_iter iter; + struct edge_group_iter_info info; + if (edge_set_empty(edges)) { return; } + + fprintf(f, " - %s:", name); + edge_set_group_iter_reset(edges, EDGE_GROUP_ITER_ALL, &iter); + while (edge_set_group_iter_next(&iter, &info)) { + fprintf(f, " %d->%d", from, info.to); + } + fprintf(f, "\n"); +} +#endif + +static bool +state_has_labeled_edge_to_eclosure_with_unanchored_end_loop(const struct fsm *nfa, + fsm_state_t s_i, struct state_set **eclosures, + fsm_state_t unanchored_end_loop, + fsm_state_t *indirect_dst) +{ + if (unanchored_end_loop == NO_STATE) { return false; } + assert(unanchored_end_loop < nfa->statecount); + + assert(s_i < nfa->statecount); + + /* The unanchored_end_loop doesn't count, here. */ + if (s_i == unanchored_end_loop) { return false; } + + /* Check whether the state has a labeled edge to a state with the + * unanchored_end_loop in its epsilon closure. */ + const struct fsm_state *s = &nfa->states[s_i]; + struct edge_group_iter iter; + struct edge_group_iter_info info; + edge_set_group_iter_reset(s->edges, EDGE_GROUP_ITER_ALL, &iter); + while (edge_set_group_iter_next(&iter, &info)) { + assert(info.to < nfa->statecount); + const struct state_set *to_eclosure = eclosures[info.to]; + + struct state_iter dst_si; + state_set_reset(to_eclosure, &dst_si); + fsm_state_t dst_s_i; + while (state_set_next(&dst_si, &dst_s_i)) { + if (dst_s_i == unanchored_end_loop) { + if (info.to != unanchored_end_loop) { + *indirect_dst = info.to; + } + + return true; + } + } + } + + return false; +} + +static bool +start_state_epsilon_closure_matches_empty_string(const struct fsm *nfa, const struct state_set *eclosure) +{ + struct state_iter si; + state_set_reset(eclosure, &si); + + fsm_state_t s_i; + while (state_set_next(&si, &s_i)) { + if (fsm_isend(nfa, s_i)) { return true; } + } + + return false; +} + +static const struct fsm_options dump_nfa_opt = { + .io = FSM_IO_STR, + .ambig = AMBIG_MULTIPLE, + .case_ranges = 1, + .consolidate_edges = 1, + .group_edges = 1, +}; + +static bool +analyze_group_nfa(const struct fsm *nfa, struct analysis_info *ainfo) +{ + if (LOG_ANALYZE_GROUP_NFA) { + fprintf(stderr, "==== %s\n", __func__); + if (LOG_ANALYZE_GROUP_NFA > 1) { + fsm_print(stderr, nfa, &dump_nfa_opt, NULL, FSM_PRINT_DOT); + } + } + + ainfo->start = NO_STATE; + ainfo->eager_match_state = NO_STATE; + + if (!fsm_getstart(nfa, &ainfo->start)) { + return false; + } + + const size_t state_count = fsm_countstates(nfa); + assert(ainfo->start < state_count); + + struct state_set **eclosures = fsm_epsilon_closure(nfa); + if (eclosures == NULL) { + return false; + } + + /* Mark any states that are reachable from any state besides the + * start state -- this means they cannot be the unanchored start + * loop, in cases where the pass below would otherwise detect + * more than one. */ + for (fsm_state_t s_i = 0; s_i < state_count; s_i++) { + if (s_i == ainfo->start) { continue; } + + struct state_iter si; + state_set_reset(nfa->states[s_i].epsilons, &si); + fsm_state_t eps_i; + while (state_set_next(&si, &eps_i)) { + /* Ignore self edges */ + if (eps_i == s_i) { continue; } + + if (!state_set_add(&ainfo->reachable_from_nonstart_state, nfa->alloc, eps_i)) { + return false; + } + } + + struct edge_group_iter egi; + struct edge_group_iter_info info; + edge_set_group_iter_reset(nfa->states[s_i].edges, EDGE_GROUP_ITER_ALL, &egi); + while (edge_set_group_iter_next(&egi, &info)) { + /* Ignore self edges */ + if (info.to == s_i) { continue; } + + if (!state_set_add(&ainfo->reachable_from_nonstart_state, nfa->alloc, info.to)) { + return false; + } + } + } + + /* Copy labeled edges from the unanchored start loop and + * its epsilon closure to ainfo->repeatable_firsts, except + * for edges leading back to the unanchored start loop. */ + if (ainfo->unanchored_start_loop != NO_STATE) { + struct state_iter si; + state_set_reset(eclosures[ainfo->unanchored_start_loop], &si); + fsm_state_t cs_i; + while (state_set_next(&si, &cs_i)) { + assert(cs_i < nfa->statecount); + const struct fsm_state *cs = &nfa->states[cs_i]; /* closure state */ + + struct edge_group_iter egi; + struct edge_group_iter_info info; + + edge_set_group_iter_reset(cs->edges, EDGE_GROUP_ITER_ALL, &egi); + while (edge_set_group_iter_next(&egi, &info)) { + if (info.to != ainfo->unanchored_start_loop) { + if (!edge_set_add_bulk(&ainfo->repeatable_firsts, + nfa->alloc, info.symbols, info.to)) { + goto alloc_fail; + } + } + } + } + } + + /* Copy labeled edges from the anchored start and its epsilon + * closure to ainfo->anchored_firsts. */ + struct state_iter si_anchored_start; + state_set_reset(ainfo->anchored_starts, &si_anchored_start); + fsm_state_t anchored_start; + while (state_set_next(&si_anchored_start, &anchored_start)) { + struct state_iter si; + state_set_reset(eclosures[anchored_start], &si); + fsm_state_t cs_i; + while (state_set_next(&si, &cs_i)) { + assert(cs_i < nfa->statecount); + const struct fsm_state *cs = &nfa->states[cs_i]; + + struct edge_group_iter egi; + struct edge_group_iter_info info; + + edge_set_group_iter_reset(cs->edges, EDGE_GROUP_ITER_ALL, &egi); + while (edge_set_group_iter_next(&egi, &info)) { + if (!edge_set_add_bulk(&ainfo->anchored_firsts, + nfa->alloc, info.symbols, info.to)) { + goto alloc_fail; + } + } + } + } + + /* If the start state always matches, set a flag noting that it will need special handling + * later. It's arguably pointless to combine "" with other regexes, because it will always + * trivially match, but otherwise it would never match. */ + ainfo->nullable = start_state_epsilon_closure_matches_empty_string(nfa, eclosures[ainfo->start]); + + /* Collect states that lead to an anchored end or eager match. */ + for (size_t s_i = 0; s_i < state_count; s_i++) { + fsm_state_t indirect_dst = NO_STATE; + if (state_has_labeled_edge_to_eclosure_with_unanchored_end_loop(nfa, s_i, eclosures, ainfo->unanchored_end_loop, &indirect_dst)) { + if (!state_set_add(&ainfo->eager_matches, nfa->alloc, s_i)) { + goto alloc_fail; + } + } + + if (indirect_dst != NO_STATE) { + if (!state_set_add(&ainfo->needs_indirect_epsilon_edge_to_eager_match_state, nfa->alloc, indirect_dst)) { + goto alloc_fail; + } + } + } + +#if LOG_ANALYZE_GROUP_NFA_RESULTS + { + fprintf(stderr, "# analysis_info start %d, usl %d, uel %d, eai %d\n", + ainfo->start, ainfo->unanchored_start_loop, ainfo->unanchored_end_loop, ainfo->end_any_inner); + dump_state_set(stderr, "anchored_ends", ainfo->anchored_ends); + dump_state_set(stderr, "eager_matches", ainfo->eager_matches); + dump_edge_set(stderr, "repeatable_firsts", ainfo->unanchored_start_loop, ainfo->repeatable_firsts); + } +#endif + + fsm_closure_free(nfa, eclosures, state_count); + + /* The unanchored start and end loop cannot be the same state. */ + assert(ainfo->unanchored_start_loop == NO_STATE + || ainfo->unanchored_start_loop != ainfo->unanchored_end_loop); + + return true; + +alloc_fail: + fprintf(stderr, "alloc fail\n"); + if (eclosures != NULL) { + fsm_closure_free(nfa, eclosures, state_count); + } + + return false; +} + +/* Replace any labeled edges on nfa->states[from_state] going to old_to + * with a new edge leading to new_to. There currently isn't a function in + * the libfsm API for this (and it shouldn't be necessary in general), but + * if it gets one later this can be replaced. */ +static bool +replace_labeled_edge(struct fsm *nfa, fsm_state_t from_state, fsm_state_t old_to, fsm_state_t new_to, + bool *found) +{ + if (old_to == NO_STATE) { + /* nothing to do */ + return true; + } + assert(new_to < nfa->statecount); + assert(from_state < nfa->statecount); + + struct fsm_state *from = &nfa->states[from_state]; + struct edge_set *old_edges = from->edges; + struct edge_set *new_edges = edge_set_new(); + + /* copy, replacing edges to old_to */ + struct edge_group_iter iter; + struct edge_group_iter_info info; + edge_set_group_iter_reset(old_edges, EDGE_GROUP_ITER_ALL, &iter); + while (edge_set_group_iter_next(&iter, &info)) { + if (info.to == old_to) { *found = true; } + if (!edge_set_add_bulk(&new_edges, nfa->alloc, info.symbols, + info.to == old_to ? new_to : info.to)) { + return false; + } + } + + edge_set_free(nfa->alloc, old_edges); + from->edges = new_edges; + return true; +} + +/* Make a couple changes to the group NFA so that it can be combined correctly: + * + * - If the group NFA has an unanchored_end_loop, add a new state, + * eager_match_state, which will be a waypoint between edges that previously + * led to the unanchored_end_loop and the global NFA's global_unanchored_end_loop + * (so it can potentially also match other group NFAs with unanchored starts). + * This state will get an eager output ID. + * + * - Set an end ID on every anchored end state, so halting on these counts as a match. */ +static bool +modify_group_nfa(struct fsm *nfa, size_t id, struct analysis_info *ainfo, size_t id_base) +{ + const bool nullable_and_end_any_inner = ainfo->nullable + && ainfo->end_any_inner != NO_STATE; + + const bool log = LOG_ANALYZE_GROUP_NFA > 0; + + /* Add the eager match state if there are eager match states + * or a nullable unanchored end. This will link to the global NFA's + * unanchored_end_loop. */ + if (!state_set_empty(ainfo->eager_matches) || nullable_and_end_any_inner) { + if (!fsm_addstate(nfa, &ainfo->eager_match_state)) { + return false; + } + if (log) { + fprintf(stderr, "%s: added eager_match_state %d\n", __func__, ainfo->eager_match_state); + } + + /* Set eager match ID on new eager_match_state. */ + const fsm_output_id_t oid = (fsm_output_id_t)(id + id_base); + if (!fsm_eager_output_set(nfa, ainfo->eager_match_state, oid)) { + return false; + } + if (log) { + fprintf(stderr, "%s: set eager_output id %d on eager_match_state %d\n", + __func__, oid, ainfo->eager_match_state); + } + + /* For every state in eager_matches, replace every edge leading to + * the unanchored_end_loop with an edge with the same labels to + * eager_match_state. + * + * If the labeled edge does not directly lead to the unanchored_end_loop, + * then add an epsilon edge from wherever it leads to eager_match_state + * instead. */ + struct state_iter si; + state_set_reset(ainfo->eager_matches, &si); + fsm_state_t ems_i; + while (state_set_next(&si, &ems_i)) { + bool found = false; + if (!replace_labeled_edge(nfa, ems_i, + ainfo->end_any_inner, ainfo->eager_match_state, &found)) { + return false; + } + + if (log) { + if (found) { + fprintf(stderr, "%s: replacing labeled edges from eager_match_state %d to unanchored_end_loop %d with edge to new eager_match_state %d\n", + __func__, ems_i, ainfo->unanchored_end_loop, ainfo->eager_match_state); + } else if (!found && ainfo->unanchored_end_loop != NO_STATE) { + fprintf(stderr, "%s: not found: labeled edges from eager_match_state %d to unanchored_end_loop %d\n", + __func__, ems_i, ainfo->unanchored_end_loop); + } + } + + /* The state must not link to the unanchored end loop anymore. + * Doing so will cause a combinatorial explosion that makes + * combining more than ~10 NFAs incredibly expensive. */ + struct edge_group_iter iter; + struct edge_group_iter_info info; + assert(ems_i < nfa->statecount); + const struct fsm_state *ems = &nfa->states[ems_i]; + edge_set_group_iter_reset(ems->edges, EDGE_GROUP_ITER_ALL, &iter); + while (edge_set_group_iter_next(&iter, &info)) { + assert(info.to != ainfo->unanchored_end_loop); + } + } + + state_set_reset(ainfo->needs_indirect_epsilon_edge_to_eager_match_state, &si); + fsm_state_t intermediate_i; + while (state_set_next(&si, &intermediate_i)) { + if (!fsm_addedge_epsilon(nfa, intermediate_i, ainfo->eager_match_state)) { return false; } + if (log) { + fprintf(stderr, "%s: adding epsilon edge from intermediate eager match state %d to new eager_match_state %d\n", + __func__, intermediate_i, ainfo->eager_match_state); + } + } + } + + /* If the group NFA matches the empty string and has an unanchored end, then + * link its unanchored start state to the eager match state. This ensures + * all inputs will match this group NFA when combined. */ + if (nullable_and_end_any_inner) { + assert(ainfo->start != NO_STATE); + assert(ainfo->eager_match_state != NO_STATE); + + if (ainfo->unanchored_start_loop != NO_STATE) { + struct fsm_state *s = &nfa->states[ainfo->unanchored_start_loop]; + if (!state_set_add(&s->epsilons, nfa->alloc, ainfo->eager_match_state)) { + return false; + } + if (log) { + fprintf(stderr, "%s: adding epsilon edge from unanchored_start_loop %d to eager_match_state %d\n", + __func__, ainfo->unanchored_start_loop, ainfo->eager_match_state); + } + } + } + + /* If there are anchored ends, set an endid on them */ + if (!state_set_empty(ainfo->anchored_ends)) { + struct state_iter si; + state_set_reset(ainfo->anchored_ends, &si); + fsm_state_t anchored_end_state; + const fsm_end_id_t end_id = (fsm_end_id_t)(id + id_base); + while (state_set_next(&si, &anchored_end_state)) { + if (!fsm_isend(nfa, anchored_end_state)) { + fprintf(stderr, "%s: anchored_end_state %d: NOT end\n", __func__, anchored_end_state); + assert(fsm_isend(nfa, anchored_end_state)); + } + + if (fsm_endid_set(nfa, anchored_end_state, end_id) < 0) { + return false; + } + if (log) { + fprintf(stderr, "%s: setting endid %d on anchored_end_state %d\n", + __func__, end_id, anchored_end_state); + } + } } + +#if LOG_AFTER_MODIFY_GROUP_NFA + fprintf(stderr, "=== after %s\n", __func__); + fsm_print(stderr, nfa, &dump_nfa_opt, NULL, FSM_PRINT_DOT); + fsm_endid_dump(stderr, nfa); + fsm_eager_output_dump(stderr, nfa); + #endif + return true; +} + +static void +rebase_analysis_info(struct analysis_info *ainfo, fsm_state_t base) +{ + if (base == 0) { return; } + +#define SHIFT(S) if (ainfo-> S != NO_STATE) { ainfo-> S += base; } + SHIFT(start); + SHIFT(unanchored_start_loop); + SHIFT(unanchored_end_loop); + SHIFT(end_any_inner); + SHIFT(eager_match_state); +#undef SHIFT + + state_set_rebase(&ainfo->anchored_ends, base); + state_set_rebase(&ainfo->anchored_starts, base); + state_set_rebase(&ainfo->eager_matches, base); + state_set_rebase(&ainfo->reachable_from_nonstart_state, base); + + edge_set_rebase(&ainfo->anchored_firsts, base); + edge_set_rebase(&ainfo->repeatable_firsts, base); + +} + +static void +free_analysis(const struct fsm_alloc *alloc, struct analysis_info *ainfo) +{ + state_set_free(ainfo->anchored_starts); + state_set_free(ainfo->anchored_ends); + state_set_free(ainfo->eager_matches); + state_set_free(ainfo->needs_indirect_epsilon_edge_to_eager_match_state); + state_set_free(ainfo->reachable_from_nonstart_state); + edge_set_free(alloc, ainfo->anchored_firsts); + edge_set_free(alloc, ainfo->repeatable_firsts); +} + +/* Combine an array of FSMs into a single FSM that attempts to match them + * all in one pass, with an extra loop so that more than one pattern with + * eager outputs can match. */ +struct fsm * +fsm_union_repeated_pattern_group(size_t nfa_count, + struct fsm **nfas, struct fsm_combined_base_pair *bases, size_t id_base) +{ + const struct fsm_alloc *alloc = nfas[0]->alloc; + const bool log = 0 || LOG_UNION_REPEATED_PATTERN_GROUP; + + struct analysis_info *ainfos = f_calloc(alloc, nfa_count, sizeof(ainfos[0])); + if (ainfos == NULL) { goto fail; } + + size_t est_total_states = 0; + for (size_t i = 0; i < nfa_count; i++) { + assert(nfas[i]); + if (nfas[i]->alloc != alloc) { + errno = EINVAL; + return NULL; + } + + /* Any NFAs passed to this function must be built with + * an re_comp flag of RE_SAVE_LINKAGE_INFO, because some + * of the info saved during construction informs + * linking. */ + if (nfas[i]->linkage_info == NULL) { + errno = EINVAL; + return NULL; + } + + const size_t count = fsm_countstates(nfas[i]); + est_total_states += count; + } + + for (size_t i = 0; i < nfa_count; i++) { + struct fsm *fsm = nfas[i]; + + struct analysis_info *ainfo = &ainfos[i]; + + /* Copy these fields over, because fsm->linkage_info will be + * freed during the call to fsm_merge below. */ + { + struct linkage_info *linkage_info = fsm->linkage_info; + + ainfo->unanchored_start_loop = linkage_info->unanchored_start_loop; + ainfo->unanchored_end_loop = linkage_info->unanchored_end_loop; + ainfo->end_any_inner = linkage_info->end_any_inner; + + /* Transfer ownership of these. */ + ainfo->anchored_starts = linkage_info->anchored_starts; + linkage_info->anchored_starts = NULL; + ainfo->anchored_ends = linkage_info->anchored_ends; + linkage_info->anchored_ends = NULL; + } + + /* Identify various states in the NFA that will be relevant to combining. */ + if (!analyze_group_nfa(fsm, ainfo)) { + goto fail; + } + + /* Change the NFA structure so it can better link into the combined FSM, + * and set endids and/or output IDs as appropriate. */ + if (!modify_group_nfa(fsm, i, &ainfos[i], id_base)) { + goto fail; + } + } + + est_total_states += 5; /* new start and end, new unanchored start and end loops */ + + struct fsm *res = fsm_new_statealloc(alloc, est_total_states); + if (res == NULL) { return NULL; } + + /* The new overall start state */ + fsm_state_t global_start; + if (!fsm_addstate(res, &global_start)) { goto fail; } + + /* States linking to the starts of unanchored and anchored + * subgraphs, respectively. Matching group NFAs with unanchored + * ends will loop back to the global_unanchored_start_loop, but + * patterns anchored at the start are only reachable via + * global_anchored_start. */ + fsm_state_t global_unanchored_start_loop, global_anchored_start; + if (!fsm_addstate(res, &global_unanchored_start_loop)) { goto fail; } + if (!fsm_addstate(res, &global_anchored_start)) { goto fail; } + + /* The unanchored end loop state, and an end state with no outgoing edges. */ + fsm_state_t global_unanchored_end_loop, global_end; + if (!fsm_addstate(res, &global_end)) { goto fail; } + if (!fsm_addstate(res, &global_unanchored_end_loop)) { goto fail; } + + if (bases != NULL) { + memset(bases, 0x00, nfa_count * sizeof(bases[0])); + } + + /* For each group NFA, link its unanchored and anchored start states + * and eager_match_state to the global ones. */ + for (size_t nfa_i = 0; nfa_i < nfa_count; nfa_i++) { + struct fsm *fsm = nfas[nfa_i]; + nfas[nfa_i] = NULL; /* transfer ownership */ + + const size_t state_count = fsm_countstates(fsm); + struct analysis_info *ainfo = &ainfos[nfa_i]; + if (ainfo->start == NO_STATE) { + fsm_free(fsm); /* no start, just discard */ + continue; + } + assert(ainfo->start < state_count); + + /* Call fsm_merge; the argument order shouldn't matter. */ + struct fsm_combine_info combine_info; + struct fsm *merged = fsm_merge(res, fsm, &combine_info); + if (merged == NULL) { goto fail; } + + if (log) { + fprintf(stderr, "merged: bases a %d and b %d\n", combine_info.base_a, combine_info.base_b); + } + + /* Update offsets if res had its state IDs shifted forward. */ + global_start += combine_info.base_a; + global_unanchored_start_loop += combine_info.base_a; + global_anchored_start += combine_info.base_a; + global_end += combine_info.base_a; + global_unanchored_end_loop += combine_info.base_a; + + /* Also update offsets for the group FSM's states. */ + rebase_analysis_info(ainfo, combine_info.base_b); + + if (bases != NULL) { + bases[nfa_i].state = combine_info.base_b; + bases[nfa_i].capture = combine_info.capture_base_b; + } + + /* Link the FSM's eager match state back to the global_unanchored_end_loop, so that after + * matching it in an unanchored way it can continue attempting to match other combined + * patterns that aren't anchored at their start. Also link it to the global end, so + * it will be retained during determinisation and minimisation. */ + if (ainfo->eager_match_state != NO_STATE) { + if (!fsm_addedge_epsilon(merged, ainfo->eager_match_state, global_unanchored_end_loop)) { + goto fail; + } + if (!fsm_addedge_epsilon(merged, ainfo->eager_match_state, global_end)) { + goto fail; + } + if (log) { + fprintf(stderr, "eager_match_state: adding epsilon EMS %d -> global_unanchored_end_loop %d and EMS %d -> global_end %d\n", + ainfo->eager_match_state, global_unanchored_end_loop, + ainfo->eager_match_state, global_end); + } + + /* If the NFA matches the empty string and is not anchored at the end, then + * add an epsilon edge from the global start directly to its eager match state. + * This ensures all inputs will match this group NFA when combined. */ + if (ainfo->nullable) { + assert(ainfo->unanchored_end_loop != NO_STATE); + if (!fsm_addedge_epsilon(merged, global_start, ainfo->eager_match_state)) { + goto fail; + } + if (log) { + fprintf(stderr, "nullable: global_start %d -> eager_match_state %d\n", + global_start, ainfo->eager_match_state); + } + } + } else { + /* If the NFA matches an end-anchored empty string, then add an epsilon edge from + * the global start to an anchored end, which has an endid. */ + if (ainfo->nullable && !state_set_empty(ainfo->anchored_ends)) { + struct state_iter si; + state_set_reset(ainfo->anchored_ends, &si); + fsm_state_t anchored_end_state; + + while (state_set_next(&si, &anchored_end_state)) { + if (!fsm_addedge_epsilon(merged, global_start, anchored_end_state)) { + goto fail; + } + if (log) { + fprintf(stderr, "nullable & anchored ends: global_start %d -> anchored_end_state %d\n", + global_start, anchored_end_state); + } + + /* It should only be necessary to link one, since that's enough + * for determinisation to carry the end id back to the start's + * epsilon closure. */ + break; + } + } + } + + struct edge_group_iter iter; + struct edge_group_iter_info info; + + /* Link the global_anchored_start to group FSM paths that are start-anchored + * and can only match at the start of input. */ + edge_set_group_iter_reset(ainfo->anchored_firsts, EDGE_GROUP_ITER_ALL, &iter); + while (edge_set_group_iter_next(&iter, &info)) { + assert(global_anchored_start < merged->statecount); + struct fsm_state *anchored_start = &merged->states[global_anchored_start]; + if (!edge_set_add_bulk(&anchored_start->edges, merged->alloc, + info.symbols, info.to)) { + goto fail; + } + if (log) { + fprintf(stderr, "anchored_firsts: adding global_anchored_start %d -> info.to %d (with same labels)\n", + global_anchored_start, info.to); + } + } + + /* Link the global_unanchored_start_loop to group FSM paths that aren't + * start-anchored. */ + edge_set_group_iter_reset(ainfo->repeatable_firsts, EDGE_GROUP_ITER_ALL, &iter); + while (edge_set_group_iter_next(&iter, &info)) { + struct fsm_state *unanchored_start = &merged->states[global_unanchored_start_loop]; + if (!edge_set_add_bulk(&unanchored_start->edges, merged->alloc, + info.symbols, info.to)) { + goto fail; + } + + if (log) { + fprintf(stderr, "repeatable_firsts: adding global_unanchored_start_loop %d -> info.to %d (same edges)\n", + global_unanchored_start_loop, info.to); + } + } + + + /* Add an epsilon edge from the global unanchored start loop to the NFA's. + * Without this, eager outputs for eager matches in the start state's epsilon + * closure may get lost during determinisation. */ + if (ainfo->unanchored_start_loop != NO_STATE) { + if (!fsm_addedge_epsilon(merged, global_unanchored_start_loop, ainfo->unanchored_start_loop)) { + goto fail; + } + + if (log) { + fprintf(stderr, "repeatable_firsts: adding an epsilon edge from global_unanchored_start_loop %d to NFA unanchored_start_loop %d\n", + global_unanchored_start_loop, ainfo->unanchored_start_loop); + } + } + + res = merged; + } + + /* Link the global start to the global unanchored start loop and anchored start states. */ + if (log) { + fprintf(stderr, "linking: global_start %d -> global_unanchored_start_loop %d and global_anchored_start %d\n", + global_start, global_unanchored_start_loop, global_anchored_start); + } + if (!fsm_addedge_epsilon(res, global_start, global_unanchored_start_loop)) { goto fail; } + if (!fsm_addedge_epsilon(res, global_start, global_anchored_start)) { goto fail; } + + /* Link the global unanchored start loop to itself, so it can + * consume and ignore input preceding each matching group NFA. */ + if (!fsm_addedge_any(res, global_unanchored_start_loop, global_unanchored_start_loop)) { goto fail; } + + /* Link the global unanchored end loop and global end. */ + if (log) { + fprintf(stderr, "linking: global_unanchored_end_loop %d -> global_end %d (and -> self)\n", global_unanchored_end_loop, global_end); + } + if (!fsm_addedge_any(res, global_unanchored_end_loop, global_unanchored_end_loop)) { goto fail; } + if (!fsm_addedge_epsilon(res, global_unanchored_end_loop, global_end)) { goto fail; } + + /* Link from the global_unanchored_end_loop to the global_unanchored_start_loop, + * so patterns with an unanchored start can follow other patterns with an unanchored + * end, possibly with other ignored input between them. */ + if (log || LOG_FSM_UNION_REPEATED_PATTERN_GROUP_OUTPUT) { + fprintf(stderr, "%s: g_start %d, g_start_loop %d, g_start_anchored %d, g_end_loop %d, g_end %d (after all merging)\n", + __func__, global_start, global_unanchored_start_loop, global_anchored_start, global_unanchored_end_loop, global_end); + fprintf(stderr, "%s: linking global_unanchored_end_loop %d to global_unanchored_start_loop %d\n", + __func__, global_unanchored_end_loop, global_unanchored_start_loop); + fprintf(stderr, "%s: setting global_start %d and end %d\n", __func__, global_start, global_end); + } + if (!fsm_addedge_epsilon(res, global_unanchored_end_loop, global_unanchored_start_loop)) { goto fail; } + + /* Link the global unanchored end loop to the global end, so + * reaching the end of input there is considered a match. */ + if (!fsm_addedge_epsilon(res, global_unanchored_end_loop, global_end)) { goto fail; } + + /* These need to be set after merging, because that clears the start state. */ + fsm_setstart(res, global_start); + fsm_setend(res, global_end, 1); + + for (size_t i = 0; i < nfa_count; i++) { + free_analysis(alloc, &ainfos[i]); + } + + if (LOG_UNION_REPEATED_PATTERN_GROUP || LOG_FSM_UNION_REPEATED_PATTERN_GROUP_OUTPUT) { + fprintf(stderr, "==== %s output (combined, pre det+min)\n", __func__); + fsm_print(stderr, res, &dump_nfa_opt, NULL, FSM_PRINT_DOT); + fsm_endid_dump(stderr, res); + fsm_eager_output_dump(stderr, res); + } + + f_free(alloc, ainfos); + return res; + +fail: + if (ainfos != NULL) { + for (size_t i = 0; i < nfa_count; i++) { + free_analysis(alloc, &ainfos[i]); + } + f_free(alloc, ainfos); + } + + return NULL; } diff --git a/src/libfsm/vacuum.c b/src/libfsm/vacuum.c new file mode 100644 index 000000000..cc819a622 --- /dev/null +++ b/src/libfsm/vacuum.c @@ -0,0 +1,40 @@ +/* + * Copyright 2024 Scott Vokes + * + * See LICENCE for the full copyright terms. + */ + +#include +#include +#include + +#include + +#include "internal.h" + +bool +fsm_vacuum(struct fsm *fsm) +{ + assert(fsm != NULL); + assert(fsm->statecount <= fsm->statealloc); + size_t nceil = fsm->statealloc; + + while ((fsm->statecount < nceil/2) && nceil > 1) { + nceil /= 2; + } + + if (nceil == fsm->statealloc) { + return true; + } + + struct fsm_state *nstates = f_realloc(fsm->alloc, + fsm->states, nceil * sizeof(nstates[0])); + if (nstates == NULL) { + return false; + } + + fsm->states = nstates; + fsm->statealloc = nceil; + + return true; +} diff --git a/src/libfsm/vm.c b/src/libfsm/vm.c index ac896d096..0eb4fa16c 100644 --- a/src/libfsm/vm.c +++ b/src/libfsm/vm.c @@ -12,11 +12,13 @@ #include #include +#include #include #include "internal.h" #include "vm/vm.h" +#include "vm/retlist.h" #include "print/ir.h" // VM state: @@ -84,32 +86,45 @@ cmp_name(int cmp) } struct fsm_dfavm * -fsm_vm_compile_with_options(const struct fsm *fsm, struct fsm_vm_compile_opts opts) +fsm_vm_compile_with_options(const struct fsm *fsm, + const struct fsm_options *opt, + struct fsm_vm_compile_opts vmopts) { static const struct dfavm_assembler_ir zero; struct dfavm_assembler_ir a; struct ir *ir; + struct ret_list retlist; struct fsm_dfavm *vm; - ir = make_ir(fsm); + assert(fsm != NULL); + assert(opt != NULL); + + ir = make_ir(fsm, opt); if (ir == NULL) { return NULL; } + if (!build_retlist(&retlist, ir)) { + free_ir(fsm, ir); + return NULL; + } + a = zero; - if (!dfavm_compile_ir(&a, ir, opts)) { + if (!dfavm_compile_ir(&a, ir, &retlist, vmopts)) { + free_retlist(&retlist); free_ir(fsm, ir); return NULL; } free_ir(fsm, ir); - vm = dfavm_compile_vm(&a, opts); + vm = dfavm_compile_vm(&a, vmopts); if (vm == NULL) { return NULL; } + free_retlist(&retlist); dfavm_opasm_finalize_op(&a); return vm; @@ -120,8 +135,19 @@ fsm_vm_compile(const struct fsm *fsm) { // static const struct fsm_vm_compile_opts defaults = { FSM_VM_COMPILE_DEFAULT_FLAGS | FSM_VM_COMPILE_PRINT_IR_PREOPT | FSM_VM_COMPILE_PRINT_IR, NULL }; static const struct fsm_vm_compile_opts defaults = { FSM_VM_COMPILE_DEFAULT_FLAGS, FSM_VM_COMPILE_VM_V1, NULL }; - - return fsm_vm_compile_with_options(fsm, defaults); + static const struct fsm_options opt_default; + + assert(fsm != NULL); + + /* + * The only fsm_option make_ir() uses is .comments, + * for whether to generate examples. + * + * The caller won't use that, because the only user-facing + * vm interfaces are to interpret for execution. So we use + * default options here. + */ + return fsm_vm_compile_with_options(fsm, &opt_default, defaults); } void diff --git a/src/libfsm/vm/Makefile b/src/libfsm/vm/Makefile index fe54811b8..ce32cc585 100644 --- a/src/libfsm/vm/Makefile +++ b/src/libfsm/vm/Makefile @@ -4,6 +4,7 @@ SRC += src/libfsm/vm/ir.c SRC += src/libfsm/vm/vm.c SRC += src/libfsm/vm/v1.c SRC += src/libfsm/vm/v2.c +SRC += src/libfsm/vm/retlist.c .for src in ${SRC:Msrc/libfsm/vm/*.c} CFLAGS.${src} += -I src # XXX: for internal.h diff --git a/src/libfsm/vm/ir.c b/src/libfsm/vm/ir.c index 8c03d94f9..4930f4bc1 100644 --- a/src/libfsm/vm/ir.c +++ b/src/libfsm/vm/ir.c @@ -18,6 +18,7 @@ #include "libfsm/internal.h" +#include "retlist.h" #include "vm.h" #include "print/ir.h" @@ -277,11 +278,11 @@ opasm_free_list(struct dfavm_assembler_ir *a, struct dfavm_op_ir *op) } static struct dfavm_op_ir * -opasm_new(struct dfavm_assembler_ir *a, enum dfavm_op_instr instr, enum dfavm_op_cmp cmp, unsigned char arg, +opasm_new(struct dfavm_assembler_ir *a, const struct ret_list *retlist, + enum dfavm_op_instr instr, enum dfavm_op_cmp cmp, unsigned char arg, const struct ir_state *ir_state) { static const struct dfavm_op_ir zero; - struct dfavm_op_ir *op; if (a->freelist != NULL) { @@ -291,66 +292,82 @@ opasm_new(struct dfavm_assembler_ir *a, enum dfavm_op_instr instr, enum dfavm_op op = pool_newop(&a->pool); } - if (op != NULL) { - *op = zero; + if (op == NULL) { + return NULL; + } + + *op = zero; - op->asm_index = a->count++; - op->index = 0; + op->asm_index = a->count++; + op->index = 0; - op->cmp = cmp; - op->instr = instr; + op->cmp = cmp; + op->instr = instr; - op->cmp_arg = arg; + op->cmp_arg = arg; - op->ir_state = ir_state; + if (ir_state != NULL) { + op->example = ir_state->example; + op->ret = ir_state->isend + ? find_ret(retlist, ir_state->endids.ids, ir_state->endids.count) + : NULL; } return op; } static struct dfavm_op_ir * -opasm_new_fetch(struct dfavm_assembler_ir *a, unsigned state, enum dfavm_op_end end, +opasm_new_fetch(struct dfavm_assembler_ir *a, const struct ret_list *retlist, + unsigned state, enum dfavm_op_end end, const struct ir_state *ir_state) { struct dfavm_op_ir *op; - op = opasm_new(a, VM_OP_FETCH, VM_CMP_ALWAYS, 0, ir_state); - if (op != NULL) { - op->u.fetch.state = state; - op->u.fetch.end_bits = end; + op = opasm_new(a, retlist, VM_OP_FETCH, VM_CMP_ALWAYS, 0, ir_state); + if (op == NULL) { + return NULL; } + op->u.fetch.state = state; + op->u.fetch.end_bits = end; + return op; } static struct dfavm_op_ir * -opasm_new_stop(struct dfavm_assembler_ir *a, enum dfavm_op_cmp cmp, unsigned char arg, enum dfavm_op_end end, +opasm_new_stop(struct dfavm_assembler_ir *a, const struct ret_list *retlist, + enum dfavm_op_cmp cmp, unsigned char arg, enum dfavm_op_end end, const struct ir_state *ir_state) { struct dfavm_op_ir *op; - op = opasm_new(a, VM_OP_STOP, cmp, arg, ir_state); - if (op != NULL) { - op->u.stop.end_bits = end; + op = opasm_new(a, retlist, VM_OP_STOP, cmp, arg, ir_state); + if (op == NULL) { + return NULL; } + op->u.stop.end_bits = end; + return op; } static struct dfavm_op_ir * -opasm_new_branch(struct dfavm_assembler_ir *a, enum dfavm_op_cmp cmp, unsigned char arg, uint32_t dest_state, +opasm_new_branch(struct dfavm_assembler_ir *a, const struct ret_list *retlist, + enum dfavm_op_cmp cmp, unsigned char arg, uint32_t dest_state, const struct ir_state *ir_state) { struct dfavm_op_ir *op; assert(dest_state < a->nstates); - op = opasm_new(a, VM_OP_BRANCH, cmp, arg, ir_state); - if (op != NULL) { - // op->u.br.dest = VM_DEST_FAR; // start with all branches as 'far' - op->u.br.dest_state = dest_state; + op = opasm_new(a, retlist, VM_OP_BRANCH, cmp, arg, ir_state); + if (op == NULL) { + return NULL; } + // op->u.br.dest = VM_DEST_FAR; // start with all branches as 'far' + op->u.br.dest_state = dest_state; + return op; } @@ -372,7 +389,8 @@ dfavm_opasm_finalize_op(struct dfavm_assembler_ir *a) } static int -cmp_mode_dests(const void *a, const void *b) { +cmp_mode_dests(const void *a, const void *b) +{ const long *va = a; const long *vb = b; @@ -476,7 +494,8 @@ analyze_table(struct dfa_table *table) } static int -xlate_table_ranges(struct dfavm_assembler_ir *a, struct dfa_table *table, struct dfavm_op_ir **opp) +xlate_table_ranges(struct dfavm_assembler_ir *a, const struct ret_list *retlist, + struct dfa_table *table, struct dfavm_op_ir **opp) { int i,lo; int count = 0; @@ -496,8 +515,8 @@ xlate_table_ranges(struct dfavm_assembler_ir *a, struct dfa_table *table, struct enum dfavm_op_cmp cmp = (i > lo+1) ? VM_CMP_LE : VM_CMP_EQ; op = (dst < 0) - ? opasm_new_stop(a, cmp, arg, VM_END_FAIL, table->ir_state) - : opasm_new_branch(a, cmp, arg, dst, table->ir_state); + ? opasm_new_stop(a, retlist, cmp, arg, VM_END_FAIL, table->ir_state) + : opasm_new_branch(a, retlist, cmp, arg, dst, table->ir_state); if (op == NULL) { return -1; @@ -514,8 +533,8 @@ xlate_table_ranges(struct dfavm_assembler_ir *a, struct dfa_table *table, struct if (lo < FSM_SIGMA_COUNT) { int64_t dst = table->tbl[lo]; *opp = (dst < 0) - ? opasm_new_stop(a, VM_CMP_ALWAYS, 0, VM_END_FAIL, table->ir_state) - : opasm_new_branch(a, VM_CMP_ALWAYS, 0, dst, table->ir_state); + ? opasm_new_stop(a, retlist, VM_CMP_ALWAYS, 0, VM_END_FAIL, table->ir_state) + : opasm_new_branch(a, retlist, VM_CMP_ALWAYS, 0, dst, table->ir_state); if (*opp == NULL) { return -1; } @@ -527,7 +546,8 @@ xlate_table_ranges(struct dfavm_assembler_ir *a, struct dfa_table *table, struct } static int -xlate_table_cases(struct dfavm_assembler_ir *a, struct dfa_table *table, struct dfavm_op_ir **opp) +xlate_table_cases(struct dfavm_assembler_ir *a, const struct ret_list *retlist, + struct dfa_table *table, struct dfavm_op_ir **opp) { int i, count = 0; int64_t mdst = table->mode.to; @@ -542,8 +562,8 @@ xlate_table_cases(struct dfavm_assembler_ir *a, struct dfa_table *table, struct } *opp = (dst < 0) - ? opasm_new_stop(a, VM_CMP_EQ, i, VM_END_FAIL, table->ir_state) - : opasm_new_branch(a, VM_CMP_EQ, i, dst, table->ir_state); + ? opasm_new_stop(a, retlist, VM_CMP_EQ, i, VM_END_FAIL, table->ir_state) + : opasm_new_branch(a, retlist, VM_CMP_EQ, i, dst, table->ir_state); if (*opp == NULL) { return -1; } @@ -552,8 +572,8 @@ xlate_table_cases(struct dfavm_assembler_ir *a, struct dfa_table *table, struct } *opp = (mdst < 0) - ? opasm_new_stop(a, VM_CMP_ALWAYS, 0, VM_END_FAIL, table->ir_state) - : opasm_new_branch(a, VM_CMP_ALWAYS, 0, mdst, table->ir_state); + ? opasm_new_stop(a, retlist, VM_CMP_ALWAYS, 0, VM_END_FAIL, table->ir_state) + : opasm_new_branch(a, retlist, VM_CMP_ALWAYS, 0, mdst, table->ir_state); if (*opp == NULL) { return -1; } @@ -564,7 +584,8 @@ xlate_table_cases(struct dfavm_assembler_ir *a, struct dfa_table *table, struct } static int -initial_translate_table(struct dfavm_assembler_ir *a, struct dfa_table *table, struct dfavm_op_ir **opp) +initial_translate_table(struct dfavm_assembler_ir *a, const struct ret_list *retlist, + struct dfa_table *table, struct dfavm_op_ir **opp) { int count, best_count; struct dfavm_op_ir *op, *best_op; @@ -592,13 +613,13 @@ initial_translate_table(struct dfavm_assembler_ir *a, struct dfa_table *table, s assert(dst >= 0); assert((size_t)dst < a->nstates); - *opp = opasm_new_stop(a, VM_CMP_NE, sym, VM_END_FAIL, table->ir_state); + *opp = opasm_new_stop(a, retlist, VM_CMP_NE, sym, VM_END_FAIL, table->ir_state); if (*opp == NULL) { return -1; } opp = &(*opp)->next; - *opp = opasm_new_branch(a, VM_CMP_ALWAYS, 0, dst, table->ir_state); + *opp = opasm_new_branch(a, retlist, VM_CMP_ALWAYS, 0, dst, table->ir_state); if (*opp == NULL) { return -1; } @@ -608,10 +629,10 @@ initial_translate_table(struct dfavm_assembler_ir *a, struct dfa_table *table, s } best_op = NULL; - best_count = xlate_table_ranges(a, table, &best_op); + best_count = xlate_table_ranges(a, retlist, table, &best_op); op = NULL; - count = xlate_table_cases(a, table, &op); + count = xlate_table_cases(a, retlist, table, &op); if (count < best_count) { opasm_free_list(a,best_op); @@ -670,7 +691,8 @@ dfa_table_init(struct dfa_table *table, long default_dest, const struct ir_state } static int -initial_translate_partial(struct dfavm_assembler_ir *a, struct ir_state *st, struct dfavm_op_ir **opp) +initial_translate_partial(struct dfavm_assembler_ir *a, const struct ret_list *retlist, + struct ir_state *st, struct dfavm_op_ir **opp) { struct dfa_table table; size_t i, ngrps; @@ -684,11 +706,12 @@ initial_translate_partial(struct dfavm_assembler_ir *a, struct ir_state *st, str group_to_table(&table, &st->u.partial.groups[i]); } - return initial_translate_table(a, &table, opp); + return initial_translate_table(a, retlist, &table, opp); } static int -initial_translate_dominant(struct dfavm_assembler_ir *a, struct ir_state *st, struct dfavm_op_ir **opp) +initial_translate_dominant(struct dfavm_assembler_ir *a, const struct ret_list *retlist, + struct ir_state *st, struct dfavm_op_ir **opp) { struct dfa_table table; size_t i, ngrps; @@ -702,11 +725,12 @@ initial_translate_dominant(struct dfavm_assembler_ir *a, struct ir_state *st, st group_to_table(&table, &st->u.dominant.groups[i]); } - return initial_translate_table(a, &table, opp); + return initial_translate_table(a, retlist, &table, opp); } static int -initial_translate_error(struct dfavm_assembler_ir *a, struct ir_state *st, struct dfavm_op_ir **opp) +initial_translate_error(struct dfavm_assembler_ir *a, const struct ret_list *retlist, + struct ir_state *st, struct dfavm_op_ir **opp) { struct dfa_table table; size_t i, ngrps; @@ -722,11 +746,13 @@ initial_translate_error(struct dfavm_assembler_ir *a, struct ir_state *st, struc group_to_table(&table, &st->u.error.groups[i]); } - return initial_translate_table(a, &table, opp); + return initial_translate_table(a, retlist, &table, opp); } static struct dfavm_op_ir * -initial_translate_state(struct dfavm_assembler_ir *a, const struct ir *ir, size_t ind) +initial_translate_state(struct dfavm_assembler_ir *a, const struct ir *ir, + const struct ret_list *retlist, + size_t ind) { struct ir_state *st; struct dfavm_op_ir **opp; @@ -735,22 +761,22 @@ initial_translate_state(struct dfavm_assembler_ir *a, const struct ir *ir, size_ opp = &a->ops[ind]; if (st->isend && st->strategy == IR_SAME && st->u.same.to == ind) { - *opp = opasm_new_stop(a, VM_CMP_ALWAYS, 0, VM_END_SUCC, st); + *opp = opasm_new_stop(a, retlist, VM_CMP_ALWAYS, 0, VM_END_SUCC, st); return a->ops[ind]; } - *opp = opasm_new_fetch(a, ind, (st->isend) ? VM_END_SUCC : VM_END_FAIL, st); + *opp = opasm_new_fetch(a, retlist, ind, (st->isend) ? VM_END_SUCC : VM_END_FAIL, st); opp = &(*opp)->next; assert(*opp == NULL); switch (st->strategy) { case IR_NONE: - *opp = opasm_new_stop(a, VM_CMP_ALWAYS, 0, VM_END_FAIL, st); + *opp = opasm_new_stop(a, retlist, VM_CMP_ALWAYS, 0, VM_END_FAIL, st); opp = &(*opp)->next; break; case IR_SAME: - *opp = opasm_new_branch(a, VM_CMP_ALWAYS, 0, st->u.same.to, st); + *opp = opasm_new_branch(a, retlist, VM_CMP_ALWAYS, 0, st->u.same.to, st); opp = &(*opp)->next; break; @@ -762,19 +788,19 @@ initial_translate_state(struct dfavm_assembler_ir *a, const struct ir *ir, size_ * intelligently. */ case IR_PARTIAL: - if (initial_translate_partial(a, st, opp) < 0) { + if (initial_translate_partial(a, retlist, st, opp) < 0) { return NULL; } break; case IR_DOMINANT: - if (initial_translate_dominant(a, st, opp) < 0) { + if (initial_translate_dominant(a, retlist, st, opp) < 0) { return NULL; } break; case IR_ERROR: - if (initial_translate_error(a, st, opp) < 0) { + if (initial_translate_error(a, retlist, st, opp) < 0) { return NULL; } break; @@ -792,20 +818,6 @@ initial_translate_state(struct dfavm_assembler_ir *a, const struct ir *ir, size_ return a->ops[ind]; } -static int -initial_translate(const struct ir *ir, struct dfavm_assembler_ir *a) -{ - size_t i,n; - - n = a->nstates; - - for (i=0; i < n; i++) { - a->ops[i] = initial_translate_state(a, ir, i); - } - - return 0; -} - static void fixup_dests(struct dfavm_assembler_ir *a) { @@ -956,7 +968,7 @@ order_basic_blocks(struct dfavm_assembler_ir *a) /* mark all states as !in_trace */ for (i=0; i < n; i++) { - a->ops[i]->in_trace = 0; + a->ops[i]->in_trace = false; } opp = &a->linked; @@ -975,7 +987,7 @@ order_basic_blocks(struct dfavm_assembler_ir *a) assert(opp != NULL); assert(*opp == NULL); - st->in_trace = 1; + st->in_trace = true; /* look for branches to states not in the trace. * Start at the last branch and work toward the first. @@ -1051,8 +1063,15 @@ print_all_states(struct dfavm_assembler_ir *a) } int -dfavm_compile_ir(struct dfavm_assembler_ir *a, const struct ir *ir, struct fsm_vm_compile_opts opts) +dfavm_compile_ir(struct dfavm_assembler_ir *a, const struct ir *ir, const struct ret_list *retlist, + struct fsm_vm_compile_opts opts) { + size_t i; + + assert(a != NULL); + assert(ir != NULL); + assert(retlist != NULL); + a->nstates = ir->n; a->start = ir->start; @@ -1068,8 +1087,8 @@ dfavm_compile_ir(struct dfavm_assembler_ir *a, const struct ir *ir, struct fsm_v return 0; } - if (initial_translate(ir, a) < 0) { - return 0; + for (i=0; i < a->nstates; i++) { + a->ops[i] = initial_translate_state(a, ir, retlist, i); } fixup_dests(a); @@ -1109,7 +1128,7 @@ dfavm_compile_ir(struct dfavm_assembler_ir *a, const struct ir *ir, struct fsm_v return 0; } - a->ops[0] = opasm_new_stop(a, VM_CMP_ALWAYS, 0, VM_END_FAIL, NULL); + a->ops[0] = opasm_new_stop(a, retlist, VM_CMP_ALWAYS, 0, VM_END_FAIL, NULL); if (a->ops[0] == NULL) { return -1; } diff --git a/src/libfsm/vm/retlist.c b/src/libfsm/vm/retlist.c new file mode 100644 index 000000000..97d4eb3f0 --- /dev/null +++ b/src/libfsm/vm/retlist.c @@ -0,0 +1,147 @@ +/* + * Copyright 2008-2024 Katherine Flavel + * + * See LICENCE for the full copyright terms. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "libfsm/internal.h" + +#include "libfsm/print/ir.h" +#include "libfsm/vm/retlist.h" + +static bool +append_ret(struct ret_list *list, + const fsm_end_id_t *ids, size_t count) +{ + const size_t low = 16; /* must be power of 2 */ + const size_t factor = 2; /* must be even */ + + assert(list != NULL); + + // TODO: alloc callbacks + if (list->count == 0) { + list->a = malloc(low * sizeof *list->a); + if (list->a == NULL) { + return false; + } + } else if (list->count >= low && (list->count & (list->count - 1)) == 0) { + void *tmp; + size_t new = list->count * factor; + if (new < list->count) { + errno = E2BIG; + perror("realloc"); + exit(EXIT_FAILURE); + } + + tmp = realloc(list->a, new * sizeof *list->a); + if (tmp == NULL) { + return false; + } + + list->a = tmp; + } + + list->a[list->count].ids = ids; + list->a[list->count].count = count; + + list->count++; + + return true; +} + +static int +cmp_ret(const void *pa, const void *pb) +{ + const struct ret *a = pa; + const struct ret *b = pb; + + if (a->count < b->count) { return -1; } + if (a->count > b->count) { return +1; } + + if (a->count == 0) { + return 0; + } + + assert(a->ids != NULL); + assert(b->ids != NULL); + + return memcmp(a->ids, b->ids, a->count * sizeof *a->ids); +} + +struct ret * +find_ret(const struct ret_list *list, + const fsm_end_id_t *ids, size_t count) +{ + struct ret key; + + key.count = count; + key.ids = ids; + + return bsearch(&key, list->a, list->count, sizeof *list->a, cmp_ret); +} + +bool +build_retlist(struct ret_list *list, const struct ir *ir) +{ + size_t i; + + assert(list != NULL); + assert(ir != NULL); + + list->count = 0; + + for (i = 0; i < ir->n; i++) { + if (!ir->states[i].isend) { + continue; + } + + if (!append_ret(list, ir->states[i].endids.ids, ir->states[i].endids.count)) { + return false; + } + } + + if (list->count > 0) { + size_t j = 0; + + /* sort for both dedup and bsearch */ + qsort(list->a, list->count, sizeof *list->a, cmp_ret); + + /* deduplicate based on endids only. + * j is the start of a run; i increments until we find + * the start of the next run */ + for (size_t i = 1; i < list->count; i++) { + assert(i > j); + if (cmp_ret(&list->a[j], &list->a[i]) == 0) { + continue; + } + + j++; + list->a[j] = list->a[i]; + } + + list->count = j + 1; + + assert(list->count > 0); + } + + return true; +} + +void +free_retlist(struct ret_list *list) +{ + if (list->count > 0) { + free(list->a); + } +} + diff --git a/src/libfsm/vm/retlist.h b/src/libfsm/vm/retlist.h new file mode 100644 index 000000000..5e67218b9 --- /dev/null +++ b/src/libfsm/vm/retlist.h @@ -0,0 +1,32 @@ +/* + * Copyright 2024 Katherine Flavel + * + * See LICENCE for the full copyright terms. + */ + +#ifndef FSM_INTERNAL_RETLIST_H +#define FSM_INTERNAL_RETLIST_H + +struct ir; + +struct ret { + size_t count; + const fsm_end_id_t *ids; +}; + +struct ret_list { + size_t count; + struct ret *a; +}; + +struct ret * +find_ret(const struct ret_list *list, const fsm_end_id_t *ids, size_t count); + +bool +build_retlist(struct ret_list *list, const struct ir *ir); + +void +free_retlist(struct ret_list *list); + +#endif + diff --git a/src/libfsm/vm/v1.c b/src/libfsm/vm/v1.c index a326b88d8..c9af2061f 100644 --- a/src/libfsm/vm/v1.c +++ b/src/libfsm/vm/v1.c @@ -12,6 +12,8 @@ #include #include +#include + #include "vm.h" enum dfavm_io_result @@ -217,7 +219,9 @@ encode_opasm_v1(const struct dfavm_vm_op *instr, size_t ninstr, size_t total_byt return ret; error: - /* XXX - cleanup */ + if (ret != NULL) { + free(ret); + } return NULL; } diff --git a/src/libfsm/vm/v2.c b/src/libfsm/vm/v2.c index c85edff98..e217f6d11 100644 --- a/src/libfsm/vm/v2.c +++ b/src/libfsm/vm/v2.c @@ -11,6 +11,8 @@ #include #include +#include + #include "vm.h" enum dfavm_vm_op_v2 { @@ -155,7 +157,10 @@ encode_opasm_v2(const struct dfavm_vm_op *instr, size_t ninstr) return ret; error: - /* XXX - cleanup */ + if (ret != NULL) { + free(ret); + } + return NULL; } diff --git a/src/libfsm/vm/vm.h b/src/libfsm/vm/vm.h index dfe99dfd7..540a19d2e 100644 --- a/src/libfsm/vm/vm.h +++ b/src/libfsm/vm/vm.h @@ -20,7 +20,8 @@ #define DFAVM_MAGIC "DFAVM$" struct ir; -struct ir_state; +struct ret; +struct ret_list; struct fsm_vm_compile_opts; struct dfavm_op_ir_pool; @@ -75,10 +76,12 @@ struct dfavm_op_ir { */ uint32_t index; - const struct ir_state *ir_state; // for void *opaque + const char *example; + const struct ret *ret; + + uint32_t num_incoming:31; // number of branches to this instruction, :31 for packing + unsigned int in_trace:1; - uint32_t num_incoming; // number of branches to this instruction - int in_trace; int cmp_arg; enum dfavm_op_cmp cmp; @@ -191,7 +194,7 @@ const char * cmp_name(int cmp); int -dfavm_compile_ir(struct dfavm_assembler_ir *a, const struct ir *ir, struct fsm_vm_compile_opts opts); +dfavm_compile_ir(struct dfavm_assembler_ir *a, const struct ir *ir, const struct ret_list *retlist, struct fsm_vm_compile_opts opts); struct fsm_dfavm * dfavm_compile_vm(const struct dfavm_assembler_ir *a, struct fsm_vm_compile_opts opts); diff --git a/src/libfsm/walk/reachable.c b/src/libfsm/walk/reachable.c index f7a961600..ec0f5ee37 100644 --- a/src/libfsm/walk/reachable.c +++ b/src/libfsm/walk/reachable.c @@ -32,7 +32,7 @@ fsm_reachable(const struct fsm *fsm, fsm_state_t state, fsm->states[i].visited = 0; } - q = f_malloc(fsm->opt->alloc, fsm->statecount * sizeof *q); + q = f_malloc(fsm->alloc, fsm->statecount * sizeof *q); if (q == NULL) { return -1; } @@ -96,7 +96,7 @@ fsm_reachable(const struct fsm *fsm, fsm_state_t state, fsm->states[i].visited = 0; } - f_free(fsm->opt->alloc, q); + f_free(fsm->alloc, q); return r; } diff --git a/src/libfsm/walk2.c b/src/libfsm/walk2.c index c321db678..6d6ab022a 100644 --- a/src/libfsm/walk2.c +++ b/src/libfsm/walk2.c @@ -9,10 +9,10 @@ #include #include #include +#include #include #include -#include #include #include #include @@ -131,7 +131,7 @@ fsm_walk2_data_free(const struct fsm *fsm, struct fsm_walk2_data *data) for (p = data->head; p != NULL; p = next) { next = p->next; - f_free(fsm->opt->alloc, p); + f_free(fsm->alloc, p); } if (data->new) { @@ -159,7 +159,7 @@ alloc_walk2_tuple(struct fsm_walk2_data *data) new_pool: - pool = f_malloc(data->new->opt->alloc, sizeof *pool); + pool = f_malloc(data->new->alloc, sizeof *pool); if (pool == NULL) { return NULL; } @@ -263,67 +263,60 @@ fsm_walk2_tuple_new(struct fsm_walk2_data *data, } if (is_end) { - size_t num_a_endids = 0, num_b_endids = 0, total_num_endids; + size_t count_a = 0, count_b = 0, count_total; if (fsm_a != NULL && fsm_isend(fsm_a,a)) { - num_a_endids = fsm_getendidcount(fsm_a, a); + count_a = fsm_endid_count(fsm_a, a); } if (fsm_b != NULL && fsm_isend(fsm_b,b)) { - num_b_endids = fsm_getendidcount(fsm_b, b); + count_b = fsm_endid_count(fsm_b, b); } - total_num_endids = num_a_endids + num_b_endids; + count_total = count_a + count_b; - if (total_num_endids > 0) { - fsm_end_id_t *endids= NULL; - enum fsm_getendids_res ret; + if (count_total > 0) { + const struct fsm_alloc *alloc; + fsm_end_id_t *ids; + int ret; - endids = calloc(total_num_endids, sizeof endids[0]); - if (endids == NULL) { + if (fsm_a != NULL) { + alloc = fsm_a->alloc; + } else if (fsm_b != NULL) { + alloc = fsm_b->alloc; + } else { + assert(!"unreached"); + } + + ids = f_malloc(alloc, count_total * sizeof *ids); + if (ids == NULL) { return NULL; } - if (num_a_endids > 0) { - size_t nwritten = 0; - ret = fsm_getendids(fsm_a, a, num_a_endids, &endids[0], &nwritten); + if (count_a > 0) { + ret = fsm_endid_get(fsm_a, a, count_a, ids); + assert(ret == 1); + } - if (ret != FSM_GETENDIDS_FOUND || nwritten != num_a_endids) { - free(endids); - errno = (ret != FSM_GETENDIDS_FOUND) ? ENOENT : EINVAL; - return NULL; - } + if (count_b > 0) { + ret = fsm_endid_get(fsm_b, b, count_b, ids + count_a); + assert(ret == 1); } - if (num_b_endids > 0) { - size_t nwritten = 0; - ret = fsm_getendids(fsm_b, b, num_b_endids, &endids[num_a_endids], &nwritten); + ret = fsm_endid_set_bulk( + data->new, + p->comb, + count_total, + &ids[0], + FSM_ENDID_BULK_REPLACE); - if (ret != FSM_GETENDIDS_FOUND || nwritten != num_b_endids) { - free(endids); - errno = (ret != FSM_GETENDIDS_FOUND) ? ENOENT : EINVAL; - return NULL; - } - } + f_free(alloc, ids); - { - enum fsm_endid_set_res ret; - - ret = fsm_endid_set_bulk( - data->new, - p->comb, - total_num_endids, - &endids[0], - FSM_ENDID_BULK_REPLACE); - if (ret == FSM_ENDID_SET_ERROR_ALLOC_FAIL) { - int errsv = errno; - free(endids); - errno = errsv; - return NULL; - } + if (!ret) { + int errsv = errno; + errno = errsv; + return NULL; } - - free(endids); } } @@ -569,12 +562,12 @@ fsm_walk2(const struct fsm *a, const struct fsm *b, data.edgemask = edgemask; data.endmask = endmask; - data.new = fsm_new(a->opt); + data.new = fsm_new(a->alloc); if (data.new == NULL) { goto error; } - data.states = tuple_set_create(data.new->opt->alloc, cmp_walk2_tuple); + data.states = tuple_set_create(data.new->alloc, cmp_walk2_tuple); if (data.states == NULL) { goto error; } @@ -603,7 +596,12 @@ fsm_walk2(const struct fsm *a, const struct fsm *b, empty: - return fsm_new(a->opt); + new = fsm_new(a->alloc); + if (new == NULL) { + return NULL; + } + + return new; error: diff --git a/src/libre/Makefile b/src/libre/Makefile index a88a92418..340aa0bcb 100644 --- a/src/libre/Makefile +++ b/src/libre/Makefile @@ -9,14 +9,20 @@ SRC += src/libre/ast_compile.c SRC += src/libre/ast_new_from_fsm.c SRC += src/libre/ast_rewrite.c SRC += src/libre/ac.c +SRC += src/libre/print.c +SRC += src/libre/re_interpolate_groups.c SRC += src/libre/re_strings.c +SRC += src/libre/re_capvm_compile.c # generated SRC += src/libre/class_name.c -.for src in ${SRC:Msrc/libre/ast_compile.c} +.for src in ${SRC:Msrc/libre/ast*.c} ${SRC:Msrc/libre/re*.c} CFLAGS.${src} += -I src # XXX: for internal.h DFLAGS.${src} += -I src # XXX: for internal.h + +CFLAGS.${src} += -std=c99 +DFLAGS.${src} += -std=c99 .endfor LIB += libre diff --git a/src/libre/ac.c b/src/libre/ac.c index 2ebdb4f7e..88ce79a44 100644 --- a/src/libre/ac.c +++ b/src/libre/ac.c @@ -13,6 +13,7 @@ #include #include +#include #include "ac.h" @@ -21,6 +22,9 @@ enum { POOL_BLOCK_SIZE = 256 }; struct trie_state { struct trie_state *children[256]; struct trie_state *fail; + /* use a state set as an endid set */ + struct state_set *endids; + fsm_state_t st; unsigned int index; unsigned int output:1; @@ -73,6 +77,7 @@ newstate(struct trie_graph *g) st->index = ++g->nstates; st->output = 0; + st->endids = NULL; return st; } @@ -86,6 +91,10 @@ cleanup_pool(struct trie_graph *g) p = g->pool; g->pool = p->next; + for (size_t i = 0; i < p->n; i++) { + state_set_free(p->states[i].endids); + } + free(p->states); free(p); } @@ -126,7 +135,7 @@ trie_create(void) } struct trie_graph * -trie_add_word(struct trie_graph *g, const char *w, size_t n) +trie_add_word(struct trie_graph *g, const char *w, size_t n, const fsm_end_id_t *endid) { struct trie_state *st; size_t i; @@ -159,6 +168,9 @@ trie_add_word(struct trie_graph *g, const char *w, size_t n) g->depth = n; } + if (endid != NULL) { + state_set_add(&st->endids, NULL, (fsm_state_t)*endid); + } return g; } @@ -278,7 +290,7 @@ trie_to_fsm_state(struct trie_state *ts, struct fsm *fsm, assert(fsm != NULL); assert(q != NULL); - if (ts->output && have_end) { + if (ts->output && have_end && state_set_empty(ts->endids)) { *q = single_end; return 1; } @@ -315,6 +327,15 @@ trie_to_fsm_state(struct trie_state *ts, struct fsm *fsm, if (ts->output) { fsm_setend(fsm, st, 1); + + struct state_iter si; + fsm_state_t state; + state_set_reset(ts->endids, &si); + while (state_set_next(&si, &state)) { + if (fsm_endid_set(fsm, st, (fsm_end_id_t) state) < 0) { + return 0; + } + } } *q = st; diff --git a/src/libre/ac.h b/src/libre/ac.h index 480c76bfb..c8a4f195f 100644 --- a/src/libre/ac.h +++ b/src/libre/ac.h @@ -7,9 +7,10 @@ #ifndef AC_H #define AC_H +#include "fsm/fsm.h" + struct fsm; struct fsm_state; -struct fsm_options; struct trie_graph; @@ -20,7 +21,8 @@ void trie_free(struct trie_graph *g); struct trie_graph * -trie_add_word(struct trie_graph *g, const char *w, size_t n); +trie_add_word(struct trie_graph *g, const char *w, size_t n, + const fsm_end_id_t *endid); int trie_add_failure_edges(struct trie_graph *g); diff --git a/src/libre/ast.c b/src/libre/ast.c index 2e1d21817..2a630a878 100644 --- a/src/libre/ast.c +++ b/src/libre/ast.c @@ -153,23 +153,13 @@ ast_free(struct ast *ast) } struct ast_count -ast_make_count(unsigned min, const struct ast_pos *start, - unsigned max, const struct ast_pos *end) +ast_make_count(unsigned min, unsigned max) { struct ast_count res; - memset(&res, 0x00, sizeof res); - res.min = min; res.max = max; - if (start != NULL) { - res.start = *start; - } - if (end != NULL) { - res.end = *end; - } - return res; } @@ -235,6 +225,7 @@ ast_expr_free(struct ast_expr_pool *pool, struct ast_expr *n) return; default: + fprintf(stderr, "n->type %d\n", n->type); assert(!"unreached"); } @@ -494,6 +485,7 @@ ast_expr_cmp(const struct ast_expr *a, const struct ast_expr *b) case AST_EXPR_GROUP: if (a->u.group.id < b->u.group.id) { return -1; } if (a->u.group.id > b->u.group.id) { return +1; } + /* .repeated flag is ignored here */ return ast_expr_cmp(a->u.group.e, b->u.group.e); @@ -646,6 +638,7 @@ ast_make_expr_alt(struct ast_expr_pool **poolp, enum re_flags re_flags) res->re_flags = re_flags; res->u.alt.alloc = 8; /* arbitrary initial value */ res->u.alt.count = 0; + res->u.alt.nullable_alt_inside_plus_repeat = 0; res->u.alt.n = calloc(res->u.alt.alloc, sizeof *res->u.alt.n); if (res->u.alt.n == NULL) { @@ -753,6 +746,7 @@ ast_make_expr_group(struct ast_expr_pool **poolp, enum re_flags re_flags, struct res->re_flags = re_flags; res->u.group.e = e; res->u.group.id = id; + res->u.group.repeated = 0; /* may be set during analysis */ return res; } @@ -770,6 +764,7 @@ ast_make_expr_anchor(struct ast_expr_pool **poolp, enum re_flags re_flags, enum res->type = AST_EXPR_ANCHOR; res->re_flags = re_flags; res->u.anchor.type = type; + res->u.anchor.is_end_nl = 0; /* may be set later */ return res; } @@ -794,8 +789,7 @@ ast_make_expr_subtract(struct ast_expr_pool **poolp, enum re_flags re_flags, str struct ast_expr * ast_make_expr_range(struct ast_expr_pool **poolp, enum re_flags re_flags, - const struct ast_endpoint *from, struct ast_pos start, - const struct ast_endpoint *to, struct ast_pos end) + const struct ast_endpoint *from, const struct ast_endpoint *to) { struct ast_expr *res; @@ -810,9 +804,7 @@ ast_make_expr_range(struct ast_expr_pool **poolp, enum re_flags re_flags, res->type = AST_EXPR_RANGE; res->re_flags = re_flags; res->u.range.from = *from; - res->u.range.start = start; res->u.range.to = *to; - res->u.range.end = end; return res; } @@ -852,7 +844,6 @@ ast_make_expr_named(struct ast_expr_pool **poolp, enum re_flags re_flags, const } } else { struct ast_endpoint from, to; - struct ast_pos pos = { 0, 0, 0 }; /* XXX: pass in pos */ from.type = AST_ENDPOINT_LITERAL; if (class->ranges[i].a <= UCHAR_MAX) { @@ -868,7 +859,7 @@ ast_make_expr_named(struct ast_expr_pool **poolp, enum re_flags re_flags, const to.u.codepoint.u = class->ranges[i].b; } - res->u.alt.n[i] = ast_make_expr_range(poolp, re_flags, &from, pos, &to, pos); + res->u.alt.n[i] = ast_make_expr_range(poolp, re_flags, &from, &to); if (res->u.alt.n[i] == NULL) { goto error; } @@ -997,13 +988,31 @@ ast_expr_is_literal(const struct ast_expr *e, *anchor_end = 1; } +done: + for (i = 0; i < count; i++) { if (nodes[i] == NULL || nodes[i]->type != AST_EXPR_LITERAL) { return 0; } - } -done: + /* mask out AST flags that we don't care about; these may be present or not */ + int ast_flags = nodes[i]->flags + & (unsigned)~(AST_FLAG_FIRST | AST_FLAG_LAST | AST_FLAG_ALWAYS_CONSUMES | AST_FLAG_CAN_CONSUME); + + /* we reject anything else */ + if (ast_flags != 0) { + return 0; + } + + /* mask out re flags that we don't handle here yet (the caller is responsible for these) */ + int re_flags = nodes[i]->re_flags + & (unsigned)~(RE_END_NL | RE_NOCAPTURE); + + /* and we don't permit any other re_flags */ + if (re_flags != 0) { + return 0; + } + } *n = count + is_end_nl; diff --git a/src/libre/ast.h b/src/libre/ast.h index 233744847..538b416de 100644 --- a/src/libre/ast.h +++ b/src/libre/ast.h @@ -7,6 +7,11 @@ #ifndef RE_AST_H #define RE_AST_H +#include +#include +#include +#include + /* * This is a duplicate of struct lx_pos, but since we're linking to * code with several distinct lexers, there isn't a clear lexer.h @@ -40,9 +45,7 @@ enum ast_expr_type { #define AST_COUNT_UNBOUNDED ((unsigned)-1) struct ast_count { unsigned min; - struct ast_pos start; unsigned max; - struct ast_pos end; }; enum ast_anchor_type { @@ -62,7 +65,9 @@ enum ast_anchor_type { * followed by nullable nodes. * * - AST_FLAG_UNSATISFIABLE - * The node caused the regex to become unsatisfiable. + * The node is unsatisfiable (can never match anything). + * This can cause AST subtrees to be pruned, or for the + * entire regex to become unsatisfiable. * * - AST_FLAG_NULLABLE * The node is not always evaluated, such as nodes that @@ -86,6 +91,13 @@ enum ast_anchor_type { * ends with the PCRE end anchor that implicitly matches a single * trailing newline. * + * - AST_FLAG_CONSTRAINED_AT_START + * The anchor needs more restrictive linkage on its start side, + * see ast_analysis's "pincer_anchors" analysis for details. + * + * - AST_FLAG_CONSTRAINED_AT_END + * End counterpart to AST_FLAG_CONSTRAINED_AT_START. + * * Not all are valid for all node types. */ enum ast_flags { @@ -98,6 +110,9 @@ enum ast_flags { AST_FLAG_ANCHORED_START = 1 << 6, AST_FLAG_ANCHORED_END = 1 << 7, AST_FLAG_END_NL = 1 << 8, + AST_FLAG_MATCHES_1NEWLINE= 1 << 9, + AST_FLAG_CONSTRAINED_AT_START = 1 << 10, + AST_FLAG_CONSTRAINED_AT_END = 1 << 11, AST_FLAG_NONE = 0x00 }; @@ -159,6 +174,8 @@ struct ast_expr { size_t count; /* used */ size_t alloc; /* allocated */ struct ast_expr **n; + int contains_empty_groups; + int nullable_alt_inside_plus_repeat; } alt; struct { @@ -172,12 +189,14 @@ struct ast_expr { struct ast_expr_repeat { struct ast_expr *e; unsigned min; - unsigned max; + unsigned max; /* can be AST_COUNT_UNBOUNDED */ + int contains_empty_groups; } repeat; struct { struct ast_expr *e; unsigned id; + int repeated; /* set during analysis */ } group; struct { @@ -192,9 +211,7 @@ struct ast_expr { struct { struct ast_endpoint from; - struct ast_pos start; struct ast_endpoint to; - struct ast_pos end; } range; struct { @@ -235,9 +252,12 @@ ast_pool_free(struct ast_expr_pool *pool); struct ast_expr_pool * ast_expr_pool_save(void); +#define AST_NO_MAX_CAPTURE_ID ((long)-1) + struct ast { struct ast_expr_pool *pool; struct ast_expr *expr; + long max_capture_id; int has_unanchored_start; int has_unanchored_end; }; @@ -251,8 +271,7 @@ void ast_free(struct ast *ast); struct ast_count -ast_make_count(unsigned min, const struct ast_pos *start, - unsigned max, const struct ast_pos *end); +ast_make_count(unsigned min, unsigned max); /* * Expressions @@ -308,18 +327,15 @@ ast_add_expr_concat(struct ast_expr *cat, struct ast_expr *node); struct ast_expr * ast_make_expr_range(struct ast_expr_pool **poolp, enum re_flags re_flags, - const struct ast_endpoint *from, struct ast_pos start, - const struct ast_endpoint *to, struct ast_pos end); + const struct ast_endpoint *from, const struct ast_endpoint *to); struct ast_expr * ast_make_expr_named(struct ast_expr_pool **poolp, enum re_flags re_flags, const struct class *class); /* XXX: exposed for sake of re(1) printing an ast; * it's not part of the API proper */ -struct fsm_options; struct ast * re_parse(enum re_dialect dialect, int (*getc)(void *opaque), void *opaque, - const struct fsm_options *opt, enum re_flags flags, struct re_err *err, int *unsatisfiable); const char * diff --git a/src/libre/ast_analysis.c b/src/libre/ast_analysis.c index 3298f62c2..399ad147d 100644 --- a/src/libre/ast_analysis.c +++ b/src/libre/ast_analysis.c @@ -13,7 +13,6 @@ #include #include "class.h" -#include "print.h" #include "ast.h" #include "ast_analysis.h" @@ -21,12 +20,15 @@ #define LOG_ANALYSIS 0 #define LOG_FIRST_ANALYSIS (0 + LOG_ANALYSIS) -#define LOG_REPEATED_GROUPS (0 + LOG_ANALYSIS) +#define LOG_REPETITION_CASES (0 + LOG_ANALYSIS) #define LOG_FORKING (0 + LOG_ANALYSIS) #define LOG_ANCHORING (0 + LOG_ANALYSIS) #define LOG_CONCAT_FLAGS (0 + LOG_ANALYSIS) #define LOG_UNANCHORED_FLAGS (0 + LOG_ANALYSIS) +#define LOG_REPETITION_CASES (0 + LOG_ANALYSIS) +#define LOG_PINCER_ANCHORS (0 + LOG_ANALYSIS) + #define LOG(LEVEL, ...) \ do { \ if ((LEVEL) <= LOG_ANALYSIS) { \ @@ -36,6 +38,12 @@ /* Mask for end-anchor flags */ #define END_ANCHOR_FLAG_MASK (AST_FLAG_ANCHORED_END | AST_FLAG_END_NL) +#define ANCHOR_FLAG_MASK (END_ANCHOR_FLAG_MASK | AST_FLAG_ANCHORED_START) + +struct capture_env { + long max_capture_id; + int use_captures; +}; static int is_nullable(const struct ast_expr *n) @@ -73,6 +81,24 @@ set_flags(struct ast_expr *n, enum ast_flags flags) n->flags |= flags; } +static int +is_unsatisfiable_or_skipped(const struct ast_expr *n) +{ + return (n->flags & AST_FLAG_UNSATISFIABLE) + || (n->type == AST_EXPR_REPEAT + && n->u.repeat.min == 0 + && n->u.repeat.max == 0); +} + +static void +prune_repeat_node(struct ast_expr *n) +{ + assert(n->type == AST_EXPR_REPEAT && n->u.repeat.min == 0); + LOG(3, "%s: setting REPEAT node %p's max count to 0\n", + __func__, (void *)n); + n->u.repeat.max = 0; +} + static enum ast_analysis_res analysis_iter(struct ast_expr *n) { @@ -104,7 +130,9 @@ analysis_iter(struct ast_expr *n) (void *)n, (void *)child); set_flags(child, AST_FLAG_NULLABLE); } - analysis_iter(child); + enum ast_analysis_res child_res = analysis_iter(child); + if (child_res != AST_ANALYSIS_OK) { return child_res; } + if (can_consume_input(child)) { any_can_consume = 1; } @@ -143,7 +171,8 @@ analysis_iter(struct ast_expr *n) for (i = 0; i < n->u.alt.count; i++) { struct ast_expr *child = n->u.alt.n[i]; - analysis_iter(child); + enum ast_analysis_res child_res = analysis_iter(child); + if (child_res != AST_ANALYSIS_OK) { return child_res; } /* spread nullability upward */ if (is_nullable(child)) { @@ -190,7 +219,8 @@ analysis_iter(struct ast_expr *n) set_flags(n, AST_FLAG_NULLABLE); } - analysis_iter(e); + enum ast_analysis_res child_res = analysis_iter(e); + if (child_res != AST_ANALYSIS_OK) { return child_res; } set_flags(n, e->flags & AST_FLAG_CAN_CONSUME); if (n->u.repeat.min > 0 && e->flags & AST_FLAG_ALWAYS_CONSUMES) { @@ -229,7 +259,8 @@ analysis_iter(struct ast_expr *n) set_flags(e, AST_FLAG_NULLABLE); } - analysis_iter(e); + enum ast_analysis_res child_res = analysis_iter(e); + if (child_res != AST_ANALYSIS_OK) { return child_res; } set_flags(n, e->flags & (AST_FLAG_CAN_CONSUME | AST_FLAG_ALWAYS_CONSUMES)); if (is_nullable(e)) { @@ -239,7 +270,7 @@ analysis_iter(struct ast_expr *n) } case AST_EXPR_ANCHOR: - /* anchor flags will be handled on the second pass */ + /* anchor flags will be handled on the next pass */ break; case AST_EXPR_SUBTRACT: @@ -267,23 +298,280 @@ analysis_iter(struct ast_expr *n) return AST_ANALYSIS_OK; } -/* Analysis for unanchored starts/ends uses three values, because some - * details decide the whole subtree is/isn't, others should defer to - * analysis elsewhere it the tree, but an overall result of undecided - * still decides yes. */ -enum unanchored_analysis_res { - UA_NO = 'N', - UA_YES = 'Y', - UA_UNDECIDED = 'U', +static int +is_only_anchors(struct ast_expr *expr) +{ + if (can_consume_input(expr)) { return 0; } + + switch (expr->type) { + case AST_EXPR_ANCHOR: + return 1; + + case AST_EXPR_CONCAT: + if (expr->u.concat.count == 0) { return 0; } + for (size_t i = 0; i < expr->u.concat.count; i++) { + if (!is_only_anchors(expr->u.concat.n[i]) + && can_consume_input(expr->u.concat.n[i])) { + return 0; + } + } + return 1; + + case AST_EXPR_ALT: + assert(expr->u.alt.count > 0); + for (size_t i = 0; i < expr->u.alt.count; i++) { + /* earlier matches will shadow later ones */ + if (is_only_anchors(expr->u.alt.n[i])) { + return 1; + } + } + return 0; + + case AST_EXPR_REPEAT: + if (expr->u.repeat.min == 0 && expr->u.repeat.max == 0) { + return 0; + } + return is_only_anchors(expr->u.repeat.e); + + case AST_EXPR_GROUP: + return is_only_anchors(expr->u.group.e); + + default: + break; + } + + return 0; +} + +static enum ast_analysis_res +analysis_iter_repetition(struct ast_expr *n, struct ast_expr *outermost_repeat_parent, + int shadowed_by_previous_alt_case, struct ast_expr *repeat_plus_ancestor) +{ + enum ast_analysis_res res = AST_ANALYSIS_OK; + + LOG(3 - LOG_REPETITION_CASES, "%s: node %p, type %s, shadowed_by_previous_alt_case %d\n", + __func__, (void *)n, ast_node_type_name(n->type), shadowed_by_previous_alt_case); + + if (shadowed_by_previous_alt_case) { + assert(outermost_repeat_parent == NULL + || outermost_repeat_parent->type == AST_EXPR_ALT); + } + + switch (n->type) { + case AST_EXPR_EMPTY: + case AST_EXPR_TOMBSTONE: + case AST_EXPR_ANCHOR: + case AST_EXPR_LITERAL: + case AST_EXPR_CODEPOINT: + case AST_EXPR_RANGE: + break; + + case AST_EXPR_CONCAT: { + /* If this CONCAT array of nodes always consumes input, then + * it cannot be repeated empty, so it cannot produce the + * special case that needs outermost_repeat_parent for + * AST_EXPR_REPEAT's case below. + * + * An example input that needs this is 'x(()x)*' for "xx", + * because the 'x' prevents the outermost group + * from repeating and matching empty again after consuming + * a run of "x"s. */ + if (always_consumes_input(n)) { + outermost_repeat_parent = NULL; + } + + for (size_t i = 0; i < n->u.concat.count; i++) { + res = analysis_iter_repetition(n->u.concat.n[i], outermost_repeat_parent, + shadowed_by_previous_alt_case, repeat_plus_ancestor); + if (res != AST_ANALYSIS_OK) { return res; } + } + break; + } + + case AST_EXPR_ALT: { + /* See AST_EXPR_GROUP below for why this matters. */ + int new_shadowed_by_previous_alt_case = shadowed_by_previous_alt_case; + + /* FIXME: check nesting of this construct */ + + LOG(3 - LOG_REPETITION_CASES, + "%s: ALT node %p, repeat_plus_ancestor %p\n", + __func__, (void *)n, (void *)repeat_plus_ancestor); + + for (size_t i = 0; i < n->u.alt.count; i++) { + /* If this is an ALT inside of a repeated subtree that contains + * a capture, this will need special handling. */ + if (outermost_repeat_parent != NULL) { + LOG(3 - LOG_REPETITION_CASES, + "%s: setting outermost_repeat_parent to %p for alt branch %zu, repeat_plus_ancestor %p\n", + __func__, (void *)n, i, (void *)repeat_plus_ancestor); + outermost_repeat_parent = n; + } + + if (is_nullable(n->u.alt.n[i]) || is_only_anchors(n->u.alt.n[i])) { + LOG(3 - LOG_REPETITION_CASES, + "%s: setting new_shadowed_by_previous_alt_case for alt branch %zu, repeat_plus_ancestor %p\n", + __func__, i, (void *)repeat_plus_ancestor); + new_shadowed_by_previous_alt_case = 1; + if (repeat_plus_ancestor != NULL) { + n->u.alt.nullable_alt_inside_plus_repeat = 1; + assert(repeat_plus_ancestor->type == AST_EXPR_REPEAT); + assert(repeat_plus_ancestor->u.repeat.min == 1); + assert(repeat_plus_ancestor->u.repeat.max == AST_COUNT_UNBOUNDED); + + /* Repetition of an alt subtree which has a capture group child that + * only contains only* anchors is not handled properly yet. This + * isn't actually _useful_, it's just something that comes up + * in fuzzing, so reject it as an unsupported PCRE construct. + * + * An example input that triggers this is '^(($)|)+$' . */ + set_flags(n, AST_FLAG_UNSATISFIABLE); + return AST_ANALYSIS_ERROR_UNSUPPORTED; + } + } + + res = analysis_iter_repetition(n->u.alt.n[i], + outermost_repeat_parent, + new_shadowed_by_previous_alt_case, + repeat_plus_ancestor); + if (res != AST_ANALYSIS_OK) { return res; } + } + break; + } + + case AST_EXPR_REPEAT: + { + struct ast_expr *child = n->u.repeat.e; + + LOG(3 - LOG_REPETITION_CASES, "%s: REPEAT node %p, min %u max %u nullable? %d, !cci %d\n", + __func__, (void *)n, n->u.repeat.min, n->u.repeat.max, + is_nullable(child), can_consume_input(child)); + + if (n->u.repeat.min == 1 && + n->u.repeat.max == AST_COUNT_UNBOUNDED) { + LOG(3 - LOG_REPETITION_CASES, "%s: setting repeat_plus_ancestor to %p\n", + __func__, (void *)n); + repeat_plus_ancestor = n; + } else { + repeat_plus_ancestor = NULL; + } + + /* Special cases for a repeated group that contains possibly empty captures, + * in order to correctly reflect their repeating one more time and capture + * at the end (but without an infinite loop). + * + * For example, '^((x?))*$' will always end up with capture groups 1 and 2 + * at the end of the input for any number of "x"s, since the outermost ()* + * can always repeat once more time, consuming nothing, and clobber the + * existing captures. We mark repeated groups so that the compiled capture + * program can move saving the captures after the repetition, instead + * behaving like `^((?:x?)*(())$`. + * + * However, if the repeated subtree always consumes input, such as with + * '^(()a)+b$', then clear any passed in outermost_repeat_parent, because + * having to consume input will prevent that extra repetition of the + * empty captures. */ + if (always_consumes_input(n)) { + res = analysis_iter_repetition(child, NULL, shadowed_by_previous_alt_case, + repeat_plus_ancestor); + } else if (outermost_repeat_parent == NULL && n->u.repeat.max > 1) { + LOG(3 - LOG_REPETITION_CASES, "%s: recursing with outermost_repeat_parent set to %p\n", + __func__, (void *)n); + res = analysis_iter_repetition(child, n, 0, + repeat_plus_ancestor); + } else { + LOG(3 - LOG_REPETITION_CASES, "%s: recursing with outermost_repeat_parent %p\n", + __func__, (void *)outermost_repeat_parent); + res = analysis_iter_repetition(child, outermost_repeat_parent, shadowed_by_previous_alt_case, + repeat_plus_ancestor); + } + if (res != AST_ANALYSIS_OK) { return res; } + break; + } + + case AST_EXPR_GROUP: + LOG(3 - LOG_REPETITION_CASES, + "%s: GROUP %p, repeat_plus_ancestor %p\n", + __func__, (void *)n, (void *)repeat_plus_ancestor); + + + if (outermost_repeat_parent != NULL && (is_nullable(n) || !can_consume_input(n))) { + int should_mark_repeated = 1; + /* If the outermost_repeat_parent is an ALT node and a previous ALT subtree + * matching the empty string is shadowing this group, then do not mark it + * as repeated, because that can lead to incorrect handling in somewhat + * contrived regexes like '^(?:|(|x))*$'. */ + if (outermost_repeat_parent->type == AST_EXPR_ALT && shadowed_by_previous_alt_case) { + LOG(3 - LOG_REPETITION_CASES, + "%s: hit group shadowed_by_previous_alt_case, skipping\n", __func__); + should_mark_repeated = 0; + } + + if (n->flags & (AST_FLAG_ANCHORED_START | AST_FLAG_ANCHORED_END)) { + LOG(3 - LOG_REPETITION_CASES, + "%s: hit repeating anchor, skipping\n", __func__); + should_mark_repeated = 0; + } + + if (should_mark_repeated) { + LOG(3 - LOG_REPETITION_CASES, "%s: setting group %u to repeated\n", + __func__, n->u.group.id); + n->u.group.repeated = 1; + assert(outermost_repeat_parent->type == AST_EXPR_REPEAT || + outermost_repeat_parent->type == AST_EXPR_ALT); + LOG(3 - LOG_REPETITION_CASES, "%s: setting contains_empty_groups on outermost_repeat_parent %p\n", + __func__, (void *)outermost_repeat_parent); + if (outermost_repeat_parent->type == AST_EXPR_REPEAT) { + outermost_repeat_parent->u.repeat.contains_empty_groups = 1; + } else if (outermost_repeat_parent->type == AST_EXPR_ALT) { + outermost_repeat_parent->u.alt.contains_empty_groups = 1; + } else { + assert(!"type mismatch"); + } + } + } + + if (repeat_plus_ancestor != NULL && (is_nullable(n) || !can_consume_input(n))) { + assert(repeat_plus_ancestor->type == AST_EXPR_REPEAT + && repeat_plus_ancestor->u.repeat.min == 1 + && repeat_plus_ancestor->u.repeat.max == AST_COUNT_UNBOUNDED); + LOG(3 - LOG_REPETITION_CASES, + "%s: not yet implemented, skipping\n", __func__); + /* return AST_ANALYSIS_ERROR_UNSUPPORTED; */ + } + + res = analysis_iter_repetition(n->u.group.e, outermost_repeat_parent, + shadowed_by_previous_alt_case, repeat_plus_ancestor); + if (res != AST_ANALYSIS_OK) { return res; } + break; + + case AST_EXPR_SUBTRACT: + res = analysis_iter_repetition(n->u.subtract.a, outermost_repeat_parent, shadowed_by_previous_alt_case, + repeat_plus_ancestor); + if (res != AST_ANALYSIS_OK) { return res; } + res = analysis_iter_repetition(n->u.subtract.b, outermost_repeat_parent, shadowed_by_previous_alt_case, + repeat_plus_ancestor); + break; + + default: + assert(!"unreached"); + } + return res; +} + +enum anchoring_analysis_res { + ANCHOR_ANALYSIS_ANCHORED = 'a', + ANCHOR_ANALYSIS_UNANCHORED = 'u', + ANCHOR_ANALYSIS_UNDECIDED = '_', }; -static enum unanchored_analysis_res +static enum anchoring_analysis_res analysis_iter_unanchored_start(const struct ast_expr *n) { if (n->flags & AST_FLAG_ANCHORED_START) { LOG(4 - LOG_UNANCHORED_FLAGS, "%s: n (%p)->flags & AST_FLAG_ANCHORED_START -> N\n", __func__, (void *)n); - return UA_NO; + return ANCHOR_ANALYSIS_ANCHORED; } LOG(4 - LOG_UNANCHORED_FLAGS, "%s: node %p, type %s\n", @@ -293,7 +581,7 @@ analysis_iter_unanchored_start(const struct ast_expr *n) case AST_EXPR_EMPTY: LOG(3 - LOG_UNANCHORED_FLAGS, "%s: %s -> U\n", __func__, ast_node_type_name(n->type)); - return UA_UNDECIDED; + return ANCHOR_ANALYSIS_UNDECIDED; case AST_EXPR_TOMBSTONE: break; @@ -302,12 +590,12 @@ analysis_iter_unanchored_start(const struct ast_expr *n) switch (n->u.anchor.type) { case AST_ANCHOR_START: LOG(3 - LOG_UNANCHORED_FLAGS, "%s: ^ anchor -> N\n", __func__); - return UA_NO; + return ANCHOR_ANALYSIS_ANCHORED; case AST_ANCHOR_END: LOG(3 - LOG_UNANCHORED_FLAGS, "%s: $ anchor -> U\n", __func__); /* will be handled by other cases */ - return UA_UNDECIDED; + return ANCHOR_ANALYSIS_UNDECIDED; default: assert(!"unreached"); @@ -323,7 +611,7 @@ analysis_iter_unanchored_start(const struct ast_expr *n) case AST_EXPR_RANGE: LOG(3 - LOG_UNANCHORED_FLAGS, "%s: %s -> Y\n", __func__, ast_node_type_name(n->type)); - return UA_YES; + return ANCHOR_ANALYSIS_UNANCHORED; case AST_EXPR_CONCAT: { size_t i; @@ -331,8 +619,8 @@ analysis_iter_unanchored_start(const struct ast_expr *n) /* has unanchored start if first non-nullable child does */ for (i = 0; i < n->u.concat.count; i++) { const struct ast_expr *child = n->u.concat.n[i]; - const enum unanchored_analysis_res child_res = analysis_iter_unanchored_start(child); - if (child_res != UA_UNDECIDED) { + const enum anchoring_analysis_res child_res = analysis_iter_unanchored_start(child); + if (child_res != ANCHOR_ANALYSIS_UNDECIDED) { LOG(3 - LOG_UNANCHORED_FLAGS, "%s: %s -> %c (child res)\n", __func__, ast_node_type_name(n->type), child_res); return child_res; @@ -350,19 +638,19 @@ analysis_iter_unanchored_start(const struct ast_expr *n) size_t i; /* if all children agree, return that result, otherwise undecided */ - const enum unanchored_analysis_res first_child_res = analysis_iter_unanchored_start(n->u.alt.n[0]); + const enum anchoring_analysis_res first_child_res = analysis_iter_unanchored_start(n->u.alt.n[0]); LOG(3 - LOG_UNANCHORED_FLAGS, "%s: ALT child 0 -- %s -> %c (child res)\n", __func__, ast_node_type_name(n->type), first_child_res); for (i = 1; i < n->u.alt.count; i++) { const struct ast_expr *child = n->u.alt.n[i]; - const enum unanchored_analysis_res child_res = analysis_iter_unanchored_start(child); + const enum anchoring_analysis_res child_res = analysis_iter_unanchored_start(child); LOG(3 - LOG_UNANCHORED_FLAGS, "%s: ALT child %zd -- %s -> %c (child res)\n", __func__, i, ast_node_type_name(n->type), child_res); if (child_res != first_child_res) { LOG(3 - LOG_UNANCHORED_FLAGS, "%s: %s -> %c (child result) contracts first, returning U\n", __func__, ast_node_type_name(n->type), child_res); - return UA_UNDECIDED; + return ANCHOR_ANALYSIS_UNDECIDED; } } @@ -375,7 +663,7 @@ analysis_iter_unanchored_start(const struct ast_expr *n) if (n->u.repeat.min == 0) { LOG(3 - LOG_UNANCHORED_FLAGS, "%s: %s -> U (repeat.min == 0)\n", __func__, ast_node_type_name(n->type)); - return UA_UNDECIDED; + return ANCHOR_ANALYSIS_UNDECIDED; } return analysis_iter_unanchored_start(n->u.repeat.e); @@ -389,16 +677,16 @@ analysis_iter_unanchored_start(const struct ast_expr *n) assert(!"unreached"); } - return UA_UNDECIDED; + return ANCHOR_ANALYSIS_UNDECIDED; } -static enum unanchored_analysis_res +static enum anchoring_analysis_res analysis_iter_unanchored_end(const struct ast_expr *n) { if (n->flags & AST_FLAG_ANCHORED_END) { LOG(4 - LOG_UNANCHORED_FLAGS, "%s: node (%p)->flags & AST_FLAG_ANCHORED_END -> N\n", __func__, (void *)n); - return UA_NO; + return ANCHOR_ANALYSIS_ANCHORED; } LOG(4 - LOG_UNANCHORED_FLAGS, "%s: node %p, type %s\n", @@ -408,7 +696,7 @@ analysis_iter_unanchored_end(const struct ast_expr *n) case AST_EXPR_EMPTY: LOG(3 - LOG_UNANCHORED_FLAGS, "%s: %s -> U\n", __func__, ast_node_type_name(n->type)); - return UA_UNDECIDED; + return ANCHOR_ANALYSIS_UNDECIDED; case AST_EXPR_TOMBSTONE: break; @@ -418,12 +706,12 @@ analysis_iter_unanchored_end(const struct ast_expr *n) case AST_ANCHOR_START: LOG(3 - LOG_UNANCHORED_FLAGS, "%s: ^ %s -> U\n", __func__, ast_node_type_name(n->type)); - return UA_UNDECIDED; + return ANCHOR_ANALYSIS_UNDECIDED; case AST_ANCHOR_END: LOG(3 - LOG_UNANCHORED_FLAGS, "%s: $ %s -> N\n", __func__, ast_node_type_name(n->type)); - return UA_NO; + return ANCHOR_ANALYSIS_ANCHORED; default: assert(!"unreached"); @@ -439,7 +727,7 @@ analysis_iter_unanchored_end(const struct ast_expr *n) case AST_EXPR_RANGE: LOG(3 - LOG_UNANCHORED_FLAGS, "%s: %s -> Y\n", __func__, ast_node_type_name(n->type)); - return UA_YES; + return ANCHOR_ANALYSIS_UNANCHORED; case AST_EXPR_CONCAT: { size_t i; @@ -447,8 +735,8 @@ analysis_iter_unanchored_end(const struct ast_expr *n) /* has unanchored end if last non-nullable child does */ for (i = n->u.concat.count; i > 0; i--) { const struct ast_expr *child = n->u.concat.n[i - 1]; - const enum unanchored_analysis_res child_res = analysis_iter_unanchored_end(child); - if (child_res != UA_UNDECIDED) { + const enum anchoring_analysis_res child_res = analysis_iter_unanchored_end(child); + if (child_res != ANCHOR_ANALYSIS_UNDECIDED) { LOG(3 - LOG_UNANCHORED_FLAGS, "%s: %s -> %c (child res)\n", __func__, ast_node_type_name(n->type), child_res); return child_res; @@ -466,19 +754,19 @@ analysis_iter_unanchored_end(const struct ast_expr *n) size_t i; /* if all children agree, return that result, otherwise undecided */ - const enum unanchored_analysis_res first_child_res = analysis_iter_unanchored_end(n->u.alt.n[0]); + const enum anchoring_analysis_res first_child_res = analysis_iter_unanchored_end(n->u.alt.n[0]); LOG(3 - LOG_UNANCHORED_FLAGS, "%s: ALT child 0 -- %s -> %c (child res)\n", __func__, ast_node_type_name(n->type), first_child_res); for (i = 1; i < n->u.alt.count; i++) { const struct ast_expr *child = n->u.alt.n[i]; - const enum unanchored_analysis_res child_res = analysis_iter_unanchored_end(child); + const enum anchoring_analysis_res child_res = analysis_iter_unanchored_end(child); LOG(3 - LOG_UNANCHORED_FLAGS, "%s: ALT child %zd -- %s -> %c (child res)\n", __func__, i, ast_node_type_name(n->type), child_res); if (child_res != first_child_res) { LOG(3 - LOG_UNANCHORED_FLAGS, "%s: %s -> %c (child result) contracts first, returning U\n", __func__, ast_node_type_name(n->type), child_res); - return UA_UNDECIDED; + return ANCHOR_ANALYSIS_UNDECIDED; } } @@ -491,7 +779,7 @@ analysis_iter_unanchored_end(const struct ast_expr *n) if (n->u.repeat.min == 0) { LOG(3 - LOG_UNANCHORED_FLAGS, "%s: %s -> U (repeat.min == 0)\n", __func__, ast_node_type_name(n->type)); - return UA_UNDECIDED; + return ANCHOR_ANALYSIS_UNDECIDED; } return analysis_iter_unanchored_end(n->u.repeat.e); @@ -505,7 +793,7 @@ analysis_iter_unanchored_end(const struct ast_expr *n) assert(!"unreached"); } - return UA_UNDECIDED; + return ANCHOR_ANALYSIS_UNDECIDED; } static void @@ -552,6 +840,47 @@ set_flags_subtree(struct ast_expr *n, enum ast_flags flags) } } +static int +can_consume_single_newline(struct ast_expr *n) +{ + if (!can_consume_input(n)) { return 0; } + + if (n->flags & AST_FLAG_MATCHES_1NEWLINE) { return 1; } + + switch (n->type) { + case AST_EXPR_LITERAL: + return n->u.literal.c == '\n'; + + case AST_EXPR_CODEPOINT: + return n->u.codepoint.u == (uint32_t)'\n'; + + case AST_EXPR_RANGE: + if ((n->u.range.from.type == AST_ENDPOINT_LITERAL) && + (n->u.range.to.type == AST_ENDPOINT_LITERAL)) { + return n->u.range.from.u.literal.c <= '\n' + && n->u.range.to.u.literal.c >= '\n'; + } else if ((n->u.range.from.type == AST_ENDPOINT_CODEPOINT) && + (n->u.range.to.type == AST_ENDPOINT_CODEPOINT)) { + return n->u.range.from.u.codepoint.u <= '\n' + && n->u.range.to.u.codepoint.u >= '\n'; + } else if (n->u.range.from.type == AST_ENDPOINT_NAMED) { + /* currently unreachable, named ranges are unsupported */ + assert(!"unreachable"); + break; + } + break; + + case AST_EXPR_SUBTRACT: + return can_consume_single_newline(n->u.subtract.a) + && !can_consume_single_newline(n->u.subtract.b); + + default: + break; + } + + return 0; +} + struct anchoring_env { enum re_flags re_flags; @@ -563,9 +892,68 @@ struct anchoring_env { /* Corresponding flag for end anchors while sweeping backward. */ int followed_by_consuming; + /* Special case for detecting '$[^a]', which matches "\n" with + * a capture group 0 of (0,1) in PCRE. */ + int followed_by_consuming_newline; + + /* Flag for tracking whether we're in a part of the subtree that + * is always before a start anchor. This influences satisfiability + * and edge cases like '()*^'. */ int before_start_anchor; + + /* Flag used to detect and reject the awkward case in '$[^a]', + * where (according to PCRE) the character class after the '$' + * should match the literal newline, but nothing else, and only + * once. Because $ is actually a zero-width assertion that + * execution is either at the end of input or a trailing + * newline, it has the rather surprising result that '$[^a]' + * will not match "x" but *will* match "x\n" (because it has a $ + * before a trailing newline, and because the newline matches + * the non-skippable [^a]). We just return an unsupported + * error for this case. */ + enum newline_after_end_anchor_state { + NAEAS_NONE, + NAEAS_WOULD_MATCH_ONCE, + } newline_after_end_anchor_state; + + int after_end_anchor; }; +/* Does the subtree match a literal '\n'? */ +static int +matches_newline(const struct ast_expr *n) +{ + switch (n->type) { + case AST_EXPR_LITERAL: + return n->u.literal.c == '\n'; + + case AST_EXPR_SUBTRACT: + return matches_newline(n->u.subtract.a) + && !matches_newline(n->u.subtract.b); + + case AST_EXPR_RANGE: + { + const struct ast_endpoint *f = &n->u.range.from; + const struct ast_endpoint *t = &n->u.range.to; + if (f->type != AST_ENDPOINT_LITERAL + || t->type != AST_ENDPOINT_LITERAL) { + /* not implemented */ + LOG(1 - LOG_ANCHORING, "%s: not implemented\n", __func__); + return 0; + } + + const int res = f->u.literal.c <= '\n' && t->u.literal.c >= '\n'; + LOG(1 - LOG_ANCHORING, "%s: RANGE res %d\n", __func__, res); + return res; + } + + default: + break; + } + + return 0; +} + /* Tree walker that analyzes the AST, marks which nodes and subtrees are * anchored at the start and/or end, and determines which subtrees are * unsatisfiable due to start anchoring. @@ -600,7 +988,7 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) case AST_ANCHOR_START: /* * If it's not possible to get here without consuming - * any input and there's a start anchor, the regex is + * any input and there's a start anchor, the subtree is * inherently unsatisfiable. */ set_flags(n, AST_FLAG_ANCHORED_START); @@ -628,8 +1016,14 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) case AST_ANCHOR_END: set_flags(n, AST_FLAG_ANCHORED_END); + env->after_end_anchor = 1; + LOG(3 - LOG_ANCHORING, + "%s: END anchor\n", __func__); if (n->u.anchor.is_end_nl && !(env->re_flags & RE_ANCHORED)) { set_flags(n, AST_FLAG_END_NL); + if (env->newline_after_end_anchor_state == NAEAS_NONE) { + env->newline_after_end_anchor_state = NAEAS_WOULD_MATCH_ONCE; + } } break; @@ -642,8 +1036,25 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) * These are the types that actually consume input. */ case AST_EXPR_LITERAL: + if (n->u.literal.c == '\n') { + set_flags(n, AST_FLAG_MATCHES_1NEWLINE); + + if (env->newline_after_end_anchor_state == NAEAS_WOULD_MATCH_ONCE) { + LOG(3 - LOG_ANCHORING, + "%s: LITERAL: rejecting non-optional newline match after $ as unsupported\n", + __func__); + set_flags(n, AST_FLAG_UNSATISFIABLE); + /* fprintf(stderr, "%s:%u: AST_ANALYSIS_ERROR_UNSUPPORTED\n", __func__, __LINE__); */ + return AST_ANALYSIS_ERROR_UNSUPPORTED; + } + } + break; + case AST_EXPR_CODEPOINT: case AST_EXPR_RANGE: + if (can_consume_single_newline(n)) { + set_flags(n, AST_FLAG_MATCHES_1NEWLINE); + } break; /* handled outside switch/case */ case AST_EXPR_CONCAT: { @@ -673,6 +1084,13 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) if (res != AST_ANALYSIS_OK && res != AST_ANALYSIS_UNSATISFIABLE) { /* unsat is handled below */ + if (res == AST_ANALYSIS_ERROR_UNSUPPORTED) { + + /* FIXME: check this */ + assert(child->flags & AST_FLAG_UNSATISFIABLE); + + set_flags(n, AST_FLAG_UNSATISFIABLE); + } return res; } @@ -703,12 +1121,12 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) env->past_always_consuming = 1; } + env->newline_after_end_anchor_state = child_env.newline_after_end_anchor_state; + env->after_end_anchor = child_env.after_end_anchor; } /* flow ANCHORED_START and ANCHORED_END flags upward */ { - int after_always_consumes = 0; - for (i = 0; i < n->u.concat.count; i++) { struct ast_expr *child = n->u.concat.n[i]; if (child->flags & AST_FLAG_ANCHORED_START) { @@ -716,20 +1134,11 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) "%s: setting ANCHORED_START due to child %zu (%p)'s analysis\n", __func__, i, (void *)child); set_flags(n, AST_FLAG_ANCHORED_START); - - if (after_always_consumes) { - LOG(3 - LOG_ANCHORING, - "%s: setting %p and child %p UNSATISFIABLE due to ^-anchored child that always consumes input\n", - __func__, (void *)n, (void *)child); - set_flags(n, AST_FLAG_UNSATISFIABLE); - set_flags(child, AST_FLAG_UNSATISFIABLE); - } } if (always_consumes_input(child)) { LOG(3 - LOG_ANCHORING, "%s: child %zd always consumes input\n", __func__, i); - after_always_consumes = 1; } } } @@ -786,17 +1195,10 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) for (i = 0; i < n->u.concat.count; i++) { struct ast_expr *child = n->u.concat.n[i]; if (after_end_anchor) { - if (always_consumes_input(child)) { - LOG(3 - LOG_ANCHORING, - "%s: after_end_anchor & ALWAYS_CONSUMES on child %p -> UNSATISFIABLE\n", - __func__, (void *)child); - set_flags(child, AST_FLAG_UNSATISFIABLE); - } - if (child->type == AST_EXPR_REPEAT && (child->flags & AST_FLAG_UNSATISFIABLE) && child->u.repeat.min == 0) { - child->u.repeat.max = 0; + prune_repeat_node(child); } } else if (!after_end_anchor && child->flags & AST_FLAG_ANCHORED_END @@ -810,6 +1212,13 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) } } + for (i = 0; i < n->u.concat.count; i++) { + struct ast_expr *child = n->u.concat.n[i]; + if (can_consume_single_newline(child)) { + set_flags(n, AST_FLAG_MATCHES_1NEWLINE); + } + } + break; } @@ -844,12 +1253,29 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) __func__, i, (void *)n->u.alt.n[i]); assert(child->flags & AST_FLAG_UNSATISFIABLE); } else if (res == AST_ANALYSIS_OK) { - all_set_past_always_consuming &= child_env.past_always_consuming; - any_sat = 1; + if (child->type == AST_EXPR_REPEAT + && /*child->u.repeat.min == 0 &&*/ child->u.repeat.max == 0) { + LOG(3 - LOG_ANCHORING, + "%s: ignoring pruned REPEAT node\n", __func__); + } else { + all_set_past_always_consuming &= child_env.past_always_consuming; + any_sat = 1; + } + } else if (res == AST_ANALYSIS_ERROR_UNSUPPORTED) { + assert(child->flags & AST_FLAG_UNSATISFIABLE); + continue; } else { return res; } + if (env->after_end_anchor + && always_consumes_input(child)) { + LOG(3 - LOG_ANCHORING, + "%s: ALT child %zd is after_end_anchor and always_consumes_input => UNSATISFIABLE\n", + __func__, i); + set_flags(child, AST_FLAG_UNSATISFIABLE); + } + if (!(child->flags & AST_FLAG_UNSATISFIABLE)) { /* ignore unsat nodes */ if (!(child->flags & AST_FLAG_ANCHORED_START)) { all_start_anchored = 0; @@ -858,6 +1284,10 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) all_end_anchored = 0; } } + + if (child->flags & AST_FLAG_MATCHES_1NEWLINE) { + set_flags(n, AST_FLAG_MATCHES_1NEWLINE); + } } if (!env->past_always_consuming && all_set_past_always_consuming) { @@ -883,7 +1313,7 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) if (all_end_anchored) { LOG(3 - LOG_ANCHORING, "%s: ALT: all_end_anchored -> ANCHORED_END\n", __func__); - /* FIXME: AST_FLAG_END_NL: need to determine how this interacts + /* TODO: AST_FLAG_END_NL: need to determine how this interacts * with alt nodes. `^(?:(a)\z|(a)$)` */ set_flags(n, AST_FLAG_ANCHORED_END); } @@ -891,8 +1321,18 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) break; } - case AST_EXPR_REPEAT: - res = analysis_iter_anchoring(env, n->u.repeat.e); + case AST_EXPR_REPEAT:; + const int orig_after_end_anchor = env->after_end_anchor; + + if (n->u.repeat.min == 0) { + /* Call the child with a copy of the anchoring env, + * since any subtree repeated zero or more times can + * always be ignored. */ + struct anchoring_env child_env = *env; + res = analysis_iter_anchoring(&child_env, n->u.repeat.e); + } else { + res = analysis_iter_anchoring(env, n->u.repeat.e); + } /* * This logic corresponds to the equivalent case for tombstone nodes @@ -911,7 +1351,7 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) if (n->u.repeat.min == 0) { LOG(3 - LOG_ANCHORING, "%s: REPEAT: UNSATISFIABLE but can be repeated 0 times, ignoring\n", __func__); - n->u.repeat.max = 0; + prune_repeat_node(n); break; } else if (n->u.repeat.min > 0) { set_flags(n, AST_FLAG_UNSATISFIABLE); @@ -920,21 +1360,63 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) return AST_ANALYSIS_UNSATISFIABLE; } } else if (res != AST_ANALYSIS_OK) { - LOG(3 - LOG_ANCHORING, - "%s: REPEAT: analysis on child returned %d\n", __func__, res); - return res; + if (n->u.repeat.min == 0) { + LOG(3 - LOG_ANCHORING, + "%s: REPEAT: analysis on child returned %d, setting repeat count to 0\n", __func__, res); + prune_repeat_node(n); + } else { + LOG(3 - LOG_ANCHORING, + "%s: REPEAT: analysis on child returned %d\n", __func__, res); + return res; + } } - if (n->u.repeat.e->flags & AST_FLAG_ANCHORED_END && n->u.repeat.min > 0) { - /* FIXME: if repeating something that is always - * anchored at the end, repeat.max could be - * capped at 1, but I have not yet found any - * inputs where that change is necessary to - * produce a correct result. */ + if (can_consume_single_newline(n->u.repeat.e)) { + set_flags(n, AST_FLAG_MATCHES_1NEWLINE); + } + + if (n->u.repeat.e->flags & AST_FLAG_ANCHORED_START && n->u.repeat.min > 0) { LOG(3 - LOG_ANCHORING, - "%s: REPEAT: repeating ANCHORED_END subtree >0 times -> ANCHORED_END\n", __func__); - set_flags(n, n->u.repeat.e->flags & END_ANCHOR_FLAG_MASK); + "%s: REPEAT: repeating ANCHORED_START subtree >0 times -> ANCHORED_START\n", __func__); + set_flags(n, AST_FLAG_ANCHORED_START); + n->u.repeat.max = 1; + } + + if (can_consume_single_newline(n->u.repeat.e)) { + set_flags(n, AST_FLAG_MATCHES_1NEWLINE); + } + + /* If the child subtree is anchored at the start and/or end, then this + * node can be repeated at most once. */ + const int child_is_anchored = n->u.repeat.e->flags & (AST_FLAG_ANCHORED_START | AST_FLAG_ANCHORED_END); + if (child_is_anchored) { + if (n->u.repeat.min >= 1) { + LOG(3 - LOG_ANCHORING, + "%s: REPEAT: copying anchor flags from child subtree since we cannot repeat it 0 times\n", __func__); + set_flags(n, n->u.repeat.e->flags & END_ANCHOR_FLAG_MASK); + } + + if (n->u.repeat.min > 1) { + LOG(3 - LOG_ANCHORING, + "%s: REPEAT: anchored, so clamping min repeat to at most once\n", __func__); + n->u.repeat.min = 1; + } + if (n->u.repeat.max > 1) { + LOG(3 - LOG_ANCHORING, + "%s: REPEAT: anchored, so clamping max repeat to at most once\n", __func__); + n->u.repeat.max = 1; + } } + + /* If there's a {0,_} repeat group after an end anchor that always consumes + * input, set its max count to 0. This should be based on the after_end_anchor + * flag value *before* analyzing the repeated subtree. */ + if (orig_after_end_anchor + && always_consumes_input(n) + && n->u.repeat.min == 0) { + prune_repeat_node(n); + } + break; case AST_EXPR_GROUP: @@ -964,6 +1446,10 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) } while(0) PROPAGATE_CHILD_FLAGS("GROUP", n, n->u.group.e); + + if (n->u.group.e->flags & AST_FLAG_MATCHES_1NEWLINE) { + set_flags(n, AST_FLAG_MATCHES_1NEWLINE); + } break; case AST_EXPR_SUBTRACT: @@ -974,6 +1460,15 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) if (n->u.subtract.a->flags & AST_FLAG_ANCHORED_END) { set_flags(n, n->u.subtract.a->flags & END_ANCHOR_FLAG_MASK); } + + if (env->newline_after_end_anchor_state == NAEAS_WOULD_MATCH_ONCE) { + LOG(3 - LOG_ANCHORING, + "%s: SUBTRACT: rejecting non-optional newline match after $ as unsupported\n", + __func__); + set_flags(n, AST_FLAG_UNSATISFIABLE); + return AST_ANALYSIS_ERROR_UNSUPPORTED; + } + if (res != AST_ANALYSIS_OK) { if (res == AST_ANALYSIS_UNSATISFIABLE) { set_flags(n, AST_FLAG_UNSATISFIABLE); @@ -991,6 +1486,10 @@ analysis_iter_anchoring(struct anchoring_env *env, struct ast_expr *n) } return res; } + if (can_consume_single_newline(n->u.repeat.e)) { + set_flags(n, AST_FLAG_MATCHES_1NEWLINE); + } + break; default: @@ -1017,9 +1516,9 @@ analysis_iter_reverse_anchoring(struct anchoring_env *env, struct ast_expr *n) enum ast_analysis_res res; /* - * Second pass, in reverse, specifically checking for end-anchored - * subtrees that are unsatisfiable because they are followed by - * nodes that always consume input. + * Second anchoring pass, in reverse, specifically checking for + * end-anchored subtrees that are unsatisfiable because they are + * followed by nodes that always consume input. * * Also check for subtrees that always consume input appearing * before a start anchor and mark them as unsatisfiable. @@ -1031,6 +1530,14 @@ analysis_iter_reverse_anchoring(struct anchoring_env *env, struct ast_expr *n) switch (n->type) { case AST_EXPR_EMPTY: + if (env->before_start_anchor) { + /* Needed for cases like '()*^' matching "". */ + LOG(3 - LOG_ANCHORING, "%s: skipping EMPTY before ^\n", __func__); + + set_flags(n, AST_FLAG_ANCHORED_START); + } + break; + case AST_EXPR_TOMBSTONE: break; @@ -1044,15 +1551,25 @@ analysis_iter_reverse_anchoring(struct anchoring_env *env, struct ast_expr *n) break; case AST_ANCHOR_END: - /* should already be set during forward pass */ - assert(n->flags & AST_FLAG_ANCHORED_END); + /* Normally this will have been set during the forward pass, + * but if it's inside an unsatisfiable CONCAT node whose earlier + * children casued it to be rejected, forward analysis will not + * have reached it. */ + set_flags(n, AST_FLAG_ANCHORED_END); if (env->followed_by_consuming) { - LOG(3 - LOG_ANCHORING, - "%s: END anchor & followed_by_consuming, setting UNSATISFIABLE\n", - __func__); - set_flags(n, AST_FLAG_UNSATISFIABLE); - return AST_ANALYSIS_UNSATISFIABLE; + if (env->followed_by_consuming_newline) { + LOG(3 - LOG_ANCHORING, + "%s: END anchor & followed_by_consuming, returning UNSUPPORTED\n", + __func__); + return AST_ANALYSIS_ERROR_UNSUPPORTED; + } else { + LOG(3 - LOG_ANCHORING, + "%s: END anchor & followed_by_consuming, setting UNSATISFIABLE\n", + __func__); + set_flags(n, AST_FLAG_UNSATISFIABLE); + return AST_ANALYSIS_UNSATISFIABLE; + } } break; @@ -1076,6 +1593,7 @@ analysis_iter_reverse_anchoring(struct anchoring_env *env, struct ast_expr *n) case AST_EXPR_CONCAT: { size_t i; + int start_anchored = 0; for (i = n->u.concat.count; i > 0; i--) { struct ast_expr *child = n->u.concat.n[i - 1]; assert(child->type != AST_EXPR_TOMBSTONE); @@ -1113,9 +1631,18 @@ analysis_iter_reverse_anchoring(struct anchoring_env *env, struct ast_expr *n) set_flags(n, AST_FLAG_UNSATISFIABLE); } } else if (res != AST_ANALYSIS_OK) { + set_flags(n, AST_FLAG_UNSATISFIABLE); + LOG(3 - LOG_ANCHORING, + "%s: CONCAT: got res of %d, bubbling up\n", __func__, res); return res; } + if (child->flags & AST_FLAG_ANCHORED_START) { + LOG(3 - LOG_ANCHORING, + "%s: CONCAT: child %zd is ANCHORED_START\n", __func__, i); + start_anchored = 1; + } + /* If we were previously not past any nodes that always * consume input, but the child's analysis set that flag, * then copy the setting if the child cannot be skipped. */ @@ -1128,13 +1655,36 @@ analysis_iter_reverse_anchoring(struct anchoring_env *env, struct ast_expr *n) env->followed_by_consuming = 1; } - if (!env->before_start_anchor && child_env.before_start_anchor - && !is_nullable(child)) { + if (!env->followed_by_consuming_newline && + (child_env.followed_by_consuming_newline + || child->flags & AST_FLAG_MATCHES_1NEWLINE)) { + LOG(3 - LOG_ANCHORING, + "%s: setting followed_by_consuming_newline due to child %p's analysis\n", + __func__, (void *)child); + env->followed_by_consuming_newline = 1; + } + + if (start_anchored) { + LOG(3 - LOG_ANCHORING, + "%s: earliest unskipped child had ANCHORED_START, bubbling up\n", + __func__); + set_flags(n, AST_FLAG_ANCHORED_START); + } + + if (!env->before_start_anchor && child_env.before_start_anchor + && !is_nullable(child)) { LOG(3 - LOG_ANCHORING, "%s: setting before_start_anchor due to child %p's analysis\n", __func__, (void *)child); env->before_start_anchor = 1; } + + if (!env->followed_by_consuming_newline && child_env.followed_by_consuming_newline) { + LOG(3 - LOG_ANCHORING, + "%s: setting followed_by_consuming_newline due to child %p's analysis\n", + __func__, (void *)child); + env->followed_by_consuming_newline = 1; + } } break; @@ -1143,6 +1693,7 @@ analysis_iter_reverse_anchoring(struct anchoring_env *env, struct ast_expr *n) case AST_EXPR_ALT: { int any_sat = 0; int all_set_followed_by_consuming = 1; + int any_set_followed_by_consuming_newline = 0; int all_set_before_start_anchor = 1; assert(n->u.alt.count > 0); @@ -1166,9 +1717,22 @@ analysis_iter_reverse_anchoring(struct anchoring_env *env, struct ast_expr *n) __func__, i, (void *)n->u.alt.n[i]); assert(child->flags & AST_FLAG_UNSATISFIABLE); } else if (res == AST_ANALYSIS_OK) { - all_set_followed_by_consuming &= child_env.followed_by_consuming; - all_set_before_start_anchor &= child_env.before_start_anchor; - any_sat = 1; + if (0 && child->type == AST_EXPR_REPEAT + && child->u.repeat.max == 0) { + LOG(3 - LOG_ANCHORING, + "%s: ignoring pruned REPEAT node\n", __func__); + } else { + all_set_followed_by_consuming &= child_env.followed_by_consuming; + all_set_before_start_anchor &= child_env.before_start_anchor; + + any_set_followed_by_consuming_newline |= child_env.followed_by_consuming_newline; + + any_sat = 1; + } + } else if (res == AST_ANALYSIS_ERROR_UNSUPPORTED) { + LOG(3 - LOG_ANCHORING, "%s: got res of UNSUPPORTED, bubbling up\n", __func__); + assert(child->flags & AST_FLAG_UNSATISFIABLE); + continue; } else { return res; } @@ -1181,11 +1745,19 @@ analysis_iter_reverse_anchoring(struct anchoring_env *env, struct ast_expr *n) env->followed_by_consuming = 1; } + if (!env->followed_by_consuming_newline && any_set_followed_by_consuming_newline) { + LOG(3 - LOG_ANCHORING, + "%s: ALT: any_set_followed_by_consuming_newline -> setting env->followed_by_consuming_newline for feature PCRE rejection\n", + __func__); + env->followed_by_consuming_newline = 1; + } + if (!env->before_start_anchor && all_set_before_start_anchor) { LOG(3 - LOG_ANCHORING, - "%s: ALT: all_set_before_start_anchor -> setting env->before_start_anchor\n", + "%s: ALT: all_set_before_start_anchor -> setting env->before_start_anchor and ANCHORED_START\n", __func__); env->before_start_anchor = 1; + set_flags(n, AST_FLAG_ANCHORED_START); } /* An ALT group is only unsatisfiable if they ALL are. */ @@ -1200,12 +1772,21 @@ analysis_iter_reverse_anchoring(struct anchoring_env *env, struct ast_expr *n) } case AST_EXPR_REPEAT: - res = analysis_iter_reverse_anchoring(env, n->u.repeat.e); + if (n->u.repeat.min == 0) { + /* Call the child with a copy of the anchoring env, + * since any subtree repeated zero or more times can + * always be ignored. */ + struct anchoring_env child_env = *env; + res = analysis_iter_reverse_anchoring(&child_env, n->u.repeat.e); + } else { + res = analysis_iter_reverse_anchoring(env, n->u.repeat.e); + } + if (res == AST_ANALYSIS_UNSATISFIABLE) { if (n->u.repeat.min == 0) { LOG(3 - LOG_ANCHORING, "%s: REPEAT: UNSATISFIABLE but can be repeated 0 times, ignoring\n", __func__); - n->u.repeat.max = 0; /* skip */ + prune_repeat_node(n); break; } else if (n->u.repeat.min > 0) { LOG(3 - LOG_ANCHORING, @@ -1218,6 +1799,18 @@ analysis_iter_reverse_anchoring(struct anchoring_env *env, struct ast_expr *n) "%s: REPEAT: analysis on child returned %d\n", __func__, res); return res; } + + /* If there's a repeat group that always matches anything before a start anchor, + * set its max count to 0. */ + if (env->before_start_anchor + && is_nullable(n) + && always_consumes_input(n)) { + LOG(3 - LOG_ANCHORING, + "%s: REPEAT: repeated group that consumes input before ^, setting max count to 0\n", + __func__); + prune_repeat_node(n); + } + break; case AST_EXPR_GROUP: @@ -1231,6 +1824,14 @@ analysis_iter_reverse_anchoring(struct anchoring_env *env, struct ast_expr *n) } return res; } + + if (n->u.group.e->flags & ANCHOR_FLAG_MASK) { + LOG(3 - LOG_ANCHORING, + "%s: bubbling up anchoring flags from %p to %p\n", + __func__, (void *)n->u.group.e, (void *)n); + set_flags(n, n->u.group.e->flags & ANCHOR_FLAG_MASK); + } + break; case AST_EXPR_SUBTRACT: @@ -1269,6 +1870,10 @@ analysis_iter_reverse_anchoring(struct anchoring_env *env, struct ast_expr *n) return AST_ANALYSIS_UNSATISFIABLE; } + if (n->flags & AST_FLAG_CAN_CONSUME && matches_newline(n)) { + env->followed_by_consuming_newline = 1; + } + return AST_ANALYSIS_OK; } @@ -1302,7 +1907,7 @@ assign_firsts(struct ast_expr *n) assign_firsts(child); if (can_consume_input(child) || (child->flags & AST_FLAG_ANCHORED_START)) { - break; + if (!is_nullable(child)) { break; } } } break; @@ -1319,9 +1924,7 @@ assign_firsts(struct ast_expr *n) } case AST_EXPR_REPEAT: - if (n->u.repeat.max > 0) { - set_flags(n, AST_FLAG_FIRST); - } + set_flags(n, AST_FLAG_FIRST); /* Don't recurse. * @@ -1333,7 +1936,7 @@ assign_firsts(struct ast_expr *n) * subexpression is compiled, the links to the global self-loop * are created, which the REPEAT node then copies. * - * It probably makes sense to not go further + * FIXME: needs tests. It probably makes sense to not go further * here because the top layer of the repeated section * should only link to the global start once. */ @@ -1384,7 +1987,7 @@ assign_lasts(struct ast_expr *n) struct ast_expr *child = n->u.concat.n[i - 1]; assign_lasts(child); if (can_consume_input(child) || (child->flags & AST_FLAG_ANCHORED_END)) { - break; + if (!is_nullable(child)) { break; } } } @@ -1402,9 +2005,7 @@ assign_lasts(struct ast_expr *n) } case AST_EXPR_REPEAT: - if (n->u.repeat.max > 0) { - set_flags(n, AST_FLAG_LAST); - } + set_flags(n, AST_FLAG_LAST); /* Don't recurse. * @@ -1416,7 +2017,7 @@ assign_lasts(struct ast_expr *n) * subexpression is compiled, the links to the global self-loop * are created, which the REPEAT node then copies. * - * It probably makes sense to not go further + * FIXME: needs tests. It probably makes sense to not go further * here because the top layer of the repeated section * should only link to the global start once. */ @@ -1437,6 +2038,461 @@ assign_lasts(struct ast_expr *n) } } +static enum ast_analysis_res +analysis_iter_captures(struct capture_env *env, struct ast_expr *n) +{ + enum ast_analysis_res res; + + switch (n->type) { + case AST_EXPR_EMPTY: + case AST_EXPR_TOMBSTONE: + break; + + case AST_EXPR_ANCHOR: + if (env->use_captures && n->u.anchor.type == AST_ANCHOR_END && !n->u.anchor.is_end_nl) { + set_flags(n, AST_FLAG_UNSATISFIABLE); + return AST_ANALYSIS_ERROR_UNSUPPORTED; + } + break; + + case AST_EXPR_LITERAL: + case AST_EXPR_CODEPOINT: + case AST_EXPR_RANGE: + break; + + case AST_EXPR_CONCAT: { + size_t i; + + for (i = 0; i < n->u.concat.count; i++) { + res = analysis_iter_captures(env, n->u.concat.n[i]); + if (res != AST_ANALYSIS_OK) { + return res; + } + } + + break; + } + + case AST_EXPR_ALT: { + size_t i; + + for (i = 0; i < n->u.alt.count; i++) { + res = analysis_iter_captures(env, n->u.alt.n[i]); + if (res != AST_ANALYSIS_OK) { + return res; + } + } + + break; + } + + case AST_EXPR_REPEAT: { + res = analysis_iter_captures(env, n->u.repeat.e); + if (res != AST_ANALYSIS_OK) { + return res; + } + +/* Set this to 1 when running the fuzzer, so that it ignores + * uninteresting failures from regexes like '.{1000000}' that use + * repetition to hit memory limits. + * + * This should be set by the build system when building for fuzzing. */ +#ifndef FUZZING_LIMITS +#define FUZZING_LIMITS 0 +#endif + +#if FUZZING_LIMITS + if ((n->u.repeat.max != AST_COUNT_UNBOUNDED && n->u.repeat.max >= 10) + || (n->u.repeat.min >= 10)) { + fprintf(stderr, "%s: rejecting regex with {count} >= 10 (%u)\n", + __func__, n->u.repeat.max); + return AST_ANALYSIS_ERROR_MEMORY; + } +#endif + + break; + } + + case AST_EXPR_GROUP: + if (env->max_capture_id == AST_NO_MAX_CAPTURE_ID + || n->u.group.id > env->max_capture_id) { + env->max_capture_id = n->u.group.id; + } + + res = analysis_iter_captures(env, n->u.group.e); + if (res != AST_ANALYSIS_OK) { + return res; + } + break; + + case AST_EXPR_SUBTRACT: + res = analysis_iter_captures(env, n->u.subtract.a); + if (res != AST_ANALYSIS_OK) { + return res; + } + + res = analysis_iter_captures(env, n->u.subtract.b); + if (res != AST_ANALYSIS_OK) { + return res; + } + break; + + default: + assert(!"unreached"); + } + + return AST_ANALYSIS_OK; +} + +struct pincer_anchors_env { + unsigned int always_consumes_or_anchored_start:1; + unsigned int always_consumes_or_anchored_end:1; + + unsigned int alloc_fail:1; + size_t used; + size_t ceil; + struct ast_expr **nodes; +}; + +enum analysis_direction { AD_FORWARD, AD_BACKWARD }; + +static int +subtree_contains_anchor(const struct ast_expr *expr, enum ast_anchor_type type) +{ + switch (expr->type) { + case AST_EXPR_EMPTY: + case AST_EXPR_LITERAL: + case AST_EXPR_CODEPOINT: + case AST_EXPR_RANGE: + case AST_EXPR_TOMBSTONE: + return 0; + + case AST_EXPR_ANCHOR: + return expr->u.anchor.type == type; + + case AST_EXPR_CONCAT: + for (size_t i = 0; i < expr->u.concat.count; i++) { + if (subtree_contains_anchor(expr->u.concat.n[i], type)) { + return 1; + } + } + return 0; + case AST_EXPR_ALT: + for (size_t i = 0; i < expr->u.alt.count; i++) { + if (subtree_contains_anchor(expr->u.alt.n[i], type)) { + return 1; + } + } + return 0; + + case AST_EXPR_REPEAT: + if (expr->u.repeat.min > 0) { + return subtree_contains_anchor(expr->u.repeat.e, type); + } + return 0; + + case AST_EXPR_GROUP: + return subtree_contains_anchor(expr->u.group.e, type); + case AST_EXPR_SUBTRACT: + return subtree_contains_anchor(expr->u.subtract.a, type); + + default: + assert(!"match fail"); + } +} + +static void +pincer_anchors_record_anchor(struct pincer_anchors_env *env, struct ast_expr *n) +{ + if (env->used == env->ceil) { + const size_t nceil = (env->ceil == 0 ? 8 : 2*env->ceil); + struct ast_expr **nnodes = realloc(env->nodes, + nceil * sizeof(nnodes[0])); + if (nnodes == NULL) { + env->alloc_fail = 1; + return; + } + env->ceil = nceil; + env->nodes = nnodes; + } + + env->nodes[env->used++] = n; +} + +/* Check for regexes with out-of-order start and end anchors, combined + * with alternate subtrees that must consume input. A minimal example of + * this is `(a|$)(b|^)` (tests/pcre-anchor/in92.re) -- Either an 'a' is + * consumed or matching must be at the end of input (with optional + * '\n'), followed by either consuming a 'b' or being at the start of + * input. This should only match "", "\n", and "ab", + * but because the anchors are out of order it takes a further analysis + * pass and extra AST_FLAG_CONSTRAINED_AT_{START,END} node flags to + * ensure that neither is linked to the unanchored start/end loops. */ +static void +analysis_iter_pincer_anchors(struct ast_expr *expr, struct pincer_anchors_env *env, enum analysis_direction dir) +{ + LOG(3 - LOG_PINCER_ANCHORS, "%s: expr %p %s\n", + __func__, (void *)expr, ast_node_type_name(expr->type)); + switch (expr->type) { + case AST_EXPR_EMPTY: + case AST_EXPR_LITERAL: + case AST_EXPR_CODEPOINT: + return; + + case AST_EXPR_ANCHOR: + if (expr->u.anchor.type == AST_ANCHOR_START) { + LOG(3 - LOG_PINCER_ANCHORS, "%s: expr %p ANCHOR_START, flags start %d, end %d, dir %s\n", + __func__, (void *)expr, + env->always_consumes_or_anchored_start, + env->always_consumes_or_anchored_end, + dir == AD_FORWARD ? "FORWARD" : "BACKWARD"); + if (dir == AD_FORWARD && env->always_consumes_or_anchored_end) { + LOG(3 - LOG_PINCER_ANCHORS, "%s: expr %p ANCHOR_START: recording CONSTRAINED_AT_END\n", + __func__, (void *)expr); + /* record for later -- these should only be set if there's an anchored *pair* */ + pincer_anchors_record_anchor(env, expr); + } + } else if (expr->u.anchor.type == AST_ANCHOR_END) { + LOG(3 - LOG_PINCER_ANCHORS, "%s: expr %p ANCHOR_END, flags start %d, end %d, dir %s\n", + __func__, (void *)expr, + env->always_consumes_or_anchored_start, + env->always_consumes_or_anchored_end, + dir == AD_FORWARD ? "FORWARD" : "BACKWARD"); + if (dir == AD_BACKWARD && env->always_consumes_or_anchored_start) { + LOG(3 - LOG_PINCER_ANCHORS, "%s: expr %p ANCHOR_END: recording CONSTRAINED_AT_START\n", + __func__, (void *)expr); + pincer_anchors_record_anchor(env, expr); + } + } + break; + + case AST_EXPR_CONCAT: + if (dir == AD_FORWARD) { + for (size_t i = 0; i < expr->u.concat.count; i++) { + analysis_iter_pincer_anchors(expr->u.concat.n[i], env, dir); + } + } else if (dir == AD_BACKWARD) { + for (size_t i = expr->u.concat.count; i > 0; i--) { + analysis_iter_pincer_anchors(expr->u.concat.n[i - 1], env, dir); + } + } + break; + case AST_EXPR_ALT: + { + int all_always_consume_or_anchored_start = 1; + int all_always_consume_or_anchored_end = 1; + int has_always_consuming = 0; + int has_non_consuming_anchored_start = 0; + int has_non_consuming_anchored_end = 0; + + if (dir == AD_FORWARD) { + for (size_t i = 0; i < expr->u.alt.count; i++) { + struct ast_expr *child = expr->u.alt.n[i]; + LOG(3 - LOG_PINCER_ANCHORS, + "%s: ALT child %zu/%zu : %p\n", + __func__, i, expr->u.alt.count, (void *)child); + if (is_unsatisfiable_or_skipped(child)) { continue; } + + analysis_iter_pincer_anchors(child, env, dir); + if (child->flags & AST_FLAG_ALWAYS_CONSUMES) { + has_always_consuming = 1; + } else { + if (child->flags & AST_FLAG_ANCHORED_START + && subtree_contains_anchor(child, AST_ANCHOR_START)) { + has_non_consuming_anchored_start = 1; + } else { + all_always_consume_or_anchored_start = 0; + } + + if (child->flags & AST_FLAG_ANCHORED_END + && subtree_contains_anchor(child, AST_ANCHOR_END)) { + has_non_consuming_anchored_end = 1; + } else { + all_always_consume_or_anchored_end = 0; + } + } + LOG(3 - LOG_PINCER_ANCHORS, + "%s: flags all_acoa_start %d, all_acoa_end %d, hac %d, hnca_start %d, hnca_end %d\n", + __func__, + all_always_consume_or_anchored_start, + all_always_consume_or_anchored_end, + has_always_consuming, + has_non_consuming_anchored_start, + has_non_consuming_anchored_end); + } + } else if (dir == AD_BACKWARD) { + for (size_t i = expr->u.alt.count; i > 0; i--) { + struct ast_expr *child = expr->u.alt.n[i - 1]; + LOG(3 - LOG_PINCER_ANCHORS, + "%s: ALT child %zu/%zu : %p\n", + __func__, i, expr->u.alt.count, (void *)child); + if (is_unsatisfiable_or_skipped(child)) { continue; } + + analysis_iter_pincer_anchors(child, env, dir); + if (child->flags & AST_FLAG_ALWAYS_CONSUMES) { + has_always_consuming = 1; + } else { + if (child->flags & AST_FLAG_ANCHORED_START + && subtree_contains_anchor(child, AST_ANCHOR_START)) { + has_non_consuming_anchored_start = 1; + } else { + all_always_consume_or_anchored_start = 0; + } + + if (child->flags & AST_FLAG_ANCHORED_END + && subtree_contains_anchor(child, AST_ANCHOR_END)) { + has_non_consuming_anchored_end = 1; + } else { + all_always_consume_or_anchored_end = 0; + } + } + LOG(3 - LOG_PINCER_ANCHORS, + "%s: flags all_acoa_start %d, all_acoa_end %d, hac %d, hnca_start %d, hnca_end %d\n", + __func__, + all_always_consume_or_anchored_start, + all_always_consume_or_anchored_end, + has_always_consuming, + has_non_consuming_anchored_start, + has_non_consuming_anchored_end); + } + } + + if (has_always_consuming + && has_non_consuming_anchored_start + && all_always_consume_or_anchored_start) { + LOG(3 - LOG_PINCER_ANCHORS, "%s: ALT %p: setting env->always_consumes_or_anchored_start\n", + __func__, (void *)expr); + env->always_consumes_or_anchored_start = 1; + LOG(3 - LOG_PINCER_ANCHORS, "%s: %p: setting AST_FLAG_CONSTRAINED_AT_START\n", + __func__, (void *)expr); + set_flags(expr, AST_FLAG_CONSTRAINED_AT_START); + } + if (has_always_consuming + && has_non_consuming_anchored_end + && all_always_consume_or_anchored_end) { + LOG(3 - LOG_PINCER_ANCHORS, "%s: ALT %p: setting env->always_consumes_or_anchored_end\n", + __func__, (void *)expr); + env->always_consumes_or_anchored_end = 1; + LOG(3 - LOG_PINCER_ANCHORS, "%s: %p: setting AST_FLAG_CONSTRAINED_AT_END\n", + __func__, (void *)expr); + set_flags(expr, AST_FLAG_CONSTRAINED_AT_END); + } + + break; + } + + case AST_EXPR_REPEAT: + analysis_iter_pincer_anchors(expr->u.repeat.e, env, dir); + break; + + case AST_EXPR_GROUP: + analysis_iter_pincer_anchors(expr->u.group.e, env, dir); + break; + case AST_EXPR_SUBTRACT: + /* It could just return here, as long as we aren't + * subtracting anything besides character sets. */ + analysis_iter_pincer_anchors(expr->u.subtract.a, env, dir); + break; + + case AST_EXPR_RANGE: + /* nothing to do */ + break; + + case AST_EXPR_TOMBSTONE: + break; + default: + assert(!"match fail"); + } +} + +static void +analysis_pincer_anchors_apply_flags_iter(struct ast_expr *expr) +{ + LOG(3 - LOG_PINCER_ANCHORS, "%s: expr %p %s\n", + __func__, (void *)expr, ast_node_type_name(expr->type)); + switch (expr->type) { + case AST_EXPR_EMPTY: + case AST_EXPR_LITERAL: + case AST_EXPR_CODEPOINT: + case AST_EXPR_ANCHOR: + break; + + case AST_EXPR_CONCAT: + for (size_t i = 0; i < expr->u.concat.count; i++) { + analysis_pincer_anchors_apply_flags_iter(expr->u.concat.n[i]); + } + break; + case AST_EXPR_ALT: + for (size_t i = 0; i < expr->u.alt.count; i++) { + analysis_pincer_anchors_apply_flags_iter(expr->u.alt.n[i]); + } + break; + + case AST_EXPR_REPEAT: + analysis_pincer_anchors_apply_flags_iter(expr->u.repeat.e); + break; + + case AST_EXPR_GROUP: + analysis_pincer_anchors_apply_flags_iter(expr->u.group.e); + break; + + case AST_EXPR_SUBTRACT: + /* it could just return here as long as we aren't + * subtracting anything besides character sets */ + analysis_pincer_anchors_apply_flags_iter(expr->u.subtract.a); + analysis_pincer_anchors_apply_flags_iter(expr->u.subtract.b); + break; + + case AST_EXPR_RANGE: + /* nothing to do */ + break; + + case AST_EXPR_TOMBSTONE: + break; + default: + assert(!"match fail"); + } +} + +static enum ast_analysis_res +analysis_pincer_anchors_apply_flags(struct ast_expr *expr, struct pincer_anchors_env *env) +{ + if (env->alloc_fail) { + return AST_ANALYSIS_ERROR_MEMORY; + } + + analysis_pincer_anchors_apply_flags_iter(expr); + + int has_start = 0; + int has_end = 0; + for (size_t i = 0; i < env->used; i++) { + const struct ast_expr *n = env->nodes[i]; + assert(n->type == AST_EXPR_ANCHOR); + if (n->u.anchor.type == AST_ANCHOR_START) { + has_start = 1; + } else if (n->u.anchor.type == AST_ANCHOR_END) { + has_end = 1; + } + } + + if (!has_start || !has_end) { + /* skip: only meaningful when constrained on both sides */ + return AST_ANALYSIS_OK; + } + + for (size_t i = 0; i < env->used; i++) { + struct ast_expr *n = env->nodes[i]; + assert(n->type == AST_EXPR_ANCHOR); + if (n->u.anchor.type == AST_ANCHOR_START) { + set_flags(n, AST_FLAG_CONSTRAINED_AT_END); + } else if (n->u.anchor.type == AST_ANCHOR_END) { + set_flags(n, AST_FLAG_CONSTRAINED_AT_START); + } + } + + return AST_ANALYSIS_OK; +} + enum ast_analysis_res ast_analysis(struct ast *ast, enum re_flags flags) { @@ -1449,14 +2505,26 @@ ast_analysis(struct ast *ast, enum re_flags flags) assert(ast->expr != NULL); /* - * First pass -- track nullability, clean up some artifacts from - * parsing. + * First pass -- track nullability, which subtrees can/always consume + * input, and clean up some artifacts from parsing. */ res = analysis_iter(ast->expr); if (res != AST_ANALYSIS_OK) { return res; } + /* Next pass, check for capture IDs. */ + { + struct capture_env env = { .max_capture_id = AST_NO_MAX_CAPTURE_ID }; + env.use_captures = !(flags & RE_NOCAPTURE); + + res = analysis_iter_captures(&env, ast->expr); + if (res != AST_ANALYSIS_OK) { + return res; + } + ast->max_capture_id = env.max_capture_id; + } + /* * Next pass: set anchoring, now that nullability info from * the first pass is in place and some other things have been @@ -1465,28 +2533,62 @@ ast_analysis(struct ast *ast, enum re_flags flags) */ { /* first anchoring analysis pass, sweeping forward */ - struct anchoring_env env = { .re_flags = flags }; + struct anchoring_env env = { + .re_flags = flags, + .newline_after_end_anchor_state = NAEAS_NONE, + }; res = analysis_iter_anchoring(&env, ast->expr); if (res != AST_ANALYSIS_OK) { return res; } /* another anchoring analysis pass, sweeping backward */ res = analysis_iter_reverse_anchoring(&env, ast->expr); if (res != AST_ANALYSIS_OK) { return res; } + } - /* - * Next passes, mark all nodes in a first and/or last - * position. This is informed by the anchoring flags, so - * that needs to happen first. - */ - assign_firsts(ast->expr); - assign_lasts(ast->expr); - - ast->has_unanchored_start = (analysis_iter_unanchored_start(ast->expr) != UA_NO); - ast->has_unanchored_end = (analysis_iter_unanchored_end(ast->expr) != UA_NO); - LOG(2 - LOG_UNANCHORED_FLAGS, - "%s: has_unanchored_start %d, has_unanchored_end %d\n", - __func__, ast->has_unanchored_start, ast->has_unanchored_end); + /* + * Next passes, mark all nodes in a first and/or last + * position. This is informed by the anchoring flags, so + * that needs to happen first. + */ + assign_firsts(ast->expr); + assign_lasts(ast->expr); + + /* Next pass, mark some cases that need special handling + * due to repetition. For example, with cases like + * ^((x?))*$ the inner capture will always need to repeat + * one more time to match () after any 'x's. + * + * This needs to happen after the anchoring passes. */ + res = analysis_iter_repetition(ast->expr, NULL, 0, NULL); + if (res != AST_ANALYSIS_OK) { return res; } + + ast->has_unanchored_start = (analysis_iter_unanchored_start(ast->expr) != ANCHOR_ANALYSIS_ANCHORED); + ast->has_unanchored_end = (analysis_iter_unanchored_end(ast->expr) != ANCHOR_ANALYSIS_ANCHORED); + + /* Based on all analysis thus far, check for another obscure + * edge case that is otherwise difficult to handle properly. + * This may set an extra flag on start/end anchor nodes. */ + { + struct pincer_anchors_env pincer_anchors_env = { .ceil = 0, }; + LOG(3 - LOG_PINCER_ANCHORS, "%s: -> analysis_iter_pincer_anchors FORWARD\n", __func__); + analysis_iter_pincer_anchors(ast->expr, &pincer_anchors_env, AD_FORWARD); + + LOG(3 - LOG_PINCER_ANCHORS, "%s: -> analysis_iter_pincer_anchors BACKWARD\n", __func__); + pincer_anchors_env.always_consumes_or_anchored_start = 0; + pincer_anchors_env.always_consumes_or_anchored_end = 0; + analysis_iter_pincer_anchors(ast->expr, &pincer_anchors_env, AD_BACKWARD); + + res = analysis_pincer_anchors_apply_flags(ast->expr, &pincer_anchors_env); + free(pincer_anchors_env.nodes); + + if (res != AST_ANALYSIS_OK) { + return res; + } } + LOG(2 - LOG_UNANCHORED_FLAGS, + "%s: has_unanchored_start %d, has_unanchored_end %d\n", + __func__, ast->has_unanchored_start, ast->has_unanchored_end); + return res; } diff --git a/src/libre/ast_analysis.h b/src/libre/ast_analysis.h index f9ec8ebbb..af3c5bf97 100644 --- a/src/libre/ast_analysis.h +++ b/src/libre/ast_analysis.h @@ -12,8 +12,6 @@ */ struct ast; -struct re_err; -struct fsm_options; enum ast_analysis_res { AST_ANALYSIS_OK, @@ -30,7 +28,8 @@ enum ast_analysis_res { AST_ANALYSIS_UNSATISFIABLE, AST_ANALYSIS_ERROR_NULL = -1, - AST_ANALYSIS_ERROR_MEMORY = -2 + AST_ANALYSIS_ERROR_MEMORY = -2, + AST_ANALYSIS_ERROR_UNSUPPORTED = -3, }; enum ast_analysis_res diff --git a/src/libre/ast_compile.c b/src/libre/ast_compile.c index 502faf8b4..e99814e6c 100644 --- a/src/libre/ast_compile.c +++ b/src/libre/ast_compile.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -15,22 +16,25 @@ #include #include +#include #include #include #include +#include +#include + #include "class.h" #include "ast.h" #include "ast_compile.h" +#include "re_capvm_compile.h" +#include "libfsm/capture.h" #include "libfsm/internal.h" /* XXX */ #define LOG_LINKAGE 0 - -#if LOG_LINKAGE -#include "print.h" -#endif +#define LOG_TRAMPOLINE 0 enum link_side { LINK_START, @@ -44,21 +48,33 @@ enum link_side { * Use the passed in start/end states (x and y) * * - LINK_GLOBAL - * Link to the global start/end state (env->start or env->end), + * Link to the global start/end state (env->start_inner or env->end_inner), * because this node has a ^ or $ anchor * * - LINK_GLOBAL_SELF_LOOP * Link to the unanchored self loop adjacent to the start/end - * state (env->start_any_loop or env->end_any_loop), because + * states (env->start_any_inner or env->end_any_inner), because * this node is in a FIRST or LAST position, but unanchored. + * + * - LINK_SKIP + * This is used to prune linkage when anchoring leads to + * unsatisfiability. */ enum link_types { LINK_TOP_DOWN, LINK_GLOBAL, LINK_GLOBAL_SELF_LOOP, + LINK_SKIP, }; +/* Call stack for AST -> NFA conversion. */ +#define DEF_COMP_STACK_CEIL 4 + +/* Special case: Don't link an epsilon edge */ +#define SKIP_EPSILON ((fsm_state_t)-1) + struct comp_env { + const struct fsm_alloc *alloc; struct fsm *fsm; enum re_flags re_flags; struct re_err *err; @@ -72,21 +88,90 @@ struct comp_env { * Also, some states in a first/last context need to link * directly to the overall start/end states, either in * place of or along with the adjacent states. + * + * The inner start and end states are considered inside of + * match group 0, outer are not. */ - fsm_state_t start; - fsm_state_t end; + fsm_state_t start_outer; + fsm_state_t start_inner; fsm_state_t start_any_loop; + fsm_state_t start_any_inner; + int have_start_any_loop; + + /* States leading to the end, with and without an unanchored + * `.*` loop that consumes any trailing characters. */ + fsm_state_t end_inner; + fsm_state_t end_outer; + fsm_state_t end_nl_inner; fsm_state_t end_any_loop; - fsm_state_t end_nl; - int has_start_any_loop; + fsm_state_t end_any_inner; int has_end_any_loop; - int has_end_nl; + int has_end_nl_inner; + + /* bitset for active capture IDs */ + uint64_t *active_capture_ids; + long max_capture_id; /* upper bound */ + + /* Evaluation stack */ + struct comp_stack { + size_t ceil; + size_t depth; /* 0 -> empty */ + + struct comp_stack_frame { + struct ast_expr *n; + fsm_state_t x; + fsm_state_t y; + unsigned step; + + union { + struct { + fsm_state_t link; + } concat; + struct { + unsigned count; + } alt; + struct { + struct fsm_subgraph subgraph; + fsm_state_t na; + fsm_state_t nz; + } repeat; + } u; + } *frames; + } stack; }; static int -comp_iter(struct comp_env *env, - fsm_state_t x, fsm_state_t y, - struct ast_expr *n, const struct ast_expr *parent); +comp_iter(struct comp_env *env, fsm_state_t x, const struct ast *ast); + +static int +eval_stack_frame(struct comp_env *env); + +static int +eval_EMPTY(struct comp_env *env); +static int +eval_CONCAT(struct comp_env *env); +static int +eval_ALT(struct comp_env *env); +static int +eval_LITERAL(struct comp_env *env); +static int +eval_CODEPOINT(struct comp_env *env); +static int +eval_REPEAT(struct comp_env *env); +static int +eval_GROUP(struct comp_env *env); +static int +eval_ANCHOR(struct comp_env *env); +static int +eval_SUBTRACT(struct comp_env *env); +static int +eval_RANGE(struct comp_env *env); +static int +eval_TOMBSTONE(struct comp_env *env); + +static int +compile_capvm_program_for_stack_end_states(struct comp_env *env, + const struct ast *ast, uint32_t *prog_id); static int utf8(uint32_t cp, char c[]) @@ -188,13 +273,17 @@ fsm_unionxy(struct fsm *a, struct fsm *b, fsm_state_t x, fsm_state_t y) static struct fsm * expr_compile(struct ast_expr *e, enum re_flags flags, - const struct fsm_options *opt, struct re_err *err) + const struct fsm_alloc *alloc, + struct re_err *err) { struct ast ast; ast.expr = e; + ast.max_capture_id = 0; + ast.has_unanchored_start = 0; + ast.has_unanchored_end = 0; - return ast_compile(&ast, flags, opt, err); + return ast_compile(&ast, flags, alloc, err); } static int @@ -208,11 +297,11 @@ addedge_literal(struct comp_env *env, enum re_flags re_flags, assert(to < env->fsm->statecount); if (re_flags & RE_ICASE) { - if (!fsm_addedge_literal(fsm, from, to, tolower((unsigned char) c))) { + if (!fsm_addedge_literal(fsm, from, to, (char)tolower((unsigned char) c))) { return 0; } - - if (!fsm_addedge_literal(fsm, from, to, toupper((unsigned char) c))) { + + if (!fsm_addedge_literal(fsm, from, to, (char)toupper((unsigned char) c))) { return 0; } } else { @@ -220,38 +309,54 @@ addedge_literal(struct comp_env *env, enum re_flags re_flags, return 0; } } - + return 1; } static int intern_start_any_loop(struct comp_env *env) { - fsm_state_t loop; + fsm_state_t loop, inner; assert(env != NULL); - if (env->has_start_any_loop) { + if (env->have_start_any_loop) { return 1; } assert(~env->re_flags & RE_ANCHORED); - assert(env->start < env->fsm->statecount); + assert(env->start_outer < env->fsm->statecount); if (!fsm_addstate(env->fsm, &loop)) { return 0; } + if (!fsm_addstate(env->fsm, &inner)) { + return 0; + } + +#if LOG_LINKAGE + fprintf(stderr, "%s: start_any: loop %d, inner: %d\n", __func__, loop, inner); +#endif + if (!fsm_addedge_any(env->fsm, loop, loop)) { return 0; } - if (!fsm_addedge_epsilon(env->fsm, env->start, loop)) { + if (!fsm_addedge_epsilon(env->fsm, env->start_outer, loop)) { + return 0; + } + if (!fsm_addedge_epsilon(env->fsm, loop, inner)) { return 0; } env->start_any_loop = loop; - env->has_start_any_loop = 1; + env->start_any_inner = inner; + env->have_start_any_loop = 1; + + if (env->fsm->linkage_info) { + env->fsm->linkage_info->unanchored_start_loop = loop; + } return 1; } @@ -259,7 +364,7 @@ intern_start_any_loop(struct comp_env *env) static int intern_end_any_loop(struct comp_env *env) { - fsm_state_t loop; + fsm_state_t loop, inner; assert(env != NULL); @@ -268,60 +373,86 @@ intern_end_any_loop(struct comp_env *env) } assert(~env->re_flags & RE_ANCHORED); - assert(env->end < env->fsm->statecount); + assert(env->end_outer < env->fsm->statecount); if (!fsm_addstate(env->fsm, &loop)) { return 0; } + if (!fsm_addstate(env->fsm, &inner)) { + return 0; + } + +#if LOG_LINKAGE + fprintf(stderr, "%s: end_any: %d, inner: %d\n", __func__, loop, inner); +#endif if (!fsm_addedge_any(env->fsm, loop, loop)) { return 0; } - if (!fsm_addedge_epsilon(env->fsm, loop, env->end)) { + if (!fsm_addedge_epsilon(env->fsm, inner, loop)) { + return 0; + } + if (!fsm_addedge_epsilon(env->fsm, loop, env->end_outer)) { return 0; } env->end_any_loop = loop; + env->end_any_inner = inner; env->has_end_any_loop = 1; + if (env->fsm->linkage_info != NULL) { + env->fsm->linkage_info->unanchored_end_loop = loop; + env->fsm->linkage_info->end_any_inner = env->end_any_inner; + } + return 1; } static int intern_end_nl(struct comp_env *env) { - /* PCRE's end anchor $ matches a single optional newline. + /* PCRE's end anchor $ matches a single optional newline, + * which should exist outside of match group 0. * - * Intern states for a `\n?` that links to the global end. */ + * Intern states for a `\n?` that links to */ assert(env != NULL); - if (env->has_end_nl) { + if (env->has_end_nl_inner) { return 1; } assert(~env->re_flags & RE_ANCHORED); assert(env->re_flags & RE_END_NL); - assert(env->end < env->fsm->statecount); + assert(~env->re_flags & RE_END_NL_DISABLE); + assert(env->end_outer < env->fsm->statecount); - fsm_state_t end_nl; - if (!fsm_addstate(env->fsm, &end_nl)) { + fsm_state_t inner; + if (!fsm_addstate(env->fsm, &inner)) { return 0; } #if LOG_LINKAGE - fprintf(stderr, "%s: end_nl: %d\n", __func__, end_nl); + fprintf(stderr, "%s: end_nl_inner: %d\n", __func__, inner); #endif - if (!fsm_addedge_epsilon(env->fsm, end_nl, env->end)) { + if (!fsm_addedge_epsilon(env->fsm, inner, env->end_outer)) { return 0; } - if (!fsm_addedge_literal(env->fsm, end_nl, env->end, (char)'\n')) { + if (!fsm_addedge_literal(env->fsm, inner, env->end_outer, (char)'\n')) { return 0; } - env->end_nl = end_nl; - env->has_end_nl = 1; + env->end_nl_inner = inner; + env->has_end_nl_inner = 1; + + if (env->fsm->linkage_info != NULL) { + if (!state_set_add(&env->fsm->linkage_info->anchored_ends, + env->fsm->alloc, env->end_outer)) { + return 0; + } + } + return 1; } @@ -338,8 +469,8 @@ can_have_backward_epsilon_edge(const struct ast_expr *e) return 0; case AST_EXPR_SUBTRACT: - /* XXX: not sure */ - return 1; + /* Single character class */ + return 0; case AST_EXPR_REPEAT: /* 0 and 1 don't have backward epsilon edges */ @@ -397,19 +528,54 @@ can_skip_concat_state_and_epsilon(const struct ast_expr *l, return 0; } +static const struct ast_expr * +get_parent_node_from_stack(const struct comp_stack *stack) +{ + if (stack->depth < 2) { return NULL; } + return stack->frames[stack->depth - 2].n; +} + static enum link_types -decide_linking(struct comp_env *env, - struct ast_expr *n, const struct ast_expr *parent, enum link_side side) +decide_linking(struct comp_env *env, fsm_state_t x, fsm_state_t y, + struct ast_expr *n, enum link_side side) { assert(n != NULL); assert(env != NULL); + (void)x; + (void)y; + + struct comp_stack *stack = &env->stack; + /* If the regex is implicitly anchored and the dialect does * not support anchoring, linking is always top-down. */ if ((env->re_flags & RE_ANCHORED)) { return LINK_TOP_DOWN; } + const struct ast_expr *parent = get_parent_node_from_stack(stack); + + /* An alt node should always pass along the top-down (x,y) pair + * unmodified, individual subtrees may override the linking. */ + if (n->type == AST_EXPR_ALT) { + return LINK_TOP_DOWN; + } + + /* Special case for anchors that are more constrained than normal -- search + * for "pincer_anchors" in ast_analysis.c for details. */ + if (n->type == AST_EXPR_ANCHOR) { + if (n->u.anchor.type == AST_ANCHOR_START) { + if (side == LINK_END && n->flags & AST_FLAG_CONSTRAINED_AT_END) { + return LINK_GLOBAL; + } + + } else if (n->u.anchor.type == AST_ANCHOR_END) { + if (side == LINK_START && n->flags & AST_FLAG_CONSTRAINED_AT_START) { + return LINK_SKIP; + } + } + } + /* parent can be NULL, if we're at the root node, but it must * never be the same node. */ assert(parent != n); @@ -467,6 +633,9 @@ print_linkage(enum link_types t) case LINK_GLOBAL_SELF_LOOP: fprintf(stderr, "[SELF_LOOP]"); break; + case LINK_SKIP: + fprintf(stderr, "[SKIP]"); + break; default: assert(!"match fail"); break; @@ -477,8 +646,9 @@ print_linkage(enum link_types t) if (!fsm_addstate(env->fsm, &(NAME))) { return 0; } #define EPSILON(FROM, TO) \ - assert((FROM) != (TO)); \ - if (!fsm_addedge_epsilon(env->fsm, (FROM), (TO))) { return 0; } + assert((FROM) != (TO) || ((FROM) == SKIP_EPSILON && (TO) == SKIP_EPSILON)); \ + if ((FROM) != SKIP_EPSILON && (TO) != SKIP_EPSILON \ + && !fsm_addedge_epsilon(env->fsm, (FROM), (TO))) { return 0; } #define ANY(FROM, TO) \ if (!fsm_addedge_any(env->fsm, (FROM), (TO))) { return 0; } @@ -486,176 +656,77 @@ print_linkage(enum link_types t) #define LITERAL(FROM, TO, C) \ if (!addedge_literal(env, n->re_flags, (FROM), (TO), ((char)C))) { return 0; } -#define RECURSE(FROM, TO, NODE, PARENT) \ - if (!comp_iter(env, (FROM), (TO), (NODE), (PARENT))) { return 0; } - -static int -comp_iter_repeated(struct comp_env *env, - fsm_state_t x, fsm_state_t y, - struct ast_expr *n) -{ - fsm_state_t a, b; - fsm_state_t na, nz; - unsigned i; - - assert(n->type == AST_EXPR_REPEAT); - const unsigned min = n->u.repeat.min; - const unsigned max = n->u.repeat.max; - struct ast_expr *e = n->u.repeat.e; - - assert(min <= max); - - if (min == 0 && max == 0) { /* {0,0} */ - EPSILON(x, y); - } else if (min == 0 && max == 1) { /* '?' */ - RECURSE(x, y, e, n); - EPSILON(x, y); - } else if (min == 1 && max == 1) { /* {1,1} */ - RECURSE(x, y, e, n); - } else if (min == 0 && max == AST_COUNT_UNBOUNDED) { /* '*' */ - NEWSTATE(na); - NEWSTATE(nz); - EPSILON(x,na); - EPSILON(nz,y); - - EPSILON(na, nz); - RECURSE(na, nz, e, n); - EPSILON(nz, na); - } else if (min == 1 && max == AST_COUNT_UNBOUNDED) { /* '+' */ - NEWSTATE(na); - NEWSTATE(nz); - EPSILON(x,na); - EPSILON(nz,y); - - RECURSE(na, nz, e, n); - EPSILON(nz, na); - } else { - /* - * Make new beginning/end states for the repeated section, - * build its NFA, and link to its head. - */ - - struct fsm_subgraph subgraph; - fsm_state_t tail; - - fsm_subgraph_start(env->fsm, &subgraph); - - NEWSTATE(na); - NEWSTATE(nz); - RECURSE(na, nz, e, n); - EPSILON(x, na); /* link head to repeated NFA head */ - - b = nz; /* set the initial tail */ - - /* can be skipped */ - if (min == 0) { - EPSILON(na, nz); - } - fsm_subgraph_stop(env->fsm, &subgraph); - tail = nz; - - if (max != AST_COUNT_UNBOUNDED) { - for (i = 1; i < max; i++) { - /* copies the original subgraph; need to set b to the - * original tail - */ - b = tail; - - if (!fsm_subgraph_duplicate(env->fsm, &subgraph, &b, &a)) { - return 0; - } - - EPSILON(nz, a); - - /* To the optional part of the repeated count */ - if (i >= min) { - EPSILON(nz, b); - } - - na = a; /* advance head for next duplication */ - nz = b; /* advance tail for concenation */ - } - } else { - for (i = 1; i < min; i++) { - /* copies the original subgraph; need to set b to the - * original tail - */ - b = tail; - - if (!fsm_subgraph_duplicate(env->fsm, &subgraph, &b, &a)) { - return 0; - } - - EPSILON(nz, a); +#define RETURN(ENV) comp_stack_pop(ENV) - na = a; /* advance head for next duplication */ - nz = b; /* advance tail for concenation */ - } - - /* back link to allow for infinite repetition */ - EPSILON(nz,na); - } +#define RECURSE(ENV, FROM, TO, NODE) \ + if (!comp_stack_push(ENV, (FROM), (TO), (NODE))) { return 0; } - /* tail to last repeated NFA tail */ - EPSILON(nz, y); - } - - return 1; -} +#define TAILCALL(ENV, FROM, TO, NODE) \ + comp_stack_tailcall(ENV, (FROM), (TO), (NODE)); static int -comp_iter(struct comp_env *env, - fsm_state_t x, fsm_state_t y, - struct ast_expr *n, const struct ast_expr *parent) +set_linking(struct comp_env *env, struct ast_expr *n, + enum link_types link_start, enum link_types link_end, + fsm_state_t *px, fsm_state_t *py) { - enum link_types link_start, link_end; - - if (n == NULL) { - return 1; - } - - link_start = decide_linking(env, n, parent, LINK_START); - link_end = decide_linking(env, n, parent, LINK_END); + fsm_state_t x = *px; + fsm_state_t y = *py; #if LOG_LINKAGE - fprintf(stderr, "%s: decide_linking %p: start ", __func__, (void *) n); + fprintf(stderr, "%s: decide_linking %p [%s]: start ", + __func__, (void *) n, ast_node_type_name(n->type)); print_linkage(link_start); fprintf(stderr, ", end "); print_linkage(link_end); fprintf(stderr, ", x %d, y %d\n", x, y); #else (void) print_linkage; + (void)n; #endif switch (link_start) { case LINK_TOP_DOWN: break; case LINK_GLOBAL: - x = env->start; + x = env->start_inner; break; case LINK_GLOBAL_SELF_LOOP: if (!intern_start_any_loop(env)) { return 0; } - assert(env->has_start_any_loop); + assert(env->have_start_any_loop); - x = env->start_any_loop; + x = env->start_any_inner; + break; + case LINK_SKIP: + x = SKIP_EPSILON; + y = SKIP_EPSILON; break; default: - assert(!"unreachable"); + assert(!"match fail"); /* these should be mutually exclusive now */ } switch (link_end) { case LINK_TOP_DOWN: break; case LINK_GLOBAL: - if (env->re_flags & RE_END_NL && (n->flags & AST_FLAG_END_NL)) { + if (env->re_flags & RE_END_NL && !(env->re_flags & RE_END_NL_DISABLE) + && (n->flags & AST_FLAG_END_NL)) { + if (!intern_end_nl(env)) { + return 0; + } + y = env->end_nl_inner; + } else if (n->type == AST_EXPR_ANCHOR + && n->u.anchor.type == AST_ANCHOR_START + && n->flags & AST_FLAG_CONSTRAINED_AT_END) { + /* the start anchor is constrained, so only + * link directly to '\n''s end here */ if (!intern_end_nl(env)) { return 0; } - y = env->end_nl; + y = env->end_nl_inner; } else { - y = env->end; + y = env->end_inner; } break; case LINK_GLOBAL_SELF_LOOP: @@ -664,243 +735,812 @@ comp_iter(struct comp_env *env, } assert(env->has_end_any_loop); - y = env->end_any_loop; + y = env->end_any_inner; + break; + case LINK_SKIP: + x = SKIP_EPSILON; + y = SKIP_EPSILON; break; default: - assert(!"unreachable"); + assert(!"match fail"); /* these should be mutually exclusive now */ } #if LOG_LINKAGE - fprintf(stderr, " ---> x: %d, y: %d\n", x, y); + fprintf(stderr, " ---> x: %d, y: %d, type: %s\n", x, y, ast_node_type_name(n->type)); #endif + *px = x; + *py = y; + return 1; +} - switch (n->type) { - case AST_EXPR_EMPTY: - /* skip these, when possible */ - EPSILON(x, y); - break; - - case AST_EXPR_CONCAT: - { - fsm_state_t curr_x; - size_t i; - - const size_t count = n->u.concat.count; - - curr_x = x; +static void +comp_stack_pop(struct comp_env *env) +{ + assert(env->stack.depth > 0); + env->stack.depth--; +} - assert(count >= 1); +static int +comp_stack_push(struct comp_env *env, fsm_state_t x, fsm_state_t y, struct ast_expr *n) +{ + struct comp_stack *stack = &env->stack; + assert(n != NULL); - if (!fsm_addstate_bulk(env->fsm, count - 1)) { + if (stack->depth == stack->ceil) { + const size_t nceil = 2*stack->ceil; + struct comp_stack_frame *nframes = f_realloc(env->alloc, + stack->frames, nceil * sizeof(stack->frames[0])); +#if LOG_LINKAGE || LOG_TRAMPOLINE + fprintf(stderr, "comp_stack_push: reallocating comp_stack, %zu -> %zu frames\n", + stack->ceil, nceil); +#endif + if (nframes == NULL) { return 0; } + stack->ceil = nceil; + stack->frames = nframes; + } - for (i = 0; i < count; i++) { - struct ast_expr *curr = n->u.concat.n[i]; + assert(stack->depth < stack->ceil); - /* If a subtree is unsatisfiable but also nullable, ignore it. */ - const enum ast_flags nullable_and_unsat = AST_FLAG_NULLABLE - | AST_FLAG_UNSATISFIABLE; - if ((curr->flags & nullable_and_unsat) == nullable_and_unsat) { - /* if necessary, link the end */ - if (i == count - 1) { - EPSILON(curr_x, y); - } - continue; - } + struct comp_stack_frame *sf = &stack->frames[stack->depth]; + memset(sf, 0x00, sizeof(*sf)); + sf->n = n; + sf->x = x; + sf->y = y; - struct ast_expr *next = i == count - 1 - ? NULL - : n->u.concat.n[i + 1]; + stack->depth++; + return 1; +} - fsm_state_t z; - if (i + 1 < count) { - if (!fsm_addstate(env->fsm, &z)) { - return 0; - } -#if LOG_LINKAGE - fprintf(stderr, "%s: added state z %d\n", __func__, z); -#endif - } else { - z = y; /* connect to right parent to close off subtree */ - } +static void +comp_stack_tailcall(struct comp_env *env, + fsm_state_t x, fsm_state_t y, struct ast_expr *n) +{ + struct comp_stack *stack = &env->stack; - /* - * If nullable, add an extra state & epsilion as a one-way gate - */ - if (!can_skip_concat_state_and_epsilon(curr, next)) { - fsm_state_t diode; + assert(stack->depth > 0); - NEWSTATE(diode); - EPSILON(curr_x, diode); - curr_x = diode; - } + /* Replace current stack frame. */ + struct comp_stack_frame *sf = &stack->frames[stack->depth - 1]; + memset(sf, 0x00, sizeof(*sf)); + sf->n = n; + sf->x = x; + sf->y = y; +} - RECURSE(curr_x, z, curr, n); +#define JUST_ONE_PROG 1 - curr_x = z; - } +static int +comp_iter(struct comp_env *env, + fsm_state_t x, const struct ast *ast) +{ + int res = 1; + assert(ast != NULL); + assert(ast->expr != NULL); - break; + struct comp_stack_frame *frames = NULL; + uint64_t *active_capture_ids = NULL; + const bool use_captures = (env->re_flags & RE_NOCAPTURE) == 0; + + frames = f_calloc(env->alloc, + DEF_COMP_STACK_CEIL, sizeof(env->stack.frames[0])); + if (frames == NULL) { + goto alloc_fail; } - case AST_EXPR_ALT: { - size_t i; - - const size_t count = n->u.alt.count; - - assert(count >= 1); + const size_t capture_id_words = (env->max_capture_id == AST_NO_MAX_CAPTURE_ID) + ? 1 /* do non-zero allocation to silence EFENCE */ + : ((env->max_capture_id)/64 + 1); + active_capture_ids = f_calloc(env->alloc, capture_id_words, + sizeof(active_capture_ids[0])); + if (active_capture_ids == NULL) { + goto alloc_fail; + } + } - for (i = 0; i < count; i++) { - /* skip unsatisfiable ALT subtrees */ - if (n->u.alt.n[i]->flags & AST_FLAG_UNSATISFIABLE) { - continue; - } + /* FIXME: this could be done in ast_compile; comp_iter is a misnomer, + * it's only ever called once. */ - /* - * CONCAT handles adding extra states and - * epsilons when necessary, so there isn't much - * more to do here. - */ - RECURSE(x, y, n->u.alt.n[i], n); - } - break; + /* Add inner and outer end states. Like start_outer and start_inner, + * these represent the boundary between match group 0 (inner) and + * states outside it (the unanchored end loop). */ + assert(env->end_inner == LINKAGE_NO_STATE); + if (!fsm_addstate(env->fsm, &env->end_inner)) { + goto alloc_fail; + } + assert(env->end_outer == LINKAGE_NO_STATE); + if (!fsm_addstate(env->fsm, &env->end_outer)) { + goto alloc_fail; + } + if (!fsm_addedge_epsilon(env->fsm, env->end_inner, env->end_outer)) { + goto alloc_fail; } - case AST_EXPR_LITERAL: - LITERAL(x, y, n->u.literal.c); - break; + fsm_setend(env->fsm, env->end_outer, 1); - case AST_EXPR_CODEPOINT: { - fsm_state_t a, b; - char c[4]; - int r, i; +#if LOG_LINKAGE + fprintf(stderr, "end: outer %d, inner %d\n", + env->end_outer, env->end_inner); +#endif - r = utf8(n->u.codepoint.u, c); - if (!r) { - if (env->err != NULL) { - env->err->e = RE_EBADCP; - env->err->cp = n->u.codepoint.u; - } +#if LOG_TRAMPOLINE + fprintf(stderr, "%s: x %d, y %d\n", __func__, x, env->end_inner); +#endif - return 0; + env->stack.ceil = DEF_COMP_STACK_CEIL; + env->stack.depth = 1; + env->stack.frames = frames; + env->active_capture_ids = active_capture_ids; + + { /* set up the first stack frame */ + struct comp_stack_frame *sf = &env->stack.frames[0]; + sf->n = ast->expr; + sf->x = x; + sf->y = env->end_inner; + sf->step = 0; + } + +#if JUST_ONE_PROG + uint32_t prog_id; + if (use_captures) { + if (!compile_capvm_program_for_stack_end_states(env, ast, &prog_id)) { + goto alloc_fail; } + } +#endif - a = x; + /* evaluate call stack until termination */ + while (res && env->stack.depth > 0) { + if (!eval_stack_frame(env)) { +#if LOG_TRAMPOLINE + fprintf(stderr, "%s: res -> 0\n", __func__); +#endif + res = 0; + break; + } + } - for (i = 0; i < r; i++) { - if (i + 1 < r) { - NEWSTATE(b); - } else { - b = y; + if (use_captures && res && env->max_capture_id != AST_NO_MAX_CAPTURE_ID) { + /* Set the active captures on the end state. */ + for (unsigned i = 0; i <= (unsigned)env->max_capture_id; i++) { + if (!u64bitset_get(env->active_capture_ids, i)) { + continue; } + if (!fsm_capture_set_active_for_end(env->fsm, i, env->end_outer)) { + goto alloc_fail; + } + } - LITERAL(a, b, c[i]); - - a = b; +#if !JUST_ONE_PROG + uint32_t prog_id; + if (!compile_capvm_program_for_stack_end_states(env, stack, ast, &prog_id)) { + goto alloc_fail; } +#endif - break; +#if LOG_TRAMPOLINE + fprintf(stderr, "%s: associated prog_id %u with state %d\n", + __func__, prog_id, stack->end_outer); +#endif + if (!fsm_capture_associate_program_with_end_state(env->fsm, + prog_id, env->end_outer)) { + goto alloc_fail; + } } - case AST_EXPR_REPEAT: - /* - * REPEAT breaks out into its own function, because - * there are several special cases - */ - if (!comp_iter_repeated(env, x, y, n)) { - return 0; - } - break; + f_free(env->alloc, env->stack.frames); + f_free(env->alloc, env->active_capture_ids); - case AST_EXPR_GROUP: - RECURSE(x, y, n->u.group.e, n); - break; + return res; - case AST_EXPR_TOMBSTONE: - /* do not link -- intentionally pruned */ - break; +alloc_fail: + /* TODO: set env->err to indicate alloc failure */ + if (frames != NULL) { + f_free(env->alloc, frames); + } + if (active_capture_ids != NULL) { + f_free(env->alloc, active_capture_ids); + } - case AST_EXPR_ANCHOR: - EPSILON(x, y); - break; + return 0; +} - case AST_EXPR_SUBTRACT: { - struct fsm *a, *b; - struct fsm *q; - enum re_flags re_flags; +static struct comp_stack_frame * +get_comp_stack_top(struct comp_env *env) +{ + struct comp_stack *stack = &env->stack; + assert(stack->depth > 0); + struct comp_stack_frame *sf = &stack->frames[stack->depth - 1]; + assert(sf->n != NULL); + return sf; +} - re_flags = n->re_flags; +static int +eval_stack_frame(struct comp_env *env) +{ + struct comp_stack_frame *sf = get_comp_stack_top(env); - /* wouldn't want to reverse twice! */ - re_flags &= ~(unsigned)RE_REVERSE; +#if LOG_TRAMPOLINE + fprintf(stderr, "%s: depth %zu/%zu, type %s, step %u\n", __func__, + stack->depth, stack->ceil, ast_node_type_name(sf->n->type), sf->step); +#endif - a = expr_compile(n->u.subtract.a, re_flags, - fsm_getoptions(env->fsm), env->err); - if (a == NULL) { + /* If this is the first time the trampoline has called this + * state, decide the linking. Some of the states below (such as + * AST_EXPR_CONCAT) can have multiple child nodes, so they will + * increment step and use it to resume where they left off as + * the trampoline returns execution to them. */ + enum link_types link_end; + if (sf->step == 0) { /* entering state */ + enum link_types link_start; + link_start = decide_linking(env, sf->x, sf->y, sf->n, LINK_START); + link_end = decide_linking(env, sf->x, sf->y, sf->n, LINK_END); + if (!set_linking(env, sf->n, link_start, link_end, &sf->x, &sf->y)) { return 0; } + } - b = expr_compile(n->u.subtract.b, re_flags, - fsm_getoptions(env->fsm), env->err); - if (b == NULL) { - fsm_free(a); - return 0; - } +#if LOG_TRAMPOLINE > 1 + fprintf(stderr, "%s: x %d, y %d\n", __func__, sf->x, sf->y); +#endif - q = fsm_subtract(a, b); - if (q == NULL) { - return 0; - } + switch (sf->n->type) { + case AST_EXPR_EMPTY: + return eval_EMPTY(env); + case AST_EXPR_CONCAT: + return eval_CONCAT(env); + case AST_EXPR_ALT: + return eval_ALT(env); + case AST_EXPR_LITERAL: + return eval_LITERAL(env); + case AST_EXPR_CODEPOINT: + return eval_CODEPOINT(env); + case AST_EXPR_REPEAT: + return eval_REPEAT(env); + case AST_EXPR_GROUP: + return eval_GROUP(env); + case AST_EXPR_ANCHOR: + return eval_ANCHOR(env); + case AST_EXPR_SUBTRACT: + return eval_SUBTRACT(env); + case AST_EXPR_RANGE: + return eval_RANGE(env); + case AST_EXPR_TOMBSTONE: + return eval_TOMBSTONE(env); + default: + assert(!"unreached"); + return 0; + } +} + +static int +eval_EMPTY(struct comp_env *env) +{ + struct comp_stack_frame *sf = get_comp_stack_top(env); +#if LOG_LINKAGE + fprintf(stderr, "eval_EMPTY: step %u, x %d -> y %d\n", + sf->step, sf->x, sf->y); +#endif + + EPSILON(sf->x, sf->y); + RETURN(env); + return 1; +} + +static int +eval_CONCAT(struct comp_env *env) +{ + struct comp_stack_frame *sf = get_comp_stack_top(env); + struct ast_expr *n = sf->n; + const size_t count = n->u.concat.count; + assert(count >= 1); + +#if LOG_LINKAGE + fprintf(stderr, "comp_iter: eval_CONCAT: x %d, y %d, step %d\n", + sf->x, sf->y, sf->step); +#endif + + if (sf->step == 0) { + sf->u.concat.link = sf->x; + } + + while (sf->step < count) { + fsm_state_t curr_x = sf->u.concat.link; + struct ast_expr *curr = n->u.concat.n[sf->step]; + + /* If a subtree is unsatisfiable but also nullable, ignore it. */ + const enum ast_flags nullable_and_unsat = AST_FLAG_NULLABLE + | AST_FLAG_UNSATISFIABLE; + if ((curr->flags & nullable_and_unsat) == nullable_and_unsat) { + sf->step++; + + /* if necessary, link the end */ + if (sf->step == count) { + EPSILON(curr_x, sf->y); + } + return 1; + } + + struct ast_expr *next = sf->step == count - 1 + ? NULL + : n->u.concat.n[sf->step + 1]; + + fsm_state_t z; + if (sf->step + 1 < count) { + if (!fsm_addstate(env->fsm, &z)) { + return 0; + } +#if LOG_LINKAGE + fprintf(stderr, "%s: added state z %d\n", __func__, z); +#endif + } else { + z = sf->y; /* connect to right parent to close off subtree */ + } + +#if LOG_LINKAGE + fprintf(stderr, "%s: curr_x %d, z %d\n", + __func__, curr_x, z); +#endif /* - * Subtraction produces quite a mess. We could trim or minimise here - * while q is self-contained, which might work out better than doing it - * in the larger FSM after merge. I'm not sure if it works out better - * overall or not. + * If nullable, add an extra state & epsilon as a one-way gate */ + if (!can_skip_concat_state_and_epsilon(curr, next)) { + fsm_state_t diode; - if (fsm_empty(q)) { - EPSILON(x, y); - break; + NEWSTATE(diode); + EPSILON(curr_x, diode); + curr_x = diode; +#if LOG_LINKAGE + fprintf(stderr, "comp_iter: added diode %d\n", diode); +#endif } - if (!fsm_unionxy(env->fsm, q, x, y)) { - return 0; +#if LOG_LINKAGE + fprintf(stderr, "comp_iter: recurse CONCAT[%u/%zu]: link %d, z %d\n", + sf->step, count, sf->u.concat.link, z); +#endif + /* Set the right side link, which will become the + * left side link for the next step (if any). */ + sf->u.concat.link = z; + sf->step++; + RECURSE(env, curr_x, z, curr); + return 1; + } + + RETURN(env); + return 1; +} + +static int +eval_ALT(struct comp_env *env) +{ + struct comp_stack_frame *sf = get_comp_stack_top(env); + const size_t count = sf->n->u.alt.count; + assert(count >= 1); + +#if LOG_LINKAGE + fprintf(stderr, "eval_ALT: step %u\n", sf->step); +#endif + + if (sf->step < count) { + struct ast_expr *n; + + /* + * CONCAT handles adding extra states and + * epsilons when necessary, so there isn't much + * more to do here. + */ +#if LOG_LINKAGE + fprintf(stderr, "eval_ALT: recurse ALT[%u/%zu]: x %d, y %d\n", + sf->step, count, sf->x, sf->y); +#endif + + n = sf->n->u.alt.n[sf->step]; + assert(n != NULL); + sf->step++; /* RECURSE can realloc the stack and make sf stale. */ + + if (!(n->flags & AST_FLAG_UNSATISFIABLE)) { + RECURSE(env, sf->x, sf->y, n); } + return 1; + } - break; + RETURN(env); + return 1; +} + +static int +eval_LITERAL(struct comp_env *env) +{ + struct comp_stack_frame *sf = get_comp_stack_top(env); + struct ast_expr *n = sf->n; +#if LOG_LINKAGE + fprintf(stderr, "%s: linking %d -> %d with literal '%c' (0x%02x)\n", + __func__, sf->x, sf->y, isprint(n->u.literal.c) ? n->u.literal.c : '.', + n->u.literal.c); +#endif + + LITERAL(sf->x, sf->y, n->u.literal.c); + + RETURN(env); + return 1; +} + +static int +eval_CODEPOINT(struct comp_env *env) +{ + struct comp_stack_frame *sf = get_comp_stack_top(env); + struct ast_expr *n = sf->n; + fsm_state_t a, b; + char c[4]; + int r, i; + + r = utf8(n->u.codepoint.u, c); + if (!r) { + if (env->err != NULL) { + env->err->e = RE_EBADCP; + env->err->cp = n->u.codepoint.u; + } + + return 0; } - case AST_EXPR_RANGE: { - unsigned int i; + a = sf->x; - if (n->u.range.from.type != AST_ENDPOINT_LITERAL || n->u.range.to.type != AST_ENDPOINT_LITERAL) { - /* not yet supported */ - return 0; + for (i = 0; i < r; i++) { + if (i + 1 < r) { + NEWSTATE(b); + } else { + b = sf->y; } - assert(n->u.range.from.u.literal.c <= n->u.range.to.u.literal.c); + LITERAL(a, b, c[i]); - if (n->u.range.from.u.literal.c == 0x00 && - n->u.range.to.u.literal.c == 0xff) - { - ANY(x, y); + a = b; + } + + RETURN(env); + return 1; +} + +static int +eval_REPEAT(struct comp_env *env) +{ + struct comp_stack_frame *sf = get_comp_stack_top(env); + fsm_state_t a, b; + unsigned i, min, max; + + assert(sf->n->type == AST_EXPR_REPEAT); + struct ast_expr_repeat *n = &sf->n->u.repeat; + + min = n->min; + max = n->max; + + assert(min <= max); + + if (min == 0 && max == 0) { /* {0,0} */ + EPSILON(sf->x, sf->y); + RETURN(env); + return 1; + } else if (min == 0 && max == 1) { /* '?' */ + EPSILON(sf->x, sf->y); + TAILCALL(env, sf->x, sf->y, n->e); + return 1; + } else if (min == 1 && max == 1) { /* {1,1} */ + TAILCALL(env, sf->x, sf->y, n->e); + return 1; + } else if (min == 0 && max == AST_COUNT_UNBOUNDED) { /* '*' */ + fsm_state_t na, nz; + NEWSTATE(na); + NEWSTATE(nz); + EPSILON(sf->x,na); + EPSILON(nz,sf->y); + + EPSILON(na, nz); + EPSILON(nz, na); + TAILCALL(env, na, nz, n->e); + return 1; + } else if (min == 1 && max == AST_COUNT_UNBOUNDED) { /* '+' */ + fsm_state_t na, nz; + NEWSTATE(na); + NEWSTATE(nz); + EPSILON(sf->x, na); + EPSILON(nz, sf->y); + + EPSILON(nz, na); + TAILCALL(env, na, nz, n->e); + return 1; + } else if (sf->step == 0) { + /* + * Make new beginning/end states for the repeated section, + * build its NFA, and link to its head. + */ + + fsm_subgraph_start(env->fsm, &sf->u.repeat.subgraph); + + sf->step++; /* resume after RECURSE */ + NEWSTATE(sf->u.repeat.na); + NEWSTATE(sf->u.repeat.nz); + RECURSE(env, sf->u.repeat.na, sf->u.repeat.nz, n->e); + return 1; + } else { + fsm_state_t tail; + assert(sf->step == 1); + EPSILON(sf->x, sf->u.repeat.na); /* link head to repeated NFA head */ + + b = sf->u.repeat.nz; /* set the initial tail */ + + /* can be skipped */ + if (min == 0) { + EPSILON(sf->u.repeat.na, sf->u.repeat.nz); + } + fsm_subgraph_stop(env->fsm, &sf->u.repeat.subgraph); + tail = sf->u.repeat.nz; + + if (max != AST_COUNT_UNBOUNDED) { + for (i = 1; i < max; i++) { + /* copies the original subgraph; need to set b to the + * original tail + */ + b = tail; + + if (!fsm_subgraph_duplicate(env->fsm, &sf->u.repeat.subgraph, &b, &a)) { + return 0; + } + + EPSILON(sf->u.repeat.nz, a); + + /* To the optional part of the repeated count */ + if (i >= min) { + EPSILON(sf->u.repeat.nz, b); + } + + sf->u.repeat.na = a; /* advance head for next duplication */ + sf->u.repeat.nz = b; /* advance tail for concenation */ + } + } else { + for (i = 1; i < min; i++) { + /* copies the original subgraph; need to set b to the + * original tail + */ + b = tail; + + if (!fsm_subgraph_duplicate(env->fsm, &sf->u.repeat.subgraph, &b, &a)) { + return 0; + } + + EPSILON(sf->u.repeat.nz, a); + + sf->u.repeat.na = a; /* advance head for next duplication */ + sf->u.repeat.nz = b; /* advance tail for concenation */ + } + + /* back link to allow for infinite repetition */ + EPSILON(sf->u.repeat.nz, sf->u.repeat.na); + } + + /* tail to last repeated NFA tail */ + EPSILON(sf->u.repeat.nz, sf->y); + RETURN(env); + return 1; + } +} + +static void +set_active_capture_ids(struct comp_env *env, unsigned id) +{ +#if LOG_LINKAGE + fprintf(stderr, "set_active_capture_ids: id %u\n", id); +#endif + assert(env->active_capture_ids != NULL); + u64bitset_set(env->active_capture_ids, id); +} + +static int +eval_GROUP(struct comp_env *env) +{ + struct comp_stack_frame *sf = get_comp_stack_top(env); + + if (env->re_flags & RE_NOCAPTURE) { + /* passthrough, disable captures */ + if (sf->step == 0) { + sf->step++; + RECURSE(env, sf->x, sf->y, sf->n->u.group.e); + } else { + RETURN(env); + } + return 1; + } + + if (sf->step == 0) { + struct ast_expr *n = sf->n; + set_active_capture_ids(env, n->u.group.id); + +#if LOG_LINKAGE + fprintf(stderr, "comp_iter: recurse GROUP: x %d, y %d\n", + sf->x, sf->y); +#endif + sf->step++; + RECURSE(env, sf->x, sf->y, n->u.group.e); + return 1; + } else { + assert(sf->step == 1); + + RETURN(env); + return 1; + } +} + +static int +eval_ANCHOR(struct comp_env *env) +{ + struct comp_stack_frame *sf = get_comp_stack_top(env); +#if 1 + +#if LOG_LINKAGE + fprintf(stderr, "%s: ignoring anchor node %p, epsilon %d -> %d\n", + __func__, (void *)sf->n, sf->x, sf->y); +#endif + EPSILON(sf->x, sf->y); + + if (env->fsm->linkage_info != NULL + && sf->x == env->start_inner + && sf->n->u.anchor.type == AST_ANCHOR_START) { + /* This state is directly linked from the global start. */ +#if LOG_LINKAGE + fprintf(stderr, "%s: adding %d to anchored_starts due to start anchor\n", + __func__, sf->y); +#endif + if (!state_set_add(&env->fsm->linkage_info->anchored_starts, + env->fsm->alloc, sf->y)) { + return 0; + } + } + +#else + switch (sf->n->u.anchor.type) { + case AST_ANCHOR_START: + if (!(sf->n->flags & AST_FLAG_FIRST)) { +#if LOG_LINKAGE + fprintf(stderr, "%s: ignoring START anchor in non-FIRST location\n", + __func__); +#endif + EPSILON(sf->x, sf->y); break; } - for (i = n->u.range.from.u.literal.c; i <= n->u.range.to.u.literal.c; i++) { - LITERAL(x, y, i); +#if LOG_LINKAGE + fprintf(stderr, "%s: START anchor %p epsilon-linking %d -> %d\n", + __func__, (void *)sf->n, env->start_inner, sf->y); +#endif + EPSILON(env->start_inner, sf->y); + if (!state_set_add(&env->fsm->linkage_info->anchored_starts, + env->fsm->alloc, y)) { + return 0; } + break; + case AST_ANCHOR_END: + if (!(sf->n->flags & AST_FLAG_LAST)) { +#if LOG_LINKAGE + fprintf(stderr, "%s: ignoring END anchor in non-LAST location\n", + __func__); +#endif + EPSILON(sf->x, sf->y); + break; + } + +#if LOG_LINKAGE + fprintf(stderr, "%s: END anchor %p epsilon-linking %d -> %d\n", + __func__, (void *)sf->n, sf->x, stack->end_inner); +#endif + EPSILON(sf->x, stack->end_inner); break; - } default: assert(!"unreached"); + return 0; } +#endif + RETURN(env); + return 1; +} + +static int +eval_SUBTRACT(struct comp_env *env) +{ + struct comp_stack_frame *sf = get_comp_stack_top(env); + + struct fsm *a, *b; + struct fsm *q; + enum re_flags re_flags = sf->n->re_flags; + + /* wouldn't want to reverse twice! */ + re_flags &= ~(unsigned)RE_REVERSE; + + /* Don't compile capture resolution programs again for the + * subtrees, just ignore capture behavior. */ + re_flags |= RE_NOCAPTURE; + + a = expr_compile(sf->n->u.subtract.a, re_flags, + env->alloc, env->err); + if (a == NULL) { + return 0; + } + + b = expr_compile(sf->n->u.subtract.b, re_flags, + env->alloc, env->err); + if (b == NULL) { + fsm_free(a); + return 0; + } + + q = fsm_subtract(a, b); + if (q == NULL) { + return 0; + } + + /* + * Subtraction produces quite a mess. We could trim or minimise here + * while q is self-contained, which might work out better than doing it + * in the larger FSM after merge. I'm not sure if it works out better + * overall or not. + */ + + if (fsm_empty(q)) { + EPSILON(sf->x, sf->y); + RETURN(env); + return 1; + } + + if (!fsm_unionxy(env->fsm, q, sf->x, sf->y)) { + return 0; + } + + RETURN(env); + return 1; +} + +static int +eval_RANGE(struct comp_env *env) +{ + struct comp_stack_frame *sf = get_comp_stack_top(env); + struct ast_expr *n = sf->n; + unsigned int i; + + if (n->u.range.from.type != AST_ENDPOINT_LITERAL || n->u.range.to.type != AST_ENDPOINT_LITERAL) { + /* not yet supported */ + return 0; + } + + assert(n->u.range.from.u.literal.c <= n->u.range.to.u.literal.c); + + if (n->u.range.from.u.literal.c == 0x00 && + n->u.range.to.u.literal.c == 0xff) + { + ANY(sf->x, sf->y); + RETURN(env); + return 1; + } + + for (i = n->u.range.from.u.literal.c; i <= n->u.range.to.u.literal.c; i++) { + LITERAL(sf->x, sf->y, i); + } + + RETURN(env); + return 1; +} + +static int +eval_TOMBSTONE(struct comp_env *env) +{ + /* do not link -- intentionally pruned */ + (void)env; + RETURN(env); return 1; } @@ -909,49 +1549,91 @@ comp_iter(struct comp_env *env, #undef NEWSTATE #undef LITERAL #undef RECURSE +#undef RETURN +#undef TAILCALL struct fsm * ast_compile(const struct ast *ast, enum re_flags re_flags, - const struct fsm_options *opt, + const struct fsm_alloc *alloc, struct re_err *err) { - fsm_state_t x, y; + /* Start states inside and outside of match group 0, + * which represents the entire matched input, but does not + * include the implied /.*?/ loop at the start or end when + * a regex is unanchored. */ + fsm_state_t start_outer, start_inner; struct fsm *fsm; assert(ast != NULL); - fsm = fsm_new(opt); + fsm = fsm_new(alloc); if (fsm == NULL) { return NULL; } - if (!fsm_addstate(fsm, &x)) { + /* TODO: move these to the call stack, for symmetry? + * Or possibly combine comp_env and stack. */ + if (!fsm_addstate(fsm, &start_outer)) { goto error; } - if (!fsm_addstate(fsm, &y)) { + if (re_flags & RE_SAVE_LINKAGE_INFO) { + struct linkage_info *li = f_malloc(alloc, sizeof(*fsm->linkage_info)); + if (li == NULL) { goto error; } + li->unanchored_start_loop = LINKAGE_NO_STATE; + li->unanchored_end_loop = LINKAGE_NO_STATE; + li->end_any_inner = LINKAGE_NO_STATE; + li->anchored_starts = NULL; + li->anchored_ends = NULL; + + fsm->linkage_info = li; + } + + if (!fsm_addstate(fsm, &start_inner)) { goto error; } - fsm_setstart(fsm, x); - fsm_setend(fsm, y, 1); + if (!fsm_addedge_epsilon(fsm, start_outer, start_inner)) { + goto error; + } + + fsm_setstart(fsm, start_outer); + +#if LOG_LINKAGE + fprintf(stderr, "start: outer %d, inner %d\n", + start_outer, start_inner); +#endif { struct comp_env env; memset(&env, 0x00, sizeof(env)); + env.alloc = fsm->alloc; env.fsm = fsm; env.re_flags = re_flags; env.err = err; - env.start = x; - env.end = y; + env.start_inner = start_inner; + env.start_outer = start_outer; + + env.end_inner = LINKAGE_NO_STATE; + env.end_outer = LINKAGE_NO_STATE; - if (!comp_iter(&env, x, y, ast->expr, NULL)) { + env.max_capture_id = ast->max_capture_id; + + if (!comp_iter(&env, start_inner, ast)) { + if (err != NULL && err->e == 0) { + err->e = RE_EBADGROUP; + } goto error; } + +#if LOG_LINKAGE + fprintf(stderr, "%s: end_inner %d, end_outer %d, end_nl_inner %d, end_any_loop %d, end_any_inner %d\n", + __func__, env.end_inner, env.end_outer, env.end_nl_inner, env.end_any_loop, env.end_any_inner); +#endif } /* XXX: @@ -981,3 +1663,25 @@ ast_compile(const struct ast *ast, return NULL; } +static int +compile_capvm_program_for_stack_end_states(struct comp_env *env, + const struct ast *ast, uint32_t *prog_id) +{ + /* compile and save program in ^, associate its id w/ end state */ + enum re_capvm_compile_ast_res res; + struct capvm_program *prog; + res = re_capvm_compile_ast(env->alloc, + ast, env->re_flags, &prog); + if (res != RE_CAPVM_COMPILE_AST_OK) { + if (env->err != NULL && env->err->e == 0 && errno != 0) { + env->err->e = RE_EERRNO; + } + return 0; + } + + if (!fsm_capture_add_program(env->fsm, prog, prog_id)) { + return 0; + } + + return 1; +} diff --git a/src/libre/ast_compile.h b/src/libre/ast_compile.h index 89e1b12a4..4fed57af4 100644 --- a/src/libre/ast_compile.h +++ b/src/libre/ast_compile.h @@ -12,12 +12,13 @@ */ struct fsm; +struct fsm_alloc; struct re_err; struct fsm * ast_compile(const struct ast *ast, enum re_flags re_flags, - const struct fsm_options *opt, + const struct fsm_alloc *alloc, struct re_err *err); #endif diff --git a/src/libre/ast_new_from_fsm.c b/src/libre/ast_new_from_fsm.c index 4b0462205..bafd998c6 100644 --- a/src/libre/ast_new_from_fsm.c +++ b/src/libre/ast_new_from_fsm.c @@ -1,6 +1,7 @@ #include #include #include +#include #include #include @@ -325,7 +326,7 @@ remove_state(struct ast_expr_pool **poolp, struct rese *state) return 0; } rep = ast_make_expr_repeat(poolp, 0, tmp, - ast_make_count(0, NULL, AST_COUNT_UNBOUNDED, NULL)); + ast_make_count(0, AST_COUNT_UNBOUNDED)); if (!rep) { ast_expr_free(*poolp, tmp); ast_expr_free(*poolp, cat); @@ -453,7 +454,8 @@ ast_expr_new_from_fsm(struct ast_expr_pool **poolp, const struct fsm *fsm) } rep = ast_make_expr_repeat( - poolp, 0, restart->expr, ast_make_count(0, NULL, AST_COUNT_UNBOUNDED, NULL)); + poolp, 0, restart->expr, + ast_make_count(0, AST_COUNT_UNBOUNDED)); if (!rep) { ast_expr_free(*poolp, cat); free_restates(*poolp, restates, numstates + 1); diff --git a/src/libre/ast_rewrite.c b/src/libre/ast_rewrite.c index d05fc0a82..139ab7bac 100644 --- a/src/libre/ast_rewrite.c +++ b/src/libre/ast_rewrite.c @@ -118,7 +118,8 @@ deduplicate_alt(struct ast_expr_pool **poolp, struct ast_expr *n) } static struct fsm * -compile_subexpr(struct ast_expr *e, enum re_flags flags) +compile_subexpr(struct ast_expr *e, const struct fsm_alloc *alloc, + enum re_flags flags) { struct fsm *fsm; struct ast ast; @@ -146,7 +147,7 @@ compile_subexpr(struct ast_expr *e, enum re_flags flags) return 0; } - fsm = ast_compile(&ast, flags | RE_ANCHORED, NULL, NULL); + fsm = ast_compile(&ast, flags | RE_ANCHORED | RE_NOCAPTURE, alloc, NULL); if (fsm == NULL) { return 0; } @@ -619,17 +620,23 @@ rewrite_subtract(struct ast_expr_pool **poolp, struct ast_expr *n, enum re_flags * sub-expressions may be written to assume particular flags * (for example subtraction of two case-insensitive sets may depend * on case-insensitity to produce an empty result). + * + * It's okay to elide the alloc hooks (provided to re_comp) + * because we immediately destroy the constructed fsm, q. + * So we won't mix up different sources of allocation into the AST. */ { + const struct fsm_alloc *alloc = NULL; + struct fsm *a, *b; struct fsm *q; - a = compile_subexpr(n->u.subtract.a, flags); + a = compile_subexpr(n->u.subtract.a, alloc, flags); if (a == NULL) { return 0; } - b = compile_subexpr(n->u.subtract.b, flags); + b = compile_subexpr(n->u.subtract.b, alloc, flags); if (b == NULL) { fsm_free(a); return 0; diff --git a/src/libre/class.h b/src/libre/class.h index 45fb09bc6..79460a9da 100644 --- a/src/libre/class.h +++ b/src/libre/class.h @@ -29,6 +29,7 @@ extern const struct class utf8_Bamum; extern const struct class utf8_Bassa_Vah; extern const struct class utf8_Batak; extern const struct class utf8_Bengali; +extern const struct class utf8_Beria_Erfe; extern const struct class utf8_Bhaiksuki; extern const struct class utf8_Bopomofo; extern const struct class utf8_Brahmi; @@ -57,6 +58,7 @@ extern const struct class utf8_Egyptian_Hieroglyphs; extern const struct class utf8_Elbasan; extern const struct class utf8_Elymaic; extern const struct class utf8_Ethiopic; +extern const struct class utf8_Garay; extern const struct class utf8_Georgian; extern const struct class utf8_Glagolitic; extern const struct class utf8_Gothic; @@ -65,6 +67,7 @@ extern const struct class utf8_Greek; extern const struct class utf8_Gujarati; extern const struct class utf8_Gunjala_Gondi; extern const struct class utf8_Gurmukhi; +extern const struct class utf8_Gurung_Khema; extern const struct class utf8_Han; extern const struct class utf8_Hangul; extern const struct class utf8_Hanifi_Rohingya; @@ -80,12 +83,14 @@ extern const struct class utf8_Javanese; extern const struct class utf8_Kaithi; extern const struct class utf8_Kannada; extern const struct class utf8_Katakana; +extern const struct class utf8_Kawi; extern const struct class utf8_Kayah_Li; extern const struct class utf8_Kharoshthi; extern const struct class utf8_Khitan_Small_Script; extern const struct class utf8_Khmer; extern const struct class utf8_Khojki; extern const struct class utf8_Khudawadi; +extern const struct class utf8_Kirat_Rai; extern const struct class utf8_Lao; extern const struct class utf8_Latin; extern const struct class utf8_Lepcha; @@ -114,9 +119,10 @@ extern const struct class utf8_Mro; extern const struct class utf8_Multani; extern const struct class utf8_Myanmar; extern const struct class utf8_Nabataean; +extern const struct class utf8_Nag_Mundari; extern const struct class utf8_Nandinagari; -extern const struct class utf8_New_Tai_Lue; extern const struct class utf8_Newa; +extern const struct class utf8_New_Tai_Lue; extern const struct class utf8_Nko; extern const struct class utf8_Nushu; extern const struct class utf8_Nyiakeng_Puachue_Hmong; @@ -131,6 +137,7 @@ extern const struct class utf8_Old_Sogdian; extern const struct class utf8_Old_South_Arabian; extern const struct class utf8_Old_Turkic; extern const struct class utf8_Old_Uyghur; +extern const struct class utf8_Ol_Onal; extern const struct class utf8_Oriya; extern const struct class utf8_Osage; extern const struct class utf8_Osmanya; @@ -147,12 +154,14 @@ extern const struct class utf8_Saurashtra; extern const struct class utf8_Sharada; extern const struct class utf8_Shavian; extern const struct class utf8_Siddham; +extern const struct class utf8_Sidetic; extern const struct class utf8_SignWriting; extern const struct class utf8_Sinhala; extern const struct class utf8_Sogdian; extern const struct class utf8_Sora_Sompeng; extern const struct class utf8_Soyombo; extern const struct class utf8_Sundanese; +extern const struct class utf8_Sunuwar; extern const struct class utf8_Syloti_Nagri; extern const struct class utf8_Syriac; extern const struct class utf8_Tagalog; @@ -160,6 +169,7 @@ extern const struct class utf8_Tagbanwa; extern const struct class utf8_Tai_Le; extern const struct class utf8_Tai_Tham; extern const struct class utf8_Tai_Viet; +extern const struct class utf8_Tai_Yo; extern const struct class utf8_Takri; extern const struct class utf8_Tamil; extern const struct class utf8_Tangsa; @@ -170,7 +180,10 @@ extern const struct class utf8_Thai; extern const struct class utf8_Tibetan; extern const struct class utf8_Tifinagh; extern const struct class utf8_Tirhuta; +extern const struct class utf8_Todhri; +extern const struct class utf8_Tolong_Siki; extern const struct class utf8_Toto; +extern const struct class utf8_Tulu_Tigalari; extern const struct class utf8_Ugaritic; extern const struct class utf8_Vai; extern const struct class utf8_Vithkuqi; diff --git a/src/libre/class/utf8_Arabic.c b/src/libre/class/utf8_Arabic.c index 360564ea3..592e06d7d 100644 --- a/src/libre/class/utf8_Arabic.c +++ b/src/libre/class/utf8_Arabic.c @@ -13,20 +13,18 @@ static const struct range ranges[] = { { 0x0671UL, 0x06DCUL }, { 0x06DEUL, 0x06FFUL }, { 0x0750UL, 0x077FUL }, - { 0x0870UL, 0x088EUL }, - { 0x0890UL, 0x0891UL }, - { 0x0898UL, 0x08E1UL }, + { 0x0870UL, 0x0891UL }, + { 0x0897UL, 0x08E1UL }, { 0x08E3UL, 0x08FFUL }, - { 0xFB50UL, 0xFBC2UL }, - { 0xFBD3UL, 0xFD3DUL }, - { 0xFD40UL, 0xFD8FUL }, - { 0xFD92UL, 0xFDC7UL }, - { 0xFDCFUL, 0xFDCFUL }, + { 0xFB50UL, 0xFD3DUL }, + { 0xFD40UL, 0xFDCFUL }, { 0xFDF0UL, 0xFDFFUL }, { 0xFE70UL, 0xFE74UL }, { 0xFE76UL, 0xFEFCUL }, { 0x10E60UL, 0x10E7EUL }, - { 0x10EFDUL, 0x10EFFUL }, + { 0x10EC2UL, 0x10EC7UL }, + { 0x10ED0UL, 0x10ED8UL }, + { 0x10EFAUL, 0x10EFFUL }, { 0x1EE00UL, 0x1EE03UL }, { 0x1EE05UL, 0x1EE1FUL }, { 0x1EE21UL, 0x1EE22UL }, diff --git a/src/libre/class/utf8_Balinese.c b/src/libre/class/utf8_Balinese.c index b86ccdd3e..2c742b725 100644 --- a/src/libre/class/utf8_Balinese.c +++ b/src/libre/class/utf8_Balinese.c @@ -4,7 +4,7 @@ static const struct range ranges[] = { { 0x1B00UL, 0x1B4CUL }, - { 0x1B50UL, 0x1B7EUL } + { 0x1B4EUL, 0x1B7FUL } }; const struct class utf8_Balinese = { diff --git a/src/libre/class/utf8_Beria_Erfe.c b/src/libre/class/utf8_Beria_Erfe.c new file mode 100644 index 000000000..3bf7b4db7 --- /dev/null +++ b/src/libre/class/utf8_Beria_Erfe.c @@ -0,0 +1,14 @@ +/* generated */ + +#include "class.h" + +static const struct range ranges[] = { + { 0x16EA0UL, 0x16EB8UL }, + { 0x16EBBUL, 0x16ED3UL } +}; + +const struct class utf8_Beria_Erfe = { + ranges, + sizeof ranges / sizeof *ranges +}; + diff --git a/src/libre/class/utf8_Common.c b/src/libre/class/utf8_Common.c index 23e3cbf4b..26c3f54e8 100644 --- a/src/libre/class/utf8_Common.c +++ b/src/libre/class/utf8_Common.c @@ -43,22 +43,20 @@ static const struct range ranges[] = { { 0x2066UL, 0x2070UL }, { 0x2074UL, 0x207EUL }, { 0x2080UL, 0x208EUL }, - { 0x20A0UL, 0x20C0UL }, + { 0x20A0UL, 0x20C1UL }, { 0x2100UL, 0x2125UL }, { 0x2127UL, 0x2129UL }, { 0x212CUL, 0x2131UL }, { 0x2133UL, 0x214DUL }, { 0x214FUL, 0x215FUL }, { 0x2189UL, 0x218BUL }, - { 0x2190UL, 0x2426UL }, + { 0x2190UL, 0x2429UL }, { 0x2440UL, 0x244AUL }, { 0x2460UL, 0x27FFUL }, { 0x2900UL, 0x2B73UL }, - { 0x2B76UL, 0x2B95UL }, - { 0x2B97UL, 0x2BFFUL }, + { 0x2B76UL, 0x2BFFUL }, { 0x2E00UL, 0x2E5DUL }, - { 0x2FF0UL, 0x2FFBUL }, - { 0x3000UL, 0x3004UL }, + { 0x2FF0UL, 0x3004UL }, { 0x3006UL, 0x3006UL }, { 0x3008UL, 0x3020UL }, { 0x3030UL, 0x3037UL }, @@ -67,7 +65,8 @@ static const struct range ranges[] = { { 0x30A0UL, 0x30A0UL }, { 0x30FBUL, 0x30FCUL }, { 0x3190UL, 0x319FUL }, - { 0x31C0UL, 0x31E3UL }, + { 0x31C0UL, 0x31E5UL }, + { 0x31EFUL, 0x31EFUL }, { 0x3220UL, 0x325FUL }, { 0x327FUL, 0x32CFUL }, { 0x32FFUL, 0x32FFUL }, @@ -101,6 +100,10 @@ static const struct range ranges[] = { { 0x101D0UL, 0x101FCUL }, { 0x102E1UL, 0x102FBUL }, { 0x1BCA0UL, 0x1BCA3UL }, + { 0x1CC00UL, 0x1CCFCUL }, + { 0x1CD00UL, 0x1CEB3UL }, + { 0x1CEBAUL, 0x1CED0UL }, + { 0x1CEE0UL, 0x1CEF0UL }, { 0x1CF50UL, 0x1CFC3UL }, { 0x1D000UL, 0x1D0F5UL }, { 0x1D100UL, 0x1D126UL }, @@ -149,11 +152,10 @@ static const struct range ranges[] = { { 0x1F240UL, 0x1F248UL }, { 0x1F250UL, 0x1F251UL }, { 0x1F260UL, 0x1F265UL }, - { 0x1F300UL, 0x1F6D7UL }, + { 0x1F300UL, 0x1F6D8UL }, { 0x1F6DCUL, 0x1F6ECUL }, { 0x1F6F0UL, 0x1F6FCUL }, - { 0x1F700UL, 0x1F776UL }, - { 0x1F77BUL, 0x1F7D9UL }, + { 0x1F700UL, 0x1F7D9UL }, { 0x1F7E0UL, 0x1F7EBUL }, { 0x1F7F0UL, 0x1F7F0UL }, { 0x1F800UL, 0x1F80BUL }, @@ -161,19 +163,20 @@ static const struct range ranges[] = { { 0x1F850UL, 0x1F859UL }, { 0x1F860UL, 0x1F887UL }, { 0x1F890UL, 0x1F8ADUL }, - { 0x1F8B0UL, 0x1F8B1UL }, - { 0x1F900UL, 0x1FA53UL }, + { 0x1F8B0UL, 0x1F8BBUL }, + { 0x1F8C0UL, 0x1F8C1UL }, + { 0x1F8D0UL, 0x1F8D8UL }, + { 0x1F900UL, 0x1FA57UL }, { 0x1FA60UL, 0x1FA6DUL }, { 0x1FA70UL, 0x1FA7CUL }, - { 0x1FA80UL, 0x1FA88UL }, - { 0x1FA90UL, 0x1FABDUL }, - { 0x1FABFUL, 0x1FAC5UL }, - { 0x1FACEUL, 0x1FADBUL }, - { 0x1FAE0UL, 0x1FAE8UL }, - { 0x1FAF0UL, 0x1FAF8UL }, + { 0x1FA80UL, 0x1FA8AUL }, + { 0x1FA8EUL, 0x1FAC6UL }, + { 0x1FAC8UL, 0x1FAC8UL }, + { 0x1FACDUL, 0x1FADCUL }, + { 0x1FADFUL, 0x1FAEAUL }, + { 0x1FAEFUL, 0x1FAF8UL }, { 0x1FB00UL, 0x1FB92UL }, - { 0x1FB94UL, 0x1FBCAUL }, - { 0x1FBF0UL, 0x1FBF9UL }, + { 0x1FB94UL, 0x1FBFAUL }, { 0xE0001UL, 0xE0001UL }, { 0xE0020UL, 0xE007FUL } }; diff --git a/src/libre/class/utf8_Cyrillic.c b/src/libre/class/utf8_Cyrillic.c index 2154d4998..822ee6438 100644 --- a/src/libre/class/utf8_Cyrillic.c +++ b/src/libre/class/utf8_Cyrillic.c @@ -5,7 +5,7 @@ static const struct range ranges[] = { { 0x0400UL, 0x0484UL }, { 0x0487UL, 0x052FUL }, - { 0x1C80UL, 0x1C88UL }, + { 0x1C80UL, 0x1C8AUL }, { 0x1D2BUL, 0x1D2BUL }, { 0x1D78UL, 0x1D78UL }, { 0x2DE0UL, 0x2DFFUL }, diff --git a/src/libre/class/utf8_Egyptian_Hieroglyphs.c b/src/libre/class/utf8_Egyptian_Hieroglyphs.c index 2698da1ed..a46b2a361 100644 --- a/src/libre/class/utf8_Egyptian_Hieroglyphs.c +++ b/src/libre/class/utf8_Egyptian_Hieroglyphs.c @@ -3,7 +3,8 @@ #include "class.h" static const struct range ranges[] = { - { 0x13000UL, 0x13455UL } + { 0x13000UL, 0x13455UL }, + { 0x13460UL, 0x143FAUL } }; const struct class utf8_Egyptian_Hieroglyphs = { diff --git a/src/libre/class/utf8_Garay.c b/src/libre/class/utf8_Garay.c new file mode 100644 index 000000000..f1841e248 --- /dev/null +++ b/src/libre/class/utf8_Garay.c @@ -0,0 +1,15 @@ +/* generated */ + +#include "class.h" + +static const struct range ranges[] = { + { 0x10D40UL, 0x10D65UL }, + { 0x10D69UL, 0x10D85UL }, + { 0x10D8EUL, 0x10D8FUL } +}; + +const struct class utf8_Garay = { + ranges, + sizeof ranges / sizeof *ranges +}; + diff --git a/src/libre/class/utf8_Gurung_Khema.c b/src/libre/class/utf8_Gurung_Khema.c new file mode 100644 index 000000000..694048f31 --- /dev/null +++ b/src/libre/class/utf8_Gurung_Khema.c @@ -0,0 +1,13 @@ +/* generated */ + +#include "class.h" + +static const struct range ranges[] = { + { 0x16100UL, 0x16139UL } +}; + +const struct class utf8_Gurung_Khema = { + ranges, + sizeof ranges / sizeof *ranges +}; + diff --git a/src/libre/class/utf8_Han.c b/src/libre/class/utf8_Han.c index b22f0ed22..0fdf663e0 100644 --- a/src/libre/class/utf8_Han.c +++ b/src/libre/class/utf8_Han.c @@ -29,15 +29,15 @@ static const struct range ranges[] = { { 0xF900UL, 0xFA6DUL }, { 0xFA70UL, 0xFAD9UL }, { 0x16FE2UL, 0x16FE3UL }, - { 0x16FF0UL, 0x16FF1UL }, + { 0x16FF0UL, 0x16FF6UL }, { 0x20000UL, 0x2A6DFUL }, - { 0x2A700UL, 0x2B739UL }, - { 0x2B740UL, 0x2B81DUL }, - { 0x2B820UL, 0x2CEA1UL }, + { 0x2A700UL, 0x2B81DUL }, + { 0x2B820UL, 0x2CEADUL }, { 0x2CEB0UL, 0x2EBE0UL }, + { 0x2EBF0UL, 0x2EE5DUL }, { 0x2F800UL, 0x2FA1DUL }, { 0x30000UL, 0x3134AUL }, - { 0x31350UL, 0x323AFUL }, + { 0x31350UL, 0x33479UL }, { 0x1720UL, 0x1734UL }, { 0x10D00UL, 0x10D27UL }, { 0x10D30UL, 0x10D39UL } diff --git a/src/libre/class/utf8_Inherited.c b/src/libre/class/utf8_Inherited.c index 8e5afc76c..46ee6739a 100644 --- a/src/libre/class/utf8_Inherited.c +++ b/src/libre/class/utf8_Inherited.c @@ -8,7 +8,8 @@ static const struct range ranges[] = { { 0x064BUL, 0x0655UL }, { 0x0670UL, 0x0670UL }, { 0x0951UL, 0x0954UL }, - { 0x1AB0UL, 0x1ACEUL }, + { 0x1AB0UL, 0x1ADDUL }, + { 0x1AE0UL, 0x1AEBUL }, { 0x1CD0UL, 0x1CD2UL }, { 0x1CD4UL, 0x1CE0UL }, { 0x1CE2UL, 0x1CE8UL }, diff --git a/src/libre/class/utf8_Kannada.c b/src/libre/class/utf8_Kannada.c index c2f9ae6e6..a4f8ebd40 100644 --- a/src/libre/class/utf8_Kannada.c +++ b/src/libre/class/utf8_Kannada.c @@ -12,7 +12,7 @@ static const struct range ranges[] = { { 0x0CC6UL, 0x0CC8UL }, { 0x0CCAUL, 0x0CCDUL }, { 0x0CD5UL, 0x0CD6UL }, - { 0x0CDDUL, 0x0CDEUL }, + { 0x0CDCUL, 0x0CDEUL }, { 0x0CE0UL, 0x0CE3UL }, { 0x0CE6UL, 0x0CEFUL }, { 0x0CF1UL, 0x0CF3UL } diff --git a/src/libre/class/utf8_Kawi.c b/src/libre/class/utf8_Kawi.c index 617398734..5fad15253 100644 --- a/src/libre/class/utf8_Kawi.c +++ b/src/libre/class/utf8_Kawi.c @@ -5,7 +5,7 @@ static const struct range ranges[] = { { 0x11F00UL, 0x11F10UL }, { 0x11F12UL, 0x11F3AUL }, - { 0x11F3EUL, 0x11F59UL } + { 0x11F3EUL, 0x11F5AUL } }; const struct class utf8_Kawi = { diff --git a/src/libre/class/utf8_Khitan_Small_Script.c b/src/libre/class/utf8_Khitan_Small_Script.c index 74741da30..f1e054219 100644 --- a/src/libre/class/utf8_Khitan_Small_Script.c +++ b/src/libre/class/utf8_Khitan_Small_Script.c @@ -4,7 +4,8 @@ static const struct range ranges[] = { { 0x16FE4UL, 0x16FE4UL }, - { 0x18B00UL, 0x18CD5UL } + { 0x18B00UL, 0x18CD5UL }, + { 0x18CFFUL, 0x18CFFUL } }; const struct class utf8_Khitan_Small_Script = { diff --git a/src/libre/class/utf8_Kirat_Rai.c b/src/libre/class/utf8_Kirat_Rai.c new file mode 100644 index 000000000..34925abfd --- /dev/null +++ b/src/libre/class/utf8_Kirat_Rai.c @@ -0,0 +1,13 @@ +/* generated */ + +#include "class.h" + +static const struct range ranges[] = { + { 0x16D40UL, 0x16D79UL } +}; + +const struct class utf8_Kirat_Rai = { + ranges, + sizeof ranges / sizeof *ranges +}; + diff --git a/src/libre/class/utf8_L.c b/src/libre/class/utf8_L.c index 97eaa3ffe..e37048bc1 100644 --- a/src/libre/class/utf8_L.c +++ b/src/libre/class/utf8_L.c @@ -53,7 +53,7 @@ static const struct range ranges[] = { { 0x0840UL, 0x0858UL }, { 0x0860UL, 0x086AUL }, { 0x0870UL, 0x0887UL }, - { 0x0889UL, 0x088EUL }, + { 0x0889UL, 0x088FUL }, { 0x08A0UL, 0x08C9UL }, { 0x0904UL, 0x0939UL }, { 0x093DUL, 0x093DUL }, @@ -119,7 +119,7 @@ static const struct range ranges[] = { { 0x0C2AUL, 0x0C39UL }, { 0x0C3DUL, 0x0C3DUL }, { 0x0C58UL, 0x0C5AUL }, - { 0x0C5DUL, 0x0C5DUL }, + { 0x0C5CUL, 0x0C5DUL }, { 0x0C60UL, 0x0C61UL }, { 0x0C80UL, 0x0C80UL }, { 0x0C85UL, 0x0C8CUL }, @@ -128,7 +128,7 @@ static const struct range ranges[] = { { 0x0CAAUL, 0x0CB3UL }, { 0x0CB5UL, 0x0CB9UL }, { 0x0CBDUL, 0x0CBDUL }, - { 0x0CDDUL, 0x0CDEUL }, + { 0x0CDCUL, 0x0CDEUL }, { 0x0CE0UL, 0x0CE1UL }, { 0x0CF1UL, 0x0CF2UL }, { 0x0D04UL, 0x0D0CUL }, @@ -228,7 +228,7 @@ static const struct range ranges[] = { { 0x1C00UL, 0x1C23UL }, { 0x1C4DUL, 0x1C4FUL }, { 0x1C5AUL, 0x1C7DUL }, - { 0x1C80UL, 0x1C88UL }, + { 0x1C80UL, 0x1C8AUL }, { 0x1C90UL, 0x1CBAUL }, { 0x1CBDUL, 0x1CBFUL }, { 0x1CE9UL, 0x1CECUL }, @@ -314,11 +314,8 @@ static const struct range ranges[] = { { 0xA6A0UL, 0xA6E5UL }, { 0xA717UL, 0xA71FUL }, { 0xA722UL, 0xA788UL }, - { 0xA78BUL, 0xA7CAUL }, - { 0xA7D0UL, 0xA7D1UL }, - { 0xA7D3UL, 0xA7D3UL }, - { 0xA7D5UL, 0xA7D9UL }, - { 0xA7F2UL, 0xA801UL }, + { 0xA78BUL, 0xA7DCUL }, + { 0xA7F1UL, 0xA801UL }, { 0xA803UL, 0xA805UL }, { 0xA807UL, 0xA80AUL }, { 0xA80CUL, 0xA822UL }, @@ -415,6 +412,7 @@ static const struct range ranges[] = { { 0x105A3UL, 0x105B1UL }, { 0x105B3UL, 0x105B9UL }, { 0x105BBUL, 0x105BCUL }, + { 0x105C0UL, 0x105F3UL }, { 0x10600UL, 0x10736UL }, { 0x10740UL, 0x10755UL }, { 0x10760UL, 0x10767UL }, @@ -433,6 +431,7 @@ static const struct range ranges[] = { { 0x108F4UL, 0x108F5UL }, { 0x10900UL, 0x10915UL }, { 0x10920UL, 0x10939UL }, + { 0x10940UL, 0x10959UL }, { 0x10980UL, 0x109B7UL }, { 0x109BEUL, 0x109BFUL }, { 0x10A00UL, 0x10A00UL }, @@ -451,8 +450,11 @@ static const struct range ranges[] = { { 0x10C80UL, 0x10CB2UL }, { 0x10CC0UL, 0x10CF2UL }, { 0x10D00UL, 0x10D23UL }, + { 0x10D4AUL, 0x10D65UL }, + { 0x10D6FUL, 0x10D85UL }, { 0x10E80UL, 0x10EA9UL }, { 0x10EB0UL, 0x10EB1UL }, + { 0x10EC2UL, 0x10EC7UL }, { 0x10F00UL, 0x10F1CUL }, { 0x10F27UL, 0x10F27UL }, { 0x10F30UL, 0x10F45UL }, @@ -491,6 +493,13 @@ static const struct range ranges[] = { { 0x1133DUL, 0x1133DUL }, { 0x11350UL, 0x11350UL }, { 0x1135DUL, 0x11361UL }, + { 0x11380UL, 0x11389UL }, + { 0x1138BUL, 0x1138BUL }, + { 0x1138EUL, 0x1138EUL }, + { 0x11390UL, 0x113B5UL }, + { 0x113B7UL, 0x113B7UL }, + { 0x113D1UL, 0x113D1UL }, + { 0x113D3UL, 0x113D3UL }, { 0x11400UL, 0x11434UL }, { 0x11447UL, 0x1144AUL }, { 0x1145FUL, 0x11461UL }, @@ -525,6 +534,7 @@ static const struct range ranges[] = { { 0x11A5CUL, 0x11A89UL }, { 0x11A9DUL, 0x11A9DUL }, { 0x11AB0UL, 0x11AF8UL }, + { 0x11BC0UL, 0x11BE0UL }, { 0x11C00UL, 0x11C08UL }, { 0x11C0AUL, 0x11C2EUL }, { 0x11C40UL, 0x11C40UL }, @@ -537,6 +547,7 @@ static const struct range ranges[] = { { 0x11D67UL, 0x11D68UL }, { 0x11D6AUL, 0x11D89UL }, { 0x11D98UL, 0x11D98UL }, + { 0x11DB0UL, 0x11DDBUL }, { 0x11EE0UL, 0x11EF2UL }, { 0x11F02UL, 0x11F02UL }, { 0x11F04UL, 0x11F10UL }, @@ -547,7 +558,9 @@ static const struct range ranges[] = { { 0x12F90UL, 0x12FF0UL }, { 0x13000UL, 0x1342FUL }, { 0x13441UL, 0x13446UL }, + { 0x13460UL, 0x143FAUL }, { 0x14400UL, 0x14646UL }, + { 0x16100UL, 0x1611DUL }, { 0x16800UL, 0x16A38UL }, { 0x16A40UL, 0x16A5EUL }, { 0x16A70UL, 0x16ABEUL }, @@ -556,17 +569,21 @@ static const struct range ranges[] = { { 0x16B40UL, 0x16B43UL }, { 0x16B63UL, 0x16B77UL }, { 0x16B7DUL, 0x16B8FUL }, + { 0x16D40UL, 0x16D6CUL }, { 0x16E40UL, 0x16E7FUL }, + { 0x16EA0UL, 0x16EB8UL }, + { 0x16EBBUL, 0x16ED3UL }, { 0x16F00UL, 0x16F4AUL }, { 0x16F50UL, 0x16F50UL }, { 0x16F93UL, 0x16F9FUL }, { 0x16FE0UL, 0x16FE1UL }, { 0x16FE3UL, 0x16FE3UL }, + { 0x16FF2UL, 0x16FF3UL }, { 0x17000UL, 0x17000UL }, - { 0x187F7UL, 0x187F7UL }, - { 0x18800UL, 0x18CD5UL }, - { 0x18D00UL, 0x18D00UL }, - { 0x18D08UL, 0x18D08UL }, + { 0x187FFUL, 0x18CD5UL }, + { 0x18CFFUL, 0x18D00UL }, + { 0x18D1EUL, 0x18D1EUL }, + { 0x18D80UL, 0x18DF2UL }, { 0x1AFF0UL, 0x1AFF3UL }, { 0x1AFF5UL, 0x1AFFBUL }, { 0x1AFFDUL, 0x1AFFEUL }, @@ -619,6 +636,14 @@ static const struct range ranges[] = { { 0x1E290UL, 0x1E2ADUL }, { 0x1E2C0UL, 0x1E2EBUL }, { 0x1E4D0UL, 0x1E4EBUL }, + { 0x1E5D0UL, 0x1E5EDUL }, + { 0x1E5F0UL, 0x1E5F0UL }, + { 0x1E6C0UL, 0x1E6DEUL }, + { 0x1E6E0UL, 0x1E6E2UL }, + { 0x1E6E4UL, 0x1E6E5UL }, + { 0x1E6E7UL, 0x1E6EDUL }, + { 0x1E6F0UL, 0x1E6F4UL }, + { 0x1E6FEUL, 0x1E6FFUL }, { 0x1E7E0UL, 0x1E7E6UL }, { 0x1E7E8UL, 0x1E7EBUL }, { 0x1E7EDUL, 0x1E7EEUL }, @@ -662,18 +687,20 @@ static const struct range ranges[] = { { 0x20000UL, 0x20000UL }, { 0x2A6DFUL, 0x2A6DFUL }, { 0x2A700UL, 0x2A700UL }, - { 0x2B739UL, 0x2B739UL }, - { 0x2B740UL, 0x2B740UL }, + { 0x2B73FUL, 0x2B740UL }, { 0x2B81DUL, 0x2B81DUL }, { 0x2B820UL, 0x2B820UL }, - { 0x2CEA1UL, 0x2CEA1UL }, + { 0x2CEADUL, 0x2CEADUL }, { 0x2CEB0UL, 0x2CEB0UL }, { 0x2EBE0UL, 0x2EBE0UL }, + { 0x2EBF0UL, 0x2EBF0UL }, + { 0x2EE5DUL, 0x2EE5DUL }, { 0x2F800UL, 0x2FA1DUL }, { 0x30000UL, 0x30000UL }, { 0x3134AUL, 0x3134AUL }, { 0x31350UL, 0x31350UL }, - { 0x323AFUL, 0x323AFUL } + { 0x323AFUL, 0x323B0UL }, + { 0x33479UL, 0x33479UL } }; const struct class utf8_L = { diff --git a/src/libre/class/utf8_Latin.c b/src/libre/class/utf8_Latin.c index ed3e4b180..362ab61a6 100644 --- a/src/libre/class/utf8_Latin.c +++ b/src/libre/class/utf8_Latin.c @@ -26,11 +26,8 @@ static const struct range ranges[] = { { 0x2160UL, 0x2188UL }, { 0x2C60UL, 0x2C7FUL }, { 0xA722UL, 0xA787UL }, - { 0xA78BUL, 0xA7CAUL }, - { 0xA7D0UL, 0xA7D1UL }, - { 0xA7D3UL, 0xA7D3UL }, - { 0xA7D5UL, 0xA7D9UL }, - { 0xA7F2UL, 0xA7FFUL }, + { 0xA78BUL, 0xA7DCUL }, + { 0xA7F1UL, 0xA7FFUL }, { 0xAB30UL, 0xAB5AUL }, { 0xAB5CUL, 0xAB64UL }, { 0xAB66UL, 0xAB69UL }, diff --git a/src/libre/class/utf8_Ll.c b/src/libre/class/utf8_Ll.c index 89de4bd0e..ad884ead6 100644 --- a/src/libre/class/utf8_Ll.c +++ b/src/libre/class/utf8_Ll.c @@ -148,7 +148,7 @@ static const struct range ranges[] = { { 0x024BUL, 0x024BUL }, { 0x024DUL, 0x024DUL }, { 0x024FUL, 0x0293UL }, - { 0x0295UL, 0x02AFUL }, + { 0x0296UL, 0x02AFUL }, { 0x0371UL, 0x0371UL }, { 0x0373UL, 0x0373UL }, { 0x0377UL, 0x0377UL }, @@ -277,6 +277,7 @@ static const struct range ranges[] = { { 0x10FDUL, 0x10FFUL }, { 0x13F8UL, 0x13FDUL }, { 0x1C80UL, 0x1C88UL }, + { 0x1C8AUL, 0x1C8AUL }, { 0x1D00UL, 0x1D2BUL }, { 0x1D6BUL, 0x1D77UL }, { 0x1D79UL, 0x1D9AUL }, @@ -607,11 +608,14 @@ static const struct range ranges[] = { { 0xA7C3UL, 0xA7C3UL }, { 0xA7C8UL, 0xA7C8UL }, { 0xA7CAUL, 0xA7CAUL }, + { 0xA7CDUL, 0xA7CDUL }, + { 0xA7CFUL, 0xA7CFUL }, { 0xA7D1UL, 0xA7D1UL }, { 0xA7D3UL, 0xA7D3UL }, { 0xA7D5UL, 0xA7D5UL }, { 0xA7D7UL, 0xA7D7UL }, { 0xA7D9UL, 0xA7D9UL }, + { 0xA7DBUL, 0xA7DBUL }, { 0xA7F6UL, 0xA7F6UL }, { 0xA7FAUL, 0xA7FAUL }, { 0xAB30UL, 0xAB5AUL }, @@ -627,8 +631,10 @@ static const struct range ranges[] = { { 0x105B3UL, 0x105B9UL }, { 0x105BBUL, 0x105BCUL }, { 0x10CC0UL, 0x10CF2UL }, + { 0x10D70UL, 0x10D85UL }, { 0x118C0UL, 0x118DFUL }, { 0x16E60UL, 0x16E7FUL }, + { 0x16EBBUL, 0x16ED3UL }, { 0x1D41AUL, 0x1D433UL }, { 0x1D44EUL, 0x1D454UL }, { 0x1D456UL, 0x1D467UL }, diff --git a/src/libre/class/utf8_Lm.c b/src/libre/class/utf8_Lm.c index 277354e6a..3052e9072 100644 --- a/src/libre/class/utf8_Lm.c +++ b/src/libre/class/utf8_Lm.c @@ -49,7 +49,7 @@ static const struct range ranges[] = { { 0xA717UL, 0xA71FUL }, { 0xA770UL, 0xA770UL }, { 0xA788UL, 0xA788UL }, - { 0xA7F2UL, 0xA7F4UL }, + { 0xA7F1UL, 0xA7F4UL }, { 0xA7F8UL, 0xA7F9UL }, { 0xA9CFUL, 0xA9CFUL }, { 0xA9E6UL, 0xA9E6UL }, @@ -63,16 +63,24 @@ static const struct range ranges[] = { { 0x10780UL, 0x10785UL }, { 0x10787UL, 0x107B0UL }, { 0x107B2UL, 0x107BAUL }, + { 0x10D4EUL, 0x10D4EUL }, + { 0x10D6FUL, 0x10D6FUL }, + { 0x10EC5UL, 0x10EC5UL }, + { 0x11DD9UL, 0x11DD9UL }, { 0x16B40UL, 0x16B43UL }, + { 0x16D40UL, 0x16D42UL }, + { 0x16D6BUL, 0x16D6CUL }, { 0x16F93UL, 0x16F9FUL }, { 0x16FE0UL, 0x16FE1UL }, { 0x16FE3UL, 0x16FE3UL }, + { 0x16FF2UL, 0x16FF3UL }, { 0x1AFF0UL, 0x1AFF3UL }, { 0x1AFF5UL, 0x1AFFBUL }, { 0x1AFFDUL, 0x1AFFEUL }, { 0x1E030UL, 0x1E06DUL }, { 0x1E137UL, 0x1E13DUL }, { 0x1E4EBUL, 0x1E4EBUL }, + { 0x1E6FFUL, 0x1E6FFUL }, { 0x1E94BUL, 0x1E94BUL } }; diff --git a/src/libre/class/utf8_Lo.c b/src/libre/class/utf8_Lo.c index 2188ecd6f..859b511dc 100644 --- a/src/libre/class/utf8_Lo.c +++ b/src/libre/class/utf8_Lo.c @@ -7,7 +7,7 @@ static const struct range ranges[] = { { 0x00BAUL, 0x00BAUL }, { 0x01BBUL, 0x01BBUL }, { 0x01C0UL, 0x01C3UL }, - { 0x0294UL, 0x0294UL }, + { 0x0294UL, 0x0295UL }, { 0x05D0UL, 0x05EAUL }, { 0x05EFUL, 0x05F2UL }, { 0x0620UL, 0x063FUL }, @@ -27,7 +27,7 @@ static const struct range ranges[] = { { 0x0840UL, 0x0858UL }, { 0x0860UL, 0x086AUL }, { 0x0870UL, 0x0887UL }, - { 0x0889UL, 0x088EUL }, + { 0x0889UL, 0x088FUL }, { 0x08A0UL, 0x08C8UL }, { 0x0904UL, 0x0939UL }, { 0x093DUL, 0x093DUL }, @@ -93,7 +93,7 @@ static const struct range ranges[] = { { 0x0C2AUL, 0x0C39UL }, { 0x0C3DUL, 0x0C3DUL }, { 0x0C58UL, 0x0C5AUL }, - { 0x0C5DUL, 0x0C5DUL }, + { 0x0C5CUL, 0x0C5DUL }, { 0x0C60UL, 0x0C61UL }, { 0x0C80UL, 0x0C80UL }, { 0x0C85UL, 0x0C8CUL }, @@ -102,7 +102,7 @@ static const struct range ranges[] = { { 0x0CAAUL, 0x0CB3UL }, { 0x0CB5UL, 0x0CB9UL }, { 0x0CBDUL, 0x0CBDUL }, - { 0x0CDDUL, 0x0CDEUL }, + { 0x0CDCUL, 0x0CDEUL }, { 0x0CE0UL, 0x0CE1UL }, { 0x0CF1UL, 0x0CF2UL }, { 0x0D04UL, 0x0D0CUL }, @@ -315,6 +315,7 @@ static const struct range ranges[] = { { 0x10450UL, 0x1049DUL }, { 0x10500UL, 0x10527UL }, { 0x10530UL, 0x10563UL }, + { 0x105C0UL, 0x105F3UL }, { 0x10600UL, 0x10736UL }, { 0x10740UL, 0x10755UL }, { 0x10760UL, 0x10767UL }, @@ -330,6 +331,7 @@ static const struct range ranges[] = { { 0x108F4UL, 0x108F5UL }, { 0x10900UL, 0x10915UL }, { 0x10920UL, 0x10939UL }, + { 0x10940UL, 0x10959UL }, { 0x10980UL, 0x109B7UL }, { 0x109BEUL, 0x109BFUL }, { 0x10A00UL, 0x10A00UL }, @@ -346,8 +348,12 @@ static const struct range ranges[] = { { 0x10B80UL, 0x10B91UL }, { 0x10C00UL, 0x10C48UL }, { 0x10D00UL, 0x10D23UL }, + { 0x10D4AUL, 0x10D4DUL }, + { 0x10D4FUL, 0x10D4FUL }, { 0x10E80UL, 0x10EA9UL }, { 0x10EB0UL, 0x10EB1UL }, + { 0x10EC2UL, 0x10EC4UL }, + { 0x10EC6UL, 0x10EC7UL }, { 0x10F00UL, 0x10F1CUL }, { 0x10F27UL, 0x10F27UL }, { 0x10F30UL, 0x10F45UL }, @@ -386,6 +392,13 @@ static const struct range ranges[] = { { 0x1133DUL, 0x1133DUL }, { 0x11350UL, 0x11350UL }, { 0x1135DUL, 0x11361UL }, + { 0x11380UL, 0x11389UL }, + { 0x1138BUL, 0x1138BUL }, + { 0x1138EUL, 0x1138EUL }, + { 0x11390UL, 0x113B5UL }, + { 0x113B7UL, 0x113B7UL }, + { 0x113D1UL, 0x113D1UL }, + { 0x113D3UL, 0x113D3UL }, { 0x11400UL, 0x11434UL }, { 0x11447UL, 0x1144AUL }, { 0x1145FUL, 0x11461UL }, @@ -419,6 +432,7 @@ static const struct range ranges[] = { { 0x11A5CUL, 0x11A89UL }, { 0x11A9DUL, 0x11A9DUL }, { 0x11AB0UL, 0x11AF8UL }, + { 0x11BC0UL, 0x11BE0UL }, { 0x11C00UL, 0x11C08UL }, { 0x11C0AUL, 0x11C2EUL }, { 0x11C40UL, 0x11C40UL }, @@ -431,6 +445,8 @@ static const struct range ranges[] = { { 0x11D67UL, 0x11D68UL }, { 0x11D6AUL, 0x11D89UL }, { 0x11D98UL, 0x11D98UL }, + { 0x11DB0UL, 0x11DD8UL }, + { 0x11DDAUL, 0x11DDBUL }, { 0x11EE0UL, 0x11EF2UL }, { 0x11F02UL, 0x11F02UL }, { 0x11F04UL, 0x11F10UL }, @@ -441,7 +457,9 @@ static const struct range ranges[] = { { 0x12F90UL, 0x12FF0UL }, { 0x13000UL, 0x1342FUL }, { 0x13441UL, 0x13446UL }, + { 0x13460UL, 0x143FAUL }, { 0x14400UL, 0x14646UL }, + { 0x16100UL, 0x1611DUL }, { 0x16800UL, 0x16A38UL }, { 0x16A40UL, 0x16A5EUL }, { 0x16A70UL, 0x16ABEUL }, @@ -449,13 +467,14 @@ static const struct range ranges[] = { { 0x16B00UL, 0x16B2FUL }, { 0x16B63UL, 0x16B77UL }, { 0x16B7DUL, 0x16B8FUL }, + { 0x16D43UL, 0x16D6AUL }, { 0x16F00UL, 0x16F4AUL }, { 0x16F50UL, 0x16F50UL }, { 0x17000UL, 0x17000UL }, - { 0x187F7UL, 0x187F7UL }, - { 0x18800UL, 0x18CD5UL }, - { 0x18D00UL, 0x18D00UL }, - { 0x18D08UL, 0x18D08UL }, + { 0x187FFUL, 0x18CD5UL }, + { 0x18CFFUL, 0x18D00UL }, + { 0x18D1EUL, 0x18D1EUL }, + { 0x18D80UL, 0x18DF2UL }, { 0x1B000UL, 0x1B122UL }, { 0x1B132UL, 0x1B132UL }, { 0x1B150UL, 0x1B152UL }, @@ -472,6 +491,14 @@ static const struct range ranges[] = { { 0x1E290UL, 0x1E2ADUL }, { 0x1E2C0UL, 0x1E2EBUL }, { 0x1E4D0UL, 0x1E4EAUL }, + { 0x1E5D0UL, 0x1E5EDUL }, + { 0x1E5F0UL, 0x1E5F0UL }, + { 0x1E6C0UL, 0x1E6DEUL }, + { 0x1E6E0UL, 0x1E6E2UL }, + { 0x1E6E4UL, 0x1E6E5UL }, + { 0x1E6E7UL, 0x1E6EDUL }, + { 0x1E6F0UL, 0x1E6F4UL }, + { 0x1E6FEUL, 0x1E6FEUL }, { 0x1E7E0UL, 0x1E7E6UL }, { 0x1E7E8UL, 0x1E7EBUL }, { 0x1E7EDUL, 0x1E7EEUL }, @@ -513,18 +540,20 @@ static const struct range ranges[] = { { 0x20000UL, 0x20000UL }, { 0x2A6DFUL, 0x2A6DFUL }, { 0x2A700UL, 0x2A700UL }, - { 0x2B739UL, 0x2B739UL }, - { 0x2B740UL, 0x2B740UL }, + { 0x2B73FUL, 0x2B740UL }, { 0x2B81DUL, 0x2B81DUL }, { 0x2B820UL, 0x2B820UL }, - { 0x2CEA1UL, 0x2CEA1UL }, + { 0x2CEADUL, 0x2CEADUL }, { 0x2CEB0UL, 0x2CEB0UL }, { 0x2EBE0UL, 0x2EBE0UL }, + { 0x2EBF0UL, 0x2EBF0UL }, + { 0x2EE5DUL, 0x2EE5DUL }, { 0x2F800UL, 0x2FA1DUL }, { 0x30000UL, 0x30000UL }, { 0x3134AUL, 0x3134AUL }, { 0x31350UL, 0x31350UL }, - { 0x323AFUL, 0x323AFUL } + { 0x323AFUL, 0x323B0UL }, + { 0x33479UL, 0x33479UL } }; const struct class utf8_Lo = { diff --git a/src/libre/class/utf8_Lu.c b/src/libre/class/utf8_Lu.c index 0e8d00299..e1f752b18 100644 --- a/src/libre/class/utf8_Lu.c +++ b/src/libre/class/utf8_Lu.c @@ -279,6 +279,7 @@ static const struct range ranges[] = { { 0x10C7UL, 0x10C7UL }, { 0x10CDUL, 0x10CDUL }, { 0x13A0UL, 0x13F5UL }, + { 0x1C89UL, 0x1C89UL }, { 0x1C90UL, 0x1CBAUL }, { 0x1CBDUL, 0x1CBFUL }, { 0x1E00UL, 0x1E00UL }, @@ -603,9 +604,15 @@ static const struct range ranges[] = { { 0xA7C2UL, 0xA7C2UL }, { 0xA7C4UL, 0xA7C7UL }, { 0xA7C9UL, 0xA7C9UL }, + { 0xA7CBUL, 0xA7CCUL }, + { 0xA7CEUL, 0xA7CEUL }, { 0xA7D0UL, 0xA7D0UL }, + { 0xA7D2UL, 0xA7D2UL }, + { 0xA7D4UL, 0xA7D4UL }, { 0xA7D6UL, 0xA7D6UL }, { 0xA7D8UL, 0xA7D8UL }, + { 0xA7DAUL, 0xA7DAUL }, + { 0xA7DCUL, 0xA7DCUL }, { 0xA7F5UL, 0xA7F5UL }, { 0xFF21UL, 0xFF3AUL }, { 0x10400UL, 0x10427UL }, @@ -615,8 +622,10 @@ static const struct range ranges[] = { { 0x1058CUL, 0x10592UL }, { 0x10594UL, 0x10595UL }, { 0x10C80UL, 0x10CB2UL }, + { 0x10D50UL, 0x10D65UL }, { 0x118A0UL, 0x118BFUL }, { 0x16E40UL, 0x16E5FUL }, + { 0x16EA0UL, 0x16EB8UL }, { 0x1D400UL, 0x1D419UL }, { 0x1D434UL, 0x1D44DUL }, { 0x1D468UL, 0x1D481UL }, diff --git a/src/libre/class/utf8_M.c b/src/libre/class/utf8_M.c index 38c00e3f9..ce4a80f2b 100644 --- a/src/libre/class/utf8_M.c +++ b/src/libre/class/utf8_M.c @@ -27,7 +27,7 @@ static const struct range ranges[] = { { 0x0825UL, 0x0827UL }, { 0x0829UL, 0x082DUL }, { 0x0859UL, 0x085BUL }, - { 0x0898UL, 0x089FUL }, + { 0x0897UL, 0x089FUL }, { 0x08CAUL, 0x08E1UL }, { 0x08E3UL, 0x0903UL }, { 0x093AUL, 0x093CUL }, @@ -139,7 +139,8 @@ static const struct range ranges[] = { { 0x1A55UL, 0x1A5EUL }, { 0x1A60UL, 0x1A7CUL }, { 0x1A7FUL, 0x1A7FUL }, - { 0x1AB0UL, 0x1ACEUL }, + { 0x1AB0UL, 0x1ADDUL }, + { 0x1AE0UL, 0x1AEBUL }, { 0x1B00UL, 0x1B04UL }, { 0x1B34UL, 0x1B44UL }, { 0x1B6BUL, 0x1B73UL }, @@ -203,8 +204,9 @@ static const struct range ranges[] = { { 0x10A3FUL, 0x10A3FUL }, { 0x10AE5UL, 0x10AE6UL }, { 0x10D24UL, 0x10D27UL }, + { 0x10D69UL, 0x10D6DUL }, { 0x10EABUL, 0x10EACUL }, - { 0x10EFDUL, 0x10EFFUL }, + { 0x10EFAUL, 0x10EFFUL }, { 0x10F46UL, 0x10F50UL }, { 0x10F82UL, 0x10F85UL }, { 0x11000UL, 0x11002UL }, @@ -235,6 +237,13 @@ static const struct range ranges[] = { { 0x11362UL, 0x11363UL }, { 0x11366UL, 0x1136CUL }, { 0x11370UL, 0x11374UL }, + { 0x113B8UL, 0x113C0UL }, + { 0x113C2UL, 0x113C2UL }, + { 0x113C5UL, 0x113C5UL }, + { 0x113C7UL, 0x113CAUL }, + { 0x113CCUL, 0x113D0UL }, + { 0x113D2UL, 0x113D2UL }, + { 0x113E1UL, 0x113E2UL }, { 0x11435UL, 0x11446UL }, { 0x1145EUL, 0x1145EUL }, { 0x114B0UL, 0x114C3UL }, @@ -259,6 +268,7 @@ static const struct range ranges[] = { { 0x11A47UL, 0x11A47UL }, { 0x11A51UL, 0x11A5BUL }, { 0x11A8AUL, 0x11A99UL }, + { 0x11B60UL, 0x11B67UL }, { 0x11C2FUL, 0x11C36UL }, { 0x11C38UL, 0x11C3FUL }, { 0x11C92UL, 0x11CA7UL }, @@ -276,8 +286,10 @@ static const struct range ranges[] = { { 0x11F03UL, 0x11F03UL }, { 0x11F34UL, 0x11F3AUL }, { 0x11F3EUL, 0x11F42UL }, + { 0x11F5AUL, 0x11F5AUL }, { 0x13440UL, 0x13440UL }, { 0x13447UL, 0x13455UL }, + { 0x1611EUL, 0x1612FUL }, { 0x16AF0UL, 0x16AF4UL }, { 0x16B30UL, 0x16B36UL }, { 0x16F4FUL, 0x16F4FUL }, @@ -310,6 +322,11 @@ static const struct range ranges[] = { { 0x1E2AEUL, 0x1E2AEUL }, { 0x1E2ECUL, 0x1E2EFUL }, { 0x1E4ECUL, 0x1E4EFUL }, + { 0x1E5EEUL, 0x1E5EFUL }, + { 0x1E6E3UL, 0x1E6E3UL }, + { 0x1E6E6UL, 0x1E6E6UL }, + { 0x1E6EEUL, 0x1E6EFUL }, + { 0x1E6F5UL, 0x1E6F5UL }, { 0x1E8D0UL, 0x1E8D6UL }, { 0x1E944UL, 0x1E94AUL }, { 0xE0100UL, 0xE01EFUL } diff --git a/src/libre/class/utf8_Mc.c b/src/libre/class/utf8_Mc.c index 752d1bed5..aa391dcb9 100644 --- a/src/libre/class/utf8_Mc.c +++ b/src/libre/class/utf8_Mc.c @@ -137,6 +137,12 @@ static const struct range ranges[] = { { 0x1134BUL, 0x1134DUL }, { 0x11357UL, 0x11357UL }, { 0x11362UL, 0x11363UL }, + { 0x113B8UL, 0x113BAUL }, + { 0x113C2UL, 0x113C2UL }, + { 0x113C5UL, 0x113C5UL }, + { 0x113C7UL, 0x113CAUL }, + { 0x113CCUL, 0x113CDUL }, + { 0x113CFUL, 0x113CFUL }, { 0x11435UL, 0x11437UL }, { 0x11440UL, 0x11441UL }, { 0x11445UL, 0x11445UL }, @@ -153,6 +159,7 @@ static const struct range ranges[] = { { 0x116ACUL, 0x116ACUL }, { 0x116AEUL, 0x116AFUL }, { 0x116B6UL, 0x116B6UL }, + { 0x1171EUL, 0x1171EUL }, { 0x11720UL, 0x11721UL }, { 0x11726UL, 0x11726UL }, { 0x1182CUL, 0x1182EUL }, @@ -168,6 +175,9 @@ static const struct range ranges[] = { { 0x11A39UL, 0x11A39UL }, { 0x11A57UL, 0x11A58UL }, { 0x11A97UL, 0x11A97UL }, + { 0x11B61UL, 0x11B61UL }, + { 0x11B65UL, 0x11B65UL }, + { 0x11B67UL, 0x11B67UL }, { 0x11C2FUL, 0x11C2FUL }, { 0x11C3EUL, 0x11C3EUL }, { 0x11CA9UL, 0x11CA9UL }, @@ -181,6 +191,7 @@ static const struct range ranges[] = { { 0x11F34UL, 0x11F35UL }, { 0x11F3EUL, 0x11F3FUL }, { 0x11F41UL, 0x11F41UL }, + { 0x1612AUL, 0x1612CUL }, { 0x16F51UL, 0x16F87UL }, { 0x16FF0UL, 0x16FF1UL }, { 0x1D165UL, 0x1D166UL }, diff --git a/src/libre/class/utf8_Mn.c b/src/libre/class/utf8_Mn.c index 2167dc311..4ad1f3ddd 100644 --- a/src/libre/class/utf8_Mn.c +++ b/src/libre/class/utf8_Mn.c @@ -27,7 +27,7 @@ static const struct range ranges[] = { { 0x0825UL, 0x0827UL }, { 0x0829UL, 0x082DUL }, { 0x0859UL, 0x085BUL }, - { 0x0898UL, 0x089FUL }, + { 0x0897UL, 0x089FUL }, { 0x08CAUL, 0x08E1UL }, { 0x08E3UL, 0x0902UL }, { 0x093AUL, 0x093AUL }, @@ -145,7 +145,8 @@ static const struct range ranges[] = { { 0x1A73UL, 0x1A7CUL }, { 0x1A7FUL, 0x1A7FUL }, { 0x1AB0UL, 0x1ABDUL }, - { 0x1ABFUL, 0x1ACEUL }, + { 0x1ABFUL, 0x1ADDUL }, + { 0x1AE0UL, 0x1AEBUL }, { 0x1B00UL, 0x1B03UL }, { 0x1B34UL, 0x1B34UL }, { 0x1B36UL, 0x1B3AUL }, @@ -225,8 +226,9 @@ static const struct range ranges[] = { { 0x10A3FUL, 0x10A3FUL }, { 0x10AE5UL, 0x10AE6UL }, { 0x10D24UL, 0x10D27UL }, + { 0x10D69UL, 0x10D6DUL }, { 0x10EABUL, 0x10EACUL }, - { 0x10EFDUL, 0x10EFFUL }, + { 0x10EFAUL, 0x10EFFUL }, { 0x10F46UL, 0x10F50UL }, { 0x10F82UL, 0x10F85UL }, { 0x11001UL, 0x11001UL }, @@ -257,6 +259,11 @@ static const struct range ranges[] = { { 0x11340UL, 0x11340UL }, { 0x11366UL, 0x1136CUL }, { 0x11370UL, 0x11374UL }, + { 0x113BBUL, 0x113C0UL }, + { 0x113CEUL, 0x113CEUL }, + { 0x113D0UL, 0x113D0UL }, + { 0x113D2UL, 0x113D2UL }, + { 0x113E1UL, 0x113E2UL }, { 0x11438UL, 0x1143FUL }, { 0x11442UL, 0x11444UL }, { 0x11446UL, 0x11446UL }, @@ -276,7 +283,8 @@ static const struct range ranges[] = { { 0x116ADUL, 0x116ADUL }, { 0x116B0UL, 0x116B5UL }, { 0x116B7UL, 0x116B7UL }, - { 0x1171DUL, 0x1171FUL }, + { 0x1171DUL, 0x1171DUL }, + { 0x1171FUL, 0x1171FUL }, { 0x11722UL, 0x11725UL }, { 0x11727UL, 0x1172BUL }, { 0x1182FUL, 0x11837UL }, @@ -295,6 +303,9 @@ static const struct range ranges[] = { { 0x11A59UL, 0x11A5BUL }, { 0x11A8AUL, 0x11A96UL }, { 0x11A98UL, 0x11A99UL }, + { 0x11B60UL, 0x11B60UL }, + { 0x11B62UL, 0x11B64UL }, + { 0x11B66UL, 0x11B66UL }, { 0x11C30UL, 0x11C36UL }, { 0x11C38UL, 0x11C3DUL }, { 0x11C3FUL, 0x11C3FUL }, @@ -315,8 +326,11 @@ static const struct range ranges[] = { { 0x11F36UL, 0x11F3AUL }, { 0x11F40UL, 0x11F40UL }, { 0x11F42UL, 0x11F42UL }, + { 0x11F5AUL, 0x11F5AUL }, { 0x13440UL, 0x13440UL }, { 0x13447UL, 0x13455UL }, + { 0x1611EUL, 0x16129UL }, + { 0x1612DUL, 0x1612FUL }, { 0x16AF0UL, 0x16AF4UL }, { 0x16B30UL, 0x16B36UL }, { 0x16F4FUL, 0x16F4FUL }, @@ -346,6 +360,11 @@ static const struct range ranges[] = { { 0x1E2AEUL, 0x1E2AEUL }, { 0x1E2ECUL, 0x1E2EFUL }, { 0x1E4ECUL, 0x1E4EFUL }, + { 0x1E5EEUL, 0x1E5EFUL }, + { 0x1E6E3UL, 0x1E6E3UL }, + { 0x1E6E6UL, 0x1E6E6UL }, + { 0x1E6EEUL, 0x1E6EFUL }, + { 0x1E6F5UL, 0x1E6F5UL }, { 0x1E8D0UL, 0x1E8D6UL }, { 0x1E944UL, 0x1E94AUL }, { 0xE0100UL, 0xE01EFUL } diff --git a/src/libre/class/utf8_Myanmar.c b/src/libre/class/utf8_Myanmar.c index deb2f419c..43a91b0d5 100644 --- a/src/libre/class/utf8_Myanmar.c +++ b/src/libre/class/utf8_Myanmar.c @@ -5,7 +5,8 @@ static const struct range ranges[] = { { 0x1000UL, 0x109FUL }, { 0xA9E0UL, 0xA9FEUL }, - { 0xAA60UL, 0xAA7FUL } + { 0xAA60UL, 0xAA7FUL }, + { 0x116D0UL, 0x116E3UL } }; const struct class utf8_Myanmar = { diff --git a/src/libre/class/utf8_N.c b/src/libre/class/utf8_N.c index 3e180474f..fa7dcc037 100644 --- a/src/libre/class/utf8_N.c +++ b/src/libre/class/utf8_N.c @@ -96,6 +96,7 @@ static const struct range ranges[] = { { 0x10BA9UL, 0x10BAFUL }, { 0x10CFAUL, 0x10CFFUL }, { 0x10D30UL, 0x10D39UL }, + { 0x10D40UL, 0x10D49UL }, { 0x10E60UL, 0x10E7EUL }, { 0x10F1DUL, 0x10F26UL }, { 0x10F51UL, 0x10F54UL }, @@ -110,20 +111,27 @@ static const struct range ranges[] = { { 0x114D0UL, 0x114D9UL }, { 0x11650UL, 0x11659UL }, { 0x116C0UL, 0x116C9UL }, + { 0x116D0UL, 0x116E3UL }, { 0x11730UL, 0x1173BUL }, { 0x118E0UL, 0x118F2UL }, { 0x11950UL, 0x11959UL }, + { 0x11BF0UL, 0x11BF9UL }, { 0x11C50UL, 0x11C6CUL }, { 0x11D50UL, 0x11D59UL }, { 0x11DA0UL, 0x11DA9UL }, + { 0x11DE0UL, 0x11DE9UL }, { 0x11F50UL, 0x11F59UL }, { 0x11FC0UL, 0x11FD4UL }, { 0x12400UL, 0x1246EUL }, + { 0x16130UL, 0x16139UL }, { 0x16A60UL, 0x16A69UL }, { 0x16AC0UL, 0x16AC9UL }, { 0x16B50UL, 0x16B59UL }, { 0x16B5BUL, 0x16B61UL }, + { 0x16D70UL, 0x16D79UL }, { 0x16E80UL, 0x16E96UL }, + { 0x16FF4UL, 0x16FF6UL }, + { 0x1CCF0UL, 0x1CCF9UL }, { 0x1D2C0UL, 0x1D2D3UL }, { 0x1D2E0UL, 0x1D2F3UL }, { 0x1D360UL, 0x1D378UL }, @@ -131,6 +139,7 @@ static const struct range ranges[] = { { 0x1E140UL, 0x1E149UL }, { 0x1E2F0UL, 0x1E2F9UL }, { 0x1E4F0UL, 0x1E4F9UL }, + { 0x1E5F1UL, 0x1E5FAUL }, { 0x1E8C7UL, 0x1E8CFUL }, { 0x1E950UL, 0x1E959UL }, { 0x1EC71UL, 0x1ECABUL }, diff --git a/src/libre/class/utf8_Nd.c b/src/libre/class/utf8_Nd.c index 684b0c73f..19822a94b 100644 --- a/src/libre/class/utf8_Nd.c +++ b/src/libre/class/utf8_Nd.c @@ -42,6 +42,7 @@ static const struct range ranges[] = { { 0xFF10UL, 0xFF19UL }, { 0x104A0UL, 0x104A9UL }, { 0x10D30UL, 0x10D39UL }, + { 0x10D40UL, 0x10D49UL }, { 0x11066UL, 0x1106FUL }, { 0x110F0UL, 0x110F9UL }, { 0x11136UL, 0x1113FUL }, @@ -51,20 +52,27 @@ static const struct range ranges[] = { { 0x114D0UL, 0x114D9UL }, { 0x11650UL, 0x11659UL }, { 0x116C0UL, 0x116C9UL }, + { 0x116D0UL, 0x116E3UL }, { 0x11730UL, 0x11739UL }, { 0x118E0UL, 0x118E9UL }, { 0x11950UL, 0x11959UL }, + { 0x11BF0UL, 0x11BF9UL }, { 0x11C50UL, 0x11C59UL }, { 0x11D50UL, 0x11D59UL }, { 0x11DA0UL, 0x11DA9UL }, + { 0x11DE0UL, 0x11DE9UL }, { 0x11F50UL, 0x11F59UL }, + { 0x16130UL, 0x16139UL }, { 0x16A60UL, 0x16A69UL }, { 0x16AC0UL, 0x16AC9UL }, { 0x16B50UL, 0x16B59UL }, + { 0x16D70UL, 0x16D79UL }, + { 0x1CCF0UL, 0x1CCF9UL }, { 0x1D7CEUL, 0x1D7FFUL }, { 0x1E140UL, 0x1E149UL }, { 0x1E2F0UL, 0x1E2F9UL }, { 0x1E4F0UL, 0x1E4F9UL }, + { 0x1E5F1UL, 0x1E5FAUL }, { 0x1E950UL, 0x1E959UL }, { 0x1FBF0UL, 0x1FBF9UL } }; diff --git a/src/libre/class/utf8_Nl.c b/src/libre/class/utf8_Nl.c index fdf83eb7f..71097a550 100644 --- a/src/libre/class/utf8_Nl.c +++ b/src/libre/class/utf8_Nl.c @@ -14,7 +14,8 @@ static const struct range ranges[] = { { 0x10341UL, 0x10341UL }, { 0x1034AUL, 0x1034AUL }, { 0x103D1UL, 0x103D5UL }, - { 0x12400UL, 0x1246EUL } + { 0x12400UL, 0x1246EUL }, + { 0x16FF4UL, 0x16FF6UL } }; const struct class utf8_Nl = { diff --git a/src/libre/class/utf8_Ol_Onal.c b/src/libre/class/utf8_Ol_Onal.c new file mode 100644 index 000000000..7223f22e5 --- /dev/null +++ b/src/libre/class/utf8_Ol_Onal.c @@ -0,0 +1,14 @@ +/* generated */ + +#include "class.h" + +static const struct range ranges[] = { + { 0x1E5D0UL, 0x1E5FAUL }, + { 0x1E5FFUL, 0x1E5FFUL } +}; + +const struct class utf8_Ol_Onal = { + ranges, + sizeof ranges / sizeof *ranges +}; + diff --git a/src/libre/class/utf8_P.c b/src/libre/class/utf8_P.c index 38e217087..664a1d28c 100644 --- a/src/libre/class/utf8_P.c +++ b/src/libre/class/utf8_P.c @@ -68,8 +68,9 @@ static const struct range ranges[] = { { 0x1A1EUL, 0x1A1FUL }, { 0x1AA0UL, 0x1AA6UL }, { 0x1AA8UL, 0x1AADUL }, + { 0x1B4EUL, 0x1B4FUL }, { 0x1B5AUL, 0x1B60UL }, - { 0x1B7DUL, 0x1B7EUL }, + { 0x1B7DUL, 0x1B7FUL }, { 0x1BFCUL, 0x1BFFUL }, { 0x1C3BUL, 0x1C3FUL }, { 0x1C7EUL, 0x1C7FUL }, @@ -148,7 +149,9 @@ static const struct range ranges[] = { { 0x10AF0UL, 0x10AF6UL }, { 0x10B39UL, 0x10B3FUL }, { 0x10B99UL, 0x10B9CUL }, + { 0x10D6EUL, 0x10D6EUL }, { 0x10EADUL, 0x10EADUL }, + { 0x10ED0UL, 0x10ED0UL }, { 0x10F55UL, 0x10F59UL }, { 0x10F86UL, 0x10F89UL }, { 0x11047UL, 0x1104DUL }, @@ -162,6 +165,8 @@ static const struct range ranges[] = { { 0x111DDUL, 0x111DFUL }, { 0x11238UL, 0x1123DUL }, { 0x112A9UL, 0x112A9UL }, + { 0x113D4UL, 0x113D5UL }, + { 0x113D7UL, 0x113D8UL }, { 0x1144BUL, 0x1144FUL }, { 0x1145AUL, 0x1145BUL }, { 0x1145DUL, 0x1145DUL }, @@ -178,6 +183,7 @@ static const struct range ranges[] = { { 0x11A9AUL, 0x11A9CUL }, { 0x11A9EUL, 0x11AA2UL }, { 0x11B00UL, 0x11B09UL }, + { 0x11BE1UL, 0x11BE1UL }, { 0x11C41UL, 0x11C45UL }, { 0x11C70UL, 0x11C71UL }, { 0x11EF7UL, 0x11EF8UL }, @@ -189,10 +195,12 @@ static const struct range ranges[] = { { 0x16AF5UL, 0x16AF5UL }, { 0x16B37UL, 0x16B3BUL }, { 0x16B44UL, 0x16B44UL }, + { 0x16D6DUL, 0x16D6FUL }, { 0x16E97UL, 0x16E9AUL }, { 0x16FE2UL, 0x16FE2UL }, { 0x1BC9FUL, 0x1BC9FUL }, { 0x1DA87UL, 0x1DA8BUL }, + { 0x1E5FFUL, 0x1E5FFUL }, { 0x1E95EUL, 0x1E95FUL } }; diff --git a/src/libre/class/utf8_Pd.c b/src/libre/class/utf8_Pd.c index 0681ece71..c15bf5021 100644 --- a/src/libre/class/utf8_Pd.c +++ b/src/libre/class/utf8_Pd.c @@ -21,6 +21,7 @@ static const struct range ranges[] = { { 0xFE58UL, 0xFE58UL }, { 0xFE63UL, 0xFE63UL }, { 0xFF0DUL, 0xFF0DUL }, + { 0x10D6EUL, 0x10D6EUL }, { 0x10EADUL, 0x10EADUL } }; diff --git a/src/libre/class/utf8_Po.c b/src/libre/class/utf8_Po.c index 95ccb96b9..4b20947c0 100644 --- a/src/libre/class/utf8_Po.c +++ b/src/libre/class/utf8_Po.c @@ -62,8 +62,9 @@ static const struct range ranges[] = { { 0x1A1EUL, 0x1A1FUL }, { 0x1AA0UL, 0x1AA6UL }, { 0x1AA8UL, 0x1AADUL }, + { 0x1B4EUL, 0x1B4FUL }, { 0x1B5AUL, 0x1B60UL }, - { 0x1B7DUL, 0x1B7EUL }, + { 0x1B7DUL, 0x1B7FUL }, { 0x1BFCUL, 0x1BFFUL }, { 0x1C3BUL, 0x1C3FUL }, { 0x1C7EUL, 0x1C7FUL }, @@ -145,6 +146,7 @@ static const struct range ranges[] = { { 0x10AF0UL, 0x10AF6UL }, { 0x10B39UL, 0x10B3FUL }, { 0x10B99UL, 0x10B9CUL }, + { 0x10ED0UL, 0x10ED0UL }, { 0x10F55UL, 0x10F59UL }, { 0x10F86UL, 0x10F89UL }, { 0x11047UL, 0x1104DUL }, @@ -158,6 +160,8 @@ static const struct range ranges[] = { { 0x111DDUL, 0x111DFUL }, { 0x11238UL, 0x1123DUL }, { 0x112A9UL, 0x112A9UL }, + { 0x113D4UL, 0x113D5UL }, + { 0x113D7UL, 0x113D8UL }, { 0x1144BUL, 0x1144FUL }, { 0x1145AUL, 0x1145BUL }, { 0x1145DUL, 0x1145DUL }, @@ -174,6 +178,7 @@ static const struct range ranges[] = { { 0x11A9AUL, 0x11A9CUL }, { 0x11A9EUL, 0x11AA2UL }, { 0x11B00UL, 0x11B09UL }, + { 0x11BE1UL, 0x11BE1UL }, { 0x11C41UL, 0x11C45UL }, { 0x11C70UL, 0x11C71UL }, { 0x11EF7UL, 0x11EF8UL }, @@ -185,10 +190,12 @@ static const struct range ranges[] = { { 0x16AF5UL, 0x16AF5UL }, { 0x16B37UL, 0x16B3BUL }, { 0x16B44UL, 0x16B44UL }, + { 0x16D6DUL, 0x16D6FUL }, { 0x16E97UL, 0x16E9AUL }, { 0x16FE2UL, 0x16FE2UL }, { 0x1BC9FUL, 0x1BC9FUL }, { 0x1DA87UL, 0x1DA8BUL }, + { 0x1E5FFUL, 0x1E5FFUL }, { 0x1E95EUL, 0x1E95FUL } }; diff --git a/src/libre/class/utf8_S.c b/src/libre/class/utf8_S.c index 5a114bea5..9b66a152c 100644 --- a/src/libre/class/utf8_S.c +++ b/src/libre/class/utf8_S.c @@ -75,7 +75,7 @@ static const struct range ranges[] = { { 0x2052UL, 0x2052UL }, { 0x207AUL, 0x207CUL }, { 0x208AUL, 0x208CUL }, - { 0x20A0UL, 0x20C0UL }, + { 0x20A0UL, 0x20C1UL }, { 0x2100UL, 0x2101UL }, { 0x2103UL, 0x2106UL }, { 0x2108UL, 0x2109UL }, @@ -93,7 +93,7 @@ static const struct range ranges[] = { { 0x218AUL, 0x218BUL }, { 0x2190UL, 0x2307UL }, { 0x230CUL, 0x2328UL }, - { 0x232BUL, 0x2426UL }, + { 0x232BUL, 0x2429UL }, { 0x2440UL, 0x244AUL }, { 0x249CUL, 0x24E9UL }, { 0x2500UL, 0x2767UL }, @@ -103,14 +103,13 @@ static const struct range ranges[] = { { 0x2999UL, 0x29D7UL }, { 0x29DCUL, 0x29FBUL }, { 0x29FEUL, 0x2B73UL }, - { 0x2B76UL, 0x2B95UL }, - { 0x2B97UL, 0x2BFFUL }, + { 0x2B76UL, 0x2BFFUL }, { 0x2CE5UL, 0x2CEAUL }, { 0x2E50UL, 0x2E51UL }, { 0x2E80UL, 0x2E99UL }, { 0x2E9BUL, 0x2EF3UL }, { 0x2F00UL, 0x2FD5UL }, - { 0x2FF0UL, 0x2FFBUL }, + { 0x2FF0UL, 0x2FFFUL }, { 0x3004UL, 0x3004UL }, { 0x3012UL, 0x3013UL }, { 0x3020UL, 0x3020UL }, @@ -119,7 +118,8 @@ static const struct range ranges[] = { { 0x309BUL, 0x309CUL }, { 0x3190UL, 0x3191UL }, { 0x3196UL, 0x319FUL }, - { 0x31C0UL, 0x31E3UL }, + { 0x31C0UL, 0x31E5UL }, + { 0x31EFUL, 0x31EFUL }, { 0x3200UL, 0x321EUL }, { 0x322AUL, 0x3247UL }, { 0x3250UL, 0x3250UL }, @@ -137,9 +137,10 @@ static const struct range ranges[] = { { 0xAB5BUL, 0xAB5BUL }, { 0xAB6AUL, 0xAB6BUL }, { 0xFB29UL, 0xFB29UL }, - { 0xFBB2UL, 0xFBC2UL }, + { 0xFBB2UL, 0xFBD2UL }, { 0xFD40UL, 0xFD4FUL }, - { 0xFDCFUL, 0xFDCFUL }, + { 0xFD90UL, 0xFD91UL }, + { 0xFDC8UL, 0xFDCFUL }, { 0xFDFCUL, 0xFDFFUL }, { 0xFE62UL, 0xFE62UL }, { 0xFE64UL, 0xFE66UL }, @@ -162,11 +163,18 @@ static const struct range ranges[] = { { 0x101D0UL, 0x101FCUL }, { 0x10877UL, 0x10878UL }, { 0x10AC8UL, 0x10AC8UL }, + { 0x10D8EUL, 0x10D8FUL }, + { 0x10ED1UL, 0x10ED8UL }, { 0x1173FUL, 0x1173FUL }, { 0x11FD5UL, 0x11FF1UL }, { 0x16B3CUL, 0x16B3FUL }, { 0x16B45UL, 0x16B45UL }, { 0x1BC9CUL, 0x1BC9CUL }, + { 0x1CC00UL, 0x1CCEFUL }, + { 0x1CCFAUL, 0x1CCFCUL }, + { 0x1CD00UL, 0x1CEB3UL }, + { 0x1CEBAUL, 0x1CED0UL }, + { 0x1CEE0UL, 0x1CEF0UL }, { 0x1CF50UL, 0x1CFC3UL }, { 0x1D000UL, 0x1D0F5UL }, { 0x1D100UL, 0x1D126UL }, @@ -211,11 +219,10 @@ static const struct range ranges[] = { { 0x1F240UL, 0x1F248UL }, { 0x1F250UL, 0x1F251UL }, { 0x1F260UL, 0x1F265UL }, - { 0x1F300UL, 0x1F6D7UL }, + { 0x1F300UL, 0x1F6D8UL }, { 0x1F6DCUL, 0x1F6ECUL }, { 0x1F6F0UL, 0x1F6FCUL }, - { 0x1F700UL, 0x1F776UL }, - { 0x1F77BUL, 0x1F7D9UL }, + { 0x1F700UL, 0x1F7D9UL }, { 0x1F7E0UL, 0x1F7EBUL }, { 0x1F7F0UL, 0x1F7F0UL }, { 0x1F800UL, 0x1F80BUL }, @@ -223,18 +230,21 @@ static const struct range ranges[] = { { 0x1F850UL, 0x1F859UL }, { 0x1F860UL, 0x1F887UL }, { 0x1F890UL, 0x1F8ADUL }, - { 0x1F8B0UL, 0x1F8B1UL }, - { 0x1F900UL, 0x1FA53UL }, + { 0x1F8B0UL, 0x1F8BBUL }, + { 0x1F8C0UL, 0x1F8C1UL }, + { 0x1F8D0UL, 0x1F8D8UL }, + { 0x1F900UL, 0x1FA57UL }, { 0x1FA60UL, 0x1FA6DUL }, { 0x1FA70UL, 0x1FA7CUL }, - { 0x1FA80UL, 0x1FA88UL }, - { 0x1FA90UL, 0x1FABDUL }, - { 0x1FABFUL, 0x1FAC5UL }, - { 0x1FACEUL, 0x1FADBUL }, - { 0x1FAE0UL, 0x1FAE8UL }, - { 0x1FAF0UL, 0x1FAF8UL }, + { 0x1FA80UL, 0x1FA8AUL }, + { 0x1FA8EUL, 0x1FAC6UL }, + { 0x1FAC8UL, 0x1FAC8UL }, + { 0x1FACDUL, 0x1FADCUL }, + { 0x1FADFUL, 0x1FAEAUL }, + { 0x1FAEFUL, 0x1FAF8UL }, { 0x1FB00UL, 0x1FB92UL }, - { 0x1FB94UL, 0x1FBCAUL } + { 0x1FB94UL, 0x1FBEFUL }, + { 0x1FBFAUL, 0x1FBFAUL } }; const struct class utf8_S = { diff --git a/src/libre/class/utf8_Sc.c b/src/libre/class/utf8_Sc.c index 0effc2b86..3bba263b8 100644 --- a/src/libre/class/utf8_Sc.c +++ b/src/libre/class/utf8_Sc.c @@ -14,7 +14,7 @@ static const struct range ranges[] = { { 0x0BF9UL, 0x0BF9UL }, { 0x0E3FUL, 0x0E3FUL }, { 0x17DBUL, 0x17DBUL }, - { 0x20A0UL, 0x20C0UL }, + { 0x20A0UL, 0x20C1UL }, { 0xA838UL, 0xA838UL }, { 0xFDFCUL, 0xFDFCUL }, { 0xFE69UL, 0xFE69UL }, diff --git a/src/libre/class/utf8_Sharada.c b/src/libre/class/utf8_Sharada.c index e51b2edff..dbfa21d3f 100644 --- a/src/libre/class/utf8_Sharada.c +++ b/src/libre/class/utf8_Sharada.c @@ -3,7 +3,8 @@ #include "class.h" static const struct range ranges[] = { - { 0x11180UL, 0x111DFUL } + { 0x11180UL, 0x111DFUL }, + { 0x11B60UL, 0x11B67UL } }; const struct class utf8_Sharada = { diff --git a/src/libre/class/utf8_Sidetic.c b/src/libre/class/utf8_Sidetic.c new file mode 100644 index 000000000..ed53b543d --- /dev/null +++ b/src/libre/class/utf8_Sidetic.c @@ -0,0 +1,13 @@ +/* generated */ + +#include "class.h" + +static const struct range ranges[] = { + { 0x10940UL, 0x10959UL } +}; + +const struct class utf8_Sidetic = { + ranges, + sizeof ranges / sizeof *ranges +}; + diff --git a/src/libre/class/utf8_Sm.c b/src/libre/class/utf8_Sm.c index 38e1624d6..e157e5f8c 100644 --- a/src/libre/class/utf8_Sm.c +++ b/src/libre/class/utf8_Sm.c @@ -56,6 +56,8 @@ static const struct range ranges[] = { { 0xFF5EUL, 0xFF5EUL }, { 0xFFE2UL, 0xFFE2UL }, { 0xFFE9UL, 0xFFECUL }, + { 0x10D8EUL, 0x10D8FUL }, + { 0x1CEF0UL, 0x1CEF0UL }, { 0x1D6C1UL, 0x1D6C1UL }, { 0x1D6DBUL, 0x1D6DBUL }, { 0x1D6FBUL, 0x1D6FBUL }, @@ -66,7 +68,8 @@ static const struct range ranges[] = { { 0x1D789UL, 0x1D789UL }, { 0x1D7A9UL, 0x1D7A9UL }, { 0x1D7C3UL, 0x1D7C3UL }, - { 0x1EEF0UL, 0x1EEF1UL } + { 0x1EEF0UL, 0x1EEF1UL }, + { 0x1F8D0UL, 0x1F8D8UL } }; const struct class utf8_Sm = { diff --git a/src/libre/class/utf8_So.c b/src/libre/class/utf8_So.c index 2ddd4cd3e..93ca7e36b 100644 --- a/src/libre/class/utf8_So.c +++ b/src/libre/class/utf8_So.c @@ -69,7 +69,7 @@ static const struct range ranges[] = { { 0x232BUL, 0x237BUL }, { 0x237DUL, 0x239AUL }, { 0x23B4UL, 0x23DBUL }, - { 0x23E2UL, 0x2426UL }, + { 0x23E2UL, 0x2429UL }, { 0x2440UL, 0x244AUL }, { 0x249CUL, 0x24E9UL }, { 0x2500UL, 0x25B6UL }, @@ -82,14 +82,13 @@ static const struct range ranges[] = { { 0x2B00UL, 0x2B2FUL }, { 0x2B45UL, 0x2B46UL }, { 0x2B4DUL, 0x2B73UL }, - { 0x2B76UL, 0x2B95UL }, - { 0x2B97UL, 0x2BFFUL }, + { 0x2B76UL, 0x2BFFUL }, { 0x2CE5UL, 0x2CEAUL }, { 0x2E50UL, 0x2E51UL }, { 0x2E80UL, 0x2E99UL }, { 0x2E9BUL, 0x2EF3UL }, { 0x2F00UL, 0x2FD5UL }, - { 0x2FF0UL, 0x2FFBUL }, + { 0x2FF0UL, 0x2FFFUL }, { 0x3004UL, 0x3004UL }, { 0x3012UL, 0x3013UL }, { 0x3020UL, 0x3020UL }, @@ -97,7 +96,8 @@ static const struct range ranges[] = { { 0x303EUL, 0x303FUL }, { 0x3190UL, 0x3191UL }, { 0x3196UL, 0x319FUL }, - { 0x31C0UL, 0x31E3UL }, + { 0x31C0UL, 0x31E5UL }, + { 0x31EFUL, 0x31EFUL }, { 0x3200UL, 0x321EUL }, { 0x322AUL, 0x3247UL }, { 0x3250UL, 0x3250UL }, @@ -110,8 +110,10 @@ static const struct range ranges[] = { { 0xA836UL, 0xA837UL }, { 0xA839UL, 0xA839UL }, { 0xAA77UL, 0xAA79UL }, + { 0xFBC3UL, 0xFBD2UL }, { 0xFD40UL, 0xFD4FUL }, - { 0xFDCFUL, 0xFDCFUL }, + { 0xFD90UL, 0xFD91UL }, + { 0xFDC8UL, 0xFDCFUL }, { 0xFDFDUL, 0xFDFFUL }, { 0xFFE4UL, 0xFFE4UL }, { 0xFFE8UL, 0xFFE8UL }, @@ -125,12 +127,18 @@ static const struct range ranges[] = { { 0x101D0UL, 0x101FCUL }, { 0x10877UL, 0x10878UL }, { 0x10AC8UL, 0x10AC8UL }, + { 0x10ED1UL, 0x10ED8UL }, { 0x1173FUL, 0x1173FUL }, { 0x11FD5UL, 0x11FDCUL }, { 0x11FE1UL, 0x11FF1UL }, { 0x16B3CUL, 0x16B3FUL }, { 0x16B45UL, 0x16B45UL }, { 0x1BC9CUL, 0x1BC9CUL }, + { 0x1CC00UL, 0x1CCEFUL }, + { 0x1CCFAUL, 0x1CCFCUL }, + { 0x1CD00UL, 0x1CEB3UL }, + { 0x1CEBAUL, 0x1CED0UL }, + { 0x1CEE0UL, 0x1CEEFUL }, { 0x1CF50UL, 0x1CFC3UL }, { 0x1D000UL, 0x1D0F5UL }, { 0x1D100UL, 0x1D126UL }, @@ -163,11 +171,10 @@ static const struct range ranges[] = { { 0x1F250UL, 0x1F251UL }, { 0x1F260UL, 0x1F265UL }, { 0x1F300UL, 0x1F3FAUL }, - { 0x1F400UL, 0x1F6D7UL }, + { 0x1F400UL, 0x1F6D8UL }, { 0x1F6DCUL, 0x1F6ECUL }, { 0x1F6F0UL, 0x1F6FCUL }, - { 0x1F700UL, 0x1F776UL }, - { 0x1F77BUL, 0x1F7D9UL }, + { 0x1F700UL, 0x1F7D9UL }, { 0x1F7E0UL, 0x1F7EBUL }, { 0x1F7F0UL, 0x1F7F0UL }, { 0x1F800UL, 0x1F80BUL }, @@ -175,18 +182,20 @@ static const struct range ranges[] = { { 0x1F850UL, 0x1F859UL }, { 0x1F860UL, 0x1F887UL }, { 0x1F890UL, 0x1F8ADUL }, - { 0x1F8B0UL, 0x1F8B1UL }, - { 0x1F900UL, 0x1FA53UL }, + { 0x1F8B0UL, 0x1F8BBUL }, + { 0x1F8C0UL, 0x1F8C1UL }, + { 0x1F900UL, 0x1FA57UL }, { 0x1FA60UL, 0x1FA6DUL }, { 0x1FA70UL, 0x1FA7CUL }, - { 0x1FA80UL, 0x1FA88UL }, - { 0x1FA90UL, 0x1FABDUL }, - { 0x1FABFUL, 0x1FAC5UL }, - { 0x1FACEUL, 0x1FADBUL }, - { 0x1FAE0UL, 0x1FAE8UL }, - { 0x1FAF0UL, 0x1FAF8UL }, + { 0x1FA80UL, 0x1FA8AUL }, + { 0x1FA8EUL, 0x1FAC6UL }, + { 0x1FAC8UL, 0x1FAC8UL }, + { 0x1FACDUL, 0x1FADCUL }, + { 0x1FADFUL, 0x1FAEAUL }, + { 0x1FAEFUL, 0x1FAF8UL }, { 0x1FB00UL, 0x1FB92UL }, - { 0x1FB94UL, 0x1FBCAUL } + { 0x1FB94UL, 0x1FBEFUL }, + { 0x1FBFAUL, 0x1FBFAUL } }; const struct class utf8_So = { diff --git a/src/libre/class/utf8_Sunuwar.c b/src/libre/class/utf8_Sunuwar.c new file mode 100644 index 000000000..ce53148d0 --- /dev/null +++ b/src/libre/class/utf8_Sunuwar.c @@ -0,0 +1,14 @@ +/* generated */ + +#include "class.h" + +static const struct range ranges[] = { + { 0x11BC0UL, 0x11BE1UL }, + { 0x11BF0UL, 0x11BF9UL } +}; + +const struct class utf8_Sunuwar = { + ranges, + sizeof ranges / sizeof *ranges +}; + diff --git a/src/libre/class/utf8_Tai_Yo.c b/src/libre/class/utf8_Tai_Yo.c new file mode 100644 index 000000000..643d73fe2 --- /dev/null +++ b/src/libre/class/utf8_Tai_Yo.c @@ -0,0 +1,15 @@ +/* generated */ + +#include "class.h" + +static const struct range ranges[] = { + { 0x1E6C0UL, 0x1E6DEUL }, + { 0x1E6E0UL, 0x1E6F5UL }, + { 0x1E6FEUL, 0x1E6FFUL } +}; + +const struct class utf8_Tai_Yo = { + ranges, + sizeof ranges / sizeof *ranges +}; + diff --git a/src/libre/class/utf8_Tangut.c b/src/libre/class/utf8_Tangut.c index cbd82c907..6bf5e5687 100644 --- a/src/libre/class/utf8_Tangut.c +++ b/src/libre/class/utf8_Tangut.c @@ -4,9 +4,9 @@ static const struct range ranges[] = { { 0x16FE0UL, 0x16FE0UL }, - { 0x17000UL, 0x187F7UL }, - { 0x18800UL, 0x18AFFUL }, - { 0x18D00UL, 0x18D08UL } + { 0x17000UL, 0x18AFFUL }, + { 0x18D00UL, 0x18D1EUL }, + { 0x18D80UL, 0x18DF2UL } }; const struct class utf8_Tangut = { diff --git a/src/libre/class/utf8_Telugu.c b/src/libre/class/utf8_Telugu.c index a2c65ae49..1b80f5e1f 100644 --- a/src/libre/class/utf8_Telugu.c +++ b/src/libre/class/utf8_Telugu.c @@ -12,7 +12,7 @@ static const struct range ranges[] = { { 0x0C4AUL, 0x0C4DUL }, { 0x0C55UL, 0x0C56UL }, { 0x0C58UL, 0x0C5AUL }, - { 0x0C5DUL, 0x0C5DUL }, + { 0x0C5CUL, 0x0C5DUL }, { 0x0C60UL, 0x0C63UL }, { 0x0C66UL, 0x0C6FUL }, { 0x0C77UL, 0x0C7FUL } diff --git a/src/libre/class/utf8_Todhri.c b/src/libre/class/utf8_Todhri.c new file mode 100644 index 000000000..732870a64 --- /dev/null +++ b/src/libre/class/utf8_Todhri.c @@ -0,0 +1,13 @@ +/* generated */ + +#include "class.h" + +static const struct range ranges[] = { + { 0x105C0UL, 0x105F3UL } +}; + +const struct class utf8_Todhri = { + ranges, + sizeof ranges / sizeof *ranges +}; + diff --git a/src/libre/class/utf8_Tolong_Siki.c b/src/libre/class/utf8_Tolong_Siki.c new file mode 100644 index 000000000..e670a957b --- /dev/null +++ b/src/libre/class/utf8_Tolong_Siki.c @@ -0,0 +1,14 @@ +/* generated */ + +#include "class.h" + +static const struct range ranges[] = { + { 0x11DB0UL, 0x11DDBUL }, + { 0x11DE0UL, 0x11DE9UL } +}; + +const struct class utf8_Tolong_Siki = { + ranges, + sizeof ranges / sizeof *ranges +}; + diff --git a/src/libre/class/utf8_Tulu_Tigalari.c b/src/libre/class/utf8_Tulu_Tigalari.c new file mode 100644 index 000000000..b611066f2 --- /dev/null +++ b/src/libre/class/utf8_Tulu_Tigalari.c @@ -0,0 +1,23 @@ +/* generated */ + +#include "class.h" + +static const struct range ranges[] = { + { 0x11380UL, 0x11389UL }, + { 0x1138BUL, 0x1138BUL }, + { 0x1138EUL, 0x1138EUL }, + { 0x11390UL, 0x113B5UL }, + { 0x113B7UL, 0x113C0UL }, + { 0x113C2UL, 0x113C2UL }, + { 0x113C5UL, 0x113C5UL }, + { 0x113C7UL, 0x113CAUL }, + { 0x113CCUL, 0x113D5UL }, + { 0x113D7UL, 0x113D8UL }, + { 0x113E1UL, 0x113E2UL } +}; + +const struct class utf8_Tulu_Tigalari = { + ranges, + sizeof ranges / sizeof *ranges +}; + diff --git a/src/libre/class/utf8_assigned.c b/src/libre/class/utf8_assigned.c index bb764f63f..32b689a26 100644 --- a/src/libre/class/utf8_assigned.c +++ b/src/libre/class/utf8_assigned.c @@ -24,9 +24,8 @@ static const struct range ranges[] = { { 0x0840UL, 0x085BUL }, { 0x085EUL, 0x085EUL }, { 0x0860UL, 0x086AUL }, - { 0x0870UL, 0x088EUL }, - { 0x0890UL, 0x0891UL }, - { 0x0898UL, 0x0983UL }, + { 0x0870UL, 0x0891UL }, + { 0x0897UL, 0x0983UL }, { 0x0985UL, 0x098CUL }, { 0x098FUL, 0x0990UL }, { 0x0993UL, 0x09A8UL }, @@ -109,7 +108,7 @@ static const struct range ranges[] = { { 0x0C4AUL, 0x0C4DUL }, { 0x0C55UL, 0x0C56UL }, { 0x0C58UL, 0x0C5AUL }, - { 0x0C5DUL, 0x0C5DUL }, + { 0x0C5CUL, 0x0C5DUL }, { 0x0C60UL, 0x0C63UL }, { 0x0C66UL, 0x0C6FUL }, { 0x0C77UL, 0x0C8CUL }, @@ -121,7 +120,7 @@ static const struct range ranges[] = { { 0x0CC6UL, 0x0CC8UL }, { 0x0CCAUL, 0x0CCDUL }, { 0x0CD5UL, 0x0CD6UL }, - { 0x0CDDUL, 0x0CDEUL }, + { 0x0CDCUL, 0x0CDEUL }, { 0x0CE0UL, 0x0CE3UL }, { 0x0CE6UL, 0x0CEFUL }, { 0x0CF1UL, 0x0CF3UL }, @@ -216,13 +215,13 @@ static const struct range ranges[] = { { 0x1A7FUL, 0x1A89UL }, { 0x1A90UL, 0x1A99UL }, { 0x1AA0UL, 0x1AADUL }, - { 0x1AB0UL, 0x1ACEUL }, + { 0x1AB0UL, 0x1ADDUL }, + { 0x1AE0UL, 0x1AEBUL }, { 0x1B00UL, 0x1B4CUL }, - { 0x1B50UL, 0x1B7EUL }, - { 0x1B80UL, 0x1BF3UL }, + { 0x1B4EUL, 0x1BF3UL }, { 0x1BFCUL, 0x1C37UL }, { 0x1C3BUL, 0x1C49UL }, - { 0x1C4DUL, 0x1C88UL }, + { 0x1C4DUL, 0x1C8AUL }, { 0x1C90UL, 0x1CBAUL }, { 0x1CBDUL, 0x1CC7UL }, { 0x1CD0UL, 0x1CFAUL }, @@ -246,14 +245,13 @@ static const struct range ranges[] = { { 0x2066UL, 0x2071UL }, { 0x2074UL, 0x208EUL }, { 0x2090UL, 0x209CUL }, - { 0x20A0UL, 0x20C0UL }, + { 0x20A0UL, 0x20C1UL }, { 0x20D0UL, 0x20F0UL }, { 0x2100UL, 0x218BUL }, - { 0x2190UL, 0x2426UL }, + { 0x2190UL, 0x2429UL }, { 0x2440UL, 0x244AUL }, { 0x2460UL, 0x2B73UL }, - { 0x2B76UL, 0x2B95UL }, - { 0x2B97UL, 0x2CF3UL }, + { 0x2B76UL, 0x2CF3UL }, { 0x2CF9UL, 0x2D25UL }, { 0x2D27UL, 0x2D27UL }, { 0x2D2DUL, 0x2D2DUL }, @@ -272,25 +270,21 @@ static const struct range ranges[] = { { 0x2E80UL, 0x2E99UL }, { 0x2E9BUL, 0x2EF3UL }, { 0x2F00UL, 0x2FD5UL }, - { 0x2FF0UL, 0x2FFBUL }, - { 0x3000UL, 0x303FUL }, + { 0x2FF0UL, 0x303FUL }, { 0x3041UL, 0x3096UL }, { 0x3099UL, 0x30FFUL }, { 0x3105UL, 0x312FUL }, { 0x3131UL, 0x318EUL }, - { 0x3190UL, 0x31E3UL }, - { 0x31F0UL, 0x321EUL }, + { 0x3190UL, 0x31E5UL }, + { 0x31EFUL, 0x321EUL }, { 0x3220UL, 0x3400UL }, { 0x4DBFUL, 0x4E00UL }, { 0x9FFFUL, 0xA48CUL }, { 0xA490UL, 0xA4C6UL }, { 0xA4D0UL, 0xA62BUL }, { 0xA640UL, 0xA6F7UL }, - { 0xA700UL, 0xA7CAUL }, - { 0xA7D0UL, 0xA7D1UL }, - { 0xA7D3UL, 0xA7D3UL }, - { 0xA7D5UL, 0xA7D9UL }, - { 0xA7F2UL, 0xA82CUL }, + { 0xA700UL, 0xA7DCUL }, + { 0xA7F1UL, 0xA82CUL }, { 0xA830UL, 0xA839UL }, { 0xA840UL, 0xA877UL }, { 0xA880UL, 0xA8C5UL }, @@ -330,10 +324,7 @@ static const struct range ranges[] = { { 0xFB3EUL, 0xFB3EUL }, { 0xFB40UL, 0xFB41UL }, { 0xFB43UL, 0xFB44UL }, - { 0xFB46UL, 0xFBC2UL }, - { 0xFBD3UL, 0xFD8FUL }, - { 0xFD92UL, 0xFDC7UL }, - { 0xFDCFUL, 0xFDCFUL }, + { 0xFB46UL, 0xFDCFUL }, { 0xFDF0UL, 0xFE19UL }, { 0xFE20UL, 0xFE52UL }, { 0xFE54UL, 0xFE66UL }, @@ -385,6 +376,7 @@ static const struct range ranges[] = { { 0x105A3UL, 0x105B1UL }, { 0x105B3UL, 0x105B9UL }, { 0x105BBUL, 0x105BCUL }, + { 0x105C0UL, 0x105F3UL }, { 0x10600UL, 0x10736UL }, { 0x10740UL, 0x10755UL }, { 0x10760UL, 0x10767UL }, @@ -403,7 +395,7 @@ static const struct range ranges[] = { { 0x108F4UL, 0x108F5UL }, { 0x108FBUL, 0x1091BUL }, { 0x1091FUL, 0x10939UL }, - { 0x1093FUL, 0x1093FUL }, + { 0x1093FUL, 0x10959UL }, { 0x10980UL, 0x109B7UL }, { 0x109BCUL, 0x109CFUL }, { 0x109D2UL, 0x10A03UL }, @@ -428,11 +420,16 @@ static const struct range ranges[] = { { 0x10CC0UL, 0x10CF2UL }, { 0x10CFAUL, 0x10D27UL }, { 0x10D30UL, 0x10D39UL }, + { 0x10D40UL, 0x10D65UL }, + { 0x10D69UL, 0x10D85UL }, + { 0x10D8EUL, 0x10D8FUL }, { 0x10E60UL, 0x10E7EUL }, { 0x10E80UL, 0x10EA9UL }, { 0x10EABUL, 0x10EADUL }, { 0x10EB0UL, 0x10EB1UL }, - { 0x10EFDUL, 0x10F27UL }, + { 0x10EC2UL, 0x10EC7UL }, + { 0x10ED0UL, 0x10ED8UL }, + { 0x10EFAUL, 0x10F27UL }, { 0x10F30UL, 0x10F59UL }, { 0x10F70UL, 0x10F89UL }, { 0x10FB0UL, 0x10FCBUL }, @@ -472,6 +469,17 @@ static const struct range ranges[] = { { 0x1135DUL, 0x11363UL }, { 0x11366UL, 0x1136CUL }, { 0x11370UL, 0x11374UL }, + { 0x11380UL, 0x11389UL }, + { 0x1138BUL, 0x1138BUL }, + { 0x1138EUL, 0x1138EUL }, + { 0x11390UL, 0x113B5UL }, + { 0x113B7UL, 0x113C0UL }, + { 0x113C2UL, 0x113C2UL }, + { 0x113C5UL, 0x113C5UL }, + { 0x113C7UL, 0x113CAUL }, + { 0x113CCUL, 0x113D5UL }, + { 0x113D7UL, 0x113D8UL }, + { 0x113E1UL, 0x113E2UL }, { 0x11400UL, 0x1145BUL }, { 0x1145DUL, 0x11461UL }, { 0x11480UL, 0x114C7UL }, @@ -483,6 +491,7 @@ static const struct range ranges[] = { { 0x11660UL, 0x1166CUL }, { 0x11680UL, 0x116B9UL }, { 0x116C0UL, 0x116C9UL }, + { 0x116D0UL, 0x116E3UL }, { 0x11700UL, 0x1171AUL }, { 0x1171DUL, 0x1172BUL }, { 0x11730UL, 0x11746UL }, @@ -503,6 +512,9 @@ static const struct range ranges[] = { { 0x11A50UL, 0x11AA2UL }, { 0x11AB0UL, 0x11AF8UL }, { 0x11B00UL, 0x11B09UL }, + { 0x11B60UL, 0x11B67UL }, + { 0x11BC0UL, 0x11BE1UL }, + { 0x11BF0UL, 0x11BF9UL }, { 0x11C00UL, 0x11C08UL }, { 0x11C0AUL, 0x11C36UL }, { 0x11C38UL, 0x11C45UL }, @@ -523,10 +535,12 @@ static const struct range ranges[] = { { 0x11D90UL, 0x11D91UL }, { 0x11D93UL, 0x11D98UL }, { 0x11DA0UL, 0x11DA9UL }, + { 0x11DB0UL, 0x11DDBUL }, + { 0x11DE0UL, 0x11DE9UL }, { 0x11EE0UL, 0x11EF8UL }, { 0x11F00UL, 0x11F10UL }, { 0x11F12UL, 0x11F3AUL }, - { 0x11F3EUL, 0x11F59UL }, + { 0x11F3EUL, 0x11F5AUL }, { 0x11FB0UL, 0x11FB0UL }, { 0x11FC0UL, 0x11FF1UL }, { 0x11FFFUL, 0x12399UL }, @@ -535,7 +549,9 @@ static const struct range ranges[] = { { 0x12480UL, 0x12543UL }, { 0x12F90UL, 0x12FF2UL }, { 0x13000UL, 0x13455UL }, + { 0x13460UL, 0x143FAUL }, { 0x14400UL, 0x14646UL }, + { 0x16100UL, 0x16139UL }, { 0x16800UL, 0x16A38UL }, { 0x16A40UL, 0x16A5EUL }, { 0x16A60UL, 0x16A69UL }, @@ -548,17 +564,20 @@ static const struct range ranges[] = { { 0x16B5BUL, 0x16B61UL }, { 0x16B63UL, 0x16B77UL }, { 0x16B7DUL, 0x16B8FUL }, + { 0x16D40UL, 0x16D79UL }, { 0x16E40UL, 0x16E9AUL }, + { 0x16EA0UL, 0x16EB8UL }, + { 0x16EBBUL, 0x16ED3UL }, { 0x16F00UL, 0x16F4AUL }, { 0x16F4FUL, 0x16F87UL }, { 0x16F8FUL, 0x16F9FUL }, { 0x16FE0UL, 0x16FE4UL }, - { 0x16FF0UL, 0x16FF1UL }, + { 0x16FF0UL, 0x16FF6UL }, { 0x17000UL, 0x17000UL }, - { 0x187F7UL, 0x187F7UL }, - { 0x18800UL, 0x18CD5UL }, - { 0x18D00UL, 0x18D00UL }, - { 0x18D08UL, 0x18D08UL }, + { 0x187FFUL, 0x18CD5UL }, + { 0x18CFFUL, 0x18D00UL }, + { 0x18D1EUL, 0x18D1EUL }, + { 0x18D80UL, 0x18DF2UL }, { 0x1AFF0UL, 0x1AFF3UL }, { 0x1AFF5UL, 0x1AFFBUL }, { 0x1AFFDUL, 0x1AFFEUL }, @@ -573,6 +592,10 @@ static const struct range ranges[] = { { 0x1BC80UL, 0x1BC88UL }, { 0x1BC90UL, 0x1BC99UL }, { 0x1BC9CUL, 0x1BCA3UL }, + { 0x1CC00UL, 0x1CCFCUL }, + { 0x1CD00UL, 0x1CEB3UL }, + { 0x1CEBAUL, 0x1CED0UL }, + { 0x1CEE0UL, 0x1CEF0UL }, { 0x1CF00UL, 0x1CF2DUL }, { 0x1CF30UL, 0x1CF46UL }, { 0x1CF50UL, 0x1CFC3UL }, @@ -624,6 +647,11 @@ static const struct range ranges[] = { { 0x1E2C0UL, 0x1E2F9UL }, { 0x1E2FFUL, 0x1E2FFUL }, { 0x1E4D0UL, 0x1E4F9UL }, + { 0x1E5D0UL, 0x1E5FAUL }, + { 0x1E5FFUL, 0x1E5FFUL }, + { 0x1E6C0UL, 0x1E6DEUL }, + { 0x1E6E0UL, 0x1E6F5UL }, + { 0x1E6FEUL, 0x1E6FFUL }, { 0x1E7E0UL, 0x1E7E6UL }, { 0x1E7E8UL, 0x1E7EBUL }, { 0x1E7EDUL, 0x1E7EEUL }, @@ -681,11 +709,10 @@ static const struct range ranges[] = { { 0x1F240UL, 0x1F248UL }, { 0x1F250UL, 0x1F251UL }, { 0x1F260UL, 0x1F265UL }, - { 0x1F300UL, 0x1F6D7UL }, + { 0x1F300UL, 0x1F6D8UL }, { 0x1F6DCUL, 0x1F6ECUL }, { 0x1F6F0UL, 0x1F6FCUL }, - { 0x1F700UL, 0x1F776UL }, - { 0x1F77BUL, 0x1F7D9UL }, + { 0x1F700UL, 0x1F7D9UL }, { 0x1F7E0UL, 0x1F7EBUL }, { 0x1F7F0UL, 0x1F7F0UL }, { 0x1F800UL, 0x1F80BUL }, @@ -693,34 +720,37 @@ static const struct range ranges[] = { { 0x1F850UL, 0x1F859UL }, { 0x1F860UL, 0x1F887UL }, { 0x1F890UL, 0x1F8ADUL }, - { 0x1F8B0UL, 0x1F8B1UL }, - { 0x1F900UL, 0x1FA53UL }, + { 0x1F8B0UL, 0x1F8BBUL }, + { 0x1F8C0UL, 0x1F8C1UL }, + { 0x1F8D0UL, 0x1F8D8UL }, + { 0x1F900UL, 0x1FA57UL }, { 0x1FA60UL, 0x1FA6DUL }, { 0x1FA70UL, 0x1FA7CUL }, - { 0x1FA80UL, 0x1FA88UL }, - { 0x1FA90UL, 0x1FABDUL }, - { 0x1FABFUL, 0x1FAC5UL }, - { 0x1FACEUL, 0x1FADBUL }, - { 0x1FAE0UL, 0x1FAE8UL }, - { 0x1FAF0UL, 0x1FAF8UL }, + { 0x1FA80UL, 0x1FA8AUL }, + { 0x1FA8EUL, 0x1FAC6UL }, + { 0x1FAC8UL, 0x1FAC8UL }, + { 0x1FACDUL, 0x1FADCUL }, + { 0x1FADFUL, 0x1FAEAUL }, + { 0x1FAEFUL, 0x1FAF8UL }, { 0x1FB00UL, 0x1FB92UL }, - { 0x1FB94UL, 0x1FBCAUL }, - { 0x1FBF0UL, 0x1FBF9UL }, + { 0x1FB94UL, 0x1FBFAUL }, { 0x20000UL, 0x20000UL }, { 0x2A6DFUL, 0x2A6DFUL }, { 0x2A700UL, 0x2A700UL }, - { 0x2B739UL, 0x2B739UL }, - { 0x2B740UL, 0x2B740UL }, + { 0x2B73FUL, 0x2B740UL }, { 0x2B81DUL, 0x2B81DUL }, { 0x2B820UL, 0x2B820UL }, - { 0x2CEA1UL, 0x2CEA1UL }, + { 0x2CEADUL, 0x2CEADUL }, { 0x2CEB0UL, 0x2CEB0UL }, { 0x2EBE0UL, 0x2EBE0UL }, + { 0x2EBF0UL, 0x2EBF0UL }, + { 0x2EE5DUL, 0x2EE5DUL }, { 0x2F800UL, 0x2FA1DUL }, { 0x30000UL, 0x30000UL }, { 0x3134AUL, 0x3134AUL }, { 0x31350UL, 0x31350UL }, - { 0x323AFUL, 0x323AFUL }, + { 0x323AFUL, 0x323B0UL }, + { 0x33479UL, 0x33479UL }, { 0xE0001UL, 0xE0001UL }, { 0xE0020UL, 0xE007FUL }, { 0xE0100UL, 0xE01EFUL }, diff --git a/src/libre/class_name.c b/src/libre/class_name.c index 30c5aaa56..e4cb7cc0f 100644 --- a/src/libre/class_name.c +++ b/src/libre/class_name.c @@ -50,6 +50,7 @@ static struct { { &utf8_Bassa_Vah, "Bassa Vah" }, { &utf8_Batak, "Batak" }, { &utf8_Bengali, "Bengali" }, + { &utf8_Beria_Erfe, "Beria Erfe" }, { &utf8_Bhaiksuki, "Bhaiksuki" }, { &utf8_Bopomofo, "Bopomofo" }, { &utf8_Brahmi, "Brahmi" }, @@ -67,7 +68,7 @@ static struct { { &utf8_Coptic, "Coptic" }, { &utf8_Cuneiform, "Cuneiform" }, { &utf8_Cypriot, "Cypriot" }, - { &utf8_Cypro_Minoan, "Cypro-Minoan" }, + { &utf8_Cypro_Minoan, "Cypro Minoan" }, { &utf8_Cyrillic, "Cyrillic" }, { &utf8_Deseret, "Deseret" }, { &utf8_Devanagari, "Devanagari" }, @@ -78,6 +79,7 @@ static struct { { &utf8_Elbasan, "Elbasan" }, { &utf8_Elymaic, "Elymaic" }, { &utf8_Ethiopic, "Ethiopic" }, + { &utf8_Garay, "Garay" }, { &utf8_Georgian, "Georgian" }, { &utf8_Glagolitic, "Glagolitic" }, { &utf8_Gothic, "Gothic" }, @@ -86,6 +88,7 @@ static struct { { &utf8_Gujarati, "Gujarati" }, { &utf8_Gunjala_Gondi, "Gunjala Gondi" }, { &utf8_Gurmukhi, "Gurmukhi" }, + { &utf8_Gurung_Khema, "Gurung Khema" }, { &utf8_Han, "Han" }, { &utf8_Hangul, "Hangul" }, { &utf8_Hanifi_Rohingya, "Hanifi Rohingya" }, @@ -101,12 +104,14 @@ static struct { { &utf8_Kaithi, "Kaithi" }, { &utf8_Kannada, "Kannada" }, { &utf8_Katakana, "Katakana" }, + { &utf8_Kawi, "Kawi" }, { &utf8_Kayah_Li, "Kayah Li" }, { &utf8_Kharoshthi, "Kharoshthi" }, { &utf8_Khitan_Small_Script, "Khitan Small Script" }, { &utf8_Khmer, "Khmer" }, { &utf8_Khojki, "Khojki" }, { &utf8_Khudawadi, "Khudawadi" }, + { &utf8_Kirat_Rai, "Kirat Rai" }, { &utf8_Lao, "Lao" }, { &utf8_Latin, "Latin" }, { &utf8_Lepcha, "Lepcha" }, @@ -135,9 +140,10 @@ static struct { { &utf8_Multani, "Multani" }, { &utf8_Myanmar, "Myanmar" }, { &utf8_Nabataean, "Nabataean" }, + { &utf8_Nag_Mundari, "Nag Mundari" }, { &utf8_Nandinagari, "Nandinagari" }, - { &utf8_New_Tai_Lue, "New Tai Lue" }, { &utf8_Newa, "Newa" }, + { &utf8_New_Tai_Lue, "New Tai Lue" }, { &utf8_Nko, "Nko" }, { &utf8_Nushu, "Nushu" }, { &utf8_Nyiakeng_Puachue_Hmong, "Nyiakeng Puachue Hmong" }, @@ -152,6 +158,7 @@ static struct { { &utf8_Old_South_Arabian, "Old South Arabian" }, { &utf8_Old_Turkic, "Old Turkic" }, { &utf8_Old_Uyghur, "Old Uyghur" }, + { &utf8_Ol_Onal, "Ol Onal" }, { &utf8_Oriya, "Oriya" }, { &utf8_Osage, "Osage" }, { &utf8_Osmanya, "Osmanya" }, @@ -168,12 +175,14 @@ static struct { { &utf8_Sharada, "Sharada" }, { &utf8_Shavian, "Shavian" }, { &utf8_Siddham, "Siddham" }, + { &utf8_Sidetic, "Sidetic" }, { &utf8_SignWriting, "SignWriting" }, { &utf8_Sinhala, "Sinhala" }, { &utf8_Sogdian, "Sogdian" }, { &utf8_Sora_Sompeng, "Sora Sompeng" }, { &utf8_Soyombo, "Soyombo" }, { &utf8_Sundanese, "Sundanese" }, + { &utf8_Sunuwar, "Sunuwar" }, { &utf8_Syloti_Nagri, "Syloti Nagri" }, { &utf8_Syriac, "Syriac" }, { &utf8_Tagalog, "Tagalog" }, @@ -181,17 +190,21 @@ static struct { { &utf8_Tai_Le, "Tai Le" }, { &utf8_Tai_Tham, "Tai Tham" }, { &utf8_Tai_Viet, "Tai Viet" }, + { &utf8_Tai_Yo, "Tai Yo" }, { &utf8_Takri, "Takri" }, { &utf8_Tamil, "Tamil" }, - { &utf8_Tangut, "Tangut" }, { &utf8_Tangsa, "Tangsa" }, + { &utf8_Tangut, "Tangut" }, { &utf8_Telugu, "Telugu" }, { &utf8_Thaana, "Thaana" }, { &utf8_Thai, "Thai" }, { &utf8_Tibetan, "Tibetan" }, { &utf8_Tifinagh, "Tifinagh" }, { &utf8_Tirhuta, "Tirhuta" }, + { &utf8_Todhri, "Todhri" }, + { &utf8_Tolong_Siki, "Tolong Siki" }, { &utf8_Toto, "Toto" }, + { &utf8_Tulu_Tigalari, "Tulu Tigalari" }, { &utf8_Ugaritic, "Ugaritic" }, { &utf8_Vai, "Vai" }, { &utf8_Vithkuqi, "Vithkuqi" }, diff --git a/src/libre/dialect/comp.h b/src/libre/dialect/comp.h index 7841f776b..4671bd921 100644 --- a/src/libre/dialect/comp.h +++ b/src/libre/dialect/comp.h @@ -15,9 +15,8 @@ typedef struct ast * re_dialect_parse_fun(re_getchar_fun *getchar, void *opaque, - const struct fsm_options *opt, enum re_flags flags, int overlap, - struct re_err *err); + struct re_err *err, struct re_pos *end); re_dialect_parse_fun parse_re_literal; re_dialect_parse_fun parse_re_glob; diff --git a/src/libre/dialect/glob/lexer.c b/src/libre/dialect/glob/lexer.c index 843cedc4e..2a4d33a29 100644 --- a/src/libre/dialect/glob/lexer.c +++ b/src/libre/dialect/glob/lexer.c @@ -10,11 +10,31 @@ static enum lx_glob_token z0(struct lx_glob_lx *lx); +static int +lx_glob_advance_end(struct lx_glob_lx *lx, int c) +{ + lx->end.byte++; + lx->end.col++; + if (c == '\n') { + lx->end.line++; + lx->end.saved_col = lx->end.col - 1; + lx->end.col = 1; + } + if (lx->push != NULL) { + if (-1 == lx->push(lx->buf_opaque, (char)c)) { + return 0; + } + } + return 1; +} + +/* This wrapper manages one character of lookahead/pushback + * and the line, column, and byte offsets. */ #if __STDC_VERSION__ >= 199901L inline #endif static int -lx_getc(struct lx_glob_lx *lx) +lx_glob_getc(struct lx_glob_lx *lx) { int c; @@ -30,18 +50,19 @@ lx_getc(struct lx_glob_lx *lx) } } - lx->end.byte++; - lx->end.col++; - - if (c == '\n') { - lx->end.line++; - lx->end.saved_col = lx->end.col - 1; - lx->end.col = 1; - } + if (!lx_glob_advance_end(lx, c)) { return EOF; } return c; } +/* This wrapper adapts calling lx_glob_getc to the interface + * in libfsm's generated code. */ +static int +fsm_getc(void *getc_opaque) +{ + return lx_glob_getc((struct lx_glob_lx *)getc_opaque); +} + #if __STDC_VERSION__ >= 199901L inline #endif @@ -50,10 +71,7 @@ lx_glob_ungetc(struct lx_glob_lx *lx, int c) { assert(lx != NULL); assert(lx->c == EOF); - lx->c = c; - - lx->end.byte--; lx->end.col--; @@ -105,6 +123,17 @@ lx_glob_dynpush(void *buf_opaque, char c) return 0; } +static void +lx_glob_dynpop(void *buf_opaque) +{ + struct lx_dynbuf *t = buf_opaque; + + assert(t != NULL); + + assert(t->p != t->a); + t->p--; +} + int lx_glob_dynclear(void *buf_opaque) { @@ -144,29 +173,28 @@ lx_glob_dynfree(void *buf_opaque) static enum lx_glob_token z0(struct lx_glob_lx *lx) { + int has_consumed_input = 0; int c; - enum { - S0, S1, S2, S3, NONE - } state; - assert(lx != NULL); if (lx->clear != NULL) { lx->clear(lx->buf_opaque); } - state = NONE; - lx->start = lx->end; - while (c = lx_getc(lx), c != EOF) { - if (state == NONE) { - state = S0; - } + void *getc_opaque = (void *)lx; + enum { + S0, S1, S2, S3 + } state; + + state = S0; + while (c = fsm_getc(getc_opaque), c != EOF) { + has_consumed_input = 1; switch (state) { - case S0: /* start */ + case S0: /* e.g. "" */ switch ((unsigned char) c) { case '*': state = S2; break; case '?': state = S3; break; @@ -175,34 +203,41 @@ z0(struct lx_glob_lx *lx) break; case S1: /* e.g. "\\x00" */ - lx_glob_ungetc(lx, c); return TOK_CHAR; + lx_glob_ungetc(lx, c); lx_glob_dynpop(lx->buf_opaque); return TOK_CHAR; case S2: /* e.g. "*" */ - lx_glob_ungetc(lx, c); return TOK_MANY; + lx_glob_ungetc(lx, c); lx_glob_dynpop(lx->buf_opaque); return TOK_MANY; - case S3: /* e.g. "?" */ - lx_glob_ungetc(lx, c); return TOK_ANY; + case S3: /* e.g. "\077" */ + lx_glob_ungetc(lx, c); lx_glob_dynpop(lx->buf_opaque); return TOK_ANY; default: ; /* unreached */ } + } + + /* end states */ + switch (state) { + case S1: return TOK_CHAR; + case S2: return TOK_MANY; + case S3: return TOK_ANY; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_glob_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; + } if (lx->push != NULL) { if (-1 == lx->push(lx->buf_opaque, (char)c)) { return TOK_ERROR; } } - } lx->lgetc = NULL; - switch (state) { - case NONE: return TOK_EOF; - case S1: return TOK_CHAR; - case S2: return TOK_MANY; - case S3: return TOK_ANY; - default: errno = EINVAL; return TOK_ERROR; - } + if (!has_consumed_input) { + return TOK_EOF; + } + return TOK_ERROR; } const char * @@ -254,6 +289,7 @@ lx_glob_init(struct lx_glob_lx *lx) lx->end.byte = 0; lx->end.line = 1; lx->end.col = 1; + (void)lx_glob_dynpop; } enum lx_glob_token diff --git a/src/libre/dialect/glob/parser.c b/src/libre/dialect/glob/parser.c index 6c039b0a4..b20798f4f 100644 --- a/src/libre/dialect/glob/parser.c +++ b/src/libre/dialect/glob/parser.c @@ -68,14 +68,6 @@ #define TOK_CLASS__nspace TOK_CLASS_NSPACE #define TOK_CLASS__ndigit TOK_CLASS_NDIGIT - /* This is a hack to work around the AST files not being able to include lexer.h. */ - #define AST_POS_OF_LX_POS(AST_POS, LX_POS) \ - do { \ - AST_POS.line = LX_POS.line; \ - AST_POS.col = LX_POS.col; \ - AST_POS.byte = LX_POS.byte; \ - } while (0) - #include "parser.h" #include "lexer.h" @@ -132,92 +124,8 @@ /* TODO: use lx's generated conveniences for the pattern buffer */ char a[512]; char *p; -#if PCRE_DIALECT - int extended_mode_comment; - enum re_flags *flags_ptr; -#endif /* PCRE_DIALECT */ }; - #define ERROR_TERMINAL (TOK_ERROR) - - #if PCRE_DIALECT - static void mark(struct re_pos *, const struct lx_pos *); - - static void - pcre_advance_lexer(struct lex_state *lex_state, struct act_state *act_state) - { - mark(&act_state->synstart, &lex_state->lx.start); - mark(&act_state->synend, &lex_state->lx.end); - act_state->lex_tok = LX_NEXT(&lex_state->lx); - } - - static enum LX_TOKEN - pcre_current_token(struct lex_state *lex_state, struct act_state *act_state) - { - enum re_flags fl = (lex_state->flags_ptr != NULL) ? lex_state->flags_ptr[0] : 0; - enum LX_TOKEN tok; - - if ((fl & RE_EXTENDED) == 0) { - tok = act_state->lex_tok; - - switch (tok) { - case TOK_WHITESPACE: - case TOK_NEWLINE: - case TOK_MAYBE_COMMENT: - return TOK_CHAR; - - default: - return tok; - } - } - - restart: - tok = act_state->lex_tok; - if (lex_state->extended_mode_comment) { - switch (tok) { - case TOK_EOF: - case TOK_ERROR: - case TOK_UNKNOWN: - /* don't eat EOF or errors */ - return tok; - - case TOK_NEWLINE: - lex_state->extended_mode_comment = 0; - /* fall through */ - default: - pcre_advance_lexer(lex_state, act_state); - goto restart; - } - } else { - switch (tok) { - case TOK_MAYBE_COMMENT: - lex_state->extended_mode_comment = 1; - /*fall through */ - case TOK_WHITESPACE: - case TOK_NEWLINE: - pcre_advance_lexer(lex_state, act_state); - goto restart; - - default: - return tok; - } - } - } - - #define CURRENT_TERMINAL (pcre_current_token(lex_state, act_state)) - #define ADVANCE_LEXER do { pcre_advance_lexer(lex_state,act_state); } while(0) - #else /* !PCRE_DIALECT */ - #define CURRENT_TERMINAL (act_state->lex_tok) - #define ADVANCE_LEXER do { mark(&act_state->synstart, &lex_state->lx.start); \ - mark(&act_state->synend, &lex_state->lx.end); \ - act_state->lex_tok = LX_NEXT(&lex_state->lx); \ - } while (0) - #endif /* PCRE_DIALECT */ - - #define SAVE_LEXER(tok) do { act_state->lex_tok_save = act_state->lex_tok; \ - act_state->lex_tok = tok; } while (0) - #define RESTORE_LEXER do { act_state->lex_tok = act_state->lex_tok_save; } while (0) - static void mark(struct re_pos *r, const struct lx_pos *pos) { @@ -288,7 +196,80 @@ return s; } -#line 292 "src/libre/dialect/glob/parser.c" + static void + advance_lexer(struct lex_state *lex_state, struct act_state *act_state) + { + mark(&act_state->synstart, &lex_state->lx.start); + mark(&act_state->synend, &lex_state->lx.end); + act_state->lex_tok = LX_NEXT(&lex_state->lx); + } + + static enum LX_TOKEN + current_token(struct lex_state *lex_state, struct act_state *act_state, + enum re_flags *flags) + { +#ifndef PCRE_DIALECT + (void) lex_state; + (void) flags; + return act_state->lex_tok; +#else + if ((*flags & RE_EXTENDED) == 0) { + switch (act_state->lex_tok) { + case TOK_WHITESPACE: + case TOK_NEWLINE: + case TOK_MAYBE_COMMENT: + return TOK_CHAR; + + default: + return act_state->lex_tok; + } + } + + int extended_mode_comment = 0; + + restart: + + if (extended_mode_comment) { + switch (act_state->lex_tok) { + case TOK_EOF: + case TOK_ERROR: + case TOK_UNKNOWN: + /* don't eat EOF or errors */ + return act_state->lex_tok; + + case TOK_NEWLINE: + extended_mode_comment = 0; + /* fall through */ + default: + advance_lexer(lex_state, act_state); + goto restart; + } + } else { + switch (act_state->lex_tok) { + case TOK_MAYBE_COMMENT: + extended_mode_comment = 1; + /* fall through */ + case TOK_WHITESPACE: + case TOK_NEWLINE: + advance_lexer(lex_state, act_state); + goto restart; + + default: + return act_state->lex_tok; + } + } +#endif + } + + #define ADVANCE_LEXER (advance_lexer(lex_state, act_state)) + #define CURRENT_TERMINAL (current_token(lex_state, act_state, flags)) + #define ERROR_TERMINAL (TOK_ERROR) + + #define SAVE_LEXER(tok) do { act_state->lex_tok_save = act_state->lex_tok; \ + act_state->lex_tok = tok; } while (0) + #define RESTORE_LEXER do { act_state->lex_tok = act_state->lex_tok_save; } while (0) + +#line 273 "src/libre/dialect/glob/parser.c" #ifndef ERROR_TERMINAL @@ -323,16 +304,16 @@ ZL2_list_Hof_Hatoms:; } /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1078 "src/libre/parser.act" +#line 1040 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcat), (ZIa))) { goto ZL1; } -#line 333 "src/libre/dialect/glob/parser.c" +#line 314 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: ast-add-concat */ - /* BEGINNING OF INLINE: 116 */ + /* BEGINNING OF INLINE: 115 */ { switch (CURRENT_TERMINAL) { case (TOK_ANY): case (TOK_MANY): case (TOK_CHAR): @@ -346,7 +327,7 @@ ZL2_list_Hof_Hatoms:; break; } } - /* END OF INLINE: 116 */ + /* END OF INLINE: 115 */ } return; ZL1:; @@ -367,24 +348,24 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 811 "src/libre/parser.act" +#line 782 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; -#line 376 "src/libre/dialect/glob/parser.c" +#line 357 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1071 "src/libre/parser.act" +#line 1033 "src/libre/parser.act" (ZIe) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIe) == NULL) { goto ZL1; } -#line 388 "src/libre/dialect/glob/parser.c" +#line 369 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -392,38 +373,38 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, case (TOK_CHAR): { t_char ZIc; + t_pos ZI111; t_pos ZI112; - t_pos ZI113; /* BEGINNING OF EXTRACT: CHAR */ { -#line 589 "src/libre/parser.act" +#line 575 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); - ZI112 = lex_state->lx.start; - ZI113 = lex_state->lx.end; + ZI111 = lex_state->lx.start; + ZI112 = lex_state->lx.end; + (void) ZI111; (void) ZI112; - (void) ZI113; ZIc = lex_state->buf.a[0]; -#line 414 "src/libre/dialect/glob/parser.c" +#line 395 "src/libre/dialect/glob/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 908 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZIe) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZIe) == NULL) { goto ZL1; } -#line 427 "src/libre/dialect/glob/parser.c" +#line 408 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -437,38 +418,38 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 811 "src/libre/parser.act" +#line 782 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; -#line 446 "src/libre/dialect/glob/parser.c" +#line 427 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1071 "src/libre/parser.act" +#line 1033 "src/libre/parser.act" (ZIg) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIg) == NULL) { goto ZL1; } -#line 458 "src/libre/dialect/glob/parser.c" +#line 439 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: ast-make-named */ /* BEGINNING OF ACTION: count-zero-or-more */ { -#line 837 "src/libre/parser.act" +#line 808 "src/libre/parser.act" - (ZIc) = ast_make_count(0, NULL, AST_COUNT_UNBOUNDED, NULL); + (ZIc) = ast_make_count(0, AST_COUNT_UNBOUNDED); -#line 467 "src/libre/dialect/glob/parser.c" +#line 448 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: count-zero-or-more */ /* BEGINNING OF ACTION: ast-make-piece */ { -#line 931 "src/libre/parser.act" +#line 897 "src/libre/parser.act" if ((ZIc).min == 0 && (ZIc).max == 0) { (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); @@ -482,7 +463,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, goto ZL1; } -#line 486 "src/libre/dialect/glob/parser.c" +#line 467 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: ast-make-piece */ } @@ -497,26 +478,26 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-atom */ { -#line 718 "src/libre/parser.act" +#line 704 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXATOM; } goto ZL2; -#line 508 "src/libre/dialect/glob/parser.c" +#line 489 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 887 "src/libre/parser.act" +#line 853 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { goto ZL2; } -#line 520 "src/libre/dialect/glob/parser.c" +#line 501 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -537,6 +518,18 @@ p_re__glob(flags flags, lex_state lex_state, act_state act_state, err err, t_ast return; } { + t_group__id ZIid; + t_ast__expr ZIe; + + /* BEGINNING OF ACTION: make-group-id */ + { +#line 881 "src/libre/parser.act" + + (ZIid) = act_state->group_id++; + +#line 531 "src/libre/dialect/glob/parser.c" + } + /* END OF ACTION: make-group-id */ /* BEGINNING OF INLINE: 119 */ { switch (CURRENT_TERMINAL) { @@ -544,17 +537,17 @@ p_re__glob(flags flags, lex_state lex_state, act_state act_state, err err, t_ast { /* BEGINNING OF ACTION: ast-make-concat */ { -#line 894 "src/libre/parser.act" +#line 860 "src/libre/parser.act" - (ZInode) = ast_make_expr_concat(act_state->poolp, *flags); - if ((ZInode) == NULL) { + (ZIe) = ast_make_expr_concat(act_state->poolp, *flags); + if ((ZIe) == NULL) { goto ZL1; } -#line 555 "src/libre/dialect/glob/parser.c" +#line 548 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: ast-make-concat */ - p_list_Hof_Hatoms (flags, lex_state, act_state, err, ZInode); + p_list_Hof_Hatoms (flags, lex_state, act_state, err, ZIe); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -565,14 +558,14 @@ p_re__glob(flags flags, lex_state lex_state, act_state act_state, err err, t_ast { /* BEGINNING OF ACTION: ast-make-empty */ { -#line 887 "src/libre/parser.act" +#line 853 "src/libre/parser.act" - (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); - if ((ZInode) == NULL) { + (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); + if ((ZIe) == NULL) { goto ZL1; } -#line 576 "src/libre/dialect/glob/parser.c" +#line 569 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -580,6 +573,18 @@ p_re__glob(flags flags, lex_state lex_state, act_state act_state, err err, t_ast } } /* END OF INLINE: 119 */ + /* BEGINNING OF ACTION: ast-make-group */ + { +#line 911 "src/libre/parser.act" + + (ZInode) = ast_make_expr_group(act_state->poolp, *flags, (ZIe), (ZIid)); + if ((ZInode) == NULL) { + goto ZL1; + } + +#line 586 "src/libre/dialect/glob/parser.c" + } + /* END OF ACTION: ast-make-group */ /* BEGINNING OF INLINE: 120 */ { { @@ -596,14 +601,14 @@ p_re__glob(flags flags, lex_state lex_state, act_state act_state, err err, t_ast { /* BEGINNING OF ACTION: err-expected-eof */ { -#line 767 "src/libre/parser.act" +#line 753 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXEOF; } goto ZL1; -#line 607 "src/libre/dialect/glob/parser.c" +#line 612 "src/libre/dialect/glob/parser.c" } /* END OF ACTION: err-expected-eof */ } @@ -621,7 +626,7 @@ ZL0:; /* BEGINNING OF TRAILER */ -#line 1090 "src/libre/parser.act" +#line 1052 "src/libre/parser.act" static int @@ -641,9 +646,8 @@ ZL0:; struct ast * DIALECT_PARSE(re_getchar_fun *f, void *opaque, - const struct fsm_options *opt, enum re_flags flags, int overlap, - struct re_err *err) + struct re_err *err, struct re_pos *end) { struct ast *ast; @@ -656,6 +660,7 @@ ZL0:; struct LX_STATE *lx; assert(f != NULL); + assert(end != NULL); ast = ast_new(); @@ -679,11 +684,6 @@ ZL0:; lex_state->buf.a = NULL; lex_state->buf.len = 0; -#if PCRE_DIALECT - lex_state->extended_mode_comment = 0; - lex_state->flags_ptr = &flags; -#endif /* PCRE_DIALECT */ - /* XXX: unneccessary since we're lexing from a string */ /* (except for pushing "[" and "]" around ::group-$dialect) */ lx->buf_opaque = &lex_state->buf; @@ -714,6 +714,9 @@ ZL0:; DIALECT_ENTRY(&flags, lex_state, act_state, err, &ast->expr); + /* we output this for reporting errors found through AST analysis */ + mark(end, &lx->end); + lx->free(lx->buf_opaque); if (err->e != RE_ESUCCESS) { @@ -779,6 +782,6 @@ ZL0:; return NULL; } -#line 783 "src/libre/dialect/glob/parser.c" +#line 786 "src/libre/dialect/glob/parser.c" /* END OF FILE */ diff --git a/src/libre/dialect/glob/parser.h b/src/libre/dialect/glob/parser.h index 4b44fa3bd..89800e6e9 100644 --- a/src/libre/dialect/glob/parser.h +++ b/src/libre/dialect/glob/parser.h @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 300 "src/libre/parser.act" +#line 281 "src/libre/parser.act" #include @@ -28,7 +28,7 @@ extern void p_re__glob(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1247 "src/libre/parser.act" +#line 1207 "src/libre/parser.act" #line 35 "src/libre/dialect/glob/parser.h" diff --git a/src/libre/dialect/glob/parser.sid b/src/libre/dialect/glob/parser.sid index 3c0699b9a..735827f37 100644 --- a/src/libre/dialect/glob/parser.sid +++ b/src/libre/dialect/glob/parser.sid @@ -15,7 +15,7 @@ ast_count; ast_class_id; !endpoint; - !group_id; + group_id; %terminals% @@ -84,7 +84,7 @@ !: () -> (:ast_count); !: (:unsigned, :pos, :unsigned, :pos) -> (:ast_count); - !: () -> (:group_id); + : () -> (:group_id); !: () -> (:char); !: () -> (:char); !: () -> (:char); @@ -97,7 +97,7 @@ : () -> (:ast_expr); !: () -> (:ast_expr); : (:ast_expr, :ast_count) -> (:ast_expr); - !: (:ast_expr, :group_id) -> (:ast_expr); + : (:ast_expr, :group_id) -> (:ast_expr); !: () -> (:re_flags); !: (:re_flags) -> (); !: (:re_flags, :re_flags) -> (); @@ -128,7 +128,6 @@ !: (:pos, :pos) -> (); !: (:pos, :pos) -> (); !: (:pos, :pos) -> (); - !: (:ast_expr, :pos, :pos) -> (); : () -> (:ast_class_id); !: () -> (:ast_class_id); @@ -165,12 +164,14 @@ }; re_glob: () -> (node :ast_expr) = { + id = ; { - node = ; - list-of-atoms(node); + e = ; + list-of-atoms(e); || - node = ; + e = ; }; + node = (e, id); { EOF; diff --git a/src/libre/dialect/like/lexer.c b/src/libre/dialect/like/lexer.c index 4f4dcbdab..2edc365a9 100644 --- a/src/libre/dialect/like/lexer.c +++ b/src/libre/dialect/like/lexer.c @@ -10,11 +10,31 @@ static enum lx_like_token z0(struct lx_like_lx *lx); +static int +lx_like_advance_end(struct lx_like_lx *lx, int c) +{ + lx->end.byte++; + lx->end.col++; + if (c == '\n') { + lx->end.line++; + lx->end.saved_col = lx->end.col - 1; + lx->end.col = 1; + } + if (lx->push != NULL) { + if (-1 == lx->push(lx->buf_opaque, (char)c)) { + return 0; + } + } + return 1; +} + +/* This wrapper manages one character of lookahead/pushback + * and the line, column, and byte offsets. */ #if __STDC_VERSION__ >= 199901L inline #endif static int -lx_getc(struct lx_like_lx *lx) +lx_like_getc(struct lx_like_lx *lx) { int c; @@ -30,18 +50,19 @@ lx_getc(struct lx_like_lx *lx) } } - lx->end.byte++; - lx->end.col++; - - if (c == '\n') { - lx->end.line++; - lx->end.saved_col = lx->end.col - 1; - lx->end.col = 1; - } + if (!lx_like_advance_end(lx, c)) { return EOF; } return c; } +/* This wrapper adapts calling lx_like_getc to the interface + * in libfsm's generated code. */ +static int +fsm_getc(void *getc_opaque) +{ + return lx_like_getc((struct lx_like_lx *)getc_opaque); +} + #if __STDC_VERSION__ >= 199901L inline #endif @@ -50,10 +71,7 @@ lx_like_ungetc(struct lx_like_lx *lx, int c) { assert(lx != NULL); assert(lx->c == EOF); - lx->c = c; - - lx->end.byte--; lx->end.col--; @@ -105,6 +123,17 @@ lx_like_dynpush(void *buf_opaque, char c) return 0; } +static void +lx_like_dynpop(void *buf_opaque) +{ + struct lx_dynbuf *t = buf_opaque; + + assert(t != NULL); + + assert(t->p != t->a); + t->p--; +} + int lx_like_dynclear(void *buf_opaque) { @@ -144,29 +173,28 @@ lx_like_dynfree(void *buf_opaque) static enum lx_like_token z0(struct lx_like_lx *lx) { + int has_consumed_input = 0; int c; - enum { - S0, S1, S2, S3, NONE - } state; - assert(lx != NULL); if (lx->clear != NULL) { lx->clear(lx->buf_opaque); } - state = NONE; - lx->start = lx->end; - while (c = lx_getc(lx), c != EOF) { - if (state == NONE) { - state = S0; - } + void *getc_opaque = (void *)lx; + enum { + S0, S1, S2, S3 + } state; + + state = S0; + while (c = fsm_getc(getc_opaque), c != EOF) { + has_consumed_input = 1; switch (state) { - case S0: /* start */ + case S0: /* e.g. "" */ switch ((unsigned char) c) { case '%': state = S2; break; case '_': state = S3; break; @@ -175,34 +203,41 @@ z0(struct lx_like_lx *lx) break; case S1: /* e.g. "\\x00" */ - lx_like_ungetc(lx, c); return TOK_CHAR; + lx_like_ungetc(lx, c); lx_like_dynpop(lx->buf_opaque); return TOK_CHAR; case S2: /* e.g. "%" */ - lx_like_ungetc(lx, c); return TOK_MANY; + lx_like_ungetc(lx, c); lx_like_dynpop(lx->buf_opaque); return TOK_MANY; case S3: /* e.g. "_" */ - lx_like_ungetc(lx, c); return TOK_ANY; + lx_like_ungetc(lx, c); lx_like_dynpop(lx->buf_opaque); return TOK_ANY; default: ; /* unreached */ } + } + + /* end states */ + switch (state) { + case S1: return TOK_CHAR; + case S2: return TOK_MANY; + case S3: return TOK_ANY; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_like_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; + } if (lx->push != NULL) { if (-1 == lx->push(lx->buf_opaque, (char)c)) { return TOK_ERROR; } } - } lx->lgetc = NULL; - switch (state) { - case NONE: return TOK_EOF; - case S1: return TOK_CHAR; - case S2: return TOK_MANY; - case S3: return TOK_ANY; - default: errno = EINVAL; return TOK_ERROR; - } + if (!has_consumed_input) { + return TOK_EOF; + } + return TOK_ERROR; } const char * @@ -254,6 +289,7 @@ lx_like_init(struct lx_like_lx *lx) lx->end.byte = 0; lx->end.line = 1; lx->end.col = 1; + (void)lx_like_dynpop; } enum lx_like_token diff --git a/src/libre/dialect/like/parser.c b/src/libre/dialect/like/parser.c index eb42f63d4..008bd2b04 100644 --- a/src/libre/dialect/like/parser.c +++ b/src/libre/dialect/like/parser.c @@ -68,14 +68,6 @@ #define TOK_CLASS__nspace TOK_CLASS_NSPACE #define TOK_CLASS__ndigit TOK_CLASS_NDIGIT - /* This is a hack to work around the AST files not being able to include lexer.h. */ - #define AST_POS_OF_LX_POS(AST_POS, LX_POS) \ - do { \ - AST_POS.line = LX_POS.line; \ - AST_POS.col = LX_POS.col; \ - AST_POS.byte = LX_POS.byte; \ - } while (0) - #include "parser.h" #include "lexer.h" @@ -132,92 +124,8 @@ /* TODO: use lx's generated conveniences for the pattern buffer */ char a[512]; char *p; -#if PCRE_DIALECT - int extended_mode_comment; - enum re_flags *flags_ptr; -#endif /* PCRE_DIALECT */ }; - #define ERROR_TERMINAL (TOK_ERROR) - - #if PCRE_DIALECT - static void mark(struct re_pos *, const struct lx_pos *); - - static void - pcre_advance_lexer(struct lex_state *lex_state, struct act_state *act_state) - { - mark(&act_state->synstart, &lex_state->lx.start); - mark(&act_state->synend, &lex_state->lx.end); - act_state->lex_tok = LX_NEXT(&lex_state->lx); - } - - static enum LX_TOKEN - pcre_current_token(struct lex_state *lex_state, struct act_state *act_state) - { - enum re_flags fl = (lex_state->flags_ptr != NULL) ? lex_state->flags_ptr[0] : 0; - enum LX_TOKEN tok; - - if ((fl & RE_EXTENDED) == 0) { - tok = act_state->lex_tok; - - switch (tok) { - case TOK_WHITESPACE: - case TOK_NEWLINE: - case TOK_MAYBE_COMMENT: - return TOK_CHAR; - - default: - return tok; - } - } - - restart: - tok = act_state->lex_tok; - if (lex_state->extended_mode_comment) { - switch (tok) { - case TOK_EOF: - case TOK_ERROR: - case TOK_UNKNOWN: - /* don't eat EOF or errors */ - return tok; - - case TOK_NEWLINE: - lex_state->extended_mode_comment = 0; - /* fall through */ - default: - pcre_advance_lexer(lex_state, act_state); - goto restart; - } - } else { - switch (tok) { - case TOK_MAYBE_COMMENT: - lex_state->extended_mode_comment = 1; - /*fall through */ - case TOK_WHITESPACE: - case TOK_NEWLINE: - pcre_advance_lexer(lex_state, act_state); - goto restart; - - default: - return tok; - } - } - } - - #define CURRENT_TERMINAL (pcre_current_token(lex_state, act_state)) - #define ADVANCE_LEXER do { pcre_advance_lexer(lex_state,act_state); } while(0) - #else /* !PCRE_DIALECT */ - #define CURRENT_TERMINAL (act_state->lex_tok) - #define ADVANCE_LEXER do { mark(&act_state->synstart, &lex_state->lx.start); \ - mark(&act_state->synend, &lex_state->lx.end); \ - act_state->lex_tok = LX_NEXT(&lex_state->lx); \ - } while (0) - #endif /* PCRE_DIALECT */ - - #define SAVE_LEXER(tok) do { act_state->lex_tok_save = act_state->lex_tok; \ - act_state->lex_tok = tok; } while (0) - #define RESTORE_LEXER do { act_state->lex_tok = act_state->lex_tok_save; } while (0) - static void mark(struct re_pos *r, const struct lx_pos *pos) { @@ -288,7 +196,80 @@ return s; } -#line 292 "src/libre/dialect/like/parser.c" + static void + advance_lexer(struct lex_state *lex_state, struct act_state *act_state) + { + mark(&act_state->synstart, &lex_state->lx.start); + mark(&act_state->synend, &lex_state->lx.end); + act_state->lex_tok = LX_NEXT(&lex_state->lx); + } + + static enum LX_TOKEN + current_token(struct lex_state *lex_state, struct act_state *act_state, + enum re_flags *flags) + { +#ifndef PCRE_DIALECT + (void) lex_state; + (void) flags; + return act_state->lex_tok; +#else + if ((*flags & RE_EXTENDED) == 0) { + switch (act_state->lex_tok) { + case TOK_WHITESPACE: + case TOK_NEWLINE: + case TOK_MAYBE_COMMENT: + return TOK_CHAR; + + default: + return act_state->lex_tok; + } + } + + int extended_mode_comment = 0; + + restart: + + if (extended_mode_comment) { + switch (act_state->lex_tok) { + case TOK_EOF: + case TOK_ERROR: + case TOK_UNKNOWN: + /* don't eat EOF or errors */ + return act_state->lex_tok; + + case TOK_NEWLINE: + extended_mode_comment = 0; + /* fall through */ + default: + advance_lexer(lex_state, act_state); + goto restart; + } + } else { + switch (act_state->lex_tok) { + case TOK_MAYBE_COMMENT: + extended_mode_comment = 1; + /* fall through */ + case TOK_WHITESPACE: + case TOK_NEWLINE: + advance_lexer(lex_state, act_state); + goto restart; + + default: + return act_state->lex_tok; + } + } +#endif + } + + #define ADVANCE_LEXER (advance_lexer(lex_state, act_state)) + #define CURRENT_TERMINAL (current_token(lex_state, act_state, flags)) + #define ERROR_TERMINAL (TOK_ERROR) + + #define SAVE_LEXER(tok) do { act_state->lex_tok_save = act_state->lex_tok; \ + act_state->lex_tok = tok; } while (0) + #define RESTORE_LEXER do { act_state->lex_tok = act_state->lex_tok_save; } while (0) + +#line 273 "src/libre/dialect/like/parser.c" #ifndef ERROR_TERMINAL @@ -323,16 +304,16 @@ ZL2_list_Hof_Hatoms:; } /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1078 "src/libre/parser.act" +#line 1040 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcat), (ZIa))) { goto ZL1; } -#line 333 "src/libre/dialect/like/parser.c" +#line 314 "src/libre/dialect/like/parser.c" } /* END OF ACTION: ast-add-concat */ - /* BEGINNING OF INLINE: 116 */ + /* BEGINNING OF INLINE: 115 */ { switch (CURRENT_TERMINAL) { case (TOK_ANY): case (TOK_MANY): case (TOK_CHAR): @@ -346,7 +327,7 @@ ZL2_list_Hof_Hatoms:; break; } } - /* END OF INLINE: 116 */ + /* END OF INLINE: 115 */ } return; ZL1:; @@ -367,24 +348,24 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 811 "src/libre/parser.act" +#line 782 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; -#line 376 "src/libre/dialect/like/parser.c" +#line 357 "src/libre/dialect/like/parser.c" } /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1071 "src/libre/parser.act" +#line 1033 "src/libre/parser.act" (ZIe) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIe) == NULL) { goto ZL1; } -#line 388 "src/libre/dialect/like/parser.c" +#line 369 "src/libre/dialect/like/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -392,38 +373,38 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, case (TOK_CHAR): { t_char ZIc; + t_pos ZI113; t_pos ZI114; - t_pos ZI115; /* BEGINNING OF EXTRACT: CHAR */ { -#line 589 "src/libre/parser.act" +#line 575 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); - ZI114 = lex_state->lx.start; - ZI115 = lex_state->lx.end; + ZI113 = lex_state->lx.start; + ZI114 = lex_state->lx.end; + (void) ZI113; (void) ZI114; - (void) ZI115; ZIc = lex_state->buf.a[0]; -#line 414 "src/libre/dialect/like/parser.c" +#line 395 "src/libre/dialect/like/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 908 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZIe) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZIe) == NULL) { goto ZL1; } -#line 427 "src/libre/dialect/like/parser.c" +#line 408 "src/libre/dialect/like/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -437,38 +418,38 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 811 "src/libre/parser.act" +#line 782 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; -#line 446 "src/libre/dialect/like/parser.c" +#line 427 "src/libre/dialect/like/parser.c" } /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1071 "src/libre/parser.act" +#line 1033 "src/libre/parser.act" (ZIg) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIg) == NULL) { goto ZL1; } -#line 458 "src/libre/dialect/like/parser.c" +#line 439 "src/libre/dialect/like/parser.c" } /* END OF ACTION: ast-make-named */ /* BEGINNING OF ACTION: count-zero-or-more */ { -#line 837 "src/libre/parser.act" +#line 808 "src/libre/parser.act" - (ZIc) = ast_make_count(0, NULL, AST_COUNT_UNBOUNDED, NULL); + (ZIc) = ast_make_count(0, AST_COUNT_UNBOUNDED); -#line 467 "src/libre/dialect/like/parser.c" +#line 448 "src/libre/dialect/like/parser.c" } /* END OF ACTION: count-zero-or-more */ /* BEGINNING OF ACTION: ast-make-piece */ { -#line 931 "src/libre/parser.act" +#line 897 "src/libre/parser.act" if ((ZIc).min == 0 && (ZIc).max == 0) { (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); @@ -482,7 +463,7 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, goto ZL1; } -#line 486 "src/libre/dialect/like/parser.c" +#line 467 "src/libre/dialect/like/parser.c" } /* END OF ACTION: ast-make-piece */ } @@ -497,26 +478,26 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-atom */ { -#line 718 "src/libre/parser.act" +#line 704 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXATOM; } goto ZL2; -#line 508 "src/libre/dialect/like/parser.c" +#line 489 "src/libre/dialect/like/parser.c" } /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 887 "src/libre/parser.act" +#line 853 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { goto ZL2; } -#line 520 "src/libre/dialect/like/parser.c" +#line 501 "src/libre/dialect/like/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -537,6 +518,18 @@ p_re__like(flags flags, lex_state lex_state, act_state act_state, err err, t_ast return; } { + t_group__id ZIid; + t_ast__expr ZIe; + + /* BEGINNING OF ACTION: make-group-id */ + { +#line 881 "src/libre/parser.act" + + (ZIid) = act_state->group_id++; + +#line 531 "src/libre/dialect/like/parser.c" + } + /* END OF ACTION: make-group-id */ /* BEGINNING OF INLINE: 119 */ { switch (CURRENT_TERMINAL) { @@ -544,17 +537,17 @@ p_re__like(flags flags, lex_state lex_state, act_state act_state, err err, t_ast { /* BEGINNING OF ACTION: ast-make-concat */ { -#line 894 "src/libre/parser.act" +#line 860 "src/libre/parser.act" - (ZInode) = ast_make_expr_concat(act_state->poolp, *flags); - if ((ZInode) == NULL) { + (ZIe) = ast_make_expr_concat(act_state->poolp, *flags); + if ((ZIe) == NULL) { goto ZL1; } -#line 555 "src/libre/dialect/like/parser.c" +#line 548 "src/libre/dialect/like/parser.c" } /* END OF ACTION: ast-make-concat */ - p_list_Hof_Hatoms (flags, lex_state, act_state, err, ZInode); + p_list_Hof_Hatoms (flags, lex_state, act_state, err, ZIe); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -565,14 +558,14 @@ p_re__like(flags flags, lex_state lex_state, act_state act_state, err err, t_ast { /* BEGINNING OF ACTION: ast-make-empty */ { -#line 887 "src/libre/parser.act" +#line 853 "src/libre/parser.act" - (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); - if ((ZInode) == NULL) { + (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); + if ((ZIe) == NULL) { goto ZL1; } -#line 576 "src/libre/dialect/like/parser.c" +#line 569 "src/libre/dialect/like/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -580,6 +573,18 @@ p_re__like(flags flags, lex_state lex_state, act_state act_state, err err, t_ast } } /* END OF INLINE: 119 */ + /* BEGINNING OF ACTION: ast-make-group */ + { +#line 911 "src/libre/parser.act" + + (ZInode) = ast_make_expr_group(act_state->poolp, *flags, (ZIe), (ZIid)); + if ((ZInode) == NULL) { + goto ZL1; + } + +#line 586 "src/libre/dialect/like/parser.c" + } + /* END OF ACTION: ast-make-group */ /* BEGINNING OF INLINE: 120 */ { { @@ -596,14 +601,14 @@ p_re__like(flags flags, lex_state lex_state, act_state act_state, err err, t_ast { /* BEGINNING OF ACTION: err-expected-eof */ { -#line 767 "src/libre/parser.act" +#line 753 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXEOF; } goto ZL1; -#line 607 "src/libre/dialect/like/parser.c" +#line 612 "src/libre/dialect/like/parser.c" } /* END OF ACTION: err-expected-eof */ } @@ -621,7 +626,7 @@ ZL0:; /* BEGINNING OF TRAILER */ -#line 1090 "src/libre/parser.act" +#line 1052 "src/libre/parser.act" static int @@ -641,9 +646,8 @@ ZL0:; struct ast * DIALECT_PARSE(re_getchar_fun *f, void *opaque, - const struct fsm_options *opt, enum re_flags flags, int overlap, - struct re_err *err) + struct re_err *err, struct re_pos *end) { struct ast *ast; @@ -656,6 +660,7 @@ ZL0:; struct LX_STATE *lx; assert(f != NULL); + assert(end != NULL); ast = ast_new(); @@ -679,11 +684,6 @@ ZL0:; lex_state->buf.a = NULL; lex_state->buf.len = 0; -#if PCRE_DIALECT - lex_state->extended_mode_comment = 0; - lex_state->flags_ptr = &flags; -#endif /* PCRE_DIALECT */ - /* XXX: unneccessary since we're lexing from a string */ /* (except for pushing "[" and "]" around ::group-$dialect) */ lx->buf_opaque = &lex_state->buf; @@ -714,6 +714,9 @@ ZL0:; DIALECT_ENTRY(&flags, lex_state, act_state, err, &ast->expr); + /* we output this for reporting errors found through AST analysis */ + mark(end, &lx->end); + lx->free(lx->buf_opaque); if (err->e != RE_ESUCCESS) { @@ -779,6 +782,6 @@ ZL0:; return NULL; } -#line 783 "src/libre/dialect/like/parser.c" +#line 786 "src/libre/dialect/like/parser.c" /* END OF FILE */ diff --git a/src/libre/dialect/like/parser.h b/src/libre/dialect/like/parser.h index 2e3c7b1dd..5294f9792 100644 --- a/src/libre/dialect/like/parser.h +++ b/src/libre/dialect/like/parser.h @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 300 "src/libre/parser.act" +#line 281 "src/libre/parser.act" #include @@ -28,7 +28,7 @@ extern void p_re__like(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1247 "src/libre/parser.act" +#line 1207 "src/libre/parser.act" #line 35 "src/libre/dialect/like/parser.h" diff --git a/src/libre/dialect/like/parser.sid b/src/libre/dialect/like/parser.sid index 26cc5c90f..309fea3b8 100644 --- a/src/libre/dialect/like/parser.sid +++ b/src/libre/dialect/like/parser.sid @@ -15,7 +15,7 @@ ast_count; ast_class_id; !endpoint; - !group_id; + group_id; %terminals% @@ -84,7 +84,7 @@ !: () -> (:ast_count); !: (:unsigned, :pos, :unsigned, :pos) -> (:ast_count); - !: () -> (:group_id); + : () -> (:group_id); !: () -> (:char); !: () -> (:char); !: () -> (:char); @@ -97,7 +97,7 @@ : () -> (:ast_expr); !: () -> (:ast_expr); : (:ast_expr, :ast_count) -> (:ast_expr); - !: (:ast_expr, :group_id) -> (:ast_expr); + : (:ast_expr, :group_id) -> (:ast_expr); !: () -> (:re_flags); !: (:re_flags) -> (); !: (:re_flags, :re_flags) -> (); @@ -128,7 +128,6 @@ !: (:pos, :pos) -> (); !: (:pos, :pos) -> (); !: (:pos, :pos) -> (); - !: (:ast_expr, :pos, :pos) -> (); : () -> (:ast_class_id); !: () -> (:ast_class_id); @@ -165,12 +164,14 @@ }; re_like: () -> (node :ast_expr) = { + id = ; { - node = ; - list-of-atoms(node); + e = ; + list-of-atoms(e); || - node = ; + e = ; }; + node = (e, id); { EOF; diff --git a/src/libre/dialect/literal/lexer.c b/src/libre/dialect/literal/lexer.c index f4ff77a37..f13bdbc6f 100644 --- a/src/libre/dialect/literal/lexer.c +++ b/src/libre/dialect/literal/lexer.c @@ -10,11 +10,31 @@ static enum lx_literal_token z0(struct lx_literal_lx *lx); +static int +lx_literal_advance_end(struct lx_literal_lx *lx, int c) +{ + lx->end.byte++; + lx->end.col++; + if (c == '\n') { + lx->end.line++; + lx->end.saved_col = lx->end.col - 1; + lx->end.col = 1; + } + if (lx->push != NULL) { + if (-1 == lx->push(lx->buf_opaque, (char)c)) { + return 0; + } + } + return 1; +} + +/* This wrapper manages one character of lookahead/pushback + * and the line, column, and byte offsets. */ #if __STDC_VERSION__ >= 199901L inline #endif static int -lx_getc(struct lx_literal_lx *lx) +lx_literal_getc(struct lx_literal_lx *lx) { int c; @@ -30,18 +50,19 @@ lx_getc(struct lx_literal_lx *lx) } } - lx->end.byte++; - lx->end.col++; - - if (c == '\n') { - lx->end.line++; - lx->end.saved_col = lx->end.col - 1; - lx->end.col = 1; - } + if (!lx_literal_advance_end(lx, c)) { return EOF; } return c; } +/* This wrapper adapts calling lx_literal_getc to the interface + * in libfsm's generated code. */ +static int +fsm_getc(void *getc_opaque) +{ + return lx_literal_getc((struct lx_literal_lx *)getc_opaque); +} + #if __STDC_VERSION__ >= 199901L inline #endif @@ -50,10 +71,7 @@ lx_literal_ungetc(struct lx_literal_lx *lx, int c) { assert(lx != NULL); assert(lx->c == EOF); - lx->c = c; - - lx->end.byte--; lx->end.col--; @@ -105,6 +123,17 @@ lx_literal_dynpush(void *buf_opaque, char c) return 0; } +static void +lx_literal_dynpop(void *buf_opaque) +{ + struct lx_dynbuf *t = buf_opaque; + + assert(t != NULL); + + assert(t->p != t->a); + t->p--; +} + int lx_literal_dynclear(void *buf_opaque) { @@ -144,52 +173,58 @@ lx_literal_dynfree(void *buf_opaque) static enum lx_literal_token z0(struct lx_literal_lx *lx) { + int has_consumed_input = 0; int c; - enum { - S0, S1, NONE - } state; - assert(lx != NULL); if (lx->clear != NULL) { lx->clear(lx->buf_opaque); } - state = NONE; - lx->start = lx->end; - while (c = lx_getc(lx), c != EOF) { - if (state == NONE) { - state = S0; - } + void *getc_opaque = (void *)lx; + enum { + S0, S1 + } state; + state = S0; + + while (c = fsm_getc(getc_opaque), c != EOF) { + has_consumed_input = 1; switch (state) { - case S0: /* start */ + case S0: /* e.g. "" */ state = S1; break; case S1: /* e.g. "" */ - lx_literal_ungetc(lx, c); return TOK_CHAR; + lx_literal_ungetc(lx, c); lx_literal_dynpop(lx->buf_opaque); return TOK_CHAR; default: ; /* unreached */ } + } + + /* end states */ + switch (state) { + case S1: return TOK_CHAR; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_literal_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; + } if (lx->push != NULL) { if (-1 == lx->push(lx->buf_opaque, (char)c)) { return TOK_ERROR; } } - } lx->lgetc = NULL; - switch (state) { - case NONE: return TOK_EOF; - case S1: return TOK_CHAR; - default: errno = EINVAL; return TOK_ERROR; - } + if (!has_consumed_input) { + return TOK_EOF; + } + return TOK_ERROR; } const char * @@ -237,6 +272,7 @@ lx_literal_init(struct lx_literal_lx *lx) lx->end.byte = 0; lx->end.line = 1; lx->end.col = 1; + (void)lx_literal_dynpop; } enum lx_literal_token diff --git a/src/libre/dialect/literal/parser.c b/src/libre/dialect/literal/parser.c index 9b85216e3..5d1dc82f7 100644 --- a/src/libre/dialect/literal/parser.c +++ b/src/libre/dialect/literal/parser.c @@ -68,14 +68,6 @@ #define TOK_CLASS__nspace TOK_CLASS_NSPACE #define TOK_CLASS__ndigit TOK_CLASS_NDIGIT - /* This is a hack to work around the AST files not being able to include lexer.h. */ - #define AST_POS_OF_LX_POS(AST_POS, LX_POS) \ - do { \ - AST_POS.line = LX_POS.line; \ - AST_POS.col = LX_POS.col; \ - AST_POS.byte = LX_POS.byte; \ - } while (0) - #include "parser.h" #include "lexer.h" @@ -132,92 +124,8 @@ /* TODO: use lx's generated conveniences for the pattern buffer */ char a[512]; char *p; -#if PCRE_DIALECT - int extended_mode_comment; - enum re_flags *flags_ptr; -#endif /* PCRE_DIALECT */ }; - #define ERROR_TERMINAL (TOK_ERROR) - - #if PCRE_DIALECT - static void mark(struct re_pos *, const struct lx_pos *); - - static void - pcre_advance_lexer(struct lex_state *lex_state, struct act_state *act_state) - { - mark(&act_state->synstart, &lex_state->lx.start); - mark(&act_state->synend, &lex_state->lx.end); - act_state->lex_tok = LX_NEXT(&lex_state->lx); - } - - static enum LX_TOKEN - pcre_current_token(struct lex_state *lex_state, struct act_state *act_state) - { - enum re_flags fl = (lex_state->flags_ptr != NULL) ? lex_state->flags_ptr[0] : 0; - enum LX_TOKEN tok; - - if ((fl & RE_EXTENDED) == 0) { - tok = act_state->lex_tok; - - switch (tok) { - case TOK_WHITESPACE: - case TOK_NEWLINE: - case TOK_MAYBE_COMMENT: - return TOK_CHAR; - - default: - return tok; - } - } - - restart: - tok = act_state->lex_tok; - if (lex_state->extended_mode_comment) { - switch (tok) { - case TOK_EOF: - case TOK_ERROR: - case TOK_UNKNOWN: - /* don't eat EOF or errors */ - return tok; - - case TOK_NEWLINE: - lex_state->extended_mode_comment = 0; - /* fall through */ - default: - pcre_advance_lexer(lex_state, act_state); - goto restart; - } - } else { - switch (tok) { - case TOK_MAYBE_COMMENT: - lex_state->extended_mode_comment = 1; - /*fall through */ - case TOK_WHITESPACE: - case TOK_NEWLINE: - pcre_advance_lexer(lex_state, act_state); - goto restart; - - default: - return tok; - } - } - } - - #define CURRENT_TERMINAL (pcre_current_token(lex_state, act_state)) - #define ADVANCE_LEXER do { pcre_advance_lexer(lex_state,act_state); } while(0) - #else /* !PCRE_DIALECT */ - #define CURRENT_TERMINAL (act_state->lex_tok) - #define ADVANCE_LEXER do { mark(&act_state->synstart, &lex_state->lx.start); \ - mark(&act_state->synend, &lex_state->lx.end); \ - act_state->lex_tok = LX_NEXT(&lex_state->lx); \ - } while (0) - #endif /* PCRE_DIALECT */ - - #define SAVE_LEXER(tok) do { act_state->lex_tok_save = act_state->lex_tok; \ - act_state->lex_tok = tok; } while (0) - #define RESTORE_LEXER do { act_state->lex_tok = act_state->lex_tok_save; } while (0) - static void mark(struct re_pos *r, const struct lx_pos *pos) { @@ -288,7 +196,80 @@ return s; } -#line 292 "src/libre/dialect/literal/parser.c" + static void + advance_lexer(struct lex_state *lex_state, struct act_state *act_state) + { + mark(&act_state->synstart, &lex_state->lx.start); + mark(&act_state->synend, &lex_state->lx.end); + act_state->lex_tok = LX_NEXT(&lex_state->lx); + } + + static enum LX_TOKEN + current_token(struct lex_state *lex_state, struct act_state *act_state, + enum re_flags *flags) + { +#ifndef PCRE_DIALECT + (void) lex_state; + (void) flags; + return act_state->lex_tok; +#else + if ((*flags & RE_EXTENDED) == 0) { + switch (act_state->lex_tok) { + case TOK_WHITESPACE: + case TOK_NEWLINE: + case TOK_MAYBE_COMMENT: + return TOK_CHAR; + + default: + return act_state->lex_tok; + } + } + + int extended_mode_comment = 0; + + restart: + + if (extended_mode_comment) { + switch (act_state->lex_tok) { + case TOK_EOF: + case TOK_ERROR: + case TOK_UNKNOWN: + /* don't eat EOF or errors */ + return act_state->lex_tok; + + case TOK_NEWLINE: + extended_mode_comment = 0; + /* fall through */ + default: + advance_lexer(lex_state, act_state); + goto restart; + } + } else { + switch (act_state->lex_tok) { + case TOK_MAYBE_COMMENT: + extended_mode_comment = 1; + /* fall through */ + case TOK_WHITESPACE: + case TOK_NEWLINE: + advance_lexer(lex_state, act_state); + goto restart; + + default: + return act_state->lex_tok; + } + } +#endif + } + + #define ADVANCE_LEXER (advance_lexer(lex_state, act_state)) + #define CURRENT_TERMINAL (current_token(lex_state, act_state, flags)) + #define ERROR_TERMINAL (TOK_ERROR) + + #define SAVE_LEXER(tok) do { act_state->lex_tok_save = act_state->lex_tok; \ + act_state->lex_tok = tok; } while (0) + #define RESTORE_LEXER do { act_state->lex_tok = act_state->lex_tok_save; } while (0) + +#line 273 "src/libre/dialect/literal/parser.c" #ifndef ERROR_TERMINAL @@ -323,16 +304,16 @@ ZL2_list_Hof_Hatoms:; } /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1078 "src/libre/parser.act" +#line 1040 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcat), (ZIa))) { goto ZL1; } -#line 333 "src/libre/dialect/literal/parser.c" +#line 314 "src/libre/dialect/literal/parser.c" } /* END OF ACTION: ast-add-concat */ - /* BEGINNING OF INLINE: 115 */ + /* BEGINNING OF INLINE: 114 */ { switch (CURRENT_TERMINAL) { case (TOK_CHAR): @@ -346,7 +327,7 @@ ZL2_list_Hof_Hatoms:; break; } } - /* END OF INLINE: 115 */ + /* END OF INLINE: 114 */ } return; ZL1:; @@ -363,6 +344,18 @@ p_re__literal(flags flags, lex_state lex_state, act_state act_state, err err, t_ return; } { + t_group__id ZIid; + t_ast__expr ZIe; + + /* BEGINNING OF ACTION: make-group-id */ + { +#line 881 "src/libre/parser.act" + + (ZIid) = act_state->group_id++; + +#line 357 "src/libre/dialect/literal/parser.c" + } + /* END OF ACTION: make-group-id */ /* BEGINNING OF INLINE: 117 */ { switch (CURRENT_TERMINAL) { @@ -370,17 +363,17 @@ p_re__literal(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: ast-make-concat */ { -#line 894 "src/libre/parser.act" +#line 860 "src/libre/parser.act" - (ZInode) = ast_make_expr_concat(act_state->poolp, *flags); - if ((ZInode) == NULL) { + (ZIe) = ast_make_expr_concat(act_state->poolp, *flags); + if ((ZIe) == NULL) { goto ZL1; } -#line 381 "src/libre/dialect/literal/parser.c" +#line 374 "src/libre/dialect/literal/parser.c" } /* END OF ACTION: ast-make-concat */ - p_list_Hof_Hatoms (flags, lex_state, act_state, err, ZInode); + p_list_Hof_Hatoms (flags, lex_state, act_state, err, ZIe); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -391,14 +384,14 @@ p_re__literal(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: ast-make-empty */ { -#line 887 "src/libre/parser.act" +#line 853 "src/libre/parser.act" - (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); - if ((ZInode) == NULL) { + (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); + if ((ZIe) == NULL) { goto ZL1; } -#line 402 "src/libre/dialect/literal/parser.c" +#line 395 "src/libre/dialect/literal/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -406,7 +399,19 @@ p_re__literal(flags flags, lex_state lex_state, act_state act_state, err err, t_ } } /* END OF INLINE: 117 */ - /* BEGINNING OF INLINE: 118 */ + /* BEGINNING OF ACTION: ast-make-group */ + { +#line 911 "src/libre/parser.act" + + (ZInode) = ast_make_expr_group(act_state->poolp, *flags, (ZIe), (ZIid)); + if ((ZInode) == NULL) { + goto ZL1; + } + +#line 412 "src/libre/dialect/literal/parser.c" + } + /* END OF ACTION: ast-make-group */ + /* BEGINNING OF INLINE: 119 */ { { switch (CURRENT_TERMINAL) { @@ -422,20 +427,20 @@ p_re__literal(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: err-expected-eof */ { -#line 767 "src/libre/parser.act" +#line 753 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXEOF; } goto ZL1; -#line 433 "src/libre/dialect/literal/parser.c" +#line 438 "src/libre/dialect/literal/parser.c" } /* END OF ACTION: err-expected-eof */ } ZL3:; } - /* END OF INLINE: 118 */ + /* END OF INLINE: 119 */ } goto ZL0; ZL1:; @@ -455,27 +460,27 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, } { t_char ZIc; + t_pos ZI111; t_pos ZI112; - t_pos ZI113; switch (CURRENT_TERMINAL) { case (TOK_CHAR): /* BEGINNING OF EXTRACT: CHAR */ { -#line 589 "src/libre/parser.act" +#line 575 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); - ZI112 = lex_state->lx.start; - ZI113 = lex_state->lx.end; + ZI111 = lex_state->lx.start; + ZI112 = lex_state->lx.end; + (void) ZI111; (void) ZI112; - (void) ZI113; ZIc = lex_state->buf.a[0]; -#line 479 "src/libre/dialect/literal/parser.c" +#line 484 "src/libre/dialect/literal/parser.c" } /* END OF EXTRACT: CHAR */ break; @@ -485,14 +490,14 @@ p_list_Hof_Hatoms_C_Catom(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 908 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode) == NULL) { goto ZL1; } -#line 496 "src/libre/dialect/literal/parser.c" +#line 501 "src/libre/dialect/literal/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -501,26 +506,26 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-atom */ { -#line 718 "src/libre/parser.act" +#line 704 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXATOM; } goto ZL2; -#line 512 "src/libre/dialect/literal/parser.c" +#line 517 "src/libre/dialect/literal/parser.c" } /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 887 "src/libre/parser.act" +#line 853 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL2; } -#line 524 "src/libre/dialect/literal/parser.c" +#line 529 "src/libre/dialect/literal/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -534,7 +539,7 @@ ZL0:; /* BEGINNING OF TRAILER */ -#line 1090 "src/libre/parser.act" +#line 1052 "src/libre/parser.act" static int @@ -554,9 +559,8 @@ ZL0:; struct ast * DIALECT_PARSE(re_getchar_fun *f, void *opaque, - const struct fsm_options *opt, enum re_flags flags, int overlap, - struct re_err *err) + struct re_err *err, struct re_pos *end) { struct ast *ast; @@ -569,6 +573,7 @@ ZL0:; struct LX_STATE *lx; assert(f != NULL); + assert(end != NULL); ast = ast_new(); @@ -592,11 +597,6 @@ ZL0:; lex_state->buf.a = NULL; lex_state->buf.len = 0; -#if PCRE_DIALECT - lex_state->extended_mode_comment = 0; - lex_state->flags_ptr = &flags; -#endif /* PCRE_DIALECT */ - /* XXX: unneccessary since we're lexing from a string */ /* (except for pushing "[" and "]" around ::group-$dialect) */ lx->buf_opaque = &lex_state->buf; @@ -627,6 +627,9 @@ ZL0:; DIALECT_ENTRY(&flags, lex_state, act_state, err, &ast->expr); + /* we output this for reporting errors found through AST analysis */ + mark(end, &lx->end); + lx->free(lx->buf_opaque); if (err->e != RE_ESUCCESS) { @@ -692,6 +695,6 @@ ZL0:; return NULL; } -#line 696 "src/libre/dialect/literal/parser.c" +#line 699 "src/libre/dialect/literal/parser.c" /* END OF FILE */ diff --git a/src/libre/dialect/literal/parser.h b/src/libre/dialect/literal/parser.h index 55fff6039..7f90a15ef 100644 --- a/src/libre/dialect/literal/parser.h +++ b/src/libre/dialect/literal/parser.h @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 300 "src/libre/parser.act" +#line 281 "src/libre/parser.act" #include @@ -28,7 +28,7 @@ extern void p_re__literal(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1247 "src/libre/parser.act" +#line 1207 "src/libre/parser.act" #line 35 "src/libre/dialect/literal/parser.h" diff --git a/src/libre/dialect/literal/parser.sid b/src/libre/dialect/literal/parser.sid index 9bf204d38..3e4d6c388 100644 --- a/src/libre/dialect/literal/parser.sid +++ b/src/libre/dialect/literal/parser.sid @@ -15,7 +15,7 @@ !ast_count; !ast_class_id; !endpoint; - !group_id; + group_id; %terminals% @@ -84,7 +84,7 @@ !: () -> (:ast_count); !: (:unsigned, :pos, :unsigned, :pos) -> (:ast_count); - !: () -> (:group_id); + : () -> (:group_id); !: () -> (:char); !: () -> (:char); !: () -> (:char); @@ -97,7 +97,7 @@ : () -> (:ast_expr); !: () -> (:ast_expr); !: (:ast_expr, :ast_count) -> (:ast_expr); - !: (:ast_expr, :group_id) -> (:ast_expr); + : (:ast_expr, :group_id) -> (:ast_expr); !: () -> (:re_flags); !: (:re_flags) -> (); !: (:re_flags, :re_flags) -> (); @@ -128,7 +128,6 @@ !: (:pos, :pos) -> (); !: (:pos, :pos) -> (); !: (:pos, :pos) -> (); - !: (:ast_expr, :pos, :pos) -> (); !: () -> (:ast_class_id); !: () -> (:ast_class_id); @@ -155,12 +154,14 @@ }; re_literal: () -> (node :ast_expr) = { + id = ; { - node = ; - list-of-atoms(node); + e = ; + list-of-atoms(e); || - node = ; + e = ; }; + node = (e, id); { EOF; diff --git a/src/libre/dialect/native/lexer.c b/src/libre/dialect/native/lexer.c index 2399683ac..b18634004 100644 --- a/src/libre/dialect/native/lexer.c +++ b/src/libre/dialect/native/lexer.c @@ -12,11 +12,31 @@ static enum lx_native_token z0(struct lx_native_lx *lx); static enum lx_native_token z1(struct lx_native_lx *lx); static enum lx_native_token z2(struct lx_native_lx *lx); +static int +lx_native_advance_end(struct lx_native_lx *lx, int c) +{ + lx->end.byte++; + lx->end.col++; + if (c == '\n') { + lx->end.line++; + lx->end.saved_col = lx->end.col - 1; + lx->end.col = 1; + } + if (lx->push != NULL) { + if (-1 == lx->push(lx->buf_opaque, (char)c)) { + return 0; + } + } + return 1; +} + +/* This wrapper manages one character of lookahead/pushback + * and the line, column, and byte offsets. */ #if __STDC_VERSION__ >= 199901L inline #endif static int -lx_getc(struct lx_native_lx *lx) +lx_native_getc(struct lx_native_lx *lx) { int c; @@ -32,18 +52,19 @@ lx_getc(struct lx_native_lx *lx) } } - lx->end.byte++; - lx->end.col++; - - if (c == '\n') { - lx->end.line++; - lx->end.saved_col = lx->end.col - 1; - lx->end.col = 1; - } + if (!lx_native_advance_end(lx, c)) { return EOF; } return c; } +/* This wrapper adapts calling lx_native_getc to the interface + * in libfsm's generated code. */ +static int +fsm_getc(void *getc_opaque) +{ + return lx_native_getc((struct lx_native_lx *)getc_opaque); +} + #if __STDC_VERSION__ >= 199901L inline #endif @@ -52,10 +73,7 @@ lx_native_ungetc(struct lx_native_lx *lx, int c) { assert(lx != NULL); assert(lx->c == EOF); - lx->c = c; - - lx->end.byte--; lx->end.col--; @@ -107,6 +125,17 @@ lx_native_dynpush(void *buf_opaque, char c) return 0; } +static void +lx_native_dynpop(void *buf_opaque) +{ + struct lx_dynbuf *t = buf_opaque; + + assert(t != NULL); + + assert(t->p != t->a); + t->p--; +} + int lx_native_dynclear(void *buf_opaque) { @@ -146,29 +175,28 @@ lx_native_dynfree(void *buf_opaque) static enum lx_native_token z0(struct lx_native_lx *lx) { + int has_consumed_input = 0; int c; - enum { - S0, S1, S2, S3, NONE - } state; - assert(lx != NULL); if (lx->clear != NULL) { lx->clear(lx->buf_opaque); } - state = NONE; - lx->start = lx->end; - while (c = lx_getc(lx), c != EOF) { - if (state == NONE) { - state = S0; - } + void *getc_opaque = (void *)lx; + enum { + S0, S1, S2, S3 + } state; + + state = S0; + while (c = fsm_getc(getc_opaque), c != EOF) { + has_consumed_input = 1; switch (state) { - case S0: /* start */ + case S0: /* e.g. "" */ switch ((unsigned char) c) { case '0': case '1': @@ -182,7 +210,9 @@ z0(struct lx_native_lx *lx) case '9': state = S1; break; case ',': state = S2; break; case '}': state = S3; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; @@ -198,69 +228,75 @@ z0(struct lx_native_lx *lx) case '7': case '8': case '9': break; - default: lx_native_ungetc(lx, c); return TOK_COUNT; + default: lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return TOK_COUNT; } break; case S2: /* e.g. "," */ - lx_native_ungetc(lx, c); return TOK_SEP; + lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return TOK_SEP; case S3: /* e.g. "}" */ - lx_native_ungetc(lx, c); return lx->z = z2, TOK_CLOSECOUNT; + lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return lx->z = z2, TOK_CLOSECOUNT; default: ; /* unreached */ } + } + + /* end states */ + switch (state) { + case S1: return TOK_COUNT; + case S2: return TOK_SEP; + case S3: return lx->z = z2, TOK_CLOSECOUNT; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; + } if (lx->push != NULL) { if (-1 == lx->push(lx->buf_opaque, (char)c)) { return TOK_ERROR; } } - } lx->lgetc = NULL; - switch (state) { - case NONE: return TOK_EOF; - case S1: return TOK_COUNT; - case S2: return TOK_SEP; - case S3: return TOK_CLOSECOUNT; - default: errno = EINVAL; return TOK_ERROR; - } + if (!has_consumed_input) { + return TOK_EOF; + } + return TOK_ERROR; } static enum lx_native_token z1(struct lx_native_lx *lx) { + int has_consumed_input = 0; int c; - enum { - S0, S1, S2, S3, S4, S5, S6, S7, S8, S9, - S10, S11, S12, S13, S14, S15, S16, S17, S18, S19, - S20, S21, S22, S23, S24, S25, S26, S27, S28, S29, - S30, S31, S32, S33, S34, S35, S36, S37, S38, S39, - S40, S41, S42, S43, S44, S45, S46, S47, S48, S49, - S50, S51, S52, S53, S54, S55, S56, NONE - } state; - assert(lx != NULL); if (lx->clear != NULL) { lx->clear(lx->buf_opaque); } - state = NONE; - lx->start = lx->end; - while (c = lx_getc(lx), c != EOF) { - if (state == NONE) { - state = S0; - } + void *getc_opaque = (void *)lx; + enum { + S0, S1, S2, S3, S4, S5, S6, S7, S8, S9, + S10, S11, S12, S13, S14, S15, S16, S17, S18, S19, + S20, S21, S22, S23, S24, S25, S26, S27, S28, S29, + S30, S31, S32, S33, S34, S35, S36, S37, S38, S39, + S40, S41, S42, S43, S44, S45, S46, S47, S48, S49, + S50, S51, S52, S53, S54, S55, S56 + } state; + state = S0; + + while (c = fsm_getc(getc_opaque), c != EOF) { + has_consumed_input = 1; switch (state) { - case S0: /* start */ + case S0: /* e.g. "" */ switch ((unsigned char) c) { case '[': state = S1; break; case '\\': state = S3; break; @@ -273,12 +309,12 @@ z1(struct lx_native_lx *lx) case S1: /* e.g. "[" */ switch ((unsigned char) c) { case ':': state = S12; break; - default: lx_native_ungetc(lx, c); return TOK_CHAR; + default: lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return TOK_CHAR; } break; case S2: /* e.g. "\\x00" */ - lx_native_ungetc(lx, c); return TOK_CHAR; + lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return TOK_CHAR; case S3: /* e.g. "\\" */ switch ((unsigned char) c) { @@ -301,25 +337,27 @@ z1(struct lx_native_lx *lx) case '5': case '6': case '7': state = S9; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S4: /* e.g. "-" */ switch ((unsigned char) c) { case ']': state = S6; break; - default: lx_native_ungetc(lx, c); return TOK_RANGE; + default: lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return TOK_RANGE; } break; case S5: /* e.g. "]" */ - lx_native_ungetc(lx, c); return lx->z = z2, TOK_CLOSEGROUP; + lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return lx->z = z2, TOK_CLOSEGROUP; case S6: /* e.g. "-]" */ - lx_native_ungetc(lx, c); return lx->z = z2, TOK_CLOSEGROUPRANGE; + lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return lx->z = z2, TOK_CLOSEGROUPRANGE; case S7: /* e.g. "\\-" */ - lx_native_ungetc(lx, c); return TOK_ESC; + lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return TOK_ESC; case S8: /* e.g. "\\x" */ switch ((unsigned char) c) { @@ -345,7 +383,9 @@ z1(struct lx_native_lx *lx) case 'd': case 'e': case 'f': state = S10; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; @@ -359,7 +399,7 @@ z1(struct lx_native_lx *lx) case '5': case '6': case '7': break; - default: lx_native_ungetc(lx, c); return TOK_OCT; + default: lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return TOK_OCT; } break; @@ -387,12 +427,14 @@ z1(struct lx_native_lx *lx) case 'd': case 'e': case 'f': state = S11; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S11: /* e.g. "\\x00" */ - lx_native_ungetc(lx, c); return TOK_HEX; + lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return TOK_HEX; case S12: /* e.g. "[:" */ switch ((unsigned char) c) { @@ -408,35 +450,45 @@ z1(struct lx_native_lx *lx) case 'w': state = S21; break; case 'u': state = S22; break; case 'p': state = S23; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S13: /* e.g. "[:d" */ switch ((unsigned char) c) { case 'i': state = S55; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S14: /* e.g. "[:s" */ switch ((unsigned char) c) { case 'p': state = S52; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S15: /* e.g. "[:h" */ switch ((unsigned char) c) { case 's': state = S14; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S16: /* e.g. "[:g" */ switch ((unsigned char) c) { case 'r': state = S49; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; @@ -444,42 +496,54 @@ z1(struct lx_native_lx *lx) switch ((unsigned char) c) { case 's': state = S41; break; case 'l': state = S42; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S18: /* e.g. "[:c" */ switch ((unsigned char) c) { case 'n': state = S38; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S19: /* e.g. "[:l" */ switch ((unsigned char) c) { case 'o': state = S37; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S20: /* e.g. "[:x" */ switch ((unsigned char) c) { case 'd': state = S13; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S21: /* e.g. "[:w" */ switch ((unsigned char) c) { case 'o': state = S35; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S22: /* e.g. "[:u" */ switch ((unsigned char) c) { case 'p': state = S32; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; @@ -487,129 +551,165 @@ z1(struct lx_native_lx *lx) switch ((unsigned char) c) { case 'r': state = S24; break; case 'u': state = S25; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S24: /* e.g. "[:pr" */ switch ((unsigned char) c) { case 'i': state = S31; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S25: /* e.g. "[:pu" */ switch ((unsigned char) c) { case 'n': state = S26; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S26: /* e.g. "[:pun" */ switch ((unsigned char) c) { case 'c': state = S27; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S27: /* e.g. "[:digi" */ switch ((unsigned char) c) { case 't': state = S28; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S28: /* e.g. "[:word" */ switch ((unsigned char) c) { case ':': state = S29; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S29: /* e.g. "[:word:" */ switch ((unsigned char) c) { case ']': state = S30; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S30: /* e.g. "[:word:]" */ - lx_native_ungetc(lx, c); return TOK_NAMED__CLASS; + lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return TOK_NAMED__CLASS; case S31: /* e.g. "[:pri" */ switch ((unsigned char) c) { case 'n': state = S27; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S32: /* e.g. "[:up" */ switch ((unsigned char) c) { case 'p': state = S33; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S33: /* e.g. "[:low" */ switch ((unsigned char) c) { case 'e': state = S34; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S34: /* e.g. "[:lowe" */ switch ((unsigned char) c) { case 'r': state = S28; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S35: /* e.g. "[:wo" */ switch ((unsigned char) c) { case 'r': state = S36; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S36: /* e.g. "[:wor" */ switch ((unsigned char) c) { case 'd': state = S28; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S37: /* e.g. "[:lo" */ switch ((unsigned char) c) { case 'w': state = S33; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S38: /* e.g. "[:cn" */ switch ((unsigned char) c) { case 't': state = S39; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S39: /* e.g. "[:cnt" */ switch ((unsigned char) c) { case 'r': state = S40; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S40: /* e.g. "[:cntr" */ switch ((unsigned char) c) { case 'l': state = S28; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S41: /* e.g. "[:as" */ switch ((unsigned char) c) { case 'c': state = S47; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; @@ -617,164 +717,200 @@ z1(struct lx_native_lx *lx) switch ((unsigned char) c) { case 'p': state = S43; break; case 'n': state = S44; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S43: /* e.g. "[:alp" */ switch ((unsigned char) c) { case 'h': state = S46; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S44: /* e.g. "[:aln" */ switch ((unsigned char) c) { case 'u': state = S45; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S45: /* e.g. "[:alnu" */ switch ((unsigned char) c) { case 'm': state = S28; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S46: /* e.g. "[:alph" */ switch ((unsigned char) c) { case 'a': state = S28; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S47: /* e.g. "[:asc" */ switch ((unsigned char) c) { case 'i': state = S48; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S48: /* e.g. "[:asci" */ switch ((unsigned char) c) { case 'i': state = S28; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S49: /* e.g. "[:gr" */ switch ((unsigned char) c) { case 'a': state = S50; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S50: /* e.g. "[:gra" */ switch ((unsigned char) c) { case 'p': state = S51; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S51: /* e.g. "[:grap" */ switch ((unsigned char) c) { case 'h': state = S28; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S52: /* e.g. "[:sp" */ switch ((unsigned char) c) { case 'a': state = S53; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S53: /* e.g. "[:spa" */ switch ((unsigned char) c) { case 'c': state = S54; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S54: /* e.g. "[:spac" */ switch ((unsigned char) c) { case 'e': state = S28; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S55: /* e.g. "[:di" */ switch ((unsigned char) c) { case 'g': state = S56; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S56: /* e.g. "[:dig" */ switch ((unsigned char) c) { case 'i': state = S27; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; default: ; /* unreached */ } - - if (lx->push != NULL) { - if (-1 == lx->push(lx->buf_opaque, (char)c)) { - return TOK_ERROR; - } - } } - lx->lgetc = NULL; - + /* end states */ switch (state) { - case NONE: return TOK_EOF; case S1: return TOK_CHAR; case S2: return TOK_CHAR; case S4: return TOK_RANGE; - case S5: return TOK_CLOSEGROUP; - case S6: return TOK_CLOSEGROUPRANGE; + case S5: return lx->z = z2, TOK_CLOSEGROUP; + case S6: return lx->z = z2, TOK_CLOSEGROUPRANGE; case S7: return TOK_ESC; case S9: return TOK_OCT; case S11: return TOK_HEX; case S30: return TOK_NAMED__CLASS; - default: errno = EINVAL; return TOK_ERROR; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } + + if (lx->push != NULL) { + if (-1 == lx->push(lx->buf_opaque, (char)c)) { + return TOK_ERROR; + } + } + + lx->lgetc = NULL; + + if (!has_consumed_input) { + return TOK_EOF; + } + return TOK_ERROR; } static enum lx_native_token z2(struct lx_native_lx *lx) { + int has_consumed_input = 0; int c; - enum { - S0, S1, S2, S3, S4, S5, S6, S7, S8, S9, - S10, S11, S12, S13, S14, S15, S16, S17, S18, S19, - S20, S21, S22, S23, NONE - } state; - assert(lx != NULL); if (lx->clear != NULL) { lx->clear(lx->buf_opaque); } - state = NONE; - lx->start = lx->end; - while (c = lx_getc(lx), c != EOF) { - if (state == NONE) { - state = S0; - } + void *getc_opaque = (void *)lx; + enum { + S0, S1, S2, S3, S4, S5, S6, S7, S8, S9, + S10, S11, S12, S13, S14, S15, S16, S17, S18, S19, + S20, S21, S22, S23 + } state; + + state = S0; + while (c = fsm_getc(getc_opaque), c != EOF) { + has_consumed_input = 1; switch (state) { - case S0: /* start */ + case S0: /* e.g. "" */ switch ((unsigned char) c) { case '\\': state = S1; break; case '{': state = S3; break; @@ -820,63 +956,63 @@ z2(struct lx_native_lx *lx) case 'v': case '{': case '|': state = S19; break; - default: lx_native_ungetc(lx, c); return TOK_CHAR; + default: lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return TOK_CHAR; } break; case S2: /* e.g. "\\x00" */ - lx_native_ungetc(lx, c); return TOK_CHAR; + lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return TOK_CHAR; case S3: /* e.g. "{" */ - lx_native_ungetc(lx, c); return lx->z = z0, TOK_OPENCOUNT; + lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return lx->z = z0, TOK_OPENCOUNT; case S4: /* e.g. "[" */ switch ((unsigned char) c) { case '^': state = S14; break; case ']': state = S15; break; - default: lx_native_ungetc(lx, c); return lx->z = z1, TOK_OPENGROUP; + default: lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return lx->z = z1, TOK_OPENGROUP; } break; case S5: /* e.g. "|" */ - lx_native_ungetc(lx, c); return TOK_ALT; + lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return TOK_ALT; case S6: /* e.g. "." */ - lx_native_ungetc(lx, c); return TOK_ANY; + lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return TOK_ANY; case S7: /* e.g. "+" */ - lx_native_ungetc(lx, c); return TOK_PLUS; + lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return TOK_PLUS; case S8: /* e.g. "*" */ - lx_native_ungetc(lx, c); return TOK_STAR; + lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return TOK_STAR; - case S9: /* e.g. "?" */ - lx_native_ungetc(lx, c); return TOK_OPT; + case S9: /* e.g. "\077" */ + lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return TOK_OPT; case S10: /* e.g. "$" */ - lx_native_ungetc(lx, c); return TOK_END; + lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return TOK_END; case S11: /* e.g. "^" */ - lx_native_ungetc(lx, c); return TOK_START; + lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return TOK_START; case S12: /* e.g. ")" */ - lx_native_ungetc(lx, c); return TOK_CLOSESUB; + lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return TOK_CLOSESUB; case S13: /* e.g. "(" */ - lx_native_ungetc(lx, c); return TOK_OPENSUB; + lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return TOK_OPENSUB; case S14: /* e.g. "[^" */ switch ((unsigned char) c) { case ']': state = S16; break; - default: lx_native_ungetc(lx, c); return lx->z = z1, TOK_OPENGROUPINV; + default: lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return lx->z = z1, TOK_OPENGROUPINV; } break; case S15: /* e.g. "[]" */ - lx_native_ungetc(lx, c); return lx->z = z1, TOK_OPENGROUPCB; + lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return lx->z = z1, TOK_OPENGROUPCB; case S16: /* e.g. "[^]" */ - lx_native_ungetc(lx, c); return lx->z = z1, TOK_OPENGROUPINVCB; + lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return lx->z = z1, TOK_OPENGROUPINVCB; case S17: /* e.g. "\\x" */ switch ((unsigned char) c) { @@ -902,7 +1038,9 @@ z2(struct lx_native_lx *lx) case 'd': case 'e': case 'f': state = S22; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; @@ -916,12 +1054,12 @@ z2(struct lx_native_lx *lx) case '5': case '6': case '7': state = S20; break; - default: lx_native_ungetc(lx, c); return TOK_OCT; + default: lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return TOK_OCT; } break; case S19: /* e.g. "\\$" */ - lx_native_ungetc(lx, c); return TOK_ESC; + lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return TOK_ESC; case S20: /* e.g. "\\00" */ switch ((unsigned char) c) { @@ -933,12 +1071,12 @@ z2(struct lx_native_lx *lx) case '5': case '6': case '7': state = S21; break; - default: lx_native_ungetc(lx, c); return TOK_OCT; + default: lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return TOK_OCT; } break; case S21: /* e.g. "\\000" */ - lx_native_ungetc(lx, c); return TOK_OCT; + lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return TOK_OCT; case S22: /* e.g. "\\x0" */ switch ((unsigned char) c) { @@ -964,32 +1102,26 @@ z2(struct lx_native_lx *lx) case 'd': case 'e': case 'f': state = S23; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S23: /* e.g. "\\x00" */ - lx_native_ungetc(lx, c); return TOK_HEX; + lx_native_ungetc(lx, c); lx_native_dynpop(lx->buf_opaque); return TOK_HEX; default: ; /* unreached */ } - - if (lx->push != NULL) { - if (-1 == lx->push(lx->buf_opaque, (char)c)) { - return TOK_ERROR; - } - } } - lx->lgetc = NULL; - + /* end states */ switch (state) { - case NONE: return TOK_EOF; case S1: return TOK_CHAR; case S2: return TOK_CHAR; - case S3: return TOK_OPENCOUNT; - case S4: return TOK_OPENGROUP; + case S3: return lx->z = z0, TOK_OPENCOUNT; + case S4: return lx->z = z1, TOK_OPENGROUP; case S5: return TOK_ALT; case S6: return TOK_ANY; case S7: return TOK_PLUS; @@ -999,16 +1131,31 @@ z2(struct lx_native_lx *lx) case S11: return TOK_START; case S12: return TOK_CLOSESUB; case S13: return TOK_OPENSUB; - case S14: return TOK_OPENGROUPINV; - case S15: return TOK_OPENGROUPCB; - case S16: return TOK_OPENGROUPINVCB; + case S14: return lx->z = z1, TOK_OPENGROUPINV; + case S15: return lx->z = z1, TOK_OPENGROUPCB; + case S16: return lx->z = z1, TOK_OPENGROUPINVCB; case S18: return TOK_OCT; case S19: return TOK_ESC; case S20: return TOK_OCT; case S21: return TOK_OCT; case S23: return TOK_HEX; - default: errno = EINVAL; return TOK_ERROR; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_native_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } + + if (lx->push != NULL) { + if (-1 == lx->push(lx->buf_opaque, (char)c)) { + return TOK_ERROR; + } + } + + lx->lgetc = NULL; + + if (!has_consumed_input) { + return TOK_EOF; + } + return TOK_ERROR; } const char * @@ -1164,6 +1311,7 @@ lx_native_init(struct lx_native_lx *lx) lx->end.byte = 0; lx->end.line = 1; lx->end.col = 1; + (void)lx_native_dynpop; } enum lx_native_token diff --git a/src/libre/dialect/native/parser.c b/src/libre/dialect/native/parser.c index 1ddf136f3..a4dfb96d9 100644 --- a/src/libre/dialect/native/parser.c +++ b/src/libre/dialect/native/parser.c @@ -68,14 +68,6 @@ #define TOK_CLASS__nspace TOK_CLASS_NSPACE #define TOK_CLASS__ndigit TOK_CLASS_NDIGIT - /* This is a hack to work around the AST files not being able to include lexer.h. */ - #define AST_POS_OF_LX_POS(AST_POS, LX_POS) \ - do { \ - AST_POS.line = LX_POS.line; \ - AST_POS.col = LX_POS.col; \ - AST_POS.byte = LX_POS.byte; \ - } while (0) - #include "parser.h" #include "lexer.h" @@ -132,92 +124,8 @@ /* TODO: use lx's generated conveniences for the pattern buffer */ char a[512]; char *p; -#if PCRE_DIALECT - int extended_mode_comment; - enum re_flags *flags_ptr; -#endif /* PCRE_DIALECT */ }; - #define ERROR_TERMINAL (TOK_ERROR) - - #if PCRE_DIALECT - static void mark(struct re_pos *, const struct lx_pos *); - - static void - pcre_advance_lexer(struct lex_state *lex_state, struct act_state *act_state) - { - mark(&act_state->synstart, &lex_state->lx.start); - mark(&act_state->synend, &lex_state->lx.end); - act_state->lex_tok = LX_NEXT(&lex_state->lx); - } - - static enum LX_TOKEN - pcre_current_token(struct lex_state *lex_state, struct act_state *act_state) - { - enum re_flags fl = (lex_state->flags_ptr != NULL) ? lex_state->flags_ptr[0] : 0; - enum LX_TOKEN tok; - - if ((fl & RE_EXTENDED) == 0) { - tok = act_state->lex_tok; - - switch (tok) { - case TOK_WHITESPACE: - case TOK_NEWLINE: - case TOK_MAYBE_COMMENT: - return TOK_CHAR; - - default: - return tok; - } - } - - restart: - tok = act_state->lex_tok; - if (lex_state->extended_mode_comment) { - switch (tok) { - case TOK_EOF: - case TOK_ERROR: - case TOK_UNKNOWN: - /* don't eat EOF or errors */ - return tok; - - case TOK_NEWLINE: - lex_state->extended_mode_comment = 0; - /* fall through */ - default: - pcre_advance_lexer(lex_state, act_state); - goto restart; - } - } else { - switch (tok) { - case TOK_MAYBE_COMMENT: - lex_state->extended_mode_comment = 1; - /*fall through */ - case TOK_WHITESPACE: - case TOK_NEWLINE: - pcre_advance_lexer(lex_state, act_state); - goto restart; - - default: - return tok; - } - } - } - - #define CURRENT_TERMINAL (pcre_current_token(lex_state, act_state)) - #define ADVANCE_LEXER do { pcre_advance_lexer(lex_state,act_state); } while(0) - #else /* !PCRE_DIALECT */ - #define CURRENT_TERMINAL (act_state->lex_tok) - #define ADVANCE_LEXER do { mark(&act_state->synstart, &lex_state->lx.start); \ - mark(&act_state->synend, &lex_state->lx.end); \ - act_state->lex_tok = LX_NEXT(&lex_state->lx); \ - } while (0) - #endif /* PCRE_DIALECT */ - - #define SAVE_LEXER(tok) do { act_state->lex_tok_save = act_state->lex_tok; \ - act_state->lex_tok = tok; } while (0) - #define RESTORE_LEXER do { act_state->lex_tok = act_state->lex_tok_save; } while (0) - static void mark(struct re_pos *r, const struct lx_pos *pos) { @@ -288,7 +196,80 @@ return s; } -#line 292 "src/libre/dialect/native/parser.c" + static void + advance_lexer(struct lex_state *lex_state, struct act_state *act_state) + { + mark(&act_state->synstart, &lex_state->lx.start); + mark(&act_state->synend, &lex_state->lx.end); + act_state->lex_tok = LX_NEXT(&lex_state->lx); + } + + static enum LX_TOKEN + current_token(struct lex_state *lex_state, struct act_state *act_state, + enum re_flags *flags) + { +#ifndef PCRE_DIALECT + (void) lex_state; + (void) flags; + return act_state->lex_tok; +#else + if ((*flags & RE_EXTENDED) == 0) { + switch (act_state->lex_tok) { + case TOK_WHITESPACE: + case TOK_NEWLINE: + case TOK_MAYBE_COMMENT: + return TOK_CHAR; + + default: + return act_state->lex_tok; + } + } + + int extended_mode_comment = 0; + + restart: + + if (extended_mode_comment) { + switch (act_state->lex_tok) { + case TOK_EOF: + case TOK_ERROR: + case TOK_UNKNOWN: + /* don't eat EOF or errors */ + return act_state->lex_tok; + + case TOK_NEWLINE: + extended_mode_comment = 0; + /* fall through */ + default: + advance_lexer(lex_state, act_state); + goto restart; + } + } else { + switch (act_state->lex_tok) { + case TOK_MAYBE_COMMENT: + extended_mode_comment = 1; + /* fall through */ + case TOK_WHITESPACE: + case TOK_NEWLINE: + advance_lexer(lex_state, act_state); + goto restart; + + default: + return act_state->lex_tok; + } + } +#endif + } + + #define ADVANCE_LEXER (advance_lexer(lex_state, act_state)) + #define CURRENT_TERMINAL (current_token(lex_state, act_state, flags)) + #define ERROR_TERMINAL (TOK_ERROR) + + #define SAVE_LEXER(tok) do { act_state->lex_tok_save = act_state->lex_tok; \ + act_state->lex_tok = tok; } while (0) + #define RESTORE_LEXER do { act_state->lex_tok = act_state->lex_tok_save; } while (0) + +#line 273 "src/libre/dialect/native/parser.c" #ifndef ERROR_TERMINAL @@ -298,27 +279,27 @@ /* BEGINNING OF FUNCTION DECLARATIONS */ static void p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags, lex_state, act_state, err, t_endpoint *, t_pos *, t_pos *); -static void p_268(flags, lex_state, act_state, err, t_pos *, t_unsigned *, t_ast__count *); +static void p_265(flags, lex_state, act_state, err, t_pos *, t_unsigned *, t_ast__count *); static void p_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms(flags, lex_state, act_state, err, t_ast__expr); -static void p_155(flags, lex_state, act_state, err); +static void p_154(flags, lex_state, act_state, err); static void p_expr_C_Clist_Hof_Hpieces(flags, lex_state, act_state, err, t_ast__expr); static void p_expr_C_Cliteral(flags, lex_state, act_state, err, t_ast__expr *); static void p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags, lex_state, act_state, err, t_ast__expr *); static void p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hclass(flags, lex_state, act_state, err, t_endpoint *, t_pos *, t_pos *); static void p_expr_C_Ccharacter_Hclass(flags, lex_state, act_state, err, t_ast__expr *); +static void p_180(flags, lex_state, act_state, err, t_ast__expr *); static void p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_Hend(flags, lex_state, act_state, err, t_endpoint *, t_pos *); -static void p_181(flags, lex_state, act_state, err, t_ast__expr *); static void p_expr_C_Cpiece(flags, lex_state, act_state, err, t_ast__expr *); static void p_expr(flags, lex_state, act_state, err, t_ast__expr *); +static void p_195(flags, lex_state, act_state, err, t_pos *, t_char *, t_ast__expr *); extern void p_re__native(flags, lex_state, act_state, err, t_ast__expr *); -static void p_196(flags, lex_state, act_state, err, t_pos *, t_char *, t_ast__expr *); static void p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint(flags, lex_state, act_state, err, t_endpoint *, t_pos *, t_pos *); static void p_expr_C_Clist_Hof_Halts(flags, lex_state, act_state, err, t_ast__expr); static void p_expr_C_Cpiece_C_Ccount(flags, lex_state, act_state, err, t_ast__count *); static void p_expr_C_Cpiece_C_Catom(flags, lex_state, act_state, err, t_ast__expr *); -static void p_249(flags, lex_state, act_state, err, t_ast__class__id *, t_pos *, t_ast__expr *); +static void p_246(flags, lex_state, act_state, err, t_ast__class__id *, t_pos *, t_ast__expr *); static void p_expr_C_Calt(flags, lex_state, act_state, err, t_ast__expr *); -static void p_253(flags, lex_state, act_state, err, t_char *, t_pos *, t_ast__expr *); +static void p_250(flags, lex_state, act_state, err, t_char *, t_pos *, t_ast__expr *); /* BEGINNING OF STATIC VARIABLES */ @@ -338,14 +319,14 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { t_char ZIc; - /* BEGINNING OF INLINE: 142 */ + /* BEGINNING OF INLINE: 141 */ { switch (CURRENT_TERMINAL) { case (TOK_CHAR): { /* BEGINNING OF EXTRACT: CHAR */ { -#line 589 "src/libre/parser.act" +#line 575 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -358,7 +339,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags ZIc = lex_state->buf.a[0]; -#line 362 "src/libre/dialect/native/parser.c" +#line 343 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; @@ -368,7 +349,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: ESC */ { -#line 405 "src/libre/parser.act" +#line 386 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -394,7 +375,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags (void) ZIstart; (void) ZIend; -#line 398 "src/libre/dialect/native/parser.c" +#line 379 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: ESC */ ADVANCE_LEXER; @@ -404,7 +385,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: HEX */ { -#line 541 "src/libre/parser.act" +#line 527 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -452,7 +433,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags ZIc = (char) (unsigned char) u; -#line 456 "src/libre/dialect/native/parser.c" +#line 437 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: HEX */ ADVANCE_LEXER; @@ -462,7 +443,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: OCT */ { -#line 498 "src/libre/parser.act" +#line 484 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -505,7 +486,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags ZIc = (char) (unsigned char) u; -#line 509 "src/libre/dialect/native/parser.c" +#line 490 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: OCT */ ADVANCE_LEXER; @@ -515,15 +496,15 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags goto ZL1; } } - /* END OF INLINE: 142 */ + /* END OF INLINE: 141 */ /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 873 "src/libre/parser.act" +#line 839 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_LITERAL; - (ZIr).u.literal.c = (unsigned char)(ZIc); + (ZIr).u.literal.c = (unsigned char) (ZIc); -#line 527 "src/libre/dialect/native/parser.c" +#line 508 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ } @@ -538,63 +519,58 @@ ZL0:; } static void -p_268(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI266, t_unsigned *ZIm, t_ast__count *ZOc) +p_265(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI263, t_unsigned *ZIm, t_ast__count *ZOc) { t_ast__count ZIc; switch (CURRENT_TERMINAL) { case (TOK_CLOSECOUNT): { - t_pos ZI218; + t_pos ZI216; t_pos ZIend; /* BEGINNING OF EXTRACT: CLOSECOUNT */ { -#line 397 "src/libre/parser.act" +#line 378 "src/libre/parser.act" - ZI218 = lex_state->lx.start; + ZI216 = lex_state->lx.start; ZIend = lex_state->lx.end; - (void) ZI218; + (void) ZI216; (void) ZIend; -#line 562 "src/libre/dialect/native/parser.c" +#line 543 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: CLOSECOUNT */ ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-count */ { -#line 791 "src/libre/parser.act" +#line 777 "src/libre/parser.act" - mark(&act_state->countstart, &(*ZI266)); + mark(&act_state->countstart, &(*ZI263)); mark(&act_state->countend, &(ZIend)); -#line 573 "src/libre/dialect/native/parser.c" +#line 554 "src/libre/dialect/native/parser.c" } /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-range */ { -#line 853 "src/libre/parser.act" - - struct ast_pos ast_start, ast_end; +#line 824 "src/libre/parser.act" if ((*ZIm) < (*ZIm)) { err->e = RE_ENEGCOUNT; err->m = (*ZIm); err->n = (*ZIm); - mark(&act_state->countstart, &(*ZI266)); + mark(&act_state->countstart, &(*ZI263)); mark(&act_state->countend, &(ZIend)); goto ZL1; } - AST_POS_OF_LX_POS(ast_start, (*ZI266)); - AST_POS_OF_LX_POS(ast_end, (ZIend)); - - (ZIc) = ast_make_count((*ZIm), &ast_start, (*ZIm), &ast_end); + (ZIc) = ast_make_count((*ZIm), (*ZIm)); -#line 598 "src/libre/dialect/native/parser.c" +#line 574 "src/libre/dialect/native/parser.c" } /* END OF ACTION: count-range */ } @@ -602,7 +578,7 @@ p_268(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 case (TOK_SEP): { t_unsigned ZIn; - t_pos ZI221; + t_pos ZI219; t_pos ZIend; ADVANCE_LEXER; @@ -610,7 +586,7 @@ p_268(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 case (TOK_COUNT): /* BEGINNING OF EXTRACT: COUNT */ { -#line 641 "src/libre/parser.act" +#line 627 "src/libre/parser.act" unsigned long u; char *e; @@ -630,7 +606,7 @@ p_268(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 ZIn = (unsigned int) u; -#line 634 "src/libre/dialect/native/parser.c" +#line 610 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: COUNT */ break; @@ -642,15 +618,15 @@ p_268(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 case (TOK_CLOSECOUNT): /* BEGINNING OF EXTRACT: CLOSECOUNT */ { -#line 397 "src/libre/parser.act" +#line 378 "src/libre/parser.act" - ZI221 = lex_state->lx.start; + ZI219 = lex_state->lx.start; ZIend = lex_state->lx.end; - (void) ZI221; + (void) ZI219; (void) ZIend; -#line 654 "src/libre/dialect/native/parser.c" +#line 630 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: CLOSECOUNT */ break; @@ -660,37 +636,32 @@ p_268(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-count */ { -#line 791 "src/libre/parser.act" +#line 777 "src/libre/parser.act" - mark(&act_state->countstart, &(*ZI266)); + mark(&act_state->countstart, &(*ZI263)); mark(&act_state->countend, &(ZIend)); -#line 669 "src/libre/dialect/native/parser.c" +#line 645 "src/libre/dialect/native/parser.c" } /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-range */ { -#line 853 "src/libre/parser.act" - - struct ast_pos ast_start, ast_end; +#line 824 "src/libre/parser.act" if ((ZIn) < (*ZIm)) { err->e = RE_ENEGCOUNT; err->m = (*ZIm); err->n = (ZIn); - mark(&act_state->countstart, &(*ZI266)); + mark(&act_state->countstart, &(*ZI263)); mark(&act_state->countend, &(ZIend)); goto ZL1; } - AST_POS_OF_LX_POS(ast_start, (*ZI266)); - AST_POS_OF_LX_POS(ast_end, (ZIend)); - - (ZIc) = ast_make_count((*ZIm), &ast_start, (ZIn), &ast_end); + (ZIc) = ast_make_count((*ZIm), (ZIn)); -#line 694 "src/libre/dialect/native/parser.c" +#line 665 "src/libre/dialect/native/parser.c" } /* END OF ACTION: count-range */ } @@ -725,13 +696,13 @@ ZL2_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms:; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1084 "src/libre/parser.act" +#line 1046 "src/libre/parser.act" if (!ast_add_expr_alt((ZIclass), (ZInode))) { goto ZL1; } -#line 735 "src/libre/dialect/native/parser.c" +#line 706 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF INLINE: expr::character-class::list-of-class-terms */ @@ -751,31 +722,31 @@ ZL1:; } static void -p_155(flags flags, lex_state lex_state, act_state act_state, err err) +p_154(flags flags, lex_state lex_state, act_state act_state, err err) { if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { return; } { - t_char ZI156; + t_char ZI155; + t_pos ZI156; t_pos ZI157; - t_pos ZI158; switch (CURRENT_TERMINAL) { case (TOK_RANGE): /* BEGINNING OF EXTRACT: RANGE */ { -#line 327 "src/libre/parser.act" +#line 308 "src/libre/parser.act" - ZI156 = '-'; - ZI157 = lex_state->lx.start; - ZI158 = lex_state->lx.end; + ZI155 = '-'; + ZI156 = lex_state->lx.start; + ZI157 = lex_state->lx.end; + (void) ZI155; (void) ZI156; (void) ZI157; - (void) ZI158; -#line 779 "src/libre/dialect/native/parser.c" +#line 750 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: RANGE */ break; @@ -789,14 +760,14 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-range */ { -#line 732 "src/libre/parser.act" +#line 718 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXRANGE; } goto ZL2; -#line 800 "src/libre/dialect/native/parser.c" +#line 771 "src/libre/dialect/native/parser.c" } /* END OF ACTION: err-expected-range */ } @@ -824,16 +795,16 @@ ZL2_expr_C_Clist_Hof_Hpieces:; } /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1078 "src/libre/parser.act" +#line 1040 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcat), (ZIa))) { goto ZL1; } -#line 834 "src/libre/dialect/native/parser.c" +#line 805 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-add-concat */ - /* BEGINNING OF INLINE: 225 */ + /* BEGINNING OF INLINE: 223 */ { switch (CURRENT_TERMINAL) { case (TOK_ANY): case (TOK_START): case (TOK_END): case (TOK_OPENSUB): @@ -849,7 +820,7 @@ ZL2_expr_C_Clist_Hof_Hpieces:; break; } } - /* END OF INLINE: 225 */ + /* END OF INLINE: 223 */ } return; ZL1:; @@ -868,30 +839,30 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err { t_char ZIc; - /* BEGINNING OF INLINE: 110 */ + /* BEGINNING OF INLINE: 109 */ { switch (CURRENT_TERMINAL) { case (TOK_CHAR): { + t_pos ZI117; t_pos ZI118; - t_pos ZI119; /* BEGINNING OF EXTRACT: CHAR */ { -#line 589 "src/libre/parser.act" +#line 575 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); - ZI118 = lex_state->lx.start; - ZI119 = lex_state->lx.end; + ZI117 = lex_state->lx.start; + ZI118 = lex_state->lx.end; + (void) ZI117; (void) ZI118; - (void) ZI119; ZIc = lex_state->buf.a[0]; -#line 895 "src/libre/dialect/native/parser.c" +#line 866 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; @@ -899,12 +870,12 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err break; case (TOK_ESC): { + t_pos ZI111; t_pos ZI112; - t_pos ZI113; /* BEGINNING OF EXTRACT: ESC */ { -#line 405 "src/libre/parser.act" +#line 386 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -924,13 +895,13 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err default: break; } - ZI112 = lex_state->lx.start; - ZI113 = lex_state->lx.end; + ZI111 = lex_state->lx.start; + ZI112 = lex_state->lx.end; + (void) ZI111; (void) ZI112; - (void) ZI113; -#line 934 "src/libre/dialect/native/parser.c" +#line 905 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: ESC */ ADVANCE_LEXER; @@ -938,12 +909,12 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err break; case (TOK_HEX): { + t_pos ZI115; t_pos ZI116; - t_pos ZI117; /* BEGINNING OF EXTRACT: HEX */ { -#line 541 "src/libre/parser.act" +#line 527 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -952,11 +923,11 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err assert(0 == strncmp(lex_state->buf.a, "\\x", 2)); assert(strlen(lex_state->buf.a) >= 2); /* pcre allows \x without a suffix */ - ZI116 = lex_state->lx.start; - ZI117 = lex_state->lx.end; + ZI115 = lex_state->lx.start; + ZI116 = lex_state->lx.end; + (void) ZI115; (void) ZI116; - (void) ZI117; errno = 0; @@ -991,7 +962,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err ZIc = (char) (unsigned char) u; -#line 995 "src/libre/dialect/native/parser.c" +#line 966 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: HEX */ ADVANCE_LEXER; @@ -999,12 +970,12 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err break; case (TOK_OCT): { + t_pos ZI113; t_pos ZI114; - t_pos ZI115; /* BEGINNING OF EXTRACT: OCT */ { -#line 498 "src/libre/parser.act" +#line 484 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -1013,11 +984,11 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err assert(0 == strncmp(lex_state->buf.a, "\\", 1)); assert(strlen(lex_state->buf.a) >= 2); - ZI114 = lex_state->lx.start; - ZI115 = lex_state->lx.end; + ZI113 = lex_state->lx.start; + ZI114 = lex_state->lx.end; + (void) ZI113; (void) ZI114; - (void) ZI115; errno = 0; @@ -1047,7 +1018,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err ZIc = (char) (unsigned char) u; -#line 1051 "src/libre/dialect/native/parser.c" +#line 1022 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: OCT */ ADVANCE_LEXER; @@ -1057,17 +1028,17 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err goto ZL1; } } - /* END OF INLINE: 110 */ + /* END OF INLINE: 109 */ /* BEGINNING OF ACTION: ast-make-literal */ { -#line 908 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode) == NULL) { goto ZL1; } -#line 1071 "src/libre/dialect/native/parser.c" +#line 1042 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -1087,30 +1058,30 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ switch (CURRENT_TERMINAL) { case (TOK_CHAR): { - t_char ZI262; - t_pos ZI263; - t_pos ZI264; + t_char ZI259; + t_pos ZI260; + t_pos ZI261; /* BEGINNING OF EXTRACT: CHAR */ { -#line 589 "src/libre/parser.act" +#line 575 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); - ZI263 = lex_state->lx.start; - ZI264 = lex_state->lx.end; + ZI260 = lex_state->lx.start; + ZI261 = lex_state->lx.end; - (void) ZI263; - (void) ZI264; + (void) ZI260; + (void) ZI261; - ZI262 = lex_state->buf.a[0]; + ZI259 = lex_state->buf.a[0]; -#line 1110 "src/libre/dialect/native/parser.c" +#line 1081 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; - p_253 (flags, lex_state, act_state, err, &ZI262, &ZI263, &ZInode); + p_250 (flags, lex_state, act_state, err, &ZI259, &ZI260, &ZInode); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -1119,43 +1090,43 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ break; case (TOK_ESC): { - t_char ZI250; - t_pos ZI251; - t_pos ZI252; + t_char ZI247; + t_pos ZI248; + t_pos ZI249; /* BEGINNING OF EXTRACT: ESC */ { -#line 405 "src/libre/parser.act" +#line 386 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); assert(lex_state->buf.a[2] == '\0'); - ZI250 = lex_state->buf.a[1]; - - switch (ZI250) { - case 'a': ZI250 = '\a'; break; - case 'b': ZI250 = '\b'; break; - case 'e': ZI250 = '\033'; break; - case 'f': ZI250 = '\f'; break; - case 'n': ZI250 = '\n'; break; - case 'r': ZI250 = '\r'; break; - case 't': ZI250 = '\t'; break; - case 'v': ZI250 = '\v'; break; + ZI247 = lex_state->buf.a[1]; + + switch (ZI247) { + case 'a': ZI247 = '\a'; break; + case 'b': ZI247 = '\b'; break; + case 'e': ZI247 = '\033'; break; + case 'f': ZI247 = '\f'; break; + case 'n': ZI247 = '\n'; break; + case 'r': ZI247 = '\r'; break; + case 't': ZI247 = '\t'; break; + case 'v': ZI247 = '\v'; break; default: break; } - ZI251 = lex_state->lx.start; - ZI252 = lex_state->lx.end; + ZI248 = lex_state->lx.start; + ZI249 = lex_state->lx.end; - (void) ZI251; - (void) ZI252; + (void) ZI248; + (void) ZI249; -#line 1155 "src/libre/dialect/native/parser.c" +#line 1126 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: ESC */ ADVANCE_LEXER; - p_253 (flags, lex_state, act_state, err, &ZI250, &ZI251, &ZInode); + p_250 (flags, lex_state, act_state, err, &ZI247, &ZI248, &ZInode); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -1164,13 +1135,13 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ break; case (TOK_HEX): { - t_char ZI258; - t_pos ZI259; - t_pos ZI260; + t_char ZI255; + t_pos ZI256; + t_pos ZI257; /* BEGINNING OF EXTRACT: HEX */ { -#line 541 "src/libre/parser.act" +#line 527 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -1179,11 +1150,11 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ assert(0 == strncmp(lex_state->buf.a, "\\x", 2)); assert(strlen(lex_state->buf.a) >= 2); /* pcre allows \x without a suffix */ - ZI259 = lex_state->lx.start; - ZI260 = lex_state->lx.end; + ZI256 = lex_state->lx.start; + ZI257 = lex_state->lx.end; - (void) ZI259; - (void) ZI260; + (void) ZI256; + (void) ZI257; errno = 0; @@ -1216,13 +1187,13 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ goto ZL1; } - ZI258 = (char) (unsigned char) u; + ZI255 = (char) (unsigned char) u; -#line 1222 "src/libre/dialect/native/parser.c" +#line 1193 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: HEX */ ADVANCE_LEXER; - p_253 (flags, lex_state, act_state, err, &ZI258, &ZI259, &ZInode); + p_250 (flags, lex_state, act_state, err, &ZI255, &ZI256, &ZInode); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -1231,31 +1202,31 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ break; case (TOK_NAMED__CLASS): { - t_ast__class__id ZI246; - t_pos ZI247; - t_pos ZI248; + t_ast__class__id ZI243; + t_pos ZI244; + t_pos ZI245; /* BEGINNING OF EXTRACT: NAMED_CLASS */ { -#line 661 "src/libre/parser.act" +#line 647 "src/libre/parser.act" - ZI246 = DIALECT_CLASS(lex_state->buf.a); - if (ZI246 == NULL) { + ZI243 = DIALECT_CLASS(lex_state->buf.a); + if (ZI243 == NULL) { /* syntax error -- unrecognized class */ goto ZL1; } - ZI247 = lex_state->lx.start; - ZI248 = lex_state->lx.end; + ZI244 = lex_state->lx.start; + ZI245 = lex_state->lx.end; - (void) ZI247; - (void) ZI248; + (void) ZI244; + (void) ZI245; -#line 1255 "src/libre/dialect/native/parser.c" +#line 1226 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: NAMED_CLASS */ ADVANCE_LEXER; - p_249 (flags, lex_state, act_state, err, &ZI246, &ZI247, &ZInode); + p_246 (flags, lex_state, act_state, err, &ZI243, &ZI244, &ZInode); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -1264,13 +1235,13 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ break; case (TOK_OCT): { - t_char ZI254; - t_pos ZI255; - t_pos ZI256; + t_char ZI251; + t_pos ZI252; + t_pos ZI253; /* BEGINNING OF EXTRACT: OCT */ { -#line 498 "src/libre/parser.act" +#line 484 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -1279,11 +1250,11 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ assert(0 == strncmp(lex_state->buf.a, "\\", 1)); assert(strlen(lex_state->buf.a) >= 2); - ZI255 = lex_state->lx.start; - ZI256 = lex_state->lx.end; + ZI252 = lex_state->lx.start; + ZI253 = lex_state->lx.end; - (void) ZI255; - (void) ZI256; + (void) ZI252; + (void) ZI253; errno = 0; @@ -1311,13 +1282,13 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ goto ZL1; } - ZI254 = (char) (unsigned char) u; + ZI251 = (char) (unsigned char) u; -#line 1317 "src/libre/dialect/native/parser.c" +#line 1288 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: OCT */ ADVANCE_LEXER; - p_253 (flags, lex_state, act_state, err, &ZI254, &ZI255, &ZInode); + p_250 (flags, lex_state, act_state, err, &ZI251, &ZI252, &ZInode); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -1354,7 +1325,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hclass(flags fl case (TOK_NAMED__CLASS): /* BEGINNING OF EXTRACT: NAMED_CLASS */ { -#line 661 "src/libre/parser.act" +#line 647 "src/libre/parser.act" ZIid = DIALECT_CLASS(lex_state->buf.a); if (ZIid == NULL) { @@ -1368,7 +1339,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hclass(flags fl (void) ZIstart; (void) ZIend; -#line 1372 "src/libre/dialect/native/parser.c" +#line 1343 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: NAMED_CLASS */ break; @@ -1378,12 +1349,12 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hclass(flags fl ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-range-endpoint-class */ { -#line 878 "src/libre/parser.act" +#line 844 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_NAMED; (ZIr).u.named.class = (ZIid); -#line 1387 "src/libre/dialect/native/parser.c" +#line 1358 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-class */ } @@ -1408,43 +1379,42 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state { t_pos ZIstart; t_ast__expr ZItmp; - t_pos ZIend; - /* BEGINNING OF INLINE: 167 */ + /* BEGINNING OF INLINE: 166 */ { switch (CURRENT_TERMINAL) { case (TOK_OPENGROUP): { - t_pos ZI168; + t_pos ZI167; /* BEGINNING OF EXTRACT: OPENGROUP */ { -#line 337 "src/libre/parser.act" +#line 318 "src/libre/parser.act" ZIstart = lex_state->lx.start; - ZI168 = lex_state->lx.end; + ZI167 = lex_state->lx.end; (void) ZIstart; - (void) ZI168; + (void) ZI167; -#line 1431 "src/libre/dialect/native/parser.c" +#line 1401 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: OPENGROUP */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 901 "src/libre/parser.act" +#line 867 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 1444 "src/libre/dialect/native/parser.c" +#line 1414 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZInode; - p_181 (flags, lex_state, act_state, err, &ZItmp); + p_180 (flags, lex_state, act_state, err, &ZItmp); p_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms (flags, lex_state, act_state, err, ZItmp); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; @@ -1454,60 +1424,60 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state break; case (TOK_OPENGROUPCB): { - t_pos ZI187; + t_pos ZI186; t_char ZIcbrak; t_ast__expr ZInode1; /* BEGINNING OF EXTRACT: OPENGROUPCB */ { -#line 353 "src/libre/parser.act" +#line 334 "src/libre/parser.act" ZIstart = lex_state->lx.start; - ZI187 = lex_state->lx.end; + ZI186 = lex_state->lx.end; (void) ZIstart; - (void) ZI187; + (void) ZI186; -#line 1472 "src/libre/dialect/native/parser.c" +#line 1442 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: OPENGROUPCB */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 901 "src/libre/parser.act" +#line 867 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 1485 "src/libre/dialect/native/parser.c" +#line 1455 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZInode; /* BEGINNING OF ACTION: make-literal-cbrak */ { -#line 919 "src/libre/parser.act" +#line 885 "src/libre/parser.act" (ZIcbrak) = ']'; -#line 1495 "src/libre/dialect/native/parser.c" +#line 1465 "src/libre/dialect/native/parser.c" } /* END OF ACTION: make-literal-cbrak */ - p_196 (flags, lex_state, act_state, err, &ZIstart, &ZIcbrak, &ZInode1); + p_195 (flags, lex_state, act_state, err, &ZIstart, &ZIcbrak, &ZInode1); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1084 "src/libre/parser.act" +#line 1046 "src/libre/parser.act" if (!ast_add_expr_alt((ZItmp), (ZInode1))) { goto ZL1; } -#line 1511 "src/libre/dialect/native/parser.c" +#line 1481 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-add-alt */ p_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms (flags, lex_state, act_state, err, ZItmp); @@ -1519,38 +1489,38 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state break; case (TOK_OPENGROUPINV): { - t_pos ZI179; + t_pos ZI178; /* BEGINNING OF EXTRACT: OPENGROUPINV */ { -#line 345 "src/libre/parser.act" +#line 326 "src/libre/parser.act" ZIstart = lex_state->lx.start; - ZI179 = lex_state->lx.end; + ZI178 = lex_state->lx.end; (void) ZIstart; - (void) ZI179; + (void) ZI178; -#line 1535 "src/libre/dialect/native/parser.c" +#line 1505 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: OPENGROUPINV */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 901 "src/libre/parser.act" +#line 867 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 1548 "src/libre/dialect/native/parser.c" +#line 1518 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZInode; /* BEGINNING OF ACTION: ast-make-invert */ { -#line 1000 "src/libre/parser.act" +#line 966 "src/libre/parser.act" struct ast_expr *any; @@ -1588,10 +1558,10 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL1; } -#line 1592 "src/libre/dialect/native/parser.c" +#line 1562 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-invert */ - p_181 (flags, lex_state, act_state, err, &ZItmp); + p_180 (flags, lex_state, act_state, err, &ZItmp); p_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms (flags, lex_state, act_state, err, ZItmp); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; @@ -1601,40 +1571,40 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state break; case (TOK_OPENGROUPINVCB): { - t_pos ZI194; + t_pos ZI193; t_char ZIcbrak; t_ast__expr ZInode1; /* BEGINNING OF EXTRACT: OPENGROUPINVCB */ { -#line 361 "src/libre/parser.act" +#line 342 "src/libre/parser.act" ZIstart = lex_state->lx.start; - ZI194 = lex_state->lx.end; + ZI193 = lex_state->lx.end; (void) ZIstart; - (void) ZI194; + (void) ZI193; -#line 1619 "src/libre/dialect/native/parser.c" +#line 1589 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: OPENGROUPINVCB */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 901 "src/libre/parser.act" +#line 867 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 1632 "src/libre/dialect/native/parser.c" +#line 1602 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZInode; /* BEGINNING OF ACTION: ast-make-invert */ { -#line 1000 "src/libre/parser.act" +#line 966 "src/libre/parser.act" struct ast_expr *any; @@ -1672,32 +1642,32 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL1; } -#line 1676 "src/libre/dialect/native/parser.c" +#line 1646 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-invert */ /* BEGINNING OF ACTION: make-literal-cbrak */ { -#line 919 "src/libre/parser.act" +#line 885 "src/libre/parser.act" (ZIcbrak) = ']'; -#line 1685 "src/libre/dialect/native/parser.c" +#line 1655 "src/libre/dialect/native/parser.c" } /* END OF ACTION: make-literal-cbrak */ - p_196 (flags, lex_state, act_state, err, &ZIstart, &ZIcbrak, &ZInode1); + p_195 (flags, lex_state, act_state, err, &ZIstart, &ZIcbrak, &ZInode1); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1084 "src/libre/parser.act" +#line 1046 "src/libre/parser.act" if (!ast_add_expr_alt((ZItmp), (ZInode1))) { goto ZL1; } -#line 1701 "src/libre/dialect/native/parser.c" +#line 1671 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-add-alt */ p_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms (flags, lex_state, act_state, err, ZItmp); @@ -1711,39 +1681,40 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL1; } } - /* END OF INLINE: 167 */ - /* BEGINNING OF INLINE: 200 */ + /* END OF INLINE: 166 */ + /* BEGINNING OF INLINE: 199 */ { switch (CURRENT_TERMINAL) { case (TOK_CLOSEGROUP): { - t_char ZI201; - t_pos ZI202; + t_char ZI200; + t_pos ZI201; + t_pos ZIend; /* BEGINNING OF EXTRACT: CLOSEGROUP */ { -#line 369 "src/libre/parser.act" +#line 350 "src/libre/parser.act" - ZI201 = ']'; - ZI202 = lex_state->lx.start; + ZI200 = ']'; + ZI201 = lex_state->lx.start; ZIend = lex_state->lx.end; + (void) ZI200; (void) ZI201; - (void) ZI202; (void) ZIend; -#line 1736 "src/libre/dialect/native/parser.c" +#line 1707 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: CLOSEGROUP */ ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-group */ { -#line 781 "src/libre/parser.act" +#line 767 "src/libre/parser.act" mark(&act_state->groupstart, &(ZIstart)); mark(&act_state->groupend, &(ZIend)); -#line 1747 "src/libre/dialect/native/parser.c" +#line 1718 "src/libre/dialect/native/parser.c" } /* END OF ACTION: mark-group */ } @@ -1751,56 +1722,57 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state case (TOK_CLOSEGROUPRANGE): { t_char ZIcrange; - t_pos ZI204; + t_pos ZI203; + t_pos ZIend; t_ast__expr ZIrange; /* BEGINNING OF EXTRACT: CLOSEGROUPRANGE */ { -#line 379 "src/libre/parser.act" +#line 360 "src/libre/parser.act" ZIcrange = '-'; - ZI204 = lex_state->lx.start; + ZI203 = lex_state->lx.start; ZIend = lex_state->lx.end; (void) ZIcrange; - (void) ZI204; + (void) ZI203; (void) ZIend; -#line 1770 "src/libre/dialect/native/parser.c" +#line 1742 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: CLOSEGROUPRANGE */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 908 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZIrange) = ast_make_expr_literal(act_state->poolp, *flags, (ZIcrange)); if ((ZIrange) == NULL) { goto ZL4; } -#line 1783 "src/libre/dialect/native/parser.c" +#line 1755 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1084 "src/libre/parser.act" +#line 1046 "src/libre/parser.act" if (!ast_add_expr_alt((ZItmp), (ZIrange))) { goto ZL4; } -#line 1794 "src/libre/dialect/native/parser.c" +#line 1766 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF ACTION: mark-group */ { -#line 781 "src/libre/parser.act" +#line 767 "src/libre/parser.act" mark(&act_state->groupstart, &(ZIstart)); mark(&act_state->groupend, &(ZIend)); -#line 1804 "src/libre/dialect/native/parser.c" +#line 1776 "src/libre/dialect/native/parser.c" } /* END OF ACTION: mark-group */ } @@ -1813,41 +1785,20 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state { /* BEGINNING OF ACTION: err-expected-closegroup */ { -#line 739 "src/libre/parser.act" +#line 725 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCLOSEGROUP; } goto ZL1; -#line 1824 "src/libre/dialect/native/parser.c" +#line 1796 "src/libre/dialect/native/parser.c" } /* END OF ACTION: err-expected-closegroup */ - ZIend = ZIstart; } ZL3:; } - /* END OF INLINE: 200 */ - /* BEGINNING OF ACTION: mark-expr */ - { -#line 796 "src/libre/parser.act" - - struct ast_pos ast_start, ast_end; - - AST_POS_OF_LX_POS(ast_start, (ZIstart)); - AST_POS_OF_LX_POS(ast_end, (ZIend)); - - mark(&act_state->groupstart, &(ZIstart)); - mark(&act_state->groupend, &(ZIend)); - -/* TODO: reinstate this, applies to an expr node in general - (ZItmp)->u.class.start = ast_start; - (ZItmp)->u.class.end = ast_end; -*/ - -#line 1849 "src/libre/dialect/native/parser.c" - } - /* END OF ACTION: mark-expr */ + /* END OF INLINE: 199 */ } goto ZL0; ZL1:; @@ -1858,120 +1809,47 @@ ZL0:; } static void -p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_Hend(flags flags, lex_state lex_state, act_state act_state, err err, t_endpoint *ZOr, t_pos *ZOend) -{ - t_endpoint ZIr; - t_pos ZIend; - - if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { - return; - } - { - /* BEGINNING OF INLINE: 148 */ - { - switch (CURRENT_TERMINAL) { - case (TOK_RANGE): - { - t_char ZIc; - t_pos ZI150; - - /* BEGINNING OF EXTRACT: RANGE */ - { -#line 327 "src/libre/parser.act" - - ZIc = '-'; - ZI150 = lex_state->lx.start; - ZIend = lex_state->lx.end; - - (void) ZIc; - (void) ZI150; - (void) ZIend; - -#line 1891 "src/libre/dialect/native/parser.c" - } - /* END OF EXTRACT: RANGE */ - ADVANCE_LEXER; - /* BEGINNING OF ACTION: ast-range-endpoint-literal */ - { -#line 873 "src/libre/parser.act" - - (ZIr).type = AST_ENDPOINT_LITERAL; - (ZIr).u.literal.c = (unsigned char)(ZIc); - -#line 1902 "src/libre/dialect/native/parser.c" - } - /* END OF ACTION: ast-range-endpoint-literal */ - } - break; - case (TOK_NAMED__CLASS): case (TOK_ESC): case (TOK_OCT): case (TOK_HEX): - case (TOK_CHAR): - { - t_pos ZI149; - - p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint (flags, lex_state, act_state, err, &ZIr, &ZI149, &ZIend); - if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { - RESTORE_LEXER; - goto ZL1; - } - } - break; - default: - goto ZL1; - } - } - /* END OF INLINE: 148 */ - } - goto ZL0; -ZL1:; - SAVE_LEXER ((ERROR_TERMINAL)); - return; -ZL0:; - *ZOr = ZIr; - *ZOend = ZIend; -} - -static void -p_181(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__expr *ZItmp) +p_180(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__expr *ZItmp) { switch (CURRENT_TERMINAL) { case (TOK_RANGE): { t_char ZIc; t_pos ZIrstart; - t_pos ZI182; + t_pos ZI181; t_ast__expr ZInode1; /* BEGINNING OF EXTRACT: RANGE */ { -#line 327 "src/libre/parser.act" +#line 308 "src/libre/parser.act" ZIc = '-'; ZIrstart = lex_state->lx.start; - ZI182 = lex_state->lx.end; + ZI181 = lex_state->lx.end; (void) ZIc; (void) ZIrstart; - (void) ZI182; + (void) ZI181; -#line 1957 "src/libre/dialect/native/parser.c" +#line 1835 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; - /* BEGINNING OF INLINE: 183 */ + /* BEGINNING OF INLINE: 182 */ { switch (CURRENT_TERMINAL) { default: { /* BEGINNING OF ACTION: ast-make-literal */ { -#line 908 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZInode1) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode1) == NULL) { goto ZL1; } -#line 1975 "src/libre/dialect/native/parser.c" +#line 1853 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -1979,35 +1857,35 @@ p_181(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp case (TOK_RANGE): { t_endpoint ZIlower; - t_char ZI184; + t_char ZI183; + t_pos ZI184; t_pos ZI185; - t_pos ZI186; t_endpoint ZIupper; t_pos ZIend; /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 873 "src/libre/parser.act" +#line 839 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_LITERAL; - (ZIlower).u.literal.c = (unsigned char)(ZIc); + (ZIlower).u.literal.c = (unsigned char) (ZIc); -#line 1996 "src/libre/dialect/native/parser.c" +#line 1874 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF EXTRACT: RANGE */ { -#line 327 "src/libre/parser.act" +#line 308 "src/libre/parser.act" - ZI184 = '-'; - ZI185 = lex_state->lx.start; - ZI186 = lex_state->lx.end; + ZI183 = '-'; + ZI184 = lex_state->lx.start; + ZI185 = lex_state->lx.end; + (void) ZI183; (void) ZI184; (void) ZI185; - (void) ZI186; -#line 2011 "src/libre/dialect/native/parser.c" +#line 1889 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; @@ -2018,14 +1896,10 @@ p_181(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp } /* BEGINNING OF ACTION: ast-make-range */ { -#line 1038 "src/libre/parser.act" +#line 1004 "src/libre/parser.act" - struct ast_pos ast_start, ast_end; unsigned char lower, upper; - AST_POS_OF_LX_POS(ast_start, (ZIrstart)); - AST_POS_OF_LX_POS(ast_end, (ZIend)); - if ((ZIlower).type != AST_ENDPOINT_LITERAL || (ZIupper).type != AST_ENDPOINT_LITERAL) { err->e = RE_EUNSUPPORTED; @@ -2046,28 +1920,28 @@ p_181(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp goto ZL1; } - (ZInode1) = ast_make_expr_range(act_state->poolp, *flags, &(ZIlower), ast_start, &(ZIupper), ast_end); + (ZInode1) = ast_make_expr_range(act_state->poolp, *flags, &(ZIlower), &(ZIupper)); if ((ZInode1) == NULL) { goto ZL1; } -#line 2055 "src/libre/dialect/native/parser.c" +#line 1929 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-range */ } break; } } - /* END OF INLINE: 183 */ + /* END OF INLINE: 182 */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1084 "src/libre/parser.act" +#line 1046 "src/libre/parser.act" if (!ast_add_expr_alt((*ZItmp), (ZInode1))) { goto ZL1; } -#line 2071 "src/libre/dialect/native/parser.c" +#line 1945 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-add-alt */ } @@ -2083,6 +1957,79 @@ ZL1:; return; } +static void +p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_Hend(flags flags, lex_state lex_state, act_state act_state, err err, t_endpoint *ZOr, t_pos *ZOend) +{ + t_endpoint ZIr; + t_pos ZIend; + + if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { + return; + } + { + /* BEGINNING OF INLINE: 147 */ + { + switch (CURRENT_TERMINAL) { + case (TOK_RANGE): + { + t_char ZIc; + t_pos ZI149; + + /* BEGINNING OF EXTRACT: RANGE */ + { +#line 308 "src/libre/parser.act" + + ZIc = '-'; + ZI149 = lex_state->lx.start; + ZIend = lex_state->lx.end; + + (void) ZIc; + (void) ZI149; + (void) ZIend; + +#line 1991 "src/libre/dialect/native/parser.c" + } + /* END OF EXTRACT: RANGE */ + ADVANCE_LEXER; + /* BEGINNING OF ACTION: ast-range-endpoint-literal */ + { +#line 839 "src/libre/parser.act" + + (ZIr).type = AST_ENDPOINT_LITERAL; + (ZIr).u.literal.c = (unsigned char) (ZIc); + +#line 2002 "src/libre/dialect/native/parser.c" + } + /* END OF ACTION: ast-range-endpoint-literal */ + } + break; + case (TOK_NAMED__CLASS): case (TOK_ESC): case (TOK_OCT): case (TOK_HEX): + case (TOK_CHAR): + { + t_pos ZI148; + + p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint (flags, lex_state, act_state, err, &ZIr, &ZI148, &ZIend); + if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { + RESTORE_LEXER; + goto ZL1; + } + } + break; + default: + goto ZL1; + } + } + /* END OF INLINE: 147 */ + } + goto ZL0; +ZL1:; + SAVE_LEXER ((ERROR_TERMINAL)); + return; +ZL0:; + *ZOr = ZIr; + *ZOend = ZIend; +} + static void p_expr_C_Cpiece(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__expr *ZOnode) { @@ -2103,7 +2050,7 @@ p_expr_C_Cpiece(flags flags, lex_state lex_state, act_state act_state, err err, } /* BEGINNING OF ACTION: ast-make-piece */ { -#line 931 "src/libre/parser.act" +#line 897 "src/libre/parser.act" if ((ZIc).min == 0 && (ZIc).max == 0) { (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); @@ -2117,7 +2064,7 @@ p_expr_C_Cpiece(flags flags, lex_state lex_state, act_state act_state, err err, goto ZL1; } -#line 2121 "src/libre/dialect/native/parser.c" +#line 2068 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-piece */ } @@ -2141,14 +2088,14 @@ p_expr(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__ex { /* BEGINNING OF ACTION: ast-make-alt */ { -#line 901 "src/libre/parser.act" +#line 867 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 2152 "src/libre/dialect/native/parser.c" +#line 2099 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-alt */ p_expr_C_Clist_Hof_Halts (flags, lex_state, act_state, err, ZInode); @@ -2162,14 +2109,14 @@ p_expr(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__ex { /* BEGINNING OF ACTION: ast-make-empty */ { -#line 887 "src/libre/parser.act" +#line 853 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 2173 "src/libre/dialect/native/parser.c" +#line 2120 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -2182,26 +2129,26 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-alts */ { -#line 725 "src/libre/parser.act" +#line 711 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXALTS; } goto ZL2; -#line 2193 "src/libre/dialect/native/parser.c" +#line 2140 "src/libre/dialect/native/parser.c" } /* END OF ACTION: err-expected-alts */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 887 "src/libre/parser.act" +#line 853 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL2; } -#line 2205 "src/libre/dialect/native/parser.c" +#line 2152 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -2213,67 +2160,8 @@ ZL0:; *ZOnode = ZInode; } -void -p_re__native(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__expr *ZOnode) -{ - t_ast__expr ZInode; - - if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { - return; - } - { - /* BEGINNING OF INLINE: 233 */ - { - { - p_expr (flags, lex_state, act_state, err, &ZInode); - if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { - RESTORE_LEXER; - goto ZL1; - } - } - } - /* END OF INLINE: 233 */ - /* BEGINNING OF INLINE: 234 */ - { - { - switch (CURRENT_TERMINAL) { - case (TOK_EOF): - break; - default: - goto ZL4; - } - ADVANCE_LEXER; - } - goto ZL3; - ZL4:; - { - /* BEGINNING OF ACTION: err-expected-eof */ - { -#line 767 "src/libre/parser.act" - - if (err->e == RE_ESUCCESS) { - err->e = RE_EXEOF; - } - goto ZL1; - -#line 2260 "src/libre/dialect/native/parser.c" - } - /* END OF ACTION: err-expected-eof */ - } - ZL3:; - } - /* END OF INLINE: 234 */ - } - goto ZL0; -ZL1:; - SAVE_LEXER ((ERROR_TERMINAL)); - return; -ZL0:; - *ZOnode = ZInode; -} - static void -p_196(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIstart, t_char *ZIcbrak, t_ast__expr *ZOnode1) +p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIstart, t_char *ZIcbrak, t_ast__expr *ZOnode1) { t_ast__expr ZInode1; @@ -2282,14 +2170,14 @@ p_196(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs { /* BEGINNING OF ACTION: ast-make-literal */ { -#line 908 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZInode1) = ast_make_expr_literal(act_state->poolp, *flags, (*ZIcbrak)); if ((ZInode1) == NULL) { goto ZL1; } -#line 2293 "src/libre/dialect/native/parser.c" +#line 2181 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -2297,36 +2185,36 @@ p_196(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs case (TOK_RANGE): { t_endpoint ZIr; - t_char ZI197; + t_char ZI196; + t_pos ZI197; t_pos ZI198; - t_pos ZI199; t_endpoint ZIupper; t_pos ZIend; t_endpoint ZIlower; /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 873 "src/libre/parser.act" +#line 839 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_LITERAL; - (ZIr).u.literal.c = (unsigned char)(*ZIcbrak); + (ZIr).u.literal.c = (unsigned char) (*ZIcbrak); -#line 2315 "src/libre/dialect/native/parser.c" +#line 2203 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF EXTRACT: RANGE */ { -#line 327 "src/libre/parser.act" +#line 308 "src/libre/parser.act" - ZI197 = '-'; - ZI198 = lex_state->lx.start; - ZI199 = lex_state->lx.end; + ZI196 = '-'; + ZI197 = lex_state->lx.start; + ZI198 = lex_state->lx.end; + (void) ZI196; (void) ZI197; (void) ZI198; - (void) ZI199; -#line 2330 "src/libre/dialect/native/parser.c" +#line 2218 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; @@ -2337,24 +2225,20 @@ p_196(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs } /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 873 "src/libre/parser.act" +#line 839 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_LITERAL; - (ZIlower).u.literal.c = (unsigned char)(*ZIcbrak); + (ZIlower).u.literal.c = (unsigned char) (*ZIcbrak); -#line 2346 "src/libre/dialect/native/parser.c" +#line 2234 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1038 "src/libre/parser.act" +#line 1004 "src/libre/parser.act" - struct ast_pos ast_start, ast_end; unsigned char lower, upper; - AST_POS_OF_LX_POS(ast_start, (*ZIstart)); - AST_POS_OF_LX_POS(ast_end, (ZIend)); - if ((ZIlower).type != AST_ENDPOINT_LITERAL || (ZIupper).type != AST_ENDPOINT_LITERAL) { err->e = RE_EUNSUPPORTED; @@ -2375,12 +2259,12 @@ p_196(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs goto ZL1; } - (ZInode1) = ast_make_expr_range(act_state->poolp, *flags, &(ZIlower), ast_start, &(ZIupper), ast_end); + (ZInode1) = ast_make_expr_range(act_state->poolp, *flags, &(ZIlower), &(ZIupper)); if ((ZInode1) == NULL) { goto ZL1; } -#line 2384 "src/libre/dialect/native/parser.c" +#line 2268 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -2396,6 +2280,83 @@ ZL0:; *ZOnode1 = ZInode1; } +void +p_re__native(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__expr *ZOnode) +{ + t_ast__expr ZInode; + + if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { + return; + } + { + t_group__id ZIid; + t_ast__expr ZIe; + + /* BEGINNING OF ACTION: make-group-id */ + { +#line 881 "src/libre/parser.act" + + (ZIid) = act_state->group_id++; + +#line 2302 "src/libre/dialect/native/parser.c" + } + /* END OF ACTION: make-group-id */ + p_expr (flags, lex_state, act_state, err, &ZIe); + if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { + RESTORE_LEXER; + goto ZL1; + } + /* BEGINNING OF ACTION: ast-make-group */ + { +#line 911 "src/libre/parser.act" + + (ZInode) = ast_make_expr_group(act_state->poolp, *flags, (ZIe), (ZIid)); + if ((ZInode) == NULL) { + goto ZL1; + } + +#line 2319 "src/libre/dialect/native/parser.c" + } + /* END OF ACTION: ast-make-group */ + /* BEGINNING OF INLINE: 231 */ + { + { + switch (CURRENT_TERMINAL) { + case (TOK_EOF): + break; + default: + goto ZL3; + } + ADVANCE_LEXER; + } + goto ZL2; + ZL3:; + { + /* BEGINNING OF ACTION: err-expected-eof */ + { +#line 753 "src/libre/parser.act" + + if (err->e == RE_ESUCCESS) { + err->e = RE_EXEOF; + } + goto ZL1; + +#line 2345 "src/libre/dialect/native/parser.c" + } + /* END OF ACTION: err-expected-eof */ + } + ZL2:; + } + /* END OF INLINE: 231 */ + } + goto ZL0; +ZL1:; + SAVE_LEXER ((ERROR_TERMINAL)); + return; +ZL0:; + *ZOnode = ZInode; +} + static void p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint(flags flags, lex_state lex_state, act_state act_state, err err, t_endpoint *ZOr, t_pos *ZOstart, t_pos *ZOend) { @@ -2407,7 +2368,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint(flags flags, lex_state lex_state, return; } { - /* BEGINNING OF INLINE: 145 */ + /* BEGINNING OF INLINE: 144 */ { switch (CURRENT_TERMINAL) { case (TOK_NAMED__CLASS): @@ -2432,7 +2393,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint(flags flags, lex_state lex_state, goto ZL1; } } - /* END OF INLINE: 145 */ + /* END OF INLINE: 144 */ } goto ZL0; ZL1:; @@ -2461,16 +2422,16 @@ ZL2_expr_C_Clist_Hof_Halts:; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1084 "src/libre/parser.act" +#line 1046 "src/libre/parser.act" if (!ast_add_expr_alt((ZIalts), (ZIa))) { goto ZL1; } -#line 2471 "src/libre/dialect/native/parser.c" +#line 2432 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-add-alt */ - /* BEGINNING OF INLINE: 231 */ + /* BEGINNING OF INLINE: 229 */ { switch (CURRENT_TERMINAL) { case (TOK_ALT): @@ -2485,21 +2446,21 @@ ZL2_expr_C_Clist_Hof_Halts:; break; } } - /* END OF INLINE: 231 */ + /* END OF INLINE: 229 */ } return; ZL1:; { /* BEGINNING OF ACTION: err-expected-alts */ { -#line 725 "src/libre/parser.act" +#line 711 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXALTS; } goto ZL4; -#line 2503 "src/libre/dialect/native/parser.c" +#line 2464 "src/libre/dialect/native/parser.c" } /* END OF ACTION: err-expected-alts */ } @@ -2518,21 +2479,21 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, switch (CURRENT_TERMINAL) { case (TOK_OPENCOUNT): { - t_pos ZI266; - t_pos ZI267; + t_pos ZI263; + t_pos ZI264; t_unsigned ZIm; /* BEGINNING OF EXTRACT: OPENCOUNT */ { -#line 389 "src/libre/parser.act" +#line 370 "src/libre/parser.act" - ZI266 = lex_state->lx.start; - ZI267 = lex_state->lx.end; + ZI263 = lex_state->lx.start; + ZI264 = lex_state->lx.end; - (void) ZI266; - (void) ZI267; + (void) ZI263; + (void) ZI264; -#line 2536 "src/libre/dialect/native/parser.c" +#line 2497 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: OPENCOUNT */ ADVANCE_LEXER; @@ -2540,7 +2501,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, case (TOK_COUNT): /* BEGINNING OF EXTRACT: COUNT */ { -#line 641 "src/libre/parser.act" +#line 627 "src/libre/parser.act" unsigned long u; char *e; @@ -2560,7 +2521,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ZIm = (unsigned int) u; -#line 2564 "src/libre/dialect/native/parser.c" +#line 2525 "src/libre/dialect/native/parser.c" } /* END OF EXTRACT: COUNT */ break; @@ -2568,7 +2529,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, goto ZL1; } ADVANCE_LEXER; - p_268 (flags, lex_state, act_state, err, &ZI266, &ZIm, &ZIc); + p_265 (flags, lex_state, act_state, err, &ZI263, &ZIm, &ZIc); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -2580,11 +2541,11 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-zero-or-one */ { -#line 845 "src/libre/parser.act" +#line 816 "src/libre/parser.act" - (ZIc) = ast_make_count(0, NULL, 1, NULL); + (ZIc) = ast_make_count(0, 1); -#line 2588 "src/libre/dialect/native/parser.c" +#line 2549 "src/libre/dialect/native/parser.c" } /* END OF ACTION: count-zero-or-one */ } @@ -2594,11 +2555,11 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-one-or-more */ { -#line 841 "src/libre/parser.act" +#line 812 "src/libre/parser.act" - (ZIc) = ast_make_count(1, NULL, AST_COUNT_UNBOUNDED, NULL); + (ZIc) = ast_make_count(1, AST_COUNT_UNBOUNDED); -#line 2602 "src/libre/dialect/native/parser.c" +#line 2563 "src/libre/dialect/native/parser.c" } /* END OF ACTION: count-one-or-more */ } @@ -2608,11 +2569,11 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-zero-or-more */ { -#line 837 "src/libre/parser.act" +#line 808 "src/libre/parser.act" - (ZIc) = ast_make_count(0, NULL, AST_COUNT_UNBOUNDED, NULL); + (ZIc) = ast_make_count(0, AST_COUNT_UNBOUNDED); -#line 2616 "src/libre/dialect/native/parser.c" +#line 2577 "src/libre/dialect/native/parser.c" } /* END OF ACTION: count-zero-or-more */ } @@ -2621,11 +2582,11 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, { /* BEGINNING OF ACTION: count-one */ { -#line 849 "src/libre/parser.act" +#line 820 "src/libre/parser.act" - (ZIc) = ast_make_count(1, NULL, 1, NULL); + (ZIc) = ast_make_count(1, 1); -#line 2629 "src/libre/dialect/native/parser.c" +#line 2590 "src/libre/dialect/native/parser.c" } /* END OF ACTION: count-one */ } @@ -2638,23 +2599,23 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-count */ { -#line 711 "src/libre/parser.act" +#line 697 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCOUNT; } goto ZL2; -#line 2649 "src/libre/dialect/native/parser.c" +#line 2610 "src/libre/dialect/native/parser.c" } /* END OF ACTION: err-expected-count */ /* BEGINNING OF ACTION: count-one */ { -#line 849 "src/libre/parser.act" +#line 820 "src/libre/parser.act" - (ZIc) = ast_make_count(1, NULL, 1, NULL); + (ZIc) = ast_make_count(1, 1); -#line 2658 "src/libre/dialect/native/parser.c" +#line 2619 "src/libre/dialect/native/parser.c" } /* END OF ACTION: count-one */ } @@ -2679,24 +2640,24 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 811 "src/libre/parser.act" +#line 782 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; -#line 2688 "src/libre/dialect/native/parser.c" +#line 2649 "src/libre/dialect/native/parser.c" } /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1071 "src/libre/parser.act" +#line 1033 "src/libre/parser.act" (ZIe) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIe) == NULL) { goto ZL1; } -#line 2700 "src/libre/dialect/native/parser.c" +#line 2661 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -2704,18 +2665,21 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e case (TOK_END): { ADVANCE_LEXER; - /* BEGINNING OF ACTION: ast-make-anchor-end */ + /* BEGINNING OF ACTION: ast-make-anchor-end-nl */ { -#line 976 "src/libre/parser.act" +#line 949 "src/libre/parser.act" (ZIe) = ast_make_expr_anchor(act_state->poolp, *flags, AST_ANCHOR_END); if ((ZIe) == NULL) { goto ZL1; } + if (!(*flags & RE_ANCHORED)) { + (ZIe)->u.anchor.is_end_nl = 1; + } -#line 2717 "src/libre/dialect/native/parser.c" +#line 2681 "src/libre/dialect/native/parser.c" } - /* END OF ACTION: ast-make-anchor-end */ + /* END OF ACTION: ast-make-anchor-end-nl */ } break; case (TOK_OPENSUB): @@ -2726,11 +2690,11 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: make-group-id */ { -#line 915 "src/libre/parser.act" +#line 881 "src/libre/parser.act" (ZIid) = act_state->group_id++; -#line 2734 "src/libre/dialect/native/parser.c" +#line 2698 "src/libre/dialect/native/parser.c" } /* END OF ACTION: make-group-id */ p_expr (flags, lex_state, act_state, err, &ZIg); @@ -2740,14 +2704,14 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e } /* BEGINNING OF ACTION: ast-make-group */ { -#line 945 "src/libre/parser.act" +#line 911 "src/libre/parser.act" (ZIe) = ast_make_expr_group(act_state->poolp, *flags, (ZIg), (ZIid)); if ((ZIe) == NULL) { goto ZL1; } -#line 2751 "src/libre/dialect/native/parser.c" +#line 2715 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-group */ switch (CURRENT_TERMINAL) { @@ -2764,14 +2728,14 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-anchor-start */ { -#line 969 "src/libre/parser.act" +#line 935 "src/libre/parser.act" (ZIe) = ast_make_expr_anchor(act_state->poolp, *flags, AST_ANCHOR_START); if ((ZIe) == NULL) { goto ZL1; } -#line 2775 "src/libre/dialect/native/parser.c" +#line 2739 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-anchor-start */ } @@ -2804,26 +2768,26 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-atom */ { -#line 718 "src/libre/parser.act" +#line 704 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXATOM; } goto ZL2; -#line 2815 "src/libre/dialect/native/parser.c" +#line 2779 "src/libre/dialect/native/parser.c" } /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 887 "src/libre/parser.act" +#line 853 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { goto ZL2; } -#line 2827 "src/libre/dialect/native/parser.c" +#line 2791 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -2836,7 +2800,7 @@ ZL0:; } static void -p_249(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__class__id *ZI246, t_pos *ZI247, t_ast__expr *ZOnode) +p_246(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__class__id *ZI243, t_pos *ZI244, t_ast__expr *ZOnode) { t_ast__expr ZInode; @@ -2845,14 +2809,14 @@ p_249(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla { /* BEGINNING OF ACTION: ast-make-named */ { -#line 1071 "src/libre/parser.act" +#line 1033 "src/libre/parser.act" - (ZInode) = ast_make_expr_named(act_state->poolp, *flags, (*ZI246)); + (ZInode) = ast_make_expr_named(act_state->poolp, *flags, (*ZI243)); if ((ZInode) == NULL) { goto ZL1; } -#line 2856 "src/libre/dialect/native/parser.c" +#line 2820 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -2865,15 +2829,15 @@ p_249(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla /* BEGINNING OF ACTION: ast-range-endpoint-class */ { -#line 878 "src/libre/parser.act" +#line 844 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_NAMED; - (ZIlower).u.named.class = (*ZI246); + (ZIlower).u.named.class = (*ZI243); -#line 2874 "src/libre/dialect/native/parser.c" +#line 2838 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-class */ - p_155 (flags, lex_state, act_state, err); + p_154 (flags, lex_state, act_state, err); p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_Hend (flags, lex_state, act_state, err, &ZIupper, &ZIend); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; @@ -2881,24 +2845,20 @@ p_249(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla } /* BEGINNING OF ACTION: mark-range */ { -#line 786 "src/libre/parser.act" +#line 772 "src/libre/parser.act" - mark(&act_state->rangestart, &(*ZI247)); + mark(&act_state->rangestart, &(*ZI244)); mark(&act_state->rangeend, &(ZIend)); -#line 2890 "src/libre/dialect/native/parser.c" +#line 2854 "src/libre/dialect/native/parser.c" } /* END OF ACTION: mark-range */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1038 "src/libre/parser.act" +#line 1004 "src/libre/parser.act" - struct ast_pos ast_start, ast_end; unsigned char lower, upper; - AST_POS_OF_LX_POS(ast_start, (*ZI247)); - AST_POS_OF_LX_POS(ast_end, (ZIend)); - if ((ZIlower).type != AST_ENDPOINT_LITERAL || (ZIupper).type != AST_ENDPOINT_LITERAL) { err->e = RE_EUNSUPPORTED; @@ -2919,12 +2879,12 @@ p_249(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla goto ZL1; } - (ZInode) = ast_make_expr_range(act_state->poolp, *flags, &(ZIlower), ast_start, &(ZIupper), ast_end); + (ZInode) = ast_make_expr_range(act_state->poolp, *flags, &(ZIlower), &(ZIupper)); if ((ZInode) == NULL) { goto ZL1; } -#line 2928 "src/libre/dialect/native/parser.c" +#line 2888 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -2951,14 +2911,14 @@ p_expr_C_Calt(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: ast-make-concat */ { -#line 894 "src/libre/parser.act" +#line 860 "src/libre/parser.act" (ZInode) = ast_make_expr_concat(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 2962 "src/libre/dialect/native/parser.c" +#line 2922 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-concat */ p_expr_C_Clist_Hof_Hpieces (flags, lex_state, act_state, err, ZInode); @@ -2976,7 +2936,7 @@ ZL0:; } static void -p_253(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI250, t_pos *ZI251, t_ast__expr *ZOnode) +p_250(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI247, t_pos *ZI248, t_ast__expr *ZOnode) { t_ast__expr ZInode; @@ -2985,14 +2945,14 @@ p_253(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI { /* BEGINNING OF ACTION: ast-make-literal */ { -#line 908 "src/libre/parser.act" +#line 874 "src/libre/parser.act" - (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (*ZI250)); + (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (*ZI247)); if ((ZInode) == NULL) { goto ZL1; } -#line 2996 "src/libre/dialect/native/parser.c" +#line 2956 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -3005,15 +2965,15 @@ p_253(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 873 "src/libre/parser.act" +#line 839 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_LITERAL; - (ZIlower).u.literal.c = (unsigned char)(*ZI250); + (ZIlower).u.literal.c = (unsigned char) (*ZI247); -#line 3014 "src/libre/dialect/native/parser.c" +#line 2974 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ - p_155 (flags, lex_state, act_state, err); + p_154 (flags, lex_state, act_state, err); p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_Hend (flags, lex_state, act_state, err, &ZIupper, &ZIend); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; @@ -3021,24 +2981,20 @@ p_253(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI } /* BEGINNING OF ACTION: mark-range */ { -#line 786 "src/libre/parser.act" +#line 772 "src/libre/parser.act" - mark(&act_state->rangestart, &(*ZI251)); + mark(&act_state->rangestart, &(*ZI248)); mark(&act_state->rangeend, &(ZIend)); -#line 3030 "src/libre/dialect/native/parser.c" +#line 2990 "src/libre/dialect/native/parser.c" } /* END OF ACTION: mark-range */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1038 "src/libre/parser.act" +#line 1004 "src/libre/parser.act" - struct ast_pos ast_start, ast_end; unsigned char lower, upper; - AST_POS_OF_LX_POS(ast_start, (*ZI251)); - AST_POS_OF_LX_POS(ast_end, (ZIend)); - if ((ZIlower).type != AST_ENDPOINT_LITERAL || (ZIupper).type != AST_ENDPOINT_LITERAL) { err->e = RE_EUNSUPPORTED; @@ -3059,12 +3015,12 @@ p_253(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI goto ZL1; } - (ZInode) = ast_make_expr_range(act_state->poolp, *flags, &(ZIlower), ast_start, &(ZIupper), ast_end); + (ZInode) = ast_make_expr_range(act_state->poolp, *flags, &(ZIlower), &(ZIupper)); if ((ZInode) == NULL) { goto ZL1; } -#line 3068 "src/libre/dialect/native/parser.c" +#line 3024 "src/libre/dialect/native/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -3082,7 +3038,7 @@ ZL0:; /* BEGINNING OF TRAILER */ -#line 1090 "src/libre/parser.act" +#line 1052 "src/libre/parser.act" static int @@ -3102,9 +3058,8 @@ ZL0:; struct ast * DIALECT_PARSE(re_getchar_fun *f, void *opaque, - const struct fsm_options *opt, enum re_flags flags, int overlap, - struct re_err *err) + struct re_err *err, struct re_pos *end) { struct ast *ast; @@ -3117,6 +3072,7 @@ ZL0:; struct LX_STATE *lx; assert(f != NULL); + assert(end != NULL); ast = ast_new(); @@ -3140,11 +3096,6 @@ ZL0:; lex_state->buf.a = NULL; lex_state->buf.len = 0; -#if PCRE_DIALECT - lex_state->extended_mode_comment = 0; - lex_state->flags_ptr = &flags; -#endif /* PCRE_DIALECT */ - /* XXX: unneccessary since we're lexing from a string */ /* (except for pushing "[" and "]" around ::group-$dialect) */ lx->buf_opaque = &lex_state->buf; @@ -3175,6 +3126,9 @@ ZL0:; DIALECT_ENTRY(&flags, lex_state, act_state, err, &ast->expr); + /* we output this for reporting errors found through AST analysis */ + mark(end, &lx->end); + lx->free(lx->buf_opaque); if (err->e != RE_ESUCCESS) { @@ -3240,6 +3194,6 @@ ZL0:; return NULL; } -#line 3244 "src/libre/dialect/native/parser.c" +#line 3198 "src/libre/dialect/native/parser.c" /* END OF FILE */ diff --git a/src/libre/dialect/native/parser.h b/src/libre/dialect/native/parser.h index 959bf3acc..5cf04f6c6 100644 --- a/src/libre/dialect/native/parser.h +++ b/src/libre/dialect/native/parser.h @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 300 "src/libre/parser.act" +#line 281 "src/libre/parser.act" #include @@ -28,7 +28,7 @@ extern void p_re__native(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1247 "src/libre/parser.act" +#line 1207 "src/libre/parser.act" #line 35 "src/libre/dialect/native/parser.h" diff --git a/src/libre/dialect/native/parser.sid b/src/libre/dialect/native/parser.sid index 457fe7a15..1ac7e5d64 100644 --- a/src/libre/dialect/native/parser.sid +++ b/src/libre/dialect/native/parser.sid @@ -106,8 +106,8 @@ !: (:re_flags) -> (); !: (:re_flags, :re_flags) -> (); : () -> (:ast_expr); - : () -> (:ast_expr); - !:() -> (:ast_expr); + !: () -> (:ast_expr); + :() -> (:ast_expr); !: (:ast_expr, :ast_expr) -> (:ast_expr); : (:ast_expr) -> (:ast_expr); : (:endpoint, :pos, :endpoint, :pos) -> (:ast_expr); @@ -134,7 +134,6 @@ : (:pos, :pos) -> (); : (:pos, :pos) -> (); : (:pos, :pos) -> (); - : (:ast_expr, :pos, :pos) -> (); : () -> (:ast_class_id); !: () -> (:ast_class_id); @@ -350,10 +349,7 @@ (start, end); ## ; - end = start; /* appease sid */ }; - - (tmp, start, end); }; piece: () -> (node :ast_expr) [ @@ -363,7 +359,7 @@ e = ; || END; - e = ; + e = ; || ANY; a = ; @@ -466,9 +462,9 @@ }; re_native: () -> (node :ast_expr) = { - { - node = expr; - }; + id = ; + e = expr; + node = (e, id); { EOF; diff --git a/src/libre/dialect/pcre/lexer.c b/src/libre/dialect/pcre/lexer.c index b26096785..da8f825ee 100644 --- a/src/libre/dialect/pcre/lexer.c +++ b/src/libre/dialect/pcre/lexer.c @@ -17,11 +17,31 @@ static enum lx_pcre_token z5(struct lx_pcre_lx *lx); static enum lx_pcre_token z6(struct lx_pcre_lx *lx); static enum lx_pcre_token z7(struct lx_pcre_lx *lx); +static int +lx_pcre_advance_end(struct lx_pcre_lx *lx, int c) +{ + lx->end.byte++; + lx->end.col++; + if (c == '\n') { + lx->end.line++; + lx->end.saved_col = lx->end.col - 1; + lx->end.col = 1; + } + if (lx->push != NULL) { + if (-1 == lx->push(lx->buf_opaque, (char)c)) { + return 0; + } + } + return 1; +} + +/* This wrapper manages one character of lookahead/pushback + * and the line, column, and byte offsets. */ #if __STDC_VERSION__ >= 199901L inline #endif static int -lx_getc(struct lx_pcre_lx *lx) +lx_pcre_getc(struct lx_pcre_lx *lx) { int c; @@ -37,18 +57,19 @@ lx_getc(struct lx_pcre_lx *lx) } } - lx->end.byte++; - lx->end.col++; - - if (c == '\n') { - lx->end.line++; - lx->end.saved_col = lx->end.col - 1; - lx->end.col = 1; - } + if (!lx_pcre_advance_end(lx, c)) { return EOF; } return c; } +/* This wrapper adapts calling lx_pcre_getc to the interface + * in libfsm's generated code. */ +static int +fsm_getc(void *getc_opaque) +{ + return lx_pcre_getc((struct lx_pcre_lx *)getc_opaque); +} + #if __STDC_VERSION__ >= 199901L inline #endif @@ -57,10 +78,7 @@ lx_pcre_ungetc(struct lx_pcre_lx *lx, int c) { assert(lx != NULL); assert(lx->c == EOF); - lx->c = c; - - lx->end.byte--; lx->end.col--; @@ -112,6 +130,17 @@ lx_pcre_dynpush(void *buf_opaque, char c) return 0; } +static void +lx_pcre_dynpop(void *buf_opaque) +{ + struct lx_dynbuf *t = buf_opaque; + + assert(t != NULL); + + assert(t->p != t->a); + t->p--; +} + int lx_pcre_dynclear(void *buf_opaque) { @@ -151,32 +180,33 @@ lx_pcre_dynfree(void *buf_opaque) static enum lx_pcre_token z0(struct lx_pcre_lx *lx) { + int has_consumed_input = 0; int c; - enum { - S0, S1, S2, S3, NONE - } state; - assert(lx != NULL); if (lx->clear != NULL) { lx->clear(lx->buf_opaque); } - state = NONE; - lx->start = lx->end; - while (c = lx_getc(lx), c != EOF) { - if (state == NONE) { - state = S0; - } + void *getc_opaque = (void *)lx; + enum { + S0, S1, S2, S3 + } state; + state = S0; + + while (c = fsm_getc(getc_opaque), c != EOF) { + has_consumed_input = 1; switch (state) { - case S0: /* start */ + case S0: /* e.g. "" */ switch ((unsigned char) c) { case '\\': state = S1; break; - case '\x00': lx->lgetc = NULL; return TOK_UNKNOWN; + case '\x00': + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; default: state = S2; break; } break; @@ -184,19 +214,30 @@ z0(struct lx_pcre_lx *lx) case S1: /* e.g. "\\" */ switch ((unsigned char) c) { case 'E': state = S3; break; - default: lx_pcre_ungetc(lx, c); return TOK_CHAR; + default: lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_CHAR; } break; - case S2: /* e.g. "a" */ - lx_pcre_ungetc(lx, c); return TOK_CHAR; + case S2: /* e.g. "\\x01" */ + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_CHAR; case S3: /* e.g. "\\E" */ - lx_pcre_ungetc(lx, c); return lx->z = z7, lx->z(lx); + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return lx->z = z7, lx->z(lx); default: ; /* unreached */ } + } + + /* end states */ + switch (state) { + case S1: return TOK_CHAR; + case S2: return TOK_CHAR; + case S3: return lx->z = z7, lx->z(lx); + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; + } switch (state) { case S3: @@ -211,45 +252,40 @@ z0(struct lx_pcre_lx *lx) break; } - } lx->lgetc = NULL; - switch (state) { - case NONE: return TOK_EOF; - case S1: return TOK_CHAR; - case S2: return TOK_CHAR; - case S3: return TOK_EOF; - default: errno = EINVAL; return TOK_ERROR; - } + if (!has_consumed_input) { + return TOK_EOF; + } + return TOK_ERROR; } static enum lx_pcre_token z1(struct lx_pcre_lx *lx) { + int has_consumed_input = 0; int c; - enum { - S0, S1, S2, S3, NONE - } state; - assert(lx != NULL); if (lx->clear != NULL) { lx->clear(lx->buf_opaque); } - state = NONE; - lx->start = lx->end; - while (c = lx_getc(lx), c != EOF) { - if (state == NONE) { - state = S0; - } + void *getc_opaque = (void *)lx; + enum { + S0, S1, S2, S3 + } state; + state = S0; + + while (c = fsm_getc(getc_opaque), c != EOF) { + has_consumed_input = 1; switch (state) { - case S0: /* start */ + case S0: /* e.g. "" */ switch ((unsigned char) c) { case '0': case '1': @@ -263,7 +299,9 @@ z1(struct lx_pcre_lx *lx) case '9': state = S1; break; case ',': state = S2; break; case '}': state = S3; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; @@ -279,67 +317,75 @@ z1(struct lx_pcre_lx *lx) case '7': case '8': case '9': break; - default: lx_pcre_ungetc(lx, c); return TOK_COUNT; + default: lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_COUNT; } break; case S2: /* e.g. "," */ - lx_pcre_ungetc(lx, c); return TOK_SEP; + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_SEP; case S3: /* e.g. "}" */ - lx_pcre_ungetc(lx, c); return lx->z = z7, TOK_CLOSECOUNT; + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return lx->z = z7, TOK_CLOSECOUNT; default: ; /* unreached */ } + } + + /* end states */ + switch (state) { + case S1: return TOK_COUNT; + case S2: return TOK_SEP; + case S3: return lx->z = z7, TOK_CLOSECOUNT; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; + } if (lx->push != NULL) { if (-1 == lx->push(lx->buf_opaque, (char)c)) { return TOK_ERROR; } } - } lx->lgetc = NULL; - switch (state) { - case NONE: return TOK_EOF; - case S1: return TOK_COUNT; - case S2: return TOK_SEP; - case S3: return TOK_CLOSECOUNT; - default: errno = EINVAL; return TOK_ERROR; - } + if (!has_consumed_input) { + return TOK_EOF; + } + return TOK_ERROR; } static enum lx_pcre_token z2(struct lx_pcre_lx *lx) { + int has_consumed_input = 0; int c; - enum { - S0, S1, S2, S3, NONE - } state; - assert(lx != NULL); if (lx->clear != NULL) { lx->clear(lx->buf_opaque); } - state = NONE; - lx->start = lx->end; - while (c = lx_getc(lx), c != EOF) { - if (state == NONE) { - state = S0; - } + void *getc_opaque = (void *)lx; + enum { + S0, S1, S2, S3 + } state; + + state = S0; + while (c = fsm_getc(getc_opaque), c != EOF) { + has_consumed_input = 1; switch (state) { - case S0: /* start */ + case S0: /* e.g. "" */ switch ((unsigned char) c) { case '\\': state = S1; break; - case '\x00': lx->lgetc = NULL; return TOK_UNKNOWN; + case '\x00': + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; default: state = S2; break; } break; @@ -347,19 +393,30 @@ z2(struct lx_pcre_lx *lx) case S1: /* e.g. "\\" */ switch ((unsigned char) c) { case 'E': state = S3; break; - default: lx_pcre_ungetc(lx, c); return TOK_CHAR; + default: lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_CHAR; } break; - case S2: /* e.g. "a" */ - lx_pcre_ungetc(lx, c); return TOK_CHAR; + case S2: /* e.g. "\\x01" */ + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_CHAR; case S3: /* e.g. "\\E" */ - lx_pcre_ungetc(lx, c); return lx->z = z3, lx->z(lx); + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return lx->z = z3, lx->z(lx); default: ; /* unreached */ } + } + + /* end states */ + switch (state) { + case S1: return TOK_CHAR; + case S2: return TOK_CHAR; + case S3: return lx->z = z3, lx->z(lx); + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; + } switch (state) { case S3: @@ -374,24 +431,30 @@ z2(struct lx_pcre_lx *lx) break; } - } lx->lgetc = NULL; - switch (state) { - case NONE: return TOK_EOF; - case S1: return TOK_CHAR; - case S2: return TOK_CHAR; - case S3: return TOK_EOF; - default: errno = EINVAL; return TOK_ERROR; - } + if (!has_consumed_input) { + return TOK_EOF; + } + return TOK_ERROR; } static enum lx_pcre_token z3(struct lx_pcre_lx *lx) { + int has_consumed_input = 0; int c; + assert(lx != NULL); + + if (lx->clear != NULL) { + lx->clear(lx->buf_opaque); + } + + lx->start = lx->end; + + void *getc_opaque = (void *)lx; enum { S0, S1, S2, S3, S4, S5, S6, S7, S8, S9, S10, S11, S12, S13, S14, S15, S16, S17, S18, S19, @@ -400,32 +463,23 @@ z3(struct lx_pcre_lx *lx) S40, S41, S42, S43, S44, S45, S46, S47, S48, S49, S50, S51, S52, S53, S54, S55, S56, S57, S58, S59, S60, S61, S62, S63, S64, S65, S66, S67, S68, S69, - S70, S71, S72, NONE + S70, S71, S72 } state; - assert(lx != NULL); - - if (lx->clear != NULL) { - lx->clear(lx->buf_opaque); - } - - state = NONE; - - lx->start = lx->end; - - while (c = lx_getc(lx), c != EOF) { - if (state == NONE) { - state = S0; - } + state = S0; + while (c = fsm_getc(getc_opaque), c != EOF) { + has_consumed_input = 1; switch (state) { - case S0: /* start */ + case S0: /* e.g. "" */ switch ((unsigned char) c) { case '\\': state = S1; break; case '[': state = S2; break; case '-': state = S4; break; case ']': state = S5; break; - case '\x00': lx->lgetc = NULL; return TOK_UNKNOWN; + case '\x00': + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; default: state = S3; break; } break; @@ -441,22 +495,8 @@ z3(struct lx_pcre_lx *lx) case 'h': case 's': case 'v': - case 'w': state = S24; break; - case 'Q': state = S55; break; - case 'E': state = S56; break; - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - case '8': - case '9': state = S57; break; - case 'x': state = S58; break; - case '0': state = S59; break; - case 'o': state = S60; break; - case 'c': state = S61; break; + case 'w': state = S23; break; + case 'c': state = S55; break; case '$': case '(': case '*': @@ -475,384 +515,492 @@ z3(struct lx_pcre_lx *lx) case 'r': case 't': case '{': - case '|': state = S63; break; - default: state = S62; break; + case '|': state = S56; break; + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': state = S57; break; + case 'Q': state = S58; break; + case 'E': state = S59; break; + case 'o': state = S60; break; + case 'x': state = S62; break; + case '0': state = S63; break; + default: state = S61; break; } break; case S2: /* e.g. "[" */ switch ((unsigned char) c) { case ':': state = S7; break; - default: lx_pcre_ungetc(lx, c); return TOK_CHAR; + default: lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_CHAR; } break; - case S3: /* e.g. "a" */ - lx_pcre_ungetc(lx, c); return TOK_CHAR; + case S3: /* e.g. "\\x01" */ + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_CHAR; case S4: /* e.g. "-" */ switch ((unsigned char) c) { case ']': state = S6; break; - default: lx_pcre_ungetc(lx, c); return TOK_RANGE; + default: lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_RANGE; } break; case S5: /* e.g. "]" */ - lx_pcre_ungetc(lx, c); return lx->z = z7, TOK_CLOSEGROUP; + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return lx->z = z7, TOK_CLOSEGROUP; case S6: /* e.g. "-]" */ - lx_pcre_ungetc(lx, c); return lx->z = z7, TOK_CLOSEGROUPRANGE; + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return lx->z = z7, TOK_CLOSEGROUPRANGE; case S7: /* e.g. "[:" */ switch ((unsigned char) c) { case 'd': state = S8; break; - case 'u': state = S9; break; - case 'w': state = S10; break; - case 'x': state = S11; break; - case 'b': state = S12; break; - case 'c': state = S13; break; - case 'l': state = S14; break; + case 'p': state = S9; break; + case 'x': state = S10; break; + case 'c': state = S11; break; + case 'l': state = S12; break; + case 'g': state = S13; break; + case 's': state = S14; break; case 'a': state = S15; break; - case 's': state = S16; break; - case 'p': state = S17; break; - case 'g': state = S18; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'b': state = S16; break; + case 'u': state = S17; break; + case 'w': state = S18; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S8: /* e.g. "[:d" */ switch ((unsigned char) c) { case 'i': state = S53; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S9: /* e.g. "[:u" */ + case S9: /* e.g. "[:p" */ switch ((unsigned char) c) { - case 'p': state = S52; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'r': state = S48; break; + case 'u': state = S49; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S10: /* e.g. "[:w" */ + case S10: /* e.g. "[:x" */ switch ((unsigned char) c) { - case 'o': state = S50; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'd': state = S8; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S11: /* e.g. "[:x" */ + case S11: /* e.g. "[:c" */ switch ((unsigned char) c) { - case 'd': state = S8; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'n': state = S45; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S12: /* e.g. "[:b" */ + case S12: /* e.g. "[:l" */ switch ((unsigned char) c) { - case 'l': state = S47; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'o': state = S44; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S13: /* e.g. "[:c" */ + case S13: /* e.g. "[:g" */ switch ((unsigned char) c) { - case 'n': state = S44; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'r': state = S41; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S14: /* e.g. "[:l" */ + case S14: /* e.g. "[:s" */ switch ((unsigned char) c) { - case 'o': state = S41; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'p': state = S38; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S15: /* e.g. "[:a" */ switch ((unsigned char) c) { - case 's': state = S33; break; - case 'l': state = S34; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 's': state = S30; break; + case 'l': state = S31; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S16: /* e.g. "[:s" */ + case S16: /* e.g. "[:b" */ switch ((unsigned char) c) { - case 'p': state = S30; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'l': state = S27; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S17: /* e.g. "[:p" */ + case S17: /* e.g. "[:u" */ switch ((unsigned char) c) { - case 'r': state = S25; break; - case 'u': state = S26; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'p': state = S24; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S18: /* e.g. "[:g" */ + case S18: /* e.g. "[:w" */ switch ((unsigned char) c) { - case 'r': state = S19; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'o': state = S19; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S19: /* e.g. "[:gr" */ + case S19: /* e.g. "[:wo" */ switch ((unsigned char) c) { - case 'a': state = S20; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'r': state = S20; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S20: /* e.g. "[:gra" */ + case S20: /* e.g. "[:wor" */ switch ((unsigned char) c) { - case 'p': state = S21; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'd': state = S21; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S21: /* e.g. "[:grap" */ + case S21: /* e.g. "[:word" */ switch ((unsigned char) c) { - case 'h': state = S22; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case ':': state = S22; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S22: /* e.g. "[:word" */ + case S22: /* e.g. "[:word:" */ switch ((unsigned char) c) { - case ':': state = S23; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case ']': state = S23; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S23: /* e.g. "[:word:" */ + case S23: /* e.g. "\\D" */ + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_NAMED__CLASS; + + case S24: /* e.g. "[:up" */ switch ((unsigned char) c) { - case ']': state = S24; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'p': state = S25; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S24: /* e.g. "\\d" */ - lx_pcre_ungetc(lx, c); return TOK_NAMED__CLASS; - - case S25: /* e.g. "[:pr" */ + case S25: /* e.g. "[:low" */ switch ((unsigned char) c) { - case 'i': state = S29; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'e': state = S26; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S26: /* e.g. "[:pu" */ + case S26: /* e.g. "[:lowe" */ switch ((unsigned char) c) { - case 'n': state = S27; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'r': state = S21; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S27: /* e.g. "[:pun" */ + case S27: /* e.g. "[:bl" */ switch ((unsigned char) c) { - case 'c': state = S28; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'a': state = S28; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S28: /* e.g. "[:digi" */ + case S28: /* e.g. "[:bla" */ switch ((unsigned char) c) { - case 't': state = S22; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'n': state = S29; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S29: /* e.g. "[:pri" */ + case S29: /* e.g. "[:blan" */ switch ((unsigned char) c) { - case 'n': state = S28; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'k': state = S21; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S30: /* e.g. "[:sp" */ + case S30: /* e.g. "[:as" */ switch ((unsigned char) c) { - case 'a': state = S31; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'c': state = S36; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S31: /* e.g. "[:spa" */ + case S31: /* e.g. "[:al" */ switch ((unsigned char) c) { - case 'c': state = S32; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'p': state = S32; break; + case 'n': state = S33; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S32: /* e.g. "[:spac" */ + case S32: /* e.g. "[:alp" */ switch ((unsigned char) c) { - case 'e': state = S22; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'h': state = S35; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S33: /* e.g. "[:as" */ + case S33: /* e.g. "[:aln" */ switch ((unsigned char) c) { - case 'c': state = S39; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'u': state = S34; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S34: /* e.g. "[:al" */ + case S34: /* e.g. "[:alnu" */ switch ((unsigned char) c) { - case 'n': state = S35; break; - case 'p': state = S36; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'm': state = S21; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S35: /* e.g. "[:aln" */ + case S35: /* e.g. "[:alph" */ switch ((unsigned char) c) { - case 'u': state = S38; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'a': state = S21; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S36: /* e.g. "[:alp" */ + case S36: /* e.g. "[:asc" */ switch ((unsigned char) c) { - case 'h': state = S37; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'i': state = S37; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S37: /* e.g. "[:alph" */ + case S37: /* e.g. "[:asci" */ switch ((unsigned char) c) { - case 'a': state = S22; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'i': state = S21; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S38: /* e.g. "[:alnu" */ + case S38: /* e.g. "[:sp" */ switch ((unsigned char) c) { - case 'm': state = S22; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'a': state = S39; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S39: /* e.g. "[:asc" */ + case S39: /* e.g. "[:spa" */ switch ((unsigned char) c) { - case 'i': state = S40; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'c': state = S40; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S40: /* e.g. "[:asci" */ + case S40: /* e.g. "[:spac" */ switch ((unsigned char) c) { - case 'i': state = S22; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'e': state = S21; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S41: /* e.g. "[:lo" */ + case S41: /* e.g. "[:gr" */ switch ((unsigned char) c) { - case 'w': state = S42; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'a': state = S42; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S42: /* e.g. "[:low" */ + case S42: /* e.g. "[:gra" */ switch ((unsigned char) c) { - case 'e': state = S43; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'p': state = S43; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S43: /* e.g. "[:lowe" */ + case S43: /* e.g. "[:grap" */ switch ((unsigned char) c) { - case 'r': state = S22; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'h': state = S21; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S44: /* e.g. "[:cn" */ + case S44: /* e.g. "[:lo" */ switch ((unsigned char) c) { - case 't': state = S45; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'w': state = S25; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S45: /* e.g. "[:cnt" */ + case S45: /* e.g. "[:cn" */ switch ((unsigned char) c) { - case 'r': state = S46; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 't': state = S46; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S46: /* e.g. "[:cntr" */ + case S46: /* e.g. "[:cnt" */ switch ((unsigned char) c) { - case 'l': state = S22; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'r': state = S47; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S47: /* e.g. "[:bl" */ + case S47: /* e.g. "[:cntr" */ switch ((unsigned char) c) { - case 'a': state = S48; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'l': state = S21; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S48: /* e.g. "[:bla" */ + case S48: /* e.g. "[:pr" */ switch ((unsigned char) c) { - case 'n': state = S49; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'i': state = S52; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S49: /* e.g. "[:blan" */ + case S49: /* e.g. "[:pu" */ switch ((unsigned char) c) { - case 'k': state = S22; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'n': state = S50; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S50: /* e.g. "[:wo" */ + case S50: /* e.g. "[:pun" */ switch ((unsigned char) c) { - case 'r': state = S51; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'c': state = S51; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S51: /* e.g. "[:wor" */ + case S51: /* e.g. "[:digi" */ switch ((unsigned char) c) { - case 'd': state = S22; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 't': state = S21; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S52: /* e.g. "[:up" */ + case S52: /* e.g. "[:pri" */ switch ((unsigned char) c) { - case 'p': state = S42; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'n': state = S51; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S53: /* e.g. "[:di" */ switch ((unsigned char) c) { case 'g': state = S54; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S54: /* e.g. "[:dig" */ switch ((unsigned char) c) { - case 'i': state = S28; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'i': state = S51; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S55: /* e.g. "\\Q" */ - lx_pcre_ungetc(lx, c); return lx->z = z2, lx->z(lx); + case S55: /* e.g. "\\c" */ + state = S72; break; - case S56: /* e.g. "\\E" */ - lx_pcre_ungetc(lx, c); return lx->z(lx); + case S56: /* e.g. "\\$" */ + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_ESC; case S57: /* e.g. "\\1" */ switch ((unsigned char) c) { @@ -866,13 +1014,29 @@ z3(struct lx_pcre_lx *lx) case '7': case '8': case '9': break; - default: lx_pcre_ungetc(lx, c); return TOK_UNSUPPORTED; + default: lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_UNSUPPORTED; } break; - case S58: /* e.g. "\\x" */ + case S58: /* e.g. "\\Q" */ + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return lx->z = z2, lx->z(lx); + + case S59: /* e.g. "\\E" */ + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return lx->z(lx); + + case S60: /* e.g. "\\o" */ switch ((unsigned char) c) { - case '{': state = S69; break; + case '{': state = S70; break; + default: lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_NOESC; + } + break; + + case S61: /* e.g. "\\\\x00" */ + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_NOESC; + + case S62: /* e.g. "\\x" */ + switch ((unsigned char) c) { + case '{': state = S66; break; case '0': case '1': case '2': @@ -894,12 +1058,12 @@ z3(struct lx_pcre_lx *lx) case 'c': case 'd': case 'e': - case 'f': state = S70; break; - default: lx_pcre_ungetc(lx, c); return TOK_HEX; + case 'f': state = S67; break; + default: lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_HEX; } break; - case S59: /* e.g. "\\0" */ + case S63: /* e.g. "\\0" */ switch ((unsigned char) c) { case '0': case '1': @@ -908,31 +1072,12 @@ z3(struct lx_pcre_lx *lx) case '4': case '5': case '6': - case '7': state = S68; break; - default: lx_pcre_ungetc(lx, c); return TOK_OCT; + case '7': state = S64; break; + default: lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_OCT; } break; - case S60: /* e.g. "\\o" */ - switch ((unsigned char) c) { - case '{': state = S65; break; - default: lx_pcre_ungetc(lx, c); return TOK_NOESC; - } - break; - - case S61: /* e.g. "\\c" */ - state = S64; break; - - case S62: /* e.g. "\\g" */ - lx_pcre_ungetc(lx, c); return TOK_NOESC; - - case S63: /* e.g. "\\a" */ - lx_pcre_ungetc(lx, c); return TOK_ESC; - - case S64: /* e.g. "\\ca" */ - lx_pcre_ungetc(lx, c); return TOK_CONTROL; - - case S65: /* e.g. "\\o{" */ + case S64: /* e.g. "\\00" */ switch ((unsigned char) c) { case '0': case '1': @@ -941,44 +1086,15 @@ z3(struct lx_pcre_lx *lx) case '4': case '5': case '6': - case '7': state = S66; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case '7': state = S65; break; + default: lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_OCT; } break; - case S66: /* e.g. "\\o{0" */ - switch ((unsigned char) c) { - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': break; - case '}': state = S67; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; - } - break; + case S65: /* e.g. "\\000" */ + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_OCT; - case S67: /* e.g. "\\000" */ - lx_pcre_ungetc(lx, c); return TOK_OCT; - - case S68: /* e.g. "\\00" */ - switch ((unsigned char) c) { - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': state = S67; break; - default: lx_pcre_ungetc(lx, c); return TOK_OCT; - } - break; - - case S69: /* e.g. "\\x{" */ + case S66: /* e.g. "\\x{" */ switch ((unsigned char) c) { case '0': case '1': @@ -1001,12 +1117,14 @@ z3(struct lx_pcre_lx *lx) case 'c': case 'd': case 'e': - case 'f': state = S72; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case 'f': state = S69; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S70: /* e.g. "\\xa" */ + case S67: /* e.g. "\\x0" */ switch ((unsigned char) c) { case '0': case '1': @@ -1029,17 +1147,17 @@ z3(struct lx_pcre_lx *lx) case 'c': case 'd': case 'e': - case 'f': state = S71; break; - default: lx_pcre_ungetc(lx, c); return TOK_HEX; + case 'f': state = S68; break; + default: lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_HEX; } break; - case S71: /* e.g. "\\xaa" */ - lx_pcre_ungetc(lx, c); return TOK_HEX; + case S68: /* e.g. "\\x00" */ + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_HEX; - case S72: /* e.g. "\\x{a" */ + case S69: /* e.g. "\\x{0" */ switch ((unsigned char) c) { - case '}': state = S71; break; + case '}': state = S68; break; case '0': case '1': case '2': @@ -1062,17 +1180,84 @@ z3(struct lx_pcre_lx *lx) case 'd': case 'e': case 'f': break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; + } + break; + + case S70: /* e.g. "\\o{" */ + switch ((unsigned char) c) { + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': state = S71; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; + case S71: /* e.g. "\\o{0" */ + switch ((unsigned char) c) { + case '}': state = S65; break; + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; + } + break; + + case S72: /* e.g. "\\c\\x00" */ + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_CONTROL; + default: ; /* unreached */ } + } + + /* end states */ + switch (state) { + case S1: return TOK_INVALID; + case S2: return TOK_CHAR; + case S3: return TOK_CHAR; + case S4: return TOK_RANGE; + case S5: return lx->z = z7, TOK_CLOSEGROUP; + case S6: return lx->z = z7, TOK_CLOSEGROUPRANGE; + case S23: return TOK_NAMED__CLASS; + case S55: return TOK_NOESC; + case S56: return TOK_ESC; + case S57: return TOK_UNSUPPORTED; + case S58: return lx->z = z2, lx->z(lx); + case S59: return TOK_EOF; + case S60: return TOK_NOESC; + case S61: return TOK_NOESC; + case S62: return TOK_HEX; + case S63: return TOK_OCT; + case S64: return TOK_OCT; + case S65: return TOK_OCT; + case S67: return TOK_HEX; + case S68: return TOK_HEX; + case S72: return TOK_CONTROL; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; + } switch (state) { - case S55: - case S56: + case S58: + case S59: break; default: @@ -1084,64 +1269,41 @@ z3(struct lx_pcre_lx *lx) break; } - } lx->lgetc = NULL; - switch (state) { - case NONE: return TOK_EOF; - case S1: return TOK_INVALID; - case S2: return TOK_CHAR; - case S3: return TOK_CHAR; - case S4: return TOK_RANGE; - case S5: return TOK_CLOSEGROUP; - case S6: return TOK_CLOSEGROUPRANGE; - case S24: return TOK_NAMED__CLASS; - case S55: return TOK_EOF; - case S56: return TOK_EOF; - case S57: return TOK_UNSUPPORTED; - case S58: return TOK_HEX; - case S59: return TOK_OCT; - case S60: return TOK_NOESC; - case S61: return TOK_NOESC; - case S62: return TOK_NOESC; - case S63: return TOK_ESC; - case S64: return TOK_CONTROL; - case S67: return TOK_OCT; - case S68: return TOK_OCT; - case S70: return TOK_HEX; - case S71: return TOK_HEX; - default: errno = EINVAL; return TOK_ERROR; - } + if (!has_consumed_input) { + return TOK_EOF; + } + return TOK_ERROR; } static enum lx_pcre_token z4(struct lx_pcre_lx *lx) { + int has_consumed_input = 0; int c; - enum { - S0, S1, S2, S3, S4, S5, S6, S7, S8, S9, - S10, NONE - } state; - assert(lx != NULL); if (lx->clear != NULL) { lx->clear(lx->buf_opaque); } - state = NONE; - lx->start = lx->end; - while (c = lx_getc(lx), c != EOF) { - if (state == NONE) { - state = S0; - } + void *getc_opaque = (void *)lx; + enum { + S0, S1, S2, S3, S4, S5, S6, S7, S8, S9, + S10 + } state; + + state = S0; + while (c = fsm_getc(getc_opaque), c != EOF) { + has_consumed_input = 1; switch (state) { - case S0: /* start */ + case S0: /* e.g. "" */ switch ((unsigned char) c) { case '+': case 'R': state = S1; break; @@ -1184,12 +1346,14 @@ z4(struct lx_pcre_lx *lx) case '-': state = S8; break; case ')': state = S9; break; case ':': state = S10; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S1: /* e.g. "R" */ - lx_pcre_ungetc(lx, c); return TOK_UNSUPPORTED; + case S1: /* e.g. "+" */ + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_UNSUPPORTED; case S2: /* e.g. "0" */ switch ((unsigned char) c) { @@ -1203,53 +1367,45 @@ z4(struct lx_pcre_lx *lx) case '7': case '8': case '9': break; - default: lx_pcre_ungetc(lx, c); return TOK_UNSUPPORTED; + default: lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_UNSUPPORTED; } break; case S3: /* e.g. "n" */ - lx_pcre_ungetc(lx, c); return TOK_FLAG__IGNORE; + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_FLAG__IGNORE; case S4: /* e.g. "x" */ switch ((unsigned char) c) { case 'x': state = S7; break; - default: lx_pcre_ungetc(lx, c); return TOK_FLAG__EXTENDED; + default: lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_FLAG__EXTENDED; } break; case S5: /* e.g. "s" */ - lx_pcre_ungetc(lx, c); return TOK_FLAG__SINGLE; + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_FLAG__SINGLE; case S6: /* e.g. "i" */ - lx_pcre_ungetc(lx, c); return TOK_FLAG__INSENSITIVE; + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_FLAG__INSENSITIVE; case S7: /* e.g. "a" */ - lx_pcre_ungetc(lx, c); return TOK_FLAG__UNKNOWN; + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_FLAG__UNKNOWN; case S8: /* e.g. "-" */ - lx_pcre_ungetc(lx, c); return TOK_NEGATE; + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_NEGATE; case S9: /* e.g. ")" */ - lx_pcre_ungetc(lx, c); return lx->z = z7, TOK_CLOSE; + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return lx->z = z7, TOK_CLOSE; case S10: /* e.g. ":" */ - lx_pcre_ungetc(lx, c); return lx->z = z7, TOK_SUB; + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return lx->z = z7, TOK_SUB; default: ; /* unreached */ } - - if (lx->push != NULL) { - if (-1 == lx->push(lx->buf_opaque, (char)c)) { - return TOK_ERROR; - } - } } - lx->lgetc = NULL; - + /* end states */ switch (state) { - case NONE: return TOK_EOF; case S1: return TOK_UNSUPPORTED; case S2: return TOK_UNSUPPORTED; case S3: return TOK_FLAG__IGNORE; @@ -1258,38 +1414,52 @@ z4(struct lx_pcre_lx *lx) case S6: return TOK_FLAG__INSENSITIVE; case S7: return TOK_FLAG__UNKNOWN; case S8: return TOK_NEGATE; - case S9: return TOK_CLOSE; - case S10: return TOK_SUB; - default: errno = EINVAL; return TOK_ERROR; + case S9: return lx->z = z7, TOK_CLOSE; + case S10: return lx->z = z7, TOK_SUB; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } + + if (lx->push != NULL) { + if (-1 == lx->push(lx->buf_opaque, (char)c)) { + return TOK_ERROR; + } + } + + lx->lgetc = NULL; + + if (!has_consumed_input) { + return TOK_EOF; + } + return TOK_ERROR; } static enum lx_pcre_token z5(struct lx_pcre_lx *lx) { + int has_consumed_input = 0; int c; - enum { - S0, S1, S2, S3, NONE - } state; - assert(lx != NULL); if (lx->clear != NULL) { lx->clear(lx->buf_opaque); } - state = NONE; - lx->start = lx->end; - while (c = lx_getc(lx), c != EOF) { - if (state == NONE) { - state = S0; - } + void *getc_opaque = (void *)lx; + enum { + S0, S1, S2, S3 + } state; + + state = S0; + while (c = fsm_getc(getc_opaque), c != EOF) { + has_consumed_input = 1; switch (state) { - case S0: /* start */ + case S0: /* e.g. "" */ switch ((unsigned char) c) { case '(': state = S2; break; case ')': state = S3; break; @@ -1297,23 +1467,34 @@ z5(struct lx_pcre_lx *lx) } break; - case S1: /* e.g. "a" */ + case S1: /* e.g. "\\x00" */ switch ((unsigned char) c) { case '(': - case ')': lx_pcre_ungetc(lx, c); return lx->z(lx); + case ')': lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return lx->z(lx); default: break; } break; case S2: /* e.g. "(" */ - lx_pcre_ungetc(lx, c); return TOK_INVALID__COMMENT; + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_INVALID__COMMENT; case S3: /* e.g. ")" */ - lx_pcre_ungetc(lx, c); return lx->z = z7, lx->z(lx); + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return lx->z = z7, lx->z(lx); default: ; /* unreached */ } + } + + /* end states */ + switch (state) { + case S1: return TOK_UNKNOWN; + case S2: return TOK_INVALID__COMMENT; + case S3: return lx->z = z7, lx->z(lx); + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; + } switch (state) { case S1: @@ -1329,24 +1510,30 @@ z5(struct lx_pcre_lx *lx) break; } - } lx->lgetc = NULL; - switch (state) { - case NONE: return TOK_EOF; - case S1: return TOK_EOF; - case S2: return TOK_INVALID__COMMENT; - case S3: return TOK_EOF; - default: errno = EINVAL; return TOK_ERROR; - } + if (!has_consumed_input) { + return TOK_EOF; + } + return TOK_ERROR; } static enum lx_pcre_token z6(struct lx_pcre_lx *lx) { + int has_consumed_input = 0; int c; + assert(lx != NULL); + + if (lx->clear != NULL) { + lx->clear(lx->buf_opaque); + } + + lx->start = lx->end; + + void *getc_opaque = (void *)lx; enum { S0, S1, S2, S3, S4, S5, S6, S7, S8, S9, S10, S11, S12, S13, S14, S15, S16, S17, S18, S19, @@ -1355,26 +1542,15 @@ z6(struct lx_pcre_lx *lx) S40, S41, S42, S43, S44, S45, S46, S47, S48, S49, S50, S51, S52, S53, S54, S55, S56, S57, S58, S59, S60, S61, S62, S63, S64, S65, S66, S67, S68, S69, - S70, S71, S72, S73, S74, S75, S76, S77, S78, NONE + S70, S71, S72, S73, S74, S75, S76, S77, S78 } state; - assert(lx != NULL); - - if (lx->clear != NULL) { - lx->clear(lx->buf_opaque); - } - - state = NONE; - - lx->start = lx->end; - - while (c = lx_getc(lx), c != EOF) { - if (state == NONE) { - state = S0; - } + state = S0; + while (c = fsm_getc(getc_opaque), c != EOF) { + has_consumed_input = 1; switch (state) { - case S0: /* start */ + case S0: /* e.g. "" */ switch ((unsigned char) c) { case ':': state = S1; break; case 'L': state = S2; break; @@ -1390,13 +1566,17 @@ z6(struct lx_pcre_lx *lx) case 'n': state = S12; break; case 'F': state = S13; break; case ')': state = S14; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S1: /* e.g. ":" */ switch ((unsigned char) c) { - case ')': lx->lgetc = NULL; return TOK_UNKNOWN; + case ')': + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; default: state = S78; break; } break; @@ -1404,7 +1584,9 @@ z6(struct lx_pcre_lx *lx) case S2: /* e.g. "L" */ switch ((unsigned char) c) { case 'F': state = S37; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; @@ -1412,35 +1594,45 @@ z6(struct lx_pcre_lx *lx) switch ((unsigned char) c) { case 'l': state = S18; break; case 'o': state = S76; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S4: /* e.g. "M" */ switch ((unsigned char) c) { case 'A': state = S74; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S5: /* e.g. "T" */ switch ((unsigned char) c) { case 'H': state = S72; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S6: /* e.g. "S" */ switch ((unsigned char) c) { case 'K': state = S70; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S7: /* e.g. "P" */ switch ((unsigned char) c) { case 'R': state = S67; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; @@ -1448,21 +1640,27 @@ z6(struct lx_pcre_lx *lx) switch ((unsigned char) c) { case 'C': state = S60; break; case 'N': state = S61; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S9: /* e.g. "a" */ switch ((unsigned char) c) { case 't': state = S56; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S10: /* e.g. "N" */ switch ((unsigned char) c) { case 'O': state = S46; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; @@ -1470,7 +1668,9 @@ z6(struct lx_pcre_lx *lx) switch ((unsigned char) c) { case 'R': state = S41; break; case 'O': state = S42; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; @@ -1478,7 +1678,9 @@ z6(struct lx_pcre_lx *lx) switch ((unsigned char) c) { case 'l': state = S18; break; case 'e': state = S19; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; @@ -1486,31 +1688,35 @@ z6(struct lx_pcre_lx *lx) switch ((unsigned char) c) { case ':': state = S1; break; case 'A': state = S15; break; - default: lx_pcre_ungetc(lx, c); return TOK_UNSUPPORTED; + default: lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_UNSUPPORTED; } break; case S14: /* e.g. ")" */ - lx_pcre_ungetc(lx, c); return lx->z = z7, lx->z(lx); + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return lx->z = z7, lx->z(lx); case S15: /* e.g. "FA" */ switch ((unsigned char) c) { case 'I': state = S16; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S16: /* e.g. "FAI" */ switch ((unsigned char) c) { case 'L': state = S17; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S17: /* e.g. "FAIL" */ switch ((unsigned char) c) { case ':': state = S1; break; - default: lx_pcre_ungetc(lx, c); return TOK_UNSUPPORTED; + default: lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_UNSUPPORTED; } break; @@ -1518,84 +1724,108 @@ z6(struct lx_pcre_lx *lx) switch ((unsigned char) c) { case 'a': case 'b': state = S36; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S19: /* e.g. "ne" */ switch ((unsigned char) c) { case 'g': state = S20; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S20: /* e.g. "neg" */ switch ((unsigned char) c) { case 'a': state = S21; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S21: /* e.g. "nega" */ switch ((unsigned char) c) { case 't': state = S22; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S22: /* e.g. "negat" */ switch ((unsigned char) c) { case 'i': state = S23; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S23: /* e.g. "negati" */ switch ((unsigned char) c) { case 'v': state = S24; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S24: /* e.g. "negativ" */ switch ((unsigned char) c) { case 'e': state = S25; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S25: /* e.g. "negative" */ switch ((unsigned char) c) { case '_': state = S26; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S26: /* e.g. "negative_" */ switch ((unsigned char) c) { case 'l': state = S27; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S27: /* e.g. "negative_l" */ switch ((unsigned char) c) { case 'o': state = S28; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S28: /* e.g. "negative_lo" */ switch ((unsigned char) c) { case 'o': state = S29; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S29: /* e.g. "negative_loo" */ switch ((unsigned char) c) { case 'k': state = S30; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; @@ -1603,338 +1833,428 @@ z6(struct lx_pcre_lx *lx) switch ((unsigned char) c) { case 'b': state = S31; break; case 'a': state = S32; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S31: /* e.g. "negative_lookb" */ switch ((unsigned char) c) { case 'e': state = S38; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S32: /* e.g. "negative_looka" */ switch ((unsigned char) c) { case 'h': state = S33; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S33: /* e.g. "negative_lookah" */ switch ((unsigned char) c) { case 'e': state = S34; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S34: /* e.g. "negative_lookahe" */ switch ((unsigned char) c) { case 'a': state = S35; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S35: /* e.g. "negative_lookahea" */ switch ((unsigned char) c) { case 'd': state = S36; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S36: /* e.g. "nla" */ switch ((unsigned char) c) { case ':': state = S37; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S37: /* e.g. "LF" */ - lx_pcre_ungetc(lx, c); return TOK_UNSUPPORTED; + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_UNSUPPORTED; case S38: /* e.g. "negative_lookbe" */ switch ((unsigned char) c) { case 'h': state = S39; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S39: /* e.g. "negative_lookbeh" */ switch ((unsigned char) c) { case 'i': state = S40; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S40: /* e.g. "negative_lookbehi" */ switch ((unsigned char) c) { case 'n': state = S35; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S41: /* e.g. "CR" */ switch ((unsigned char) c) { case 'L': state = S2; break; - default: lx_pcre_ungetc(lx, c); return TOK_UNSUPPORTED; + default: lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_UNSUPPORTED; } break; case S42: /* e.g. "CO" */ switch ((unsigned char) c) { case 'M': state = S43; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S43: /* e.g. "COM" */ switch ((unsigned char) c) { case 'M': state = S44; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S44: /* e.g. "COMM" */ switch ((unsigned char) c) { case 'I': state = S45; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S45: /* e.g. "ACCEP" */ switch ((unsigned char) c) { case 'T': state = S17; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S46: /* e.g. "NO" */ switch ((unsigned char) c) { case '_': state = S47; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S47: /* e.g. "NO_" */ switch ((unsigned char) c) { case 'S': state = S48; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S48: /* e.g. "NO_S" */ switch ((unsigned char) c) { case 'T': state = S49; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S49: /* e.g. "NO_ST" */ switch ((unsigned char) c) { case 'A': state = S50; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S50: /* e.g. "NO_STA" */ switch ((unsigned char) c) { case 'R': state = S51; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S51: /* e.g. "NO_STAR" */ switch ((unsigned char) c) { case 'T': state = S52; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S52: /* e.g. "NO_START" */ switch ((unsigned char) c) { case '_': state = S53; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S53: /* e.g. "NO_START_" */ switch ((unsigned char) c) { case 'O': state = S54; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S54: /* e.g. "NO_START_O" */ switch ((unsigned char) c) { case 'P': state = S55; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S55: /* e.g. "NO_START_OP" */ switch ((unsigned char) c) { case 'T': state = S37; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S56: /* e.g. "at" */ switch ((unsigned char) c) { case 'o': state = S57; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S57: /* e.g. "ato" */ switch ((unsigned char) c) { case 'm': state = S58; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S58: /* e.g. "atom" */ switch ((unsigned char) c) { case 'i': state = S59; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S59: /* e.g. "atomi" */ switch ((unsigned char) c) { case 'c': state = S36; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S60: /* e.g. "AC" */ switch ((unsigned char) c) { case 'C': state = S65; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S61: /* e.g. "AN" */ switch ((unsigned char) c) { case 'Y': state = S62; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S62: /* e.g. "ANY" */ switch ((unsigned char) c) { case 'C': state = S63; break; - default: lx_pcre_ungetc(lx, c); return TOK_UNSUPPORTED; + default: lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_UNSUPPORTED; } break; case S63: /* e.g. "ANYC" */ switch ((unsigned char) c) { case 'R': state = S64; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S64: /* e.g. "ANYCR" */ switch ((unsigned char) c) { case 'L': state = S2; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S65: /* e.g. "ACC" */ switch ((unsigned char) c) { case 'E': state = S66; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S66: /* e.g. "ACCE" */ switch ((unsigned char) c) { case 'P': state = S45; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S67: /* e.g. "PR" */ switch ((unsigned char) c) { case 'U': state = S68; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S68: /* e.g. "PRU" */ switch ((unsigned char) c) { case 'N': state = S69; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S69: /* e.g. "PRUN" */ switch ((unsigned char) c) { case 'E': state = S17; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S70: /* e.g. "SK" */ switch ((unsigned char) c) { case 'I': state = S71; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S71: /* e.g. "SKI" */ switch ((unsigned char) c) { case 'P': state = S17; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S72: /* e.g. "TH" */ switch ((unsigned char) c) { case 'E': state = S73; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S73: /* e.g. "THE" */ switch ((unsigned char) c) { case 'N': state = S17; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S74: /* e.g. "MA" */ switch ((unsigned char) c) { case 'R': state = S75; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S75: /* e.g. "MAR" */ switch ((unsigned char) c) { case 'K': state = S17; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S76: /* e.g. "po" */ switch ((unsigned char) c) { case 's': state = S77; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S77: /* e.g. "pos" */ switch ((unsigned char) c) { case 'i': state = S21; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S78: /* e.g. ":a" */ + case S78: /* e.g. ":\\x00" */ switch ((unsigned char) c) { - case ')': lx_pcre_ungetc(lx, c); return TOK_UNSUPPORTED; + case ')': lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_UNSUPPORTED; default: break; } break; @@ -1942,6 +2262,21 @@ z6(struct lx_pcre_lx *lx) default: ; /* unreached */ } + } + + /* end states */ + switch (state) { + case S13: return TOK_UNSUPPORTED; + case S14: return lx->z = z7, lx->z(lx); + case S17: return TOK_UNSUPPORTED; + case S37: return TOK_UNSUPPORTED; + case S41: return TOK_UNSUPPORTED; + case S62: return TOK_UNSUPPORTED; + case S78: return TOK_UNSUPPORTED; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; + } switch (state) { case S14: @@ -1956,53 +2291,44 @@ z6(struct lx_pcre_lx *lx) break; } - } lx->lgetc = NULL; - switch (state) { - case NONE: return TOK_EOF; - case S13: return TOK_UNSUPPORTED; - case S14: return TOK_EOF; - case S17: return TOK_UNSUPPORTED; - case S37: return TOK_UNSUPPORTED; - case S41: return TOK_UNSUPPORTED; - case S62: return TOK_UNSUPPORTED; - case S78: return TOK_UNSUPPORTED; - default: errno = EINVAL; return TOK_ERROR; - } + if (!has_consumed_input) { + return TOK_EOF; + } + return TOK_ERROR; } static enum lx_pcre_token z7(struct lx_pcre_lx *lx) { + int has_consumed_input = 0; int c; - enum { - S0, S1, S2, S3, S4, S5, S6, S7, S8, S9, - S10, S11, S12, S13, S14, S15, S16, S17, S18, S19, - S20, S21, S22, S23, S24, S25, S26, S27, S28, S29, - S30, S31, S32, S33, S34, S35, S36, S37, S38, S39, - S40, S41, S42, S43, S44, S45, S46, S47, S48, NONE - } state; - assert(lx != NULL); if (lx->clear != NULL) { lx->clear(lx->buf_opaque); } - state = NONE; - lx->start = lx->end; - while (c = lx_getc(lx), c != EOF) { - if (state == NONE) { - state = S0; - } + void *getc_opaque = (void *)lx; + enum { + S0, S1, S2, S3, S4, S5, S6, S7, S8, S9, + S10, S11, S12, S13, S14, S15, S16, S17, S18, S19, + S20, S21, S22, S23, S24, S25, S26, S27, S28, S29, + S30, S31, S32, S33, S34, S35, S36, S37, S38, S39, + S40, S41, S42, S43, S44, S45, S46, S47, S48 + } state; + + state = S0; + while (c = fsm_getc(getc_opaque), c != EOF) { + has_consumed_input = 1; switch (state) { - case S0: /* start */ + case S0: /* e.g. "" */ switch ((unsigned char) c) { case '\\': state = S2; break; case '\n': @@ -2023,13 +2349,15 @@ z7(struct lx_pcre_lx *lx) case '$': state = S14; break; case '^': state = S15; break; case ')': state = S16; break; - case '\x00': lx->lgetc = NULL; return TOK_UNKNOWN; + case '\x00': + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; default: state = S1; break; } break; - case S1: /* e.g. "a" */ - lx_pcre_ungetc(lx, c); return TOK_CHAR; + case S1: /* e.g. "\\x01" */ + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_CHAR; case S2: /* e.g. "\\" */ switch ((unsigned char) c) { @@ -2042,24 +2370,7 @@ z7(struct lx_pcre_lx *lx) case 'X': case 'b': case 'g': - case 'k': state = S20; break; - case 'Q': state = S28; break; - case 'o': state = S29; break; - case 'c': state = S30; break; - case 'x': state = S32; break; - case '0': state = S33; break; - case 'R': state = S34; break; - case 'D': - case 'H': - case 'N': - case 'S': - case 'V': - case 'W': - case 'd': - case 'h': - case 's': - case 'v': - case 'w': state = S35; break; + case 'k': state = S21; break; case '$': case '(': case ')': @@ -2077,8 +2388,9 @@ z7(struct lx_pcre_lx *lx) case 'r': case 't': case '{': - case '|': state = S36; break; - case 'E': state = S37; break; + case '|': state = S28; break; + case 'E': state = S29; break; + case 'z': state = S30; break; case '1': case '2': case '3': @@ -2087,29 +2399,45 @@ z7(struct lx_pcre_lx *lx) case '6': case '7': case '8': - case '9': state = S38; break; - case 'z': state = S39; break; - default: state = S31; break; + case '9': state = S31; break; + case 'Q': state = S32; break; + case 'c': state = S33; break; + case 'o': state = S34; break; + case 'x': state = S36; break; + case 'D': + case 'H': + case 'N': + case 'S': + case 'V': + case 'W': + case 'd': + case 'h': + case 's': + case 'v': + case 'w': state = S37; break; + case 'R': state = S38; break; + case '0': state = S39; break; + default: state = S35; break; } break; case S3: /* e.g. "\\x0a" */ - lx_pcre_ungetc(lx, c); return TOK_NEWLINE; + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_NEWLINE; case S4: /* e.g. "\\x09" */ - lx_pcre_ungetc(lx, c); return TOK_WHITESPACE; + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_WHITESPACE; case S5: /* e.g. "#" */ - lx_pcre_ungetc(lx, c); return TOK_MAYBE_COMMENT; + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_MAYBE_COMMENT; case S6: /* e.g. "{" */ - lx_pcre_ungetc(lx, c); return lx->z = z1, TOK_OPENCOUNT; + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return lx->z = z1, TOK_OPENCOUNT; case S7: /* e.g. "[" */ switch ((unsigned char) c) { case '^': state = S25; break; case ']': state = S26; break; - default: lx_pcre_ungetc(lx, c); return lx->z = z3, TOK_OPENGROUP; + default: lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return lx->z = z3, TOK_OPENGROUP; } break; @@ -2117,66 +2445,56 @@ z7(struct lx_pcre_lx *lx) switch ((unsigned char) c) { case '?': state = S17; break; case '*': state = S18; break; - default: lx_pcre_ungetc(lx, c); return TOK_OPEN; + default: lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_OPEN; } break; case S9: /* e.g. "|" */ - lx_pcre_ungetc(lx, c); return TOK_ALT; + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_ALT; case S10: /* e.g. "." */ - lx_pcre_ungetc(lx, c); return TOK_ANY; + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_ANY; case S11: /* e.g. "+" */ - lx_pcre_ungetc(lx, c); return TOK_PLUS; + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_PLUS; case S12: /* e.g. "*" */ - lx_pcre_ungetc(lx, c); return TOK_STAR; + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_STAR; - case S13: /* e.g. "?" */ - lx_pcre_ungetc(lx, c); return TOK_OPT; + case S13: /* e.g. "\077" */ + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_OPT; case S14: /* e.g. "$" */ - lx_pcre_ungetc(lx, c); return TOK_END__NL; + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_END__NL; case S15: /* e.g. "^" */ - lx_pcre_ungetc(lx, c); return TOK_START; + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_START; case S16: /* e.g. ")" */ - lx_pcre_ungetc(lx, c); return TOK_CLOSE; + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_CLOSE; - case S17: /* e.g. "(?" */ + case S17: /* e.g. "(\077" */ switch ((unsigned char) c) { case '#': state = S19; break; + case '<': state = S20; break; case '!': case '&': - case '=': state = S20; break; - case 'P': state = S21; break; - case '<': state = S22; break; - default: lx_pcre_ungetc(lx, c); return lx->z = z4, TOK_FLAGS; + case '=': state = S21; break; + case 'P': state = S22; break; + default: lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return lx->z = z4, TOK_FLAGS; } break; case S18: /* e.g. "(*" */ - lx_pcre_ungetc(lx, c); return lx->z = z6, lx->z(lx); + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return lx->z = z6, lx->z(lx); - case S19: /* e.g. "(?#" */ - lx_pcre_ungetc(lx, c); return lx->z = z5, lx->z(lx); + case S19: /* e.g. "(\077#" */ + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return lx->z = z5, lx->z(lx); - case S20: /* e.g. "\\b" */ - lx_pcre_ungetc(lx, c); return TOK_UNSUPPORTED; - - case S21: /* e.g. "(?P" */ - switch ((unsigned char) c) { - case '>': state = S20; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; - } - break; - - case S22: /* e.g. "(?<" */ + case S20: /* e.g. "(\077<" */ switch ((unsigned char) c) { case '!': - case '=': state = S20; break; + case '=': state = S21; break; case 'A': case 'B': case 'C': @@ -2230,11 +2548,25 @@ z7(struct lx_pcre_lx *lx) case 'x': case 'y': case 'z': state = S23; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; + } + break; + + case S21: /* e.g. "\\B" */ + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_UNSUPPORTED; + + case S22: /* e.g. "(\077P" */ + switch ((unsigned char) c) { + case '>': state = S21; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S23: /* e.g. "(?': state = S24; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S24: /* e.g. "(?" */ - lx_pcre_ungetc(lx, c); return TOK_OPENCAPTURE; + case S24: /* e.g. "(\077" */ + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_OPENCAPTURE; case S25: /* e.g. "[^" */ switch ((unsigned char) c) { case ']': state = S27; break; - default: lx_pcre_ungetc(lx, c); return lx->z = z3, TOK_OPENGROUPINV; + default: lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return lx->z = z3, TOK_OPENGROUPINV; } break; case S26: /* e.g. "[]" */ - lx_pcre_ungetc(lx, c); return lx->z = z3, TOK_OPENGROUPCB; + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return lx->z = z3, TOK_OPENGROUPCB; case S27: /* e.g. "[^]" */ - lx_pcre_ungetc(lx, c); return lx->z = z3, TOK_OPENGROUPINVCB; + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return lx->z = z3, TOK_OPENGROUPINVCB; - case S28: /* e.g. "\\Q" */ - lx_pcre_ungetc(lx, c); return lx->z = z0, lx->z(lx); + case S28: /* e.g. "\\$" */ + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_ESC; - case S29: /* e.g. "\\o" */ + case S29: /* e.g. "\\E" */ + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return lx->z(lx); + + case S30: /* e.g. "\\z" */ + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_END; + + case S31: /* e.g. "\\1" */ switch ((unsigned char) c) { - case '{': state = S47; break; - default: lx_pcre_ungetc(lx, c); return TOK_NOESC; + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': break; + default: lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_UNSUPPORTED; } break; - case S30: /* e.g. "\\c" */ - state = S46; break; + case S32: /* e.g. "\\Q" */ + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return lx->z = z0, lx->z(lx); - case S31: /* e.g. "\\i" */ - lx_pcre_ungetc(lx, c); return TOK_NOESC; + case S33: /* e.g. "\\c" */ + state = S48; break; - case S32: /* e.g. "\\x" */ + case S34: /* e.g. "\\o" */ + switch ((unsigned char) c) { + case '{': state = S46; break; + default: lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_NOESC; + } + break; + + case S35: /* e.g. "\\\\x00" */ + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_NOESC; + + case S36: /* e.g. "\\x" */ switch ((unsigned char) c) { case '{': state = S42; break; case '0': @@ -2361,37 +2720,17 @@ z7(struct lx_pcre_lx *lx) case 'd': case 'e': case 'f': state = S43; break; - default: lx_pcre_ungetc(lx, c); return TOK_HEX; + default: lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_HEX; } break; - case S33: /* e.g. "\\0" */ - switch ((unsigned char) c) { - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': state = S40; break; - default: lx_pcre_ungetc(lx, c); return TOK_OCT; - } - break; - - case S34: /* e.g. "\\R" */ - lx_pcre_ungetc(lx, c); return TOK_EOL; - - case S35: /* e.g. "\\d" */ - lx_pcre_ungetc(lx, c); return TOK_NAMED__CLASS; - - case S36: /* e.g. "\\a" */ - lx_pcre_ungetc(lx, c); return TOK_ESC; + case S37: /* e.g. "\\D" */ + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_NAMED__CLASS; - case S37: /* e.g. "\\E" */ - lx_pcre_ungetc(lx, c); return lx->z(lx); + case S38: /* e.g. "\\R" */ + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_EOL; - case S38: /* e.g. "\\1" */ + case S39: /* e.g. "\\0" */ switch ((unsigned char) c) { case '0': case '1': @@ -2400,16 +2739,11 @@ z7(struct lx_pcre_lx *lx) case '4': case '5': case '6': - case '7': - case '8': - case '9': break; - default: lx_pcre_ungetc(lx, c); return TOK_UNSUPPORTED; + case '7': state = S40; break; + default: lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_OCT; } break; - case S39: /* e.g. "\\z" */ - lx_pcre_ungetc(lx, c); return TOK_END; - case S40: /* e.g. "\\00" */ switch ((unsigned char) c) { case '0': @@ -2420,12 +2754,12 @@ z7(struct lx_pcre_lx *lx) case '5': case '6': case '7': state = S41; break; - default: lx_pcre_ungetc(lx, c); return TOK_OCT; + default: lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_OCT; } break; case S41: /* e.g. "\\000" */ - lx_pcre_ungetc(lx, c); return TOK_OCT; + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_OCT; case S42: /* e.g. "\\x{" */ switch ((unsigned char) c) { @@ -2451,11 +2785,13 @@ z7(struct lx_pcre_lx *lx) case 'd': case 'e': case 'f': state = S45; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S43: /* e.g. "\\xa" */ + case S43: /* e.g. "\\x0" */ switch ((unsigned char) c) { case '0': case '1': @@ -2479,14 +2815,14 @@ z7(struct lx_pcre_lx *lx) case 'd': case 'e': case 'f': state = S44; break; - default: lx_pcre_ungetc(lx, c); return TOK_HEX; + default: lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_HEX; } break; - case S44: /* e.g. "\\xaa" */ - lx_pcre_ungetc(lx, c); return TOK_HEX; + case S44: /* e.g. "\\x00" */ + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_HEX; - case S45: /* e.g. "\\x{a" */ + case S45: /* e.g. "\\x{0" */ switch ((unsigned char) c) { case '}': state = S44; break; case '0': @@ -2511,14 +2847,13 @@ z7(struct lx_pcre_lx *lx) case 'd': case 'e': case 'f': break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S46: /* e.g. "\\ca" */ - lx_pcre_ungetc(lx, c); return TOK_CONTROL; - - case S47: /* e.g. "\\o{" */ + case S46: /* e.g. "\\o{" */ switch ((unsigned char) c) { case '0': case '1': @@ -2527,12 +2862,14 @@ z7(struct lx_pcre_lx *lx) case '4': case '5': case '6': - case '7': state = S48; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + case '7': state = S47; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S48: /* e.g. "\\o{0" */ + case S47: /* e.g. "\\o{0" */ switch ((unsigned char) c) { case '}': state = S41; break; case '0': @@ -2543,43 +2880,29 @@ z7(struct lx_pcre_lx *lx) case '5': case '6': case '7': break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - default: - ; /* unreached */ - } - - switch (state) { - case S18: - case S19: - case S28: - case S37: - break; + case S48: /* e.g. "\\c\\x00" */ + lx_pcre_ungetc(lx, c); lx_pcre_dynpop(lx->buf_opaque); return TOK_CONTROL; default: - if (lx->push != NULL) { - if (-1 == lx->push(lx->buf_opaque, (char)c)) { - return TOK_ERROR; - } - } - break; - + ; /* unreached */ } } - lx->lgetc = NULL; - + /* end states */ switch (state) { - case NONE: return TOK_EOF; case S1: return TOK_CHAR; case S2: return TOK_INVALID; case S3: return TOK_NEWLINE; case S4: return TOK_WHITESPACE; case S5: return TOK_MAYBE_COMMENT; - case S6: return TOK_OPENCOUNT; - case S7: return TOK_OPENGROUP; + case S6: return lx->z = z1, TOK_OPENCOUNT; + case S7: return lx->z = z3, TOK_OPENGROUP; case S8: return TOK_OPEN; case S9: return TOK_ALT; case S10: return TOK_ANY; @@ -2589,33 +2912,59 @@ z7(struct lx_pcre_lx *lx) case S14: return TOK_END__NL; case S15: return TOK_START; case S16: return TOK_CLOSE; - case S17: return TOK_FLAGS; - case S18: return TOK_EOF; - case S19: return TOK_EOF; - case S20: return TOK_UNSUPPORTED; + case S17: return lx->z = z4, TOK_FLAGS; + case S18: return lx->z = z6, lx->z(lx); + case S19: return lx->z = z5, lx->z(lx); + case S21: return TOK_UNSUPPORTED; case S24: return TOK_OPENCAPTURE; - case S25: return TOK_OPENGROUPINV; - case S26: return TOK_OPENGROUPCB; - case S27: return TOK_OPENGROUPINVCB; - case S28: return TOK_EOF; - case S29: return TOK_NOESC; - case S30: return TOK_NOESC; - case S31: return TOK_NOESC; - case S32: return TOK_HEX; - case S33: return TOK_OCT; - case S34: return TOK_EOL; - case S35: return TOK_NAMED__CLASS; - case S36: return TOK_ESC; - case S37: return TOK_EOF; - case S38: return TOK_UNSUPPORTED; - case S39: return TOK_END; + case S25: return lx->z = z3, TOK_OPENGROUPINV; + case S26: return lx->z = z3, TOK_OPENGROUPCB; + case S27: return lx->z = z3, TOK_OPENGROUPINVCB; + case S28: return TOK_ESC; + case S29: return TOK_EOF; + case S30: return TOK_END; + case S31: return TOK_UNSUPPORTED; + case S32: return lx->z = z0, lx->z(lx); + case S33: return TOK_NOESC; + case S34: return TOK_NOESC; + case S35: return TOK_NOESC; + case S36: return TOK_HEX; + case S37: return TOK_NAMED__CLASS; + case S38: return TOK_EOL; + case S39: return TOK_OCT; case S40: return TOK_OCT; case S41: return TOK_OCT; case S43: return TOK_HEX; case S44: return TOK_HEX; - case S46: return TOK_CONTROL; - default: errno = EINVAL; return TOK_ERROR; + case S48: return TOK_CONTROL; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_pcre_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } + + switch (state) { + case S18: + case S19: + case S29: + case S32: + break; + + default: + if (lx->push != NULL) { + if (-1 == lx->push(lx->buf_opaque, (char)c)) { + return TOK_ERROR; + } + } + break; + + } + + lx->lgetc = NULL; + + if (!has_consumed_input) { + return TOK_EOF; + } + return TOK_ERROR; } const char * @@ -3092,6 +3441,7 @@ lx_pcre_init(struct lx_pcre_lx *lx) lx->end.byte = 0; lx->end.line = 1; lx->end.col = 1; + (void)lx_pcre_dynpop; } enum lx_pcre_token diff --git a/src/libre/dialect/pcre/parser.c b/src/libre/dialect/pcre/parser.c index bccd5bbba..e59fee020 100644 --- a/src/libre/dialect/pcre/parser.c +++ b/src/libre/dialect/pcre/parser.c @@ -68,14 +68,6 @@ #define TOK_CLASS__nspace TOK_CLASS_NSPACE #define TOK_CLASS__ndigit TOK_CLASS_NDIGIT - /* This is a hack to work around the AST files not being able to include lexer.h. */ - #define AST_POS_OF_LX_POS(AST_POS, LX_POS) \ - do { \ - AST_POS.line = LX_POS.line; \ - AST_POS.col = LX_POS.col; \ - AST_POS.byte = LX_POS.byte; \ - } while (0) - #include "parser.h" #include "lexer.h" @@ -132,92 +124,8 @@ /* TODO: use lx's generated conveniences for the pattern buffer */ char a[512]; char *p; -#if PCRE_DIALECT - int extended_mode_comment; - enum re_flags *flags_ptr; -#endif /* PCRE_DIALECT */ }; - #define ERROR_TERMINAL (TOK_ERROR) - - #if PCRE_DIALECT - static void mark(struct re_pos *, const struct lx_pos *); - - static void - pcre_advance_lexer(struct lex_state *lex_state, struct act_state *act_state) - { - mark(&act_state->synstart, &lex_state->lx.start); - mark(&act_state->synend, &lex_state->lx.end); - act_state->lex_tok = LX_NEXT(&lex_state->lx); - } - - static enum LX_TOKEN - pcre_current_token(struct lex_state *lex_state, struct act_state *act_state) - { - enum re_flags fl = (lex_state->flags_ptr != NULL) ? lex_state->flags_ptr[0] : 0; - enum LX_TOKEN tok; - - if ((fl & RE_EXTENDED) == 0) { - tok = act_state->lex_tok; - - switch (tok) { - case TOK_WHITESPACE: - case TOK_NEWLINE: - case TOK_MAYBE_COMMENT: - return TOK_CHAR; - - default: - return tok; - } - } - - restart: - tok = act_state->lex_tok; - if (lex_state->extended_mode_comment) { - switch (tok) { - case TOK_EOF: - case TOK_ERROR: - case TOK_UNKNOWN: - /* don't eat EOF or errors */ - return tok; - - case TOK_NEWLINE: - lex_state->extended_mode_comment = 0; - /* fall through */ - default: - pcre_advance_lexer(lex_state, act_state); - goto restart; - } - } else { - switch (tok) { - case TOK_MAYBE_COMMENT: - lex_state->extended_mode_comment = 1; - /*fall through */ - case TOK_WHITESPACE: - case TOK_NEWLINE: - pcre_advance_lexer(lex_state, act_state); - goto restart; - - default: - return tok; - } - } - } - - #define CURRENT_TERMINAL (pcre_current_token(lex_state, act_state)) - #define ADVANCE_LEXER do { pcre_advance_lexer(lex_state,act_state); } while(0) - #else /* !PCRE_DIALECT */ - #define CURRENT_TERMINAL (act_state->lex_tok) - #define ADVANCE_LEXER do { mark(&act_state->synstart, &lex_state->lx.start); \ - mark(&act_state->synend, &lex_state->lx.end); \ - act_state->lex_tok = LX_NEXT(&lex_state->lx); \ - } while (0) - #endif /* PCRE_DIALECT */ - - #define SAVE_LEXER(tok) do { act_state->lex_tok_save = act_state->lex_tok; \ - act_state->lex_tok = tok; } while (0) - #define RESTORE_LEXER do { act_state->lex_tok = act_state->lex_tok_save; } while (0) - static void mark(struct re_pos *r, const struct lx_pos *pos) { @@ -288,7 +196,80 @@ return s; } -#line 292 "src/libre/dialect/pcre/parser.c" + static void + advance_lexer(struct lex_state *lex_state, struct act_state *act_state) + { + mark(&act_state->synstart, &lex_state->lx.start); + mark(&act_state->synend, &lex_state->lx.end); + act_state->lex_tok = LX_NEXT(&lex_state->lx); + } + + static enum LX_TOKEN + current_token(struct lex_state *lex_state, struct act_state *act_state, + enum re_flags *flags) + { +#ifndef PCRE_DIALECT + (void) lex_state; + (void) flags; + return act_state->lex_tok; +#else + if ((*flags & RE_EXTENDED) == 0) { + switch (act_state->lex_tok) { + case TOK_WHITESPACE: + case TOK_NEWLINE: + case TOK_MAYBE_COMMENT: + return TOK_CHAR; + + default: + return act_state->lex_tok; + } + } + + int extended_mode_comment = 0; + + restart: + + if (extended_mode_comment) { + switch (act_state->lex_tok) { + case TOK_EOF: + case TOK_ERROR: + case TOK_UNKNOWN: + /* don't eat EOF or errors */ + return act_state->lex_tok; + + case TOK_NEWLINE: + extended_mode_comment = 0; + /* fall through */ + default: + advance_lexer(lex_state, act_state); + goto restart; + } + } else { + switch (act_state->lex_tok) { + case TOK_MAYBE_COMMENT: + extended_mode_comment = 1; + /* fall through */ + case TOK_WHITESPACE: + case TOK_NEWLINE: + advance_lexer(lex_state, act_state); + goto restart; + + default: + return act_state->lex_tok; + } + } +#endif + } + + #define ADVANCE_LEXER (advance_lexer(lex_state, act_state)) + #define CURRENT_TERMINAL (current_token(lex_state, act_state, flags)) + #define ERROR_TERMINAL (TOK_ERROR) + + #define SAVE_LEXER(tok) do { act_state->lex_tok_save = act_state->lex_tok; \ + act_state->lex_tok = tok; } while (0) + #define RESTORE_LEXER do { act_state->lex_tok = act_state->lex_tok_save; } while (0) + +#line 273 "src/libre/dialect/pcre/parser.c" #ifndef ERROR_TERMINAL @@ -301,25 +282,25 @@ static void p_expr_C_Cflags_C_Cflag__set(flags, lex_state, act_state, err, t_re_ static void p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags, lex_state, act_state, err, t_endpoint *, t_pos *, t_pos *); static void p_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms(flags, lex_state, act_state, err, t_ast__expr); static void p_expr_C_Clist_Hof_Hpieces(flags, lex_state, act_state, err, t_ast__expr); -static void p_296(flags, lex_state, act_state, err, t_ast__class__id *, t_pos *, t_ast__expr *); -static void p_169(flags, lex_state, act_state, err); +static void p_293(flags, lex_state, act_state, err, t_ast__class__id *, t_pos *, t_ast__expr *); +static void p_168(flags, lex_state, act_state, err); static void p_expr_C_Cliteral(flags, lex_state, act_state, err, t_ast__expr *); static void p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags, lex_state, act_state, err, t_ast__expr *); static void p_expr_C_Ccomment(flags, lex_state, act_state, err); static void p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hclass(flags, lex_state, act_state, err, t_endpoint *, t_pos *, t_pos *); static void p_expr_C_Ccharacter_Hclass(flags, lex_state, act_state, err, t_ast__expr *); static void p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_Hend(flags, lex_state, act_state, err, t_endpoint *, t_pos *); +static void p_317(flags, lex_state, act_state, err, t_char *, t_pos *, t_ast__expr *); static void p_expr_C_Cpiece(flags, lex_state, act_state, err, t_ast__expr *); -static void p_320(flags, lex_state, act_state, err, t_char *, t_pos *, t_ast__expr *); +static void p_320(flags, lex_state, act_state, err, t_pos *, t_unsigned *, t_ast__count *); static void p_expr(flags, lex_state, act_state, err, t_ast__expr *); -static void p_195(flags, lex_state, act_state, err, t_ast__expr *); -static void p_323(flags, lex_state, act_state, err, t_pos *, t_unsigned *, t_ast__count *); -static void p_324(flags, lex_state, act_state, err, t_pos *, t_unsigned *, t_ast__count *); +static void p_321(flags, lex_state, act_state, err, t_pos *, t_unsigned *, t_ast__count *); +static void p_194(flags, lex_state, act_state, err, t_ast__expr *); static void p_expr_C_Cflags(flags, lex_state, act_state, err, t_ast__expr *); static void p_expr_C_Cpiece_C_Clist_Hof_Hcounts(flags, lex_state, act_state, err, t_ast__expr, t_ast__expr *); static void p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint(flags, lex_state, act_state, err, t_endpoint *, t_pos *, t_pos *); static void p_class_Hnamed(flags, lex_state, act_state, err, t_ast__expr *, t_pos *, t_pos *); -static void p_210(flags, lex_state, act_state, err, t_pos *, t_char *, t_ast__expr *); +static void p_209(flags, lex_state, act_state, err, t_pos *, t_char *, t_ast__expr *); static void p_expr_C_Clist_Hof_Halts(flags, lex_state, act_state, err, t_ast__expr); static void p_expr_C_Cpiece_C_Ccount(flags, lex_state, act_state, err, t_ast__count *); extern void p_re__pcre(flags, lex_state, act_state, err, t_ast__expr *); @@ -344,21 +325,21 @@ p_expr_C_Cflags_C_Cflag__set(flags flags, lex_state lex_state, act_state act_sta /* BEGINNING OF EXTRACT: FLAG_EXTENDED */ { -#line 679 "src/libre/parser.act" +#line 665 "src/libre/parser.act" ZIc = RE_EXTENDED; -#line 352 "src/libre/dialect/pcre/parser.c" +#line 333 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: FLAG_EXTENDED */ ADVANCE_LEXER; /* BEGINNING OF ACTION: re-flag-union */ { -#line 829 "src/libre/parser.act" +#line 800 "src/libre/parser.act" (ZIo) = (ZIi) | (ZIc); -#line 362 "src/libre/dialect/pcre/parser.c" +#line 343 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: re-flag-union */ } @@ -375,21 +356,21 @@ p_expr_C_Cflags_C_Cflag__set(flags flags, lex_state lex_state, act_state act_sta /* BEGINNING OF EXTRACT: FLAG_INSENSITIVE */ { -#line 675 "src/libre/parser.act" +#line 661 "src/libre/parser.act" ZIc = RE_ICASE; -#line 383 "src/libre/dialect/pcre/parser.c" +#line 364 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: FLAG_INSENSITIVE */ ADVANCE_LEXER; /* BEGINNING OF ACTION: re-flag-union */ { -#line 829 "src/libre/parser.act" +#line 800 "src/libre/parser.act" (ZIo) = (ZIi) | (ZIc); -#line 393 "src/libre/dialect/pcre/parser.c" +#line 374 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: re-flag-union */ } @@ -400,21 +381,21 @@ p_expr_C_Cflags_C_Cflag__set(flags flags, lex_state lex_state, act_state act_sta /* BEGINNING OF EXTRACT: FLAG_SINGLE */ { -#line 683 "src/libre/parser.act" +#line 669 "src/libre/parser.act" ZIc = RE_SINGLE; -#line 408 "src/libre/dialect/pcre/parser.c" +#line 389 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: FLAG_SINGLE */ ADVANCE_LEXER; /* BEGINNING OF ACTION: re-flag-union */ { -#line 829 "src/libre/parser.act" +#line 800 "src/libre/parser.act" (ZIo) = (ZIi) | (ZIc); -#line 418 "src/libre/dialect/pcre/parser.c" +#line 399 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: re-flag-union */ } @@ -425,14 +406,14 @@ p_expr_C_Cflags_C_Cflag__set(flags flags, lex_state lex_state, act_state act_sta ZIo = ZIi; /* BEGINNING OF ACTION: err-unknown-flag */ { -#line 753 "src/libre/parser.act" +#line 739 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EFLAG; } goto ZL1; -#line 436 "src/libre/dialect/pcre/parser.c" +#line 417 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-unknown-flag */ } @@ -463,14 +444,14 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { t_char ZIc; - /* BEGINNING OF INLINE: 156 */ + /* BEGINNING OF INLINE: 155 */ { switch (CURRENT_TERMINAL) { case (TOK_CHAR): { /* BEGINNING OF EXTRACT: CHAR */ { -#line 589 "src/libre/parser.act" +#line 575 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); @@ -483,7 +464,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags ZIc = lex_state->buf.a[0]; -#line 487 "src/libre/dialect/pcre/parser.c" +#line 468 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; @@ -493,7 +474,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: CONTROL */ { -#line 461 "src/libre/parser.act" +#line 442 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] == 'c'); @@ -501,16 +482,21 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags assert(lex_state->buf.a[3] == '\0'); ZIc = lex_state->buf.a[2]; - /* from http://www.pcre.org/current/doc/html/pcre2pattern.html#SEC5 + + /* + * From http://www.pcre.org/current/doc/html/pcre2pattern.html#SEC5 * - * The precise effect of \cx on ASCII characters is as follows: if x is a lower case letter, it is converted to - * upper case. Then bit 6 of the character (hex 40) is inverted. Thus \cA to \cZ become hex 01 to hex 1A (A is - * 41, Z is 5A), but \c{ becomes hex 3B ({ is 7B), and \c; becomes hex 7B (; is 3B). If the code unit following - * \c has a value less than 32 or greater than 126, a compile-time error occurs. + * The precise effect of \cx on ASCII characters is as follows: + * if x is a lower case letter, it is converted to upper case. + * Then bit 6 of the character (hex 40) is inverted. + * Thus \cA to \cZ become hex 01 to hex 1A (A is 41, Z is 5A), + * but \c{ becomes hex 3B ({ is 7B), and \c; becomes hex 7B (; is 3B). + * If the code unit following \c has a value less than 32 + * or greater than 126, a compile-time error occurs. */ { - unsigned char cc = (unsigned char)ZIc; + unsigned char cc = (unsigned char) ZIc; if (cc > 126 || cc < 32) { goto ZL1; } @@ -530,20 +516,20 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags (void) ZIstart; (void) ZIend; -#line 534 "src/libre/dialect/pcre/parser.c" +#line 520 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CONTROL */ ADVANCE_LEXER; /* BEGINNING OF ACTION: err-unsupported */ { -#line 774 "src/libre/parser.act" +#line 760 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EUNSUPPORTED; } goto ZL1; -#line 547 "src/libre/dialect/pcre/parser.c" +#line 533 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-unsupported */ } @@ -552,7 +538,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: ESC */ { -#line 405 "src/libre/parser.act" +#line 386 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -578,7 +564,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags (void) ZIstart; (void) ZIend; -#line 582 "src/libre/dialect/pcre/parser.c" +#line 568 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: ESC */ ADVANCE_LEXER; @@ -588,7 +574,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: HEX */ { -#line 541 "src/libre/parser.act" +#line 527 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -636,7 +622,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags ZIc = (char) (unsigned char) u; -#line 640 "src/libre/dialect/pcre/parser.c" +#line 626 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: HEX */ ADVANCE_LEXER; @@ -646,7 +632,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: OCT */ { -#line 498 "src/libre/parser.act" +#line 484 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -689,7 +675,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags ZIc = (char) (unsigned char) u; -#line 693 "src/libre/dialect/pcre/parser.c" +#line 679 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: OCT */ ADVANCE_LEXER; @@ -699,7 +685,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags { /* BEGINNING OF EXTRACT: UNSUPPORTED */ { -#line 445 "src/libre/parser.act" +#line 426 "src/libre/parser.act" /* handle \1-\9 back references */ if (lex_state->buf.a[0] == '\\' && lex_state->buf.a[1] != '\0' && lex_state->buf.a[2] == '\0') { @@ -715,20 +701,20 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags (void) ZIstart; (void) ZIend; -#line 719 "src/libre/dialect/pcre/parser.c" +#line 705 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: UNSUPPORTED */ ADVANCE_LEXER; /* BEGINNING OF ACTION: err-unsupported */ { -#line 774 "src/libre/parser.act" +#line 760 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EUNSUPPORTED; } goto ZL1; -#line 732 "src/libre/dialect/pcre/parser.c" +#line 718 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-unsupported */ } @@ -737,15 +723,15 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hliteral(flags goto ZL1; } } - /* END OF INLINE: 156 */ + /* END OF INLINE: 155 */ /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 873 "src/libre/parser.act" +#line 839 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_LITERAL; - (ZIr).u.literal.c = (unsigned char)(ZIc); + (ZIr).u.literal.c = (unsigned char) (ZIc); -#line 749 "src/libre/dialect/pcre/parser.c" +#line 735 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ } @@ -776,13 +762,13 @@ ZL2_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms:; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1084 "src/libre/parser.act" +#line 1046 "src/libre/parser.act" if (!ast_add_expr_alt((ZIclass), (ZInode))) { goto ZL1; } -#line 786 "src/libre/dialect/pcre/parser.c" +#line 772 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF INLINE: expr::character-class::list-of-class-terms */ @@ -809,7 +795,7 @@ p_expr_C_Clist_Hof_Hpieces(flags flags, lex_state lex_state, act_state act_state } ZL2_expr_C_Clist_Hof_Hpieces:; { - /* BEGINNING OF INLINE: 273 */ + /* BEGINNING OF INLINE: 271 */ { switch (CURRENT_TERMINAL) { case (TOK_INVALID__COMMENT): @@ -836,13 +822,13 @@ ZL2_expr_C_Clist_Hof_Hpieces:; } /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1078 "src/libre/parser.act" +#line 1040 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcat), (ZIa))) { goto ZL1; } -#line 846 "src/libre/dialect/pcre/parser.c" +#line 832 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-concat */ } @@ -851,8 +837,8 @@ ZL2_expr_C_Clist_Hof_Hpieces:; goto ZL1; } } - /* END OF INLINE: 273 */ - /* BEGINNING OF INLINE: 274 */ + /* END OF INLINE: 271 */ + /* BEGINNING OF INLINE: 272 */ { switch (CURRENT_TERMINAL) { case (TOK_ANY): case (TOK_EOL): case (TOK_START): case (TOK_END): @@ -870,7 +856,7 @@ ZL2_expr_C_Clist_Hof_Hpieces:; break; } } - /* END OF INLINE: 274 */ + /* END OF INLINE: 272 */ } return; ZL1:; @@ -879,7 +865,7 @@ ZL1:; } static void -p_296(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__class__id *ZI293, t_pos *ZI294, t_ast__expr *ZOnode) +p_293(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__class__id *ZI290, t_pos *ZI291, t_ast__expr *ZOnode) { t_ast__expr ZInode; @@ -888,14 +874,14 @@ p_296(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla { /* BEGINNING OF ACTION: ast-make-named */ { -#line 1071 "src/libre/parser.act" +#line 1033 "src/libre/parser.act" - (ZInode) = ast_make_expr_named(act_state->poolp, *flags, (*ZI293)); + (ZInode) = ast_make_expr_named(act_state->poolp, *flags, (*ZI290)); if ((ZInode) == NULL) { goto ZL1; } -#line 899 "src/libre/dialect/pcre/parser.c" +#line 885 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -908,15 +894,15 @@ p_296(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla /* BEGINNING OF ACTION: ast-range-endpoint-class */ { -#line 878 "src/libre/parser.act" +#line 844 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_NAMED; - (ZIlower).u.named.class = (*ZI293); + (ZIlower).u.named.class = (*ZI290); -#line 917 "src/libre/dialect/pcre/parser.c" +#line 903 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-range-endpoint-class */ - p_169 (flags, lex_state, act_state, err); + p_168 (flags, lex_state, act_state, err); p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_Hend (flags, lex_state, act_state, err, &ZIupper, &ZIend); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; @@ -924,24 +910,20 @@ p_296(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla } /* BEGINNING OF ACTION: mark-range */ { -#line 786 "src/libre/parser.act" +#line 772 "src/libre/parser.act" - mark(&act_state->rangestart, &(*ZI294)); + mark(&act_state->rangestart, &(*ZI291)); mark(&act_state->rangeend, &(ZIend)); -#line 933 "src/libre/dialect/pcre/parser.c" +#line 919 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: mark-range */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1038 "src/libre/parser.act" +#line 1004 "src/libre/parser.act" - struct ast_pos ast_start, ast_end; unsigned char lower, upper; - AST_POS_OF_LX_POS(ast_start, (*ZI294)); - AST_POS_OF_LX_POS(ast_end, (ZIend)); - if ((ZIlower).type != AST_ENDPOINT_LITERAL || (ZIupper).type != AST_ENDPOINT_LITERAL) { err->e = RE_EUNSUPPORTED; @@ -962,12 +944,12 @@ p_296(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__cla goto ZL1; } - (ZInode) = ast_make_expr_range(act_state->poolp, *flags, &(ZIlower), ast_start, &(ZIupper), ast_end); + (ZInode) = ast_make_expr_range(act_state->poolp, *flags, &(ZIlower), &(ZIupper)); if ((ZInode) == NULL) { goto ZL1; } -#line 971 "src/libre/dialect/pcre/parser.c" +#line 953 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -984,31 +966,31 @@ ZL0:; } static void -p_169(flags flags, lex_state lex_state, act_state act_state, err err) +p_168(flags flags, lex_state lex_state, act_state act_state, err err) { if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { return; } { - t_char ZI170; + t_char ZI169; + t_pos ZI170; t_pos ZI171; - t_pos ZI172; switch (CURRENT_TERMINAL) { case (TOK_RANGE): /* BEGINNING OF EXTRACT: RANGE */ { -#line 327 "src/libre/parser.act" +#line 308 "src/libre/parser.act" - ZI170 = '-'; - ZI171 = lex_state->lx.start; - ZI172 = lex_state->lx.end; + ZI169 = '-'; + ZI170 = lex_state->lx.start; + ZI171 = lex_state->lx.end; + (void) ZI169; (void) ZI170; (void) ZI171; - (void) ZI172; -#line 1012 "src/libre/dialect/pcre/parser.c" +#line 994 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: RANGE */ break; @@ -1022,14 +1004,14 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-range */ { -#line 732 "src/libre/parser.act" +#line 718 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXRANGE; } goto ZL2; -#line 1033 "src/libre/dialect/pcre/parser.c" +#line 1015 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-range */ } @@ -1051,30 +1033,30 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err { t_char ZIc; - /* BEGINNING OF INLINE: 112 */ + /* BEGINNING OF INLINE: 111 */ { switch (CURRENT_TERMINAL) { case (TOK_CHAR): { + t_pos ZI121; t_pos ZI122; - t_pos ZI123; /* BEGINNING OF EXTRACT: CHAR */ { -#line 589 "src/libre/parser.act" +#line 575 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); - ZI122 = lex_state->lx.start; - ZI123 = lex_state->lx.end; + ZI121 = lex_state->lx.start; + ZI122 = lex_state->lx.end; + (void) ZI121; (void) ZI122; - (void) ZI123; ZIc = lex_state->buf.a[0]; -#line 1078 "src/libre/dialect/pcre/parser.c" +#line 1060 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; @@ -1082,12 +1064,12 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err break; case (TOK_CONTROL): { + t_pos ZI123; t_pos ZI124; - t_pos ZI125; /* BEGINNING OF EXTRACT: CONTROL */ { -#line 461 "src/libre/parser.act" +#line 442 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] == 'c'); @@ -1095,16 +1077,21 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err assert(lex_state->buf.a[3] == '\0'); ZIc = lex_state->buf.a[2]; - /* from http://www.pcre.org/current/doc/html/pcre2pattern.html#SEC5 + + /* + * From http://www.pcre.org/current/doc/html/pcre2pattern.html#SEC5 * - * The precise effect of \cx on ASCII characters is as follows: if x is a lower case letter, it is converted to - * upper case. Then bit 6 of the character (hex 40) is inverted. Thus \cA to \cZ become hex 01 to hex 1A (A is - * 41, Z is 5A), but \c{ becomes hex 3B ({ is 7B), and \c; becomes hex 7B (; is 3B). If the code unit following - * \c has a value less than 32 or greater than 126, a compile-time error occurs. + * The precise effect of \cx on ASCII characters is as follows: + * if x is a lower case letter, it is converted to upper case. + * Then bit 6 of the character (hex 40) is inverted. + * Thus \cA to \cZ become hex 01 to hex 1A (A is 41, Z is 5A), + * but \c{ becomes hex 3B ({ is 7B), and \c; becomes hex 7B (; is 3B). + * If the code unit following \c has a value less than 32 + * or greater than 126, a compile-time error occurs. */ { - unsigned char cc = (unsigned char)ZIc; + unsigned char cc = (unsigned char) ZIc; if (cc > 126 || cc < 32) { goto ZL1; } @@ -1118,13 +1105,13 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err ZIc = cc; } - ZI124 = lex_state->lx.start; - ZI125 = lex_state->lx.end; + ZI123 = lex_state->lx.start; + ZI124 = lex_state->lx.end; + (void) ZI123; (void) ZI124; - (void) ZI125; -#line 1128 "src/libre/dialect/pcre/parser.c" +#line 1115 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CONTROL */ ADVANCE_LEXER; @@ -1132,12 +1119,12 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err break; case (TOK_ESC): { + t_pos ZI113; t_pos ZI114; - t_pos ZI115; /* BEGINNING OF EXTRACT: ESC */ { -#line 405 "src/libre/parser.act" +#line 386 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -1157,13 +1144,13 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err default: break; } - ZI114 = lex_state->lx.start; - ZI115 = lex_state->lx.end; + ZI113 = lex_state->lx.start; + ZI114 = lex_state->lx.end; + (void) ZI113; (void) ZI114; - (void) ZI115; -#line 1167 "src/libre/dialect/pcre/parser.c" +#line 1154 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: ESC */ ADVANCE_LEXER; @@ -1171,12 +1158,12 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err break; case (TOK_HEX): { + t_pos ZI119; t_pos ZI120; - t_pos ZI121; /* BEGINNING OF EXTRACT: HEX */ { -#line 541 "src/libre/parser.act" +#line 527 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -1185,11 +1172,11 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err assert(0 == strncmp(lex_state->buf.a, "\\x", 2)); assert(strlen(lex_state->buf.a) >= 2); /* pcre allows \x without a suffix */ - ZI120 = lex_state->lx.start; - ZI121 = lex_state->lx.end; + ZI119 = lex_state->lx.start; + ZI120 = lex_state->lx.end; + (void) ZI119; (void) ZI120; - (void) ZI121; errno = 0; @@ -1224,7 +1211,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err ZIc = (char) (unsigned char) u; -#line 1228 "src/libre/dialect/pcre/parser.c" +#line 1215 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: HEX */ ADVANCE_LEXER; @@ -1232,12 +1219,12 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err break; case (TOK_NOESC): { + t_pos ZI115; t_pos ZI116; - t_pos ZI117; /* BEGINNING OF EXTRACT: NOESC */ { -#line 431 "src/libre/parser.act" +#line 412 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -1245,13 +1232,13 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err ZIc = lex_state->buf.a[1]; - ZI116 = lex_state->lx.start; - ZI117 = lex_state->lx.end; + ZI115 = lex_state->lx.start; + ZI116 = lex_state->lx.end; + (void) ZI115; (void) ZI116; - (void) ZI117; -#line 1255 "src/libre/dialect/pcre/parser.c" +#line 1242 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: NOESC */ ADVANCE_LEXER; @@ -1259,12 +1246,12 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err break; case (TOK_OCT): { + t_pos ZI117; t_pos ZI118; - t_pos ZI119; /* BEGINNING OF EXTRACT: OCT */ { -#line 498 "src/libre/parser.act" +#line 484 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -1273,11 +1260,11 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err assert(0 == strncmp(lex_state->buf.a, "\\", 1)); assert(strlen(lex_state->buf.a) >= 2); - ZI118 = lex_state->lx.start; - ZI119 = lex_state->lx.end; + ZI117 = lex_state->lx.start; + ZI118 = lex_state->lx.end; + (void) ZI117; (void) ZI118; - (void) ZI119; errno = 0; @@ -1307,7 +1294,7 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err ZIc = (char) (unsigned char) u; -#line 1311 "src/libre/dialect/pcre/parser.c" +#line 1298 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: OCT */ ADVANCE_LEXER; @@ -1315,12 +1302,12 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err break; case (TOK_UNSUPPORTED): { + t_pos ZI125; t_pos ZI126; - t_pos ZI127; /* BEGINNING OF EXTRACT: UNSUPPORTED */ { -#line 445 "src/libre/parser.act" +#line 426 "src/libre/parser.act" /* handle \1-\9 back references */ if (lex_state->buf.a[0] == '\\' && lex_state->buf.a[1] != '\0' && lex_state->buf.a[2] == '\0') { @@ -1330,26 +1317,26 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err ZIc = lex_state->buf.a[0]; } - ZI126 = lex_state->lx.start; - ZI127 = lex_state->lx.end; + ZI125 = lex_state->lx.start; + ZI126 = lex_state->lx.end; + (void) ZI125; (void) ZI126; - (void) ZI127; -#line 1340 "src/libre/dialect/pcre/parser.c" +#line 1327 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: UNSUPPORTED */ ADVANCE_LEXER; /* BEGINNING OF ACTION: err-unsupported */ { -#line 774 "src/libre/parser.act" +#line 760 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EUNSUPPORTED; } goto ZL1; -#line 1353 "src/libre/dialect/pcre/parser.c" +#line 1340 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-unsupported */ } @@ -1358,17 +1345,17 @@ p_expr_C_Cliteral(flags flags, lex_state lex_state, act_state act_state, err err goto ZL1; } } - /* END OF INLINE: 112 */ + /* END OF INLINE: 111 */ /* BEGINNING OF ACTION: ast-make-literal */ { -#line 908 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode) == NULL) { goto ZL1; } -#line 1372 "src/libre/dialect/pcre/parser.c" +#line 1359 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -1388,30 +1375,30 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ switch (CURRENT_TERMINAL) { case (TOK_CHAR): { - t_char ZI309; - t_pos ZI310; - t_pos ZI311; + t_char ZI306; + t_pos ZI307; + t_pos ZI308; /* BEGINNING OF EXTRACT: CHAR */ { -#line 589 "src/libre/parser.act" +#line 575 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); - ZI310 = lex_state->lx.start; - ZI311 = lex_state->lx.end; + ZI307 = lex_state->lx.start; + ZI308 = lex_state->lx.end; - (void) ZI310; - (void) ZI311; + (void) ZI307; + (void) ZI308; - ZI309 = lex_state->buf.a[0]; + ZI306 = lex_state->buf.a[0]; -#line 1411 "src/libre/dialect/pcre/parser.c" +#line 1398 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; - p_320 (flags, lex_state, act_state, err, &ZI309, &ZI310, &ZInode); + p_317 (flags, lex_state, act_state, err, &ZI306, &ZI307, &ZInode); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -1420,30 +1407,35 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ break; case (TOK_CONTROL): { - t_char ZI313; - t_pos ZI314; - t_pos ZI315; + t_char ZI310; + t_pos ZI311; + t_pos ZI312; /* BEGINNING OF EXTRACT: CONTROL */ { -#line 461 "src/libre/parser.act" +#line 442 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] == 'c'); assert(lex_state->buf.a[2] != '\0'); assert(lex_state->buf.a[3] == '\0'); - ZI313 = lex_state->buf.a[2]; - /* from http://www.pcre.org/current/doc/html/pcre2pattern.html#SEC5 + ZI310 = lex_state->buf.a[2]; + + /* + * From http://www.pcre.org/current/doc/html/pcre2pattern.html#SEC5 * - * The precise effect of \cx on ASCII characters is as follows: if x is a lower case letter, it is converted to - * upper case. Then bit 6 of the character (hex 40) is inverted. Thus \cA to \cZ become hex 01 to hex 1A (A is - * 41, Z is 5A), but \c{ becomes hex 3B ({ is 7B), and \c; becomes hex 7B (; is 3B). If the code unit following - * \c has a value less than 32 or greater than 126, a compile-time error occurs. + * The precise effect of \cx on ASCII characters is as follows: + * if x is a lower case letter, it is converted to upper case. + * Then bit 6 of the character (hex 40) is inverted. + * Thus \cA to \cZ become hex 01 to hex 1A (A is 41, Z is 5A), + * but \c{ becomes hex 3B ({ is 7B), and \c; becomes hex 7B (; is 3B). + * If the code unit following \c has a value less than 32 + * or greater than 126, a compile-time error occurs. */ { - unsigned char cc = (unsigned char)ZI313; + unsigned char cc = (unsigned char) ZI310; if (cc > 126 || cc < 32) { goto ZL1; } @@ -1454,32 +1446,32 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ cc ^= 0x40; - ZI313 = cc; + ZI310 = cc; } - ZI314 = lex_state->lx.start; - ZI315 = lex_state->lx.end; + ZI311 = lex_state->lx.start; + ZI312 = lex_state->lx.end; - (void) ZI314; - (void) ZI315; + (void) ZI311; + (void) ZI312; -#line 1467 "src/libre/dialect/pcre/parser.c" +#line 1459 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CONTROL */ ADVANCE_LEXER; /* BEGINNING OF ACTION: err-unsupported */ { -#line 774 "src/libre/parser.act" +#line 760 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EUNSUPPORTED; } goto ZL1; -#line 1480 "src/libre/dialect/pcre/parser.c" +#line 1472 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-unsupported */ - p_320 (flags, lex_state, act_state, err, &ZI313, &ZI314, &ZInode); + p_317 (flags, lex_state, act_state, err, &ZI310, &ZI311, &ZInode); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -1488,43 +1480,43 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ break; case (TOK_ESC): { - t_char ZI297; - t_pos ZI298; - t_pos ZI299; + t_char ZI294; + t_pos ZI295; + t_pos ZI296; /* BEGINNING OF EXTRACT: ESC */ { -#line 405 "src/libre/parser.act" +#line 386 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); assert(lex_state->buf.a[2] == '\0'); - ZI297 = lex_state->buf.a[1]; - - switch (ZI297) { - case 'a': ZI297 = '\a'; break; - case 'b': ZI297 = '\b'; break; - case 'e': ZI297 = '\033'; break; - case 'f': ZI297 = '\f'; break; - case 'n': ZI297 = '\n'; break; - case 'r': ZI297 = '\r'; break; - case 't': ZI297 = '\t'; break; - case 'v': ZI297 = '\v'; break; + ZI294 = lex_state->buf.a[1]; + + switch (ZI294) { + case 'a': ZI294 = '\a'; break; + case 'b': ZI294 = '\b'; break; + case 'e': ZI294 = '\033'; break; + case 'f': ZI294 = '\f'; break; + case 'n': ZI294 = '\n'; break; + case 'r': ZI294 = '\r'; break; + case 't': ZI294 = '\t'; break; + case 'v': ZI294 = '\v'; break; default: break; } - ZI298 = lex_state->lx.start; - ZI299 = lex_state->lx.end; + ZI295 = lex_state->lx.start; + ZI296 = lex_state->lx.end; - (void) ZI298; - (void) ZI299; + (void) ZI295; + (void) ZI296; -#line 1524 "src/libre/dialect/pcre/parser.c" +#line 1516 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: ESC */ ADVANCE_LEXER; - p_320 (flags, lex_state, act_state, err, &ZI297, &ZI298, &ZInode); + p_317 (flags, lex_state, act_state, err, &ZI294, &ZI295, &ZInode); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -1533,13 +1525,13 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ break; case (TOK_HEX): { - t_char ZI305; - t_pos ZI306; - t_pos ZI307; + t_char ZI302; + t_pos ZI303; + t_pos ZI304; /* BEGINNING OF EXTRACT: HEX */ { -#line 541 "src/libre/parser.act" +#line 527 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -1548,11 +1540,11 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ assert(0 == strncmp(lex_state->buf.a, "\\x", 2)); assert(strlen(lex_state->buf.a) >= 2); /* pcre allows \x without a suffix */ - ZI306 = lex_state->lx.start; - ZI307 = lex_state->lx.end; + ZI303 = lex_state->lx.start; + ZI304 = lex_state->lx.end; - (void) ZI306; - (void) ZI307; + (void) ZI303; + (void) ZI304; errno = 0; @@ -1585,13 +1577,13 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ goto ZL1; } - ZI305 = (char) (unsigned char) u; + ZI302 = (char) (unsigned char) u; -#line 1591 "src/libre/dialect/pcre/parser.c" +#line 1583 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: HEX */ ADVANCE_LEXER; - p_320 (flags, lex_state, act_state, err, &ZI305, &ZI306, &ZInode); + p_317 (flags, lex_state, act_state, err, &ZI302, &ZI303, &ZInode); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -1600,31 +1592,31 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ break; case (TOK_NAMED__CLASS): { - t_ast__class__id ZI293; - t_pos ZI294; - t_pos ZI295; + t_ast__class__id ZI290; + t_pos ZI291; + t_pos ZI292; /* BEGINNING OF EXTRACT: NAMED_CLASS */ { -#line 661 "src/libre/parser.act" +#line 647 "src/libre/parser.act" - ZI293 = DIALECT_CLASS(lex_state->buf.a); - if (ZI293 == NULL) { + ZI290 = DIALECT_CLASS(lex_state->buf.a); + if (ZI290 == NULL) { /* syntax error -- unrecognized class */ goto ZL1; } - ZI294 = lex_state->lx.start; - ZI295 = lex_state->lx.end; + ZI291 = lex_state->lx.start; + ZI292 = lex_state->lx.end; - (void) ZI294; - (void) ZI295; + (void) ZI291; + (void) ZI292; -#line 1624 "src/libre/dialect/pcre/parser.c" +#line 1616 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: NAMED_CLASS */ ADVANCE_LEXER; - p_296 (flags, lex_state, act_state, err, &ZI293, &ZI294, &ZInode); + p_293 (flags, lex_state, act_state, err, &ZI290, &ZI291, &ZInode); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -1634,12 +1626,12 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ case (TOK_NOESC): { t_char ZIc; + t_pos ZI134; t_pos ZI135; - t_pos ZI136; /* BEGINNING OF EXTRACT: NOESC */ { -#line 431 "src/libre/parser.act" +#line 412 "src/libre/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -1647,39 +1639,39 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ ZIc = lex_state->buf.a[1]; - ZI135 = lex_state->lx.start; - ZI136 = lex_state->lx.end; + ZI134 = lex_state->lx.start; + ZI135 = lex_state->lx.end; + (void) ZI134; (void) ZI135; - (void) ZI136; -#line 1657 "src/libre/dialect/pcre/parser.c" +#line 1649 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: NOESC */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 908 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode) == NULL) { goto ZL1; } -#line 1670 "src/libre/dialect/pcre/parser.c" +#line 1662 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ } break; case (TOK_OCT): { - t_char ZI301; - t_pos ZI302; - t_pos ZI303; + t_char ZI298; + t_pos ZI299; + t_pos ZI300; /* BEGINNING OF EXTRACT: OCT */ { -#line 498 "src/libre/parser.act" +#line 484 "src/libre/parser.act" unsigned long u; char *s, *e; @@ -1688,11 +1680,11 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ assert(0 == strncmp(lex_state->buf.a, "\\", 1)); assert(strlen(lex_state->buf.a) >= 2); - ZI302 = lex_state->lx.start; - ZI303 = lex_state->lx.end; + ZI299 = lex_state->lx.start; + ZI300 = lex_state->lx.end; - (void) ZI302; - (void) ZI303; + (void) ZI299; + (void) ZI300; errno = 0; @@ -1720,13 +1712,13 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ goto ZL1; } - ZI301 = (char) (unsigned char) u; + ZI298 = (char) (unsigned char) u; -#line 1726 "src/libre/dialect/pcre/parser.c" +#line 1718 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: OCT */ ADVANCE_LEXER; - p_320 (flags, lex_state, act_state, err, &ZI301, &ZI302, &ZInode); + p_317 (flags, lex_state, act_state, err, &ZI298, &ZI299, &ZInode); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -1735,45 +1727,45 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ break; case (TOK_UNSUPPORTED): { - t_char ZI317; - t_pos ZI318; - t_pos ZI319; + t_char ZI314; + t_pos ZI315; + t_pos ZI316; /* BEGINNING OF EXTRACT: UNSUPPORTED */ { -#line 445 "src/libre/parser.act" +#line 426 "src/libre/parser.act" /* handle \1-\9 back references */ if (lex_state->buf.a[0] == '\\' && lex_state->buf.a[1] != '\0' && lex_state->buf.a[2] == '\0') { - ZI317 = lex_state->buf.a[1]; + ZI314 = lex_state->buf.a[1]; } else { - ZI317 = lex_state->buf.a[0]; + ZI314 = lex_state->buf.a[0]; } - ZI318 = lex_state->lx.start; - ZI319 = lex_state->lx.end; + ZI315 = lex_state->lx.start; + ZI316 = lex_state->lx.end; - (void) ZI318; - (void) ZI319; + (void) ZI315; + (void) ZI316; -#line 1761 "src/libre/dialect/pcre/parser.c" +#line 1753 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: UNSUPPORTED */ ADVANCE_LEXER; /* BEGINNING OF ACTION: err-unsupported */ { -#line 774 "src/libre/parser.act" +#line 760 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EUNSUPPORTED; } goto ZL1; -#line 1774 "src/libre/dialect/pcre/parser.c" +#line 1766 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-unsupported */ - p_320 (flags, lex_state, act_state, err, &ZI317, &ZI318, &ZInode); + p_317 (flags, lex_state, act_state, err, &ZI314, &ZI315, &ZInode); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -1809,14 +1801,14 @@ p_expr_C_Ccomment(flags flags, lex_state lex_state, act_state act_state, err err ADVANCE_LEXER; /* BEGINNING OF ACTION: err-invalid-comment */ { -#line 697 "src/libre/parser.act" +#line 683 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EBADCOMMENT; } goto ZL1; -#line 1820 "src/libre/dialect/pcre/parser.c" +#line 1812 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-invalid-comment */ } @@ -1843,7 +1835,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hclass(flags fl case (TOK_NAMED__CLASS): /* BEGINNING OF EXTRACT: NAMED_CLASS */ { -#line 661 "src/libre/parser.act" +#line 647 "src/libre/parser.act" ZIid = DIALECT_CLASS(lex_state->buf.a); if (ZIid == NULL) { @@ -1857,7 +1849,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hclass(flags fl (void) ZIstart; (void) ZIend; -#line 1861 "src/libre/dialect/pcre/parser.c" +#line 1853 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: NAMED_CLASS */ break; @@ -1867,12 +1859,12 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_C_Crange_Hendpoint_Hclass(flags fl ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-range-endpoint-class */ { -#line 878 "src/libre/parser.act" +#line 844 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_NAMED; (ZIr).u.named.class = (ZIid); -#line 1876 "src/libre/dialect/pcre/parser.c" +#line 1868 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-range-endpoint-class */ } @@ -1897,43 +1889,42 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state { t_pos ZIstart; t_ast__expr ZItmp; - t_pos ZIend; - /* BEGINNING OF INLINE: 181 */ + /* BEGINNING OF INLINE: 180 */ { switch (CURRENT_TERMINAL) { case (TOK_OPENGROUP): { - t_pos ZI182; + t_pos ZI181; /* BEGINNING OF EXTRACT: OPENGROUP */ { -#line 337 "src/libre/parser.act" +#line 318 "src/libre/parser.act" ZIstart = lex_state->lx.start; - ZI182 = lex_state->lx.end; + ZI181 = lex_state->lx.end; (void) ZIstart; - (void) ZI182; + (void) ZI181; -#line 1920 "src/libre/dialect/pcre/parser.c" +#line 1911 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: OPENGROUP */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 901 "src/libre/parser.act" +#line 867 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 1933 "src/libre/dialect/pcre/parser.c" +#line 1924 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZInode; - p_195 (flags, lex_state, act_state, err, &ZItmp); + p_194 (flags, lex_state, act_state, err, &ZItmp); p_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms (flags, lex_state, act_state, err, ZItmp); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; @@ -1943,60 +1934,60 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state break; case (TOK_OPENGROUPCB): { - t_pos ZI201; + t_pos ZI200; t_char ZIcbrak; t_ast__expr ZInode1; /* BEGINNING OF EXTRACT: OPENGROUPCB */ { -#line 353 "src/libre/parser.act" +#line 334 "src/libre/parser.act" ZIstart = lex_state->lx.start; - ZI201 = lex_state->lx.end; + ZI200 = lex_state->lx.end; (void) ZIstart; - (void) ZI201; + (void) ZI200; -#line 1961 "src/libre/dialect/pcre/parser.c" +#line 1952 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: OPENGROUPCB */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 901 "src/libre/parser.act" +#line 867 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 1974 "src/libre/dialect/pcre/parser.c" +#line 1965 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZInode; /* BEGINNING OF ACTION: make-literal-cbrak */ { -#line 919 "src/libre/parser.act" +#line 885 "src/libre/parser.act" (ZIcbrak) = ']'; -#line 1984 "src/libre/dialect/pcre/parser.c" +#line 1975 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: make-literal-cbrak */ - p_210 (flags, lex_state, act_state, err, &ZIstart, &ZIcbrak, &ZInode1); + p_209 (flags, lex_state, act_state, err, &ZIstart, &ZIcbrak, &ZInode1); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1084 "src/libre/parser.act" +#line 1046 "src/libre/parser.act" if (!ast_add_expr_alt((ZItmp), (ZInode1))) { goto ZL1; } -#line 2000 "src/libre/dialect/pcre/parser.c" +#line 1991 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ p_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms (flags, lex_state, act_state, err, ZItmp); @@ -2008,38 +1999,38 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state break; case (TOK_OPENGROUPINV): { - t_pos ZI193; + t_pos ZI192; /* BEGINNING OF EXTRACT: OPENGROUPINV */ { -#line 345 "src/libre/parser.act" +#line 326 "src/libre/parser.act" ZIstart = lex_state->lx.start; - ZI193 = lex_state->lx.end; + ZI192 = lex_state->lx.end; (void) ZIstart; - (void) ZI193; + (void) ZI192; -#line 2024 "src/libre/dialect/pcre/parser.c" +#line 2015 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: OPENGROUPINV */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 901 "src/libre/parser.act" +#line 867 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 2037 "src/libre/dialect/pcre/parser.c" +#line 2028 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZInode; /* BEGINNING OF ACTION: ast-make-invert */ { -#line 1000 "src/libre/parser.act" +#line 966 "src/libre/parser.act" struct ast_expr *any; @@ -2077,10 +2068,10 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL1; } -#line 2081 "src/libre/dialect/pcre/parser.c" +#line 2072 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-invert */ - p_195 (flags, lex_state, act_state, err, &ZItmp); + p_194 (flags, lex_state, act_state, err, &ZItmp); p_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms (flags, lex_state, act_state, err, ZItmp); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; @@ -2090,40 +2081,40 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state break; case (TOK_OPENGROUPINVCB): { - t_pos ZI208; + t_pos ZI207; t_char ZIcbrak; t_ast__expr ZInode1; /* BEGINNING OF EXTRACT: OPENGROUPINVCB */ { -#line 361 "src/libre/parser.act" +#line 342 "src/libre/parser.act" ZIstart = lex_state->lx.start; - ZI208 = lex_state->lx.end; + ZI207 = lex_state->lx.end; (void) ZIstart; - (void) ZI208; + (void) ZI207; -#line 2108 "src/libre/dialect/pcre/parser.c" +#line 2099 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: OPENGROUPINVCB */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 901 "src/libre/parser.act" +#line 867 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 2121 "src/libre/dialect/pcre/parser.c" +#line 2112 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZInode; /* BEGINNING OF ACTION: ast-make-invert */ { -#line 1000 "src/libre/parser.act" +#line 966 "src/libre/parser.act" struct ast_expr *any; @@ -2161,32 +2152,32 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL1; } -#line 2165 "src/libre/dialect/pcre/parser.c" +#line 2156 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-invert */ /* BEGINNING OF ACTION: make-literal-cbrak */ { -#line 919 "src/libre/parser.act" +#line 885 "src/libre/parser.act" (ZIcbrak) = ']'; -#line 2174 "src/libre/dialect/pcre/parser.c" +#line 2165 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: make-literal-cbrak */ - p_210 (flags, lex_state, act_state, err, &ZIstart, &ZIcbrak, &ZInode1); + p_209 (flags, lex_state, act_state, err, &ZIstart, &ZIcbrak, &ZInode1); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1084 "src/libre/parser.act" +#line 1046 "src/libre/parser.act" if (!ast_add_expr_alt((ZItmp), (ZInode1))) { goto ZL1; } -#line 2190 "src/libre/dialect/pcre/parser.c" +#line 2181 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ p_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms (flags, lex_state, act_state, err, ZItmp); @@ -2200,39 +2191,40 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state goto ZL1; } } - /* END OF INLINE: 181 */ - /* BEGINNING OF INLINE: 214 */ + /* END OF INLINE: 180 */ + /* BEGINNING OF INLINE: 213 */ { switch (CURRENT_TERMINAL) { case (TOK_CLOSEGROUP): { - t_char ZI215; - t_pos ZI216; + t_char ZI214; + t_pos ZI215; + t_pos ZIend; /* BEGINNING OF EXTRACT: CLOSEGROUP */ { -#line 369 "src/libre/parser.act" +#line 350 "src/libre/parser.act" - ZI215 = ']'; - ZI216 = lex_state->lx.start; + ZI214 = ']'; + ZI215 = lex_state->lx.start; ZIend = lex_state->lx.end; + (void) ZI214; (void) ZI215; - (void) ZI216; (void) ZIend; -#line 2225 "src/libre/dialect/pcre/parser.c" +#line 2217 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CLOSEGROUP */ ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-group */ { -#line 781 "src/libre/parser.act" +#line 767 "src/libre/parser.act" mark(&act_state->groupstart, &(ZIstart)); mark(&act_state->groupend, &(ZIend)); -#line 2236 "src/libre/dialect/pcre/parser.c" +#line 2228 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: mark-group */ } @@ -2240,56 +2232,57 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state case (TOK_CLOSEGROUPRANGE): { t_char ZIcrange; - t_pos ZI218; + t_pos ZI217; + t_pos ZIend; t_ast__expr ZIrange; /* BEGINNING OF EXTRACT: CLOSEGROUPRANGE */ { -#line 379 "src/libre/parser.act" +#line 360 "src/libre/parser.act" ZIcrange = '-'; - ZI218 = lex_state->lx.start; + ZI217 = lex_state->lx.start; ZIend = lex_state->lx.end; (void) ZIcrange; - (void) ZI218; + (void) ZI217; (void) ZIend; -#line 2259 "src/libre/dialect/pcre/parser.c" +#line 2252 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CLOSEGROUPRANGE */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 908 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZIrange) = ast_make_expr_literal(act_state->poolp, *flags, (ZIcrange)); if ((ZIrange) == NULL) { goto ZL4; } -#line 2272 "src/libre/dialect/pcre/parser.c" +#line 2265 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1084 "src/libre/parser.act" +#line 1046 "src/libre/parser.act" if (!ast_add_expr_alt((ZItmp), (ZIrange))) { goto ZL4; } -#line 2283 "src/libre/dialect/pcre/parser.c" +#line 2276 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF ACTION: mark-group */ { -#line 781 "src/libre/parser.act" +#line 767 "src/libre/parser.act" mark(&act_state->groupstart, &(ZIstart)); mark(&act_state->groupend, &(ZIend)); -#line 2293 "src/libre/dialect/pcre/parser.c" +#line 2286 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: mark-group */ } @@ -2302,41 +2295,20 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state { /* BEGINNING OF ACTION: err-expected-closegroup */ { -#line 739 "src/libre/parser.act" +#line 725 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCLOSEGROUP; } goto ZL1; -#line 2313 "src/libre/dialect/pcre/parser.c" +#line 2306 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-closegroup */ - ZIend = ZIstart; } ZL3:; } - /* END OF INLINE: 214 */ - /* BEGINNING OF ACTION: mark-expr */ - { -#line 796 "src/libre/parser.act" - - struct ast_pos ast_start, ast_end; - - AST_POS_OF_LX_POS(ast_start, (ZIstart)); - AST_POS_OF_LX_POS(ast_end, (ZIend)); - - mark(&act_state->groupstart, &(ZIstart)); - mark(&act_state->groupend, &(ZIend)); - -/* TODO: reinstate this, applies to an expr node in general - (ZItmp)->u.class.start = ast_start; - (ZItmp)->u.class.end = ast_end; -*/ - -#line 2338 "src/libre/dialect/pcre/parser.c" - } - /* END OF ACTION: mark-expr */ + /* END OF INLINE: 213 */ } goto ZL0; ZL1:; @@ -2356,38 +2328,38 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_Hend(flags flags, lex_state lex_st return; } { - /* BEGINNING OF INLINE: 162 */ + /* BEGINNING OF INLINE: 161 */ { switch (CURRENT_TERMINAL) { case (TOK_RANGE): { t_char ZIc; - t_pos ZI164; + t_pos ZI163; /* BEGINNING OF EXTRACT: RANGE */ { -#line 327 "src/libre/parser.act" +#line 308 "src/libre/parser.act" ZIc = '-'; - ZI164 = lex_state->lx.start; + ZI163 = lex_state->lx.start; ZIend = lex_state->lx.end; (void) ZIc; - (void) ZI164; + (void) ZI163; (void) ZIend; -#line 2380 "src/libre/dialect/pcre/parser.c" +#line 2352 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 873 "src/libre/parser.act" +#line 839 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_LITERAL; - (ZIr).u.literal.c = (unsigned char)(ZIc); + (ZIr).u.literal.c = (unsigned char) (ZIc); -#line 2391 "src/libre/dialect/pcre/parser.c" +#line 2363 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ } @@ -2395,9 +2367,9 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_Hend(flags flags, lex_state lex_st case (TOK_NAMED__CLASS): case (TOK_ESC): case (TOK_CONTROL): case (TOK_OCT): case (TOK_HEX): case (TOK_CHAR): case (TOK_UNSUPPORTED): { - t_pos ZI163; + t_pos ZI162; - p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint (flags, lex_state, act_state, err, &ZIr, &ZI163, &ZIend); + p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint (flags, lex_state, act_state, err, &ZIr, &ZI162, &ZIend); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -2408,7 +2380,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_Hend(flags flags, lex_state lex_st goto ZL1; } } - /* END OF INLINE: 162 */ + /* END OF INLINE: 161 */ } goto ZL0; ZL1:; @@ -2420,98 +2392,25 @@ ZL0:; } static void -p_expr_C_Cpiece(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__expr *ZOnode) +p_317(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI314, t_pos *ZI315, t_ast__expr *ZOnode) { t_ast__expr ZInode; - if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { - return; - } - { - t_ast__expr ZIe; - - p_expr_C_Cpiece_C_Catom (flags, lex_state, act_state, err, &ZIe); - /* BEGINNING OF INLINE: 267 */ + switch (CURRENT_TERMINAL) { + default: { - switch (CURRENT_TERMINAL) { - case (TOK_OPT): case (TOK_PLUS): case (TOK_STAR): case (TOK_OPENCOUNT): - { - p_expr_C_Cpiece_C_Clist_Hof_Hcounts (flags, lex_state, act_state, err, ZIe, &ZInode); - if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { - RESTORE_LEXER; - goto ZL1; - } - } - break; - default: - { - t_ast__count ZIc; - - /* BEGINNING OF ACTION: count-one */ - { -#line 849 "src/libre/parser.act" - - (ZIc) = ast_make_count(1, NULL, 1, NULL); - -#line 2457 "src/libre/dialect/pcre/parser.c" - } - /* END OF ACTION: count-one */ - /* BEGINNING OF ACTION: ast-make-piece */ - { -#line 931 "src/libre/parser.act" + /* BEGINNING OF ACTION: ast-make-literal */ + { +#line 874 "src/libre/parser.act" - if ((ZIc).min == 0 && (ZIc).max == 0) { - (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); - } else if ((ZIc).min == 1 && (ZIc).max == 1) { - (ZInode) = (ZIe); - } else { - (ZInode) = ast_make_expr_repeat(act_state->poolp, *flags, (ZIe), (ZIc)); - } + (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (*ZI314)); if ((ZInode) == NULL) { - err->e = RE_EXEOF; goto ZL1; } -#line 2476 "src/libre/dialect/pcre/parser.c" - } - /* END OF ACTION: ast-make-piece */ - } - break; - case (ERROR_TERMINAL): - RESTORE_LEXER; - goto ZL1; +#line 2412 "src/libre/dialect/pcre/parser.c" } - } - /* END OF INLINE: 267 */ - } - goto ZL0; -ZL1:; - SAVE_LEXER ((ERROR_TERMINAL)); - return; -ZL0:; - *ZOnode = ZInode; -} - -static void -p_320(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI317, t_pos *ZI318, t_ast__expr *ZOnode) -{ - t_ast__expr ZInode; - - switch (CURRENT_TERMINAL) { - default: - { - /* BEGINNING OF ACTION: ast-make-literal */ - { -#line 908 "src/libre/parser.act" - - (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (*ZI317)); - if ((ZInode) == NULL) { - goto ZL1; - } - -#line 2513 "src/libre/dialect/pcre/parser.c" - } - /* END OF ACTION: ast-make-literal */ + /* END OF ACTION: ast-make-literal */ } break; case (TOK_RANGE): @@ -2522,15 +2421,15 @@ p_320(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 873 "src/libre/parser.act" +#line 839 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_LITERAL; - (ZIlower).u.literal.c = (unsigned char)(*ZI317); + (ZIlower).u.literal.c = (unsigned char) (*ZI314); -#line 2531 "src/libre/dialect/pcre/parser.c" +#line 2430 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ - p_169 (flags, lex_state, act_state, err); + p_168 (flags, lex_state, act_state, err); p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_Hend (flags, lex_state, act_state, err, &ZIupper, &ZIend); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; @@ -2538,24 +2437,20 @@ p_320(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI } /* BEGINNING OF ACTION: mark-range */ { -#line 786 "src/libre/parser.act" +#line 772 "src/libre/parser.act" - mark(&act_state->rangestart, &(*ZI318)); + mark(&act_state->rangestart, &(*ZI315)); mark(&act_state->rangeend, &(ZIend)); -#line 2547 "src/libre/dialect/pcre/parser.c" +#line 2446 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: mark-range */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1038 "src/libre/parser.act" +#line 1004 "src/libre/parser.act" - struct ast_pos ast_start, ast_end; unsigned char lower, upper; - AST_POS_OF_LX_POS(ast_start, (*ZI318)); - AST_POS_OF_LX_POS(ast_end, (ZIend)); - if ((ZIlower).type != AST_ENDPOINT_LITERAL || (ZIupper).type != AST_ENDPOINT_LITERAL) { err->e = RE_EUNSUPPORTED; @@ -2576,12 +2471,12 @@ p_320(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI goto ZL1; } - (ZInode) = ast_make_expr_range(act_state->poolp, *flags, &(ZIlower), ast_start, &(ZIupper), ast_end); + (ZInode) = ast_make_expr_range(act_state->poolp, *flags, &(ZIlower), &(ZIupper)); if ((ZInode) == NULL) { goto ZL1; } -#line 2585 "src/libre/dialect/pcre/parser.c" +#line 2480 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -2598,7 +2493,7 @@ ZL0:; } static void -p_expr(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__expr *ZOnode) +p_expr_C_Cpiece(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__expr *ZOnode) { t_ast__expr ZInode; @@ -2606,271 +2501,123 @@ p_expr(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__ex return; } { - /* BEGINNING OF ACTION: ast-make-alt */ - { -#line 901 "src/libre/parser.act" - - (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); - if ((ZInode) == NULL) { - goto ZL1; - } - -#line 2619 "src/libre/dialect/pcre/parser.c" - } - /* END OF ACTION: ast-make-alt */ - p_expr_C_Clist_Hof_Halts (flags, lex_state, act_state, err, ZInode); - if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { - RESTORE_LEXER; - goto ZL1; - } - } - goto ZL0; -ZL1:; - { - /* BEGINNING OF ACTION: err-expected-alts */ - { -#line 725 "src/libre/parser.act" - - if (err->e == RE_ESUCCESS) { - err->e = RE_EXALTS; - } - goto ZL2; - -#line 2640 "src/libre/dialect/pcre/parser.c" - } - /* END OF ACTION: err-expected-alts */ - /* BEGINNING OF ACTION: ast-make-empty */ - { -#line 887 "src/libre/parser.act" - - (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); - if ((ZInode) == NULL) { - goto ZL2; - } - -#line 2652 "src/libre/dialect/pcre/parser.c" - } - /* END OF ACTION: ast-make-empty */ - } - goto ZL0; -ZL2:; - SAVE_LEXER ((ERROR_TERMINAL)); - return; -ZL0:; - *ZOnode = ZInode; -} + t_ast__expr ZIe; -static void -p_195(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__expr *ZItmp) -{ - switch (CURRENT_TERMINAL) { - case (TOK_RANGE): + p_expr_C_Cpiece_C_Catom (flags, lex_state, act_state, err, &ZIe); + /* BEGINNING OF INLINE: 265 */ { - t_char ZIc; - t_pos ZIrstart; - t_pos ZI196; - t_ast__expr ZInode1; - - /* BEGINNING OF EXTRACT: RANGE */ - { -#line 327 "src/libre/parser.act" - - ZIc = '-'; - ZIrstart = lex_state->lx.start; - ZI196 = lex_state->lx.end; + switch (CURRENT_TERMINAL) { + case (TOK_OPT): case (TOK_PLUS): case (TOK_STAR): case (TOK_OPENCOUNT): + { + p_expr_C_Cpiece_C_Clist_Hof_Hcounts (flags, lex_state, act_state, err, ZIe, &ZInode); + if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { + RESTORE_LEXER; + goto ZL1; + } + } + break; + default: + { + t_ast__count ZIc; - (void) ZIc; - (void) ZIrstart; - (void) ZI196; - -#line 2687 "src/libre/dialect/pcre/parser.c" - } - /* END OF EXTRACT: RANGE */ - ADVANCE_LEXER; - /* BEGINNING OF INLINE: 197 */ - { - switch (CURRENT_TERMINAL) { - default: + /* BEGINNING OF ACTION: count-one */ { - /* BEGINNING OF ACTION: ast-make-literal */ - { -#line 908 "src/libre/parser.act" +#line 820 "src/libre/parser.act" - (ZInode1) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); - if ((ZInode1) == NULL) { - goto ZL1; - } + (ZIc) = ast_make_count(1, 1); -#line 2705 "src/libre/dialect/pcre/parser.c" - } - /* END OF ACTION: ast-make-literal */ +#line 2530 "src/libre/dialect/pcre/parser.c" } - break; - case (TOK_RANGE): + /* END OF ACTION: count-one */ + /* BEGINNING OF ACTION: ast-make-piece */ { - t_endpoint ZIlower; - t_char ZI198; - t_pos ZI199; - t_pos ZI200; - t_endpoint ZIupper; - t_pos ZIend; - - /* BEGINNING OF ACTION: ast-range-endpoint-literal */ - { -#line 873 "src/libre/parser.act" - - (ZIlower).type = AST_ENDPOINT_LITERAL; - (ZIlower).u.literal.c = (unsigned char)(ZIc); - -#line 2726 "src/libre/dialect/pcre/parser.c" - } - /* END OF ACTION: ast-range-endpoint-literal */ - /* BEGINNING OF EXTRACT: RANGE */ - { -#line 327 "src/libre/parser.act" - - ZI198 = '-'; - ZI199 = lex_state->lx.start; - ZI200 = lex_state->lx.end; - - (void) ZI198; - (void) ZI199; - (void) ZI200; - -#line 2741 "src/libre/dialect/pcre/parser.c" - } - /* END OF EXTRACT: RANGE */ - ADVANCE_LEXER; - p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_Hend (flags, lex_state, act_state, err, &ZIupper, &ZIend); - if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { - RESTORE_LEXER; - goto ZL1; - } - /* BEGINNING OF ACTION: ast-make-range */ - { -#line 1038 "src/libre/parser.act" - - struct ast_pos ast_start, ast_end; - unsigned char lower, upper; +#line 897 "src/libre/parser.act" - AST_POS_OF_LX_POS(ast_start, (ZIrstart)); - AST_POS_OF_LX_POS(ast_end, (ZIend)); - - if ((ZIlower).type != AST_ENDPOINT_LITERAL || - (ZIupper).type != AST_ENDPOINT_LITERAL) { - err->e = RE_EUNSUPPORTED; - goto ZL1; - } - - lower = (ZIlower).u.literal.c; - upper = (ZIupper).u.literal.c; - - if (lower > upper) { - char a[5], b[5]; - - assert(sizeof err->set >= 1 + sizeof a + 1 + sizeof b + 1 + 1); - - sprintf(err->set, "%s-%s", - escchar(a, sizeof a, lower), escchar(b, sizeof b, upper)); - err->e = RE_ENEGRANGE; - goto ZL1; + if ((ZIc).min == 0 && (ZIc).max == 0) { + (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); + } else if ((ZIc).min == 1 && (ZIc).max == 1) { + (ZInode) = (ZIe); + } else { + (ZInode) = ast_make_expr_repeat(act_state->poolp, *flags, (ZIe), (ZIc)); } - - (ZInode1) = ast_make_expr_range(act_state->poolp, *flags, &(ZIlower), ast_start, &(ZIupper), ast_end); - if ((ZInode1) == NULL) { + if ((ZInode) == NULL) { + err->e = RE_EXEOF; goto ZL1; } -#line 2785 "src/libre/dialect/pcre/parser.c" - } - /* END OF ACTION: ast-make-range */ +#line 2549 "src/libre/dialect/pcre/parser.c" } - break; + /* END OF ACTION: ast-make-piece */ } + break; + case (ERROR_TERMINAL): + RESTORE_LEXER; + goto ZL1; } - /* END OF INLINE: 197 */ - /* BEGINNING OF ACTION: ast-add-alt */ - { -#line 1084 "src/libre/parser.act" - - if (!ast_add_expr_alt((*ZItmp), (ZInode1))) { - goto ZL1; - } - -#line 2801 "src/libre/dialect/pcre/parser.c" - } - /* END OF ACTION: ast-add-alt */ } - break; - case (ERROR_TERMINAL): - return; - default: - break; + /* END OF INLINE: 265 */ } - return; + goto ZL0; ZL1:; SAVE_LEXER ((ERROR_TERMINAL)); return; +ZL0:; + *ZOnode = ZInode; } static void -p_323(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI321, t_unsigned *ZIm, t_ast__count *ZOc) +p_320(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI318, t_unsigned *ZIm, t_ast__count *ZOc) { t_ast__count ZIc; switch (CURRENT_TERMINAL) { case (TOK_CLOSECOUNT): { - t_pos ZI258; + t_pos ZI256; t_pos ZIend; /* BEGINNING OF EXTRACT: CLOSECOUNT */ { -#line 397 "src/libre/parser.act" +#line 378 "src/libre/parser.act" - ZI258 = lex_state->lx.start; + ZI256 = lex_state->lx.start; ZIend = lex_state->lx.end; - (void) ZI258; + (void) ZI256; (void) ZIend; -#line 2838 "src/libre/dialect/pcre/parser.c" +#line 2590 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CLOSECOUNT */ ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-count */ { -#line 791 "src/libre/parser.act" +#line 777 "src/libre/parser.act" - mark(&act_state->countstart, &(*ZI321)); + mark(&act_state->countstart, &(*ZI318)); mark(&act_state->countend, &(ZIend)); -#line 2849 "src/libre/dialect/pcre/parser.c" +#line 2601 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-range */ { -#line 853 "src/libre/parser.act" - - struct ast_pos ast_start, ast_end; +#line 824 "src/libre/parser.act" if ((*ZIm) < (*ZIm)) { err->e = RE_ENEGCOUNT; err->m = (*ZIm); err->n = (*ZIm); - mark(&act_state->countstart, &(*ZI321)); + mark(&act_state->countstart, &(*ZI318)); mark(&act_state->countend, &(ZIend)); goto ZL1; } - AST_POS_OF_LX_POS(ast_start, (*ZI321)); - AST_POS_OF_LX_POS(ast_end, (ZIend)); - - (ZIc) = ast_make_count((*ZIm), &ast_start, (*ZIm), &ast_end); + (ZIc) = ast_make_count((*ZIm), (*ZIm)); -#line 2874 "src/libre/dialect/pcre/parser.c" +#line 2621 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-range */ } @@ -2878,7 +2625,7 @@ p_323(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 case (TOK_SEP): { ADVANCE_LEXER; - p_324 (flags, lex_state, act_state, err, ZI321, ZIm, &ZIc); + p_321 (flags, lex_state, act_state, err, ZI318, ZIm, &ZIc); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -2899,73 +2646,131 @@ ZL0:; } static void -p_324(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI321, t_unsigned *ZIm, t_ast__count *ZOc) +p_expr(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__expr *ZOnode) +{ + t_ast__expr ZInode; + + if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { + return; + } + { + /* BEGINNING OF ACTION: ast-make-alt */ + { +#line 867 "src/libre/parser.act" + + (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); + if ((ZInode) == NULL) { + goto ZL1; + } + +#line 2667 "src/libre/dialect/pcre/parser.c" + } + /* END OF ACTION: ast-make-alt */ + p_expr_C_Clist_Hof_Halts (flags, lex_state, act_state, err, ZInode); + if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { + RESTORE_LEXER; + goto ZL1; + } + } + goto ZL0; +ZL1:; + { + /* BEGINNING OF ACTION: err-expected-alts */ + { +#line 711 "src/libre/parser.act" + + if (err->e == RE_ESUCCESS) { + err->e = RE_EXALTS; + } + goto ZL2; + +#line 2688 "src/libre/dialect/pcre/parser.c" + } + /* END OF ACTION: err-expected-alts */ + /* BEGINNING OF ACTION: ast-make-empty */ + { +#line 853 "src/libre/parser.act" + + (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); + if ((ZInode) == NULL) { + goto ZL2; + } + +#line 2700 "src/libre/dialect/pcre/parser.c" + } + /* END OF ACTION: ast-make-empty */ + } + goto ZL0; +ZL2:; + SAVE_LEXER ((ERROR_TERMINAL)); + return; +ZL0:; + *ZOnode = ZInode; +} + +static void +p_321(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI318, t_unsigned *ZIm, t_ast__count *ZOc) { t_ast__count ZIc; switch (CURRENT_TERMINAL) { case (TOK_CLOSECOUNT): { - t_pos ZI263; + t_pos ZI261; t_pos ZIend; t_unsigned ZIn; /* BEGINNING OF EXTRACT: CLOSECOUNT */ { -#line 397 "src/libre/parser.act" +#line 378 "src/libre/parser.act" - ZI263 = lex_state->lx.start; + ZI261 = lex_state->lx.start; ZIend = lex_state->lx.end; - (void) ZI263; + (void) ZI261; (void) ZIend; -#line 2924 "src/libre/dialect/pcre/parser.c" +#line 2734 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CLOSECOUNT */ ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-count */ { -#line 791 "src/libre/parser.act" +#line 777 "src/libre/parser.act" - mark(&act_state->countstart, &(*ZI321)); + mark(&act_state->countstart, &(*ZI318)); mark(&act_state->countend, &(ZIend)); -#line 2935 "src/libre/dialect/pcre/parser.c" +#line 2745 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-unbounded */ { -#line 833 "src/libre/parser.act" +#line 804 "src/libre/parser.act" (ZIn) = AST_COUNT_UNBOUNDED; -#line 2944 "src/libre/dialect/pcre/parser.c" +#line 2754 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-unbounded */ /* BEGINNING OF ACTION: count-range */ { -#line 853 "src/libre/parser.act" - - struct ast_pos ast_start, ast_end; +#line 824 "src/libre/parser.act" if ((ZIn) < (*ZIm)) { err->e = RE_ENEGCOUNT; err->m = (*ZIm); err->n = (ZIn); - mark(&act_state->countstart, &(*ZI321)); + mark(&act_state->countstart, &(*ZI318)); mark(&act_state->countend, &(ZIend)); goto ZL1; } - AST_POS_OF_LX_POS(ast_start, (*ZI321)); - AST_POS_OF_LX_POS(ast_end, (ZIend)); - - (ZIc) = ast_make_count((*ZIm), &ast_start, (ZIn), &ast_end); + (ZIc) = ast_make_count((*ZIm), (ZIn)); -#line 2969 "src/libre/dialect/pcre/parser.c" +#line 2774 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-range */ } @@ -2973,12 +2778,12 @@ p_324(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 case (TOK_COUNT): { t_unsigned ZIn; - t_pos ZI261; + t_pos ZI259; t_pos ZIend; /* BEGINNING OF EXTRACT: COUNT */ { -#line 641 "src/libre/parser.act" +#line 627 "src/libre/parser.act" unsigned long u; char *e; @@ -2998,7 +2803,7 @@ p_324(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 ZIn = (unsigned int) u; -#line 3002 "src/libre/dialect/pcre/parser.c" +#line 2807 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: COUNT */ ADVANCE_LEXER; @@ -3006,15 +2811,15 @@ p_324(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 case (TOK_CLOSECOUNT): /* BEGINNING OF EXTRACT: CLOSECOUNT */ { -#line 397 "src/libre/parser.act" +#line 378 "src/libre/parser.act" - ZI261 = lex_state->lx.start; + ZI259 = lex_state->lx.start; ZIend = lex_state->lx.end; - (void) ZI261; + (void) ZI259; (void) ZIend; -#line 3018 "src/libre/dialect/pcre/parser.c" +#line 2823 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: CLOSECOUNT */ break; @@ -3024,37 +2829,32 @@ p_324(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI3 ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-count */ { -#line 791 "src/libre/parser.act" +#line 777 "src/libre/parser.act" - mark(&act_state->countstart, &(*ZI321)); + mark(&act_state->countstart, &(*ZI318)); mark(&act_state->countend, &(ZIend)); -#line 3033 "src/libre/dialect/pcre/parser.c" +#line 2838 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-range */ { -#line 853 "src/libre/parser.act" - - struct ast_pos ast_start, ast_end; +#line 824 "src/libre/parser.act" if ((ZIn) < (*ZIm)) { err->e = RE_ENEGCOUNT; err->m = (*ZIm); err->n = (ZIn); - mark(&act_state->countstart, &(*ZI321)); + mark(&act_state->countstart, &(*ZI318)); mark(&act_state->countend, &(ZIend)); goto ZL1; } - AST_POS_OF_LX_POS(ast_start, (*ZI321)); - AST_POS_OF_LX_POS(ast_end, (ZIend)); - - (ZIc) = ast_make_count((*ZIm), &ast_start, (ZIn), &ast_end); + (ZIc) = ast_make_count((*ZIm), (ZIn)); -#line 3058 "src/libre/dialect/pcre/parser.c" +#line 2858 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-range */ } @@ -3072,6 +2872,155 @@ ZL0:; *ZOc = ZIc; } +static void +p_194(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__expr *ZItmp) +{ + switch (CURRENT_TERMINAL) { + case (TOK_RANGE): + { + t_char ZIc; + t_pos ZIrstart; + t_pos ZI195; + t_ast__expr ZInode1; + + /* BEGINNING OF EXTRACT: RANGE */ + { +#line 308 "src/libre/parser.act" + + ZIc = '-'; + ZIrstart = lex_state->lx.start; + ZI195 = lex_state->lx.end; + + (void) ZIc; + (void) ZIrstart; + (void) ZI195; + +#line 2899 "src/libre/dialect/pcre/parser.c" + } + /* END OF EXTRACT: RANGE */ + ADVANCE_LEXER; + /* BEGINNING OF INLINE: 196 */ + { + switch (CURRENT_TERMINAL) { + default: + { + /* BEGINNING OF ACTION: ast-make-literal */ + { +#line 874 "src/libre/parser.act" + + (ZInode1) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); + if ((ZInode1) == NULL) { + goto ZL1; + } + +#line 2917 "src/libre/dialect/pcre/parser.c" + } + /* END OF ACTION: ast-make-literal */ + } + break; + case (TOK_RANGE): + { + t_endpoint ZIlower; + t_char ZI197; + t_pos ZI198; + t_pos ZI199; + t_endpoint ZIupper; + t_pos ZIend; + + /* BEGINNING OF ACTION: ast-range-endpoint-literal */ + { +#line 839 "src/libre/parser.act" + + (ZIlower).type = AST_ENDPOINT_LITERAL; + (ZIlower).u.literal.c = (unsigned char) (ZIc); + +#line 2938 "src/libre/dialect/pcre/parser.c" + } + /* END OF ACTION: ast-range-endpoint-literal */ + /* BEGINNING OF EXTRACT: RANGE */ + { +#line 308 "src/libre/parser.act" + + ZI197 = '-'; + ZI198 = lex_state->lx.start; + ZI199 = lex_state->lx.end; + + (void) ZI197; + (void) ZI198; + (void) ZI199; + +#line 2953 "src/libre/dialect/pcre/parser.c" + } + /* END OF EXTRACT: RANGE */ + ADVANCE_LEXER; + p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint_Hend (flags, lex_state, act_state, err, &ZIupper, &ZIend); + if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { + RESTORE_LEXER; + goto ZL1; + } + /* BEGINNING OF ACTION: ast-make-range */ + { +#line 1004 "src/libre/parser.act" + + unsigned char lower, upper; + + if ((ZIlower).type != AST_ENDPOINT_LITERAL || + (ZIupper).type != AST_ENDPOINT_LITERAL) { + err->e = RE_EUNSUPPORTED; + goto ZL1; + } + + lower = (ZIlower).u.literal.c; + upper = (ZIupper).u.literal.c; + + if (lower > upper) { + char a[5], b[5]; + + assert(sizeof err->set >= 1 + sizeof a + 1 + sizeof b + 1 + 1); + + sprintf(err->set, "%s-%s", + escchar(a, sizeof a, lower), escchar(b, sizeof b, upper)); + err->e = RE_ENEGRANGE; + goto ZL1; + } + + (ZInode1) = ast_make_expr_range(act_state->poolp, *flags, &(ZIlower), &(ZIupper)); + if ((ZInode1) == NULL) { + goto ZL1; + } + +#line 2993 "src/libre/dialect/pcre/parser.c" + } + /* END OF ACTION: ast-make-range */ + } + break; + } + } + /* END OF INLINE: 196 */ + /* BEGINNING OF ACTION: ast-add-alt */ + { +#line 1046 "src/libre/parser.act" + + if (!ast_add_expr_alt((*ZItmp), (ZInode1))) { + goto ZL1; + } + +#line 3009 "src/libre/dialect/pcre/parser.c" + } + /* END OF ACTION: ast-add-alt */ + } + break; + case (ERROR_TERMINAL): + return; + default: + break; + } + return; +ZL1:; + SAVE_LEXER ((ERROR_TERMINAL)); + return; +} + static void p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__expr *ZOnode) { @@ -3095,23 +3044,23 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, ADVANCE_LEXER; /* BEGINNING OF ACTION: re-flag-none */ { -#line 825 "src/libre/parser.act" +#line 796 "src/libre/parser.act" (ZIempty__pos) = RE_FLAGS_NONE; -#line 3103 "src/libre/dialect/pcre/parser.c" +#line 3052 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: re-flag-none */ /* BEGINNING OF ACTION: re-flag-none */ { -#line 825 "src/libre/parser.act" +#line 796 "src/libre/parser.act" (ZIempty__neg) = RE_FLAGS_NONE; -#line 3112 "src/libre/dialect/pcre/parser.c" +#line 3061 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: re-flag-none */ - /* BEGINNING OF INLINE: 233 */ + /* BEGINNING OF INLINE: 231 */ { switch (CURRENT_TERMINAL) { case (TOK_FLAG__IGNORE): case (TOK_FLAG__UNKNOWN): case (TOK_FLAG__INSENSITIVE): case (TOK_FLAG__EXTENDED): @@ -3131,8 +3080,8 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, break; } } - /* END OF INLINE: 233 */ - /* BEGINNING OF INLINE: 235 */ + /* END OF INLINE: 231 */ + /* BEGINNING OF INLINE: 233 */ { switch (CURRENT_TERMINAL) { case (TOK_NEGATE): @@ -3152,8 +3101,8 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, break; } } - /* END OF INLINE: 235 */ - /* BEGINNING OF INLINE: 238 */ + /* END OF INLINE: 233 */ + /* BEGINNING OF INLINE: 236 */ { switch (CURRENT_TERMINAL) { case (TOK_CLOSE): @@ -3161,7 +3110,7 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-mask-re-flags */ { -#line 960 "src/libre/parser.act" +#line 926 "src/libre/parser.act" /* * Note: in cases like `(?i-i)`, the negative is @@ -3170,19 +3119,19 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, *flags |= (ZIpos); *flags &= ~(ZIneg); -#line 3174 "src/libre/dialect/pcre/parser.c" +#line 3123 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-mask-re-flags */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 887 "src/libre/parser.act" +#line 853 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL5; } -#line 3186 "src/libre/dialect/pcre/parser.c" +#line 3135 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -3195,16 +3144,16 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-get-re-flags */ { -#line 952 "src/libre/parser.act" +#line 918 "src/libre/parser.act" (ZIflags) = *flags; -#line 3203 "src/libre/dialect/pcre/parser.c" +#line 3152 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-get-re-flags */ /* BEGINNING OF ACTION: ast-mask-re-flags */ { -#line 960 "src/libre/parser.act" +#line 926 "src/libre/parser.act" /* * Note: in cases like `(?i-i)`, the negative is @@ -3213,7 +3162,7 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, *flags |= (ZIpos); *flags &= ~(ZIneg); -#line 3217 "src/libre/dialect/pcre/parser.c" +#line 3166 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-mask-re-flags */ p_expr (flags, lex_state, act_state, err, &ZIe); @@ -3223,11 +3172,11 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, } /* BEGINNING OF ACTION: ast-set-re-flags */ { -#line 956 "src/libre/parser.act" +#line 922 "src/libre/parser.act" *flags = (ZIflags); -#line 3231 "src/libre/dialect/pcre/parser.c" +#line 3180 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-set-re-flags */ switch (CURRENT_TERMINAL) { @@ -3248,32 +3197,32 @@ p_expr_C_Cflags(flags flags, lex_state lex_state, act_state act_state, err err, { /* BEGINNING OF ACTION: err-expected-closeflags */ { -#line 760 "src/libre/parser.act" +#line 746 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCLOSEFLAGS; } goto ZL1; -#line 3259 "src/libre/dialect/pcre/parser.c" +#line 3208 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-closeflags */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 887 "src/libre/parser.act" +#line 853 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 3271 "src/libre/dialect/pcre/parser.c" +#line 3220 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-empty */ } ZL4:; } - /* END OF INLINE: 238 */ + /* END OF INLINE: 236 */ } goto ZL0; ZL1:; @@ -3301,7 +3250,7 @@ p_expr_C_Cpiece_C_Clist_Hof_Hcounts(flags flags, lex_state lex_state, act_state } /* BEGINNING OF ACTION: ast-make-piece */ { -#line 931 "src/libre/parser.act" +#line 897 "src/libre/parser.act" if ((ZIc).min == 0 && (ZIc).max == 0) { (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); @@ -3315,10 +3264,10 @@ p_expr_C_Cpiece_C_Clist_Hof_Hcounts(flags flags, lex_state lex_state, act_state goto ZL1; } -#line 3319 "src/libre/dialect/pcre/parser.c" +#line 3268 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-piece */ - /* BEGINNING OF INLINE: 266 */ + /* BEGINNING OF INLINE: 264 */ { switch (CURRENT_TERMINAL) { case (TOK_OPT): @@ -3326,14 +3275,14 @@ p_expr_C_Cpiece_C_Clist_Hof_Hcounts(flags flags, lex_state lex_state, act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: err-unsupported */ { -#line 774 "src/libre/parser.act" +#line 760 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EUNSUPPORTED; } goto ZL1; -#line 3337 "src/libre/dialect/pcre/parser.c" +#line 3286 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-unsupported */ } @@ -3343,14 +3292,14 @@ p_expr_C_Cpiece_C_Clist_Hof_Hcounts(flags flags, lex_state lex_state, act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: err-unsupported */ { -#line 774 "src/libre/parser.act" +#line 760 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EUNSUPPORTED; } goto ZL1; -#line 3354 "src/libre/dialect/pcre/parser.c" +#line 3303 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-unsupported */ } @@ -3359,7 +3308,7 @@ p_expr_C_Cpiece_C_Clist_Hof_Hcounts(flags flags, lex_state lex_state, act_state break; } } - /* END OF INLINE: 266 */ + /* END OF INLINE: 264 */ } goto ZL0; ZL1:; @@ -3380,7 +3329,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint(flags flags, lex_state lex_state, return; } { - /* BEGINNING OF INLINE: 159 */ + /* BEGINNING OF INLINE: 158 */ { switch (CURRENT_TERMINAL) { case (TOK_NAMED__CLASS): @@ -3406,7 +3355,7 @@ p_expr_C_Ccharacter_Hclass_C_Crange_Hendpoint(flags flags, lex_state lex_state, goto ZL1; } } - /* END OF INLINE: 159 */ + /* END OF INLINE: 158 */ } goto ZL0; ZL1:; @@ -3435,7 +3384,7 @@ p_class_Hnamed(flags flags, lex_state lex_state, act_state act_state, err err, t case (TOK_NAMED__CLASS): /* BEGINNING OF EXTRACT: NAMED_CLASS */ { -#line 661 "src/libre/parser.act" +#line 647 "src/libre/parser.act" ZIid = DIALECT_CLASS(lex_state->buf.a); if (ZIid == NULL) { @@ -3449,7 +3398,7 @@ p_class_Hnamed(flags flags, lex_state lex_state, act_state act_state, err err, t (void) ZIstart; (void) ZIend; -#line 3453 "src/libre/dialect/pcre/parser.c" +#line 3402 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: NAMED_CLASS */ break; @@ -3459,14 +3408,14 @@ p_class_Hnamed(flags flags, lex_state lex_state, act_state act_state, err err, t ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-named */ { -#line 1071 "src/libre/parser.act" +#line 1033 "src/libre/parser.act" (ZInode) = ast_make_expr_named(act_state->poolp, *flags, (ZIid)); if ((ZInode) == NULL) { goto ZL1; } -#line 3470 "src/libre/dialect/pcre/parser.c" +#line 3419 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -3481,7 +3430,7 @@ ZL0:; } static void -p_210(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIstart, t_char *ZIcbrak, t_ast__expr *ZOnode1) +p_209(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIstart, t_char *ZIcbrak, t_ast__expr *ZOnode1) { t_ast__expr ZInode1; @@ -3490,14 +3439,14 @@ p_210(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs { /* BEGINNING OF ACTION: ast-make-literal */ { -#line 908 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZInode1) = ast_make_expr_literal(act_state->poolp, *flags, (*ZIcbrak)); if ((ZInode1) == NULL) { goto ZL1; } -#line 3501 "src/libre/dialect/pcre/parser.c" +#line 3450 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -3505,36 +3454,36 @@ p_210(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs case (TOK_RANGE): { t_endpoint ZIr; - t_char ZI211; + t_char ZI210; + t_pos ZI211; t_pos ZI212; - t_pos ZI213; t_endpoint ZIupper; t_pos ZIend; t_endpoint ZIlower; /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 873 "src/libre/parser.act" +#line 839 "src/libre/parser.act" (ZIr).type = AST_ENDPOINT_LITERAL; - (ZIr).u.literal.c = (unsigned char)(*ZIcbrak); + (ZIr).u.literal.c = (unsigned char) (*ZIcbrak); -#line 3523 "src/libre/dialect/pcre/parser.c" +#line 3472 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF EXTRACT: RANGE */ { -#line 327 "src/libre/parser.act" +#line 308 "src/libre/parser.act" - ZI211 = '-'; - ZI212 = lex_state->lx.start; - ZI213 = lex_state->lx.end; + ZI210 = '-'; + ZI211 = lex_state->lx.start; + ZI212 = lex_state->lx.end; + (void) ZI210; (void) ZI211; (void) ZI212; - (void) ZI213; -#line 3538 "src/libre/dialect/pcre/parser.c" +#line 3487 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; @@ -3545,24 +3494,20 @@ p_210(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs } /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 873 "src/libre/parser.act" +#line 839 "src/libre/parser.act" (ZIlower).type = AST_ENDPOINT_LITERAL; - (ZIlower).u.literal.c = (unsigned char)(*ZIcbrak); + (ZIlower).u.literal.c = (unsigned char) (*ZIcbrak); -#line 3554 "src/libre/dialect/pcre/parser.c" +#line 3503 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1038 "src/libre/parser.act" +#line 1004 "src/libre/parser.act" - struct ast_pos ast_start, ast_end; unsigned char lower, upper; - AST_POS_OF_LX_POS(ast_start, (*ZIstart)); - AST_POS_OF_LX_POS(ast_end, (ZIend)); - if ((ZIlower).type != AST_ENDPOINT_LITERAL || (ZIupper).type != AST_ENDPOINT_LITERAL) { err->e = RE_EUNSUPPORTED; @@ -3583,12 +3528,12 @@ p_210(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZIs goto ZL1; } - (ZInode1) = ast_make_expr_range(act_state->poolp, *flags, &(ZIlower), ast_start, &(ZIupper), ast_end); + (ZInode1) = ast_make_expr_range(act_state->poolp, *flags, &(ZIlower), &(ZIupper)); if ((ZInode1) == NULL) { goto ZL1; } -#line 3592 "src/libre/dialect/pcre/parser.c" +#line 3537 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -3621,16 +3566,16 @@ ZL2_expr_C_Clist_Hof_Halts:; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1084 "src/libre/parser.act" +#line 1046 "src/libre/parser.act" if (!ast_add_expr_alt((ZIalts), (ZIa))) { goto ZL1; } -#line 3631 "src/libre/dialect/pcre/parser.c" +#line 3576 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ - /* BEGINNING OF INLINE: 280 */ + /* BEGINNING OF INLINE: 278 */ { switch (CURRENT_TERMINAL) { case (TOK_ALT): @@ -3645,21 +3590,21 @@ ZL2_expr_C_Clist_Hof_Halts:; break; } } - /* END OF INLINE: 280 */ + /* END OF INLINE: 278 */ } return; ZL1:; { /* BEGINNING OF ACTION: err-expected-alts */ { -#line 725 "src/libre/parser.act" +#line 711 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXALTS; } goto ZL4; -#line 3663 "src/libre/dialect/pcre/parser.c" +#line 3608 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-alts */ } @@ -3678,21 +3623,21 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, switch (CURRENT_TERMINAL) { case (TOK_OPENCOUNT): { - t_pos ZI321; - t_pos ZI322; + t_pos ZI318; + t_pos ZI319; t_unsigned ZIm; /* BEGINNING OF EXTRACT: OPENCOUNT */ { -#line 389 "src/libre/parser.act" +#line 370 "src/libre/parser.act" - ZI321 = lex_state->lx.start; - ZI322 = lex_state->lx.end; + ZI318 = lex_state->lx.start; + ZI319 = lex_state->lx.end; - (void) ZI321; - (void) ZI322; + (void) ZI318; + (void) ZI319; -#line 3696 "src/libre/dialect/pcre/parser.c" +#line 3641 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: OPENCOUNT */ ADVANCE_LEXER; @@ -3700,7 +3645,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, case (TOK_COUNT): /* BEGINNING OF EXTRACT: COUNT */ { -#line 641 "src/libre/parser.act" +#line 627 "src/libre/parser.act" unsigned long u; char *e; @@ -3720,7 +3665,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ZIm = (unsigned int) u; -#line 3724 "src/libre/dialect/pcre/parser.c" +#line 3669 "src/libre/dialect/pcre/parser.c" } /* END OF EXTRACT: COUNT */ break; @@ -3728,7 +3673,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, goto ZL1; } ADVANCE_LEXER; - p_323 (flags, lex_state, act_state, err, &ZI321, &ZIm, &ZIc); + p_320 (flags, lex_state, act_state, err, &ZI318, &ZIm, &ZIc); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -3740,11 +3685,11 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-zero-or-one */ { -#line 845 "src/libre/parser.act" +#line 816 "src/libre/parser.act" - (ZIc) = ast_make_count(0, NULL, 1, NULL); + (ZIc) = ast_make_count(0, 1); -#line 3748 "src/libre/dialect/pcre/parser.c" +#line 3693 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-zero-or-one */ } @@ -3754,11 +3699,11 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-one-or-more */ { -#line 841 "src/libre/parser.act" +#line 812 "src/libre/parser.act" - (ZIc) = ast_make_count(1, NULL, AST_COUNT_UNBOUNDED, NULL); + (ZIc) = ast_make_count(1, AST_COUNT_UNBOUNDED); -#line 3762 "src/libre/dialect/pcre/parser.c" +#line 3707 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-one-or-more */ } @@ -3768,11 +3713,11 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-zero-or-more */ { -#line 837 "src/libre/parser.act" +#line 808 "src/libre/parser.act" - (ZIc) = ast_make_count(0, NULL, AST_COUNT_UNBOUNDED, NULL); + (ZIc) = ast_make_count(0, AST_COUNT_UNBOUNDED); -#line 3776 "src/libre/dialect/pcre/parser.c" +#line 3721 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-zero-or-more */ } @@ -3787,23 +3732,23 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-count */ { -#line 711 "src/libre/parser.act" +#line 697 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCOUNT; } goto ZL2; -#line 3798 "src/libre/dialect/pcre/parser.c" +#line 3743 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-count */ /* BEGINNING OF ACTION: count-one */ { -#line 849 "src/libre/parser.act" +#line 820 "src/libre/parser.act" - (ZIc) = ast_make_count(1, NULL, 1, NULL); + (ZIc) = ast_make_count(1, 1); -#line 3807 "src/libre/dialect/pcre/parser.c" +#line 3752 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: count-one */ } @@ -3824,71 +3769,65 @@ p_re__pcre(flags flags, lex_state lex_state, act_state act_state, err err, t_ast return; } { - /* BEGINNING OF INLINE: 282 */ - { - { - t_group__id ZIid; - t_ast__expr ZIe; + t_group__id ZIid; + t_ast__expr ZIe; - /* BEGINNING OF ACTION: make-group-id */ - { -#line 915 "src/libre/parser.act" + /* BEGINNING OF ACTION: make-group-id */ + { +#line 881 "src/libre/parser.act" (ZIid) = act_state->group_id++; -#line 3840 "src/libre/dialect/pcre/parser.c" - } - /* END OF ACTION: make-group-id */ - p_expr (flags, lex_state, act_state, err, &ZIe); - if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { - RESTORE_LEXER; - goto ZL1; - } - /* BEGINNING OF ACTION: ast-make-group */ - { -#line 945 "src/libre/parser.act" +#line 3782 "src/libre/dialect/pcre/parser.c" + } + /* END OF ACTION: make-group-id */ + p_expr (flags, lex_state, act_state, err, &ZIe); + if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { + RESTORE_LEXER; + goto ZL1; + } + /* BEGINNING OF ACTION: ast-make-group */ + { +#line 911 "src/libre/parser.act" (ZInode) = ast_make_expr_group(act_state->poolp, *flags, (ZIe), (ZIid)); if ((ZInode) == NULL) { goto ZL1; } -#line 3857 "src/libre/dialect/pcre/parser.c" - } - /* END OF ACTION: ast-make-group */ - } +#line 3799 "src/libre/dialect/pcre/parser.c" } - /* END OF INLINE: 282 */ - /* BEGINNING OF INLINE: 283 */ + /* END OF ACTION: ast-make-group */ + /* BEGINNING OF INLINE: 280 */ { { switch (CURRENT_TERMINAL) { case (TOK_EOF): break; default: - goto ZL4; + goto ZL3; } ADVANCE_LEXER; } - goto ZL3; - ZL4:; + goto ZL2; + ZL3:; { /* BEGINNING OF ACTION: err-expected-eof */ { -#line 767 "src/libre/parser.act" +#line 753 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXEOF; } goto ZL1; -#line 3886 "src/libre/dialect/pcre/parser.c" +#line 3825 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-eof */ } - ZL3:; + ZL2:; } - /* END OF INLINE: 283 */ + /* END OF INLINE: 280 */ } goto ZL0; ZL1:; @@ -3911,24 +3850,24 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 811 "src/libre/parser.act" +#line 782 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; -#line 3920 "src/libre/dialect/pcre/parser.c" +#line 3859 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1071 "src/libre/parser.act" +#line 1033 "src/libre/parser.act" (ZIe) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIe) == NULL) { goto ZL1; } -#line 3932 "src/libre/dialect/pcre/parser.c" +#line 3871 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -3938,14 +3877,14 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-anchor-end */ { -#line 976 "src/libre/parser.act" +#line 942 "src/libre/parser.act" (ZIe) = ast_make_expr_anchor(act_state->poolp, *flags, AST_ANCHOR_END); if ((ZIe) == NULL) { goto ZL1; } -#line 3949 "src/libre/dialect/pcre/parser.c" +#line 3888 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-anchor-end */ } @@ -3955,7 +3894,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-anchor-end-nl */ { -#line 983 "src/libre/parser.act" +#line 949 "src/libre/parser.act" (ZIe) = ast_make_expr_anchor(act_state->poolp, *flags, AST_ANCHOR_END); if ((ZIe) == NULL) { @@ -3965,7 +3904,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e (ZIe)->u.anchor.is_end_nl = 1; } -#line 3969 "src/libre/dialect/pcre/parser.c" +#line 3908 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-anchor-end-nl */ } @@ -3983,134 +3922,134 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: class-bsr */ { -#line 816 "src/libre/parser.act" +#line 787 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIclass__bsr) = &class_bsr; -#line 3992 "src/libre/dialect/pcre/parser.c" +#line 3931 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: class-bsr */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1071 "src/libre/parser.act" +#line 1033 "src/libre/parser.act" (ZIbsr) = ast_make_expr_named(act_state->poolp, *flags, (ZIclass__bsr)); if ((ZIbsr) == NULL) { goto ZL1; } -#line 4004 "src/libre/dialect/pcre/parser.c" +#line 3943 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-named */ /* BEGINNING OF ACTION: ast-make-concat */ { -#line 894 "src/libre/parser.act" +#line 860 "src/libre/parser.act" (ZIcrlf) = ast_make_expr_concat(act_state->poolp, *flags); if ((ZIcrlf) == NULL) { goto ZL1; } -#line 4016 "src/libre/dialect/pcre/parser.c" +#line 3955 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-concat */ /* BEGINNING OF ACTION: make-literal-cr */ { -#line 923 "src/libre/parser.act" +#line 889 "src/libre/parser.act" (ZIcr) = '\r'; -#line 4025 "src/libre/dialect/pcre/parser.c" +#line 3964 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: make-literal-cr */ /* BEGINNING OF ACTION: ast-make-literal */ { -#line 908 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZIecr) = ast_make_expr_literal(act_state->poolp, *flags, (ZIcr)); if ((ZIecr) == NULL) { goto ZL1; } -#line 4037 "src/libre/dialect/pcre/parser.c" +#line 3976 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: make-literal-nl */ { -#line 927 "src/libre/parser.act" +#line 893 "src/libre/parser.act" (ZInl) = '\n'; -#line 4046 "src/libre/dialect/pcre/parser.c" +#line 3985 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: make-literal-nl */ /* BEGINNING OF ACTION: ast-make-literal */ { -#line 908 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZIenl) = ast_make_expr_literal(act_state->poolp, *flags, (ZInl)); if ((ZIenl) == NULL) { goto ZL1; } -#line 4058 "src/libre/dialect/pcre/parser.c" +#line 3997 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1078 "src/libre/parser.act" +#line 1040 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcrlf), (ZIecr))) { goto ZL1; } -#line 4069 "src/libre/dialect/pcre/parser.c" +#line 4008 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-concat */ /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1078 "src/libre/parser.act" +#line 1040 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcrlf), (ZIenl))) { goto ZL1; } -#line 4080 "src/libre/dialect/pcre/parser.c" +#line 4019 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-concat */ /* BEGINNING OF ACTION: ast-make-alt */ { -#line 901 "src/libre/parser.act" +#line 867 "src/libre/parser.act" (ZIe) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZIe) == NULL) { goto ZL1; } -#line 4092 "src/libre/dialect/pcre/parser.c" +#line 4031 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-alt */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1084 "src/libre/parser.act" +#line 1046 "src/libre/parser.act" if (!ast_add_expr_alt((ZIe), (ZIcrlf))) { goto ZL1; } -#line 4103 "src/libre/dialect/pcre/parser.c" +#line 4042 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1084 "src/libre/parser.act" +#line 1046 "src/libre/parser.act" if (!ast_add_expr_alt((ZIe), (ZIbsr))) { goto ZL1; } -#line 4114 "src/libre/dialect/pcre/parser.c" +#line 4053 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ } @@ -4124,20 +4063,20 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-get-re-flags */ { -#line 952 "src/libre/parser.act" +#line 918 "src/libre/parser.act" (ZIflags) = *flags; -#line 4132 "src/libre/dialect/pcre/parser.c" +#line 4071 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-get-re-flags */ /* BEGINNING OF ACTION: make-group-id */ { -#line 915 "src/libre/parser.act" +#line 881 "src/libre/parser.act" (ZIid) = act_state->group_id++; -#line 4141 "src/libre/dialect/pcre/parser.c" +#line 4080 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: make-group-id */ p_expr (flags, lex_state, act_state, err, &ZIg); @@ -4147,23 +4086,23 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e } /* BEGINNING OF ACTION: ast-set-re-flags */ { -#line 956 "src/libre/parser.act" +#line 922 "src/libre/parser.act" *flags = (ZIflags); -#line 4155 "src/libre/dialect/pcre/parser.c" +#line 4094 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-set-re-flags */ /* BEGINNING OF ACTION: ast-make-group */ { -#line 945 "src/libre/parser.act" +#line 911 "src/libre/parser.act" (ZIe) = ast_make_expr_group(act_state->poolp, *flags, (ZIg), (ZIid)); if ((ZIe) == NULL) { goto ZL1; } -#line 4167 "src/libre/dialect/pcre/parser.c" +#line 4106 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-group */ switch (CURRENT_TERMINAL) { @@ -4180,14 +4119,14 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-anchor-start */ { -#line 969 "src/libre/parser.act" +#line 935 "src/libre/parser.act" (ZIe) = ast_make_expr_anchor(act_state->poolp, *flags, AST_ANCHOR_START); if ((ZIe) == NULL) { goto ZL1; } -#line 4191 "src/libre/dialect/pcre/parser.c" +#line 4130 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-anchor-start */ } @@ -4239,26 +4178,26 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-atom */ { -#line 718 "src/libre/parser.act" +#line 704 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXATOM; } goto ZL2; -#line 4250 "src/libre/dialect/pcre/parser.c" +#line 4189 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 887 "src/libre/parser.act" +#line 853 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { goto ZL2; } -#line 4262 "src/libre/dialect/pcre/parser.c" +#line 4201 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -4284,14 +4223,14 @@ p_expr_C_Calt(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: ast-make-concat */ { -#line 894 "src/libre/parser.act" +#line 860 "src/libre/parser.act" (ZInode) = ast_make_expr_concat(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 4295 "src/libre/dialect/pcre/parser.c" +#line 4234 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-concat */ p_expr_C_Clist_Hof_Hpieces (flags, lex_state, act_state, err, ZInode); @@ -4305,14 +4244,14 @@ p_expr_C_Calt(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: ast-make-empty */ { -#line 887 "src/libre/parser.act" +#line 853 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 4316 "src/libre/dialect/pcre/parser.c" +#line 4255 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -4348,47 +4287,27 @@ p_expr_C_Ctype(flags flags, lex_state lex_state, act_state act_state, err err, t } /* BEGINNING OF ACTION: ast-make-alt */ { -#line 901 "src/libre/parser.act" +#line 867 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 4359 "src/libre/dialect/pcre/parser.c" +#line 4298 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-make-alt */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1084 "src/libre/parser.act" +#line 1046 "src/libre/parser.act" if (!ast_add_expr_alt((ZInode), (ZIclass))) { goto ZL1; } -#line 4370 "src/libre/dialect/pcre/parser.c" +#line 4309 "src/libre/dialect/pcre/parser.c" } /* END OF ACTION: ast-add-alt */ - /* BEGINNING OF ACTION: mark-expr */ - { -#line 796 "src/libre/parser.act" - - struct ast_pos ast_start, ast_end; - - AST_POS_OF_LX_POS(ast_start, (ZIstart)); - AST_POS_OF_LX_POS(ast_end, (ZIend)); - - mark(&act_state->groupstart, &(ZIstart)); - mark(&act_state->groupend, &(ZIend)); - -/* TODO: reinstate this, applies to an expr node in general - (ZInode)->u.class.start = ast_start; - (ZInode)->u.class.end = ast_end; -*/ - -#line 4390 "src/libre/dialect/pcre/parser.c" - } - /* END OF ACTION: mark-expr */ } goto ZL0; ZL1:; @@ -4400,7 +4319,7 @@ ZL0:; /* BEGINNING OF TRAILER */ -#line 1090 "src/libre/parser.act" +#line 1052 "src/libre/parser.act" static int @@ -4420,9 +4339,8 @@ ZL0:; struct ast * DIALECT_PARSE(re_getchar_fun *f, void *opaque, - const struct fsm_options *opt, enum re_flags flags, int overlap, - struct re_err *err) + struct re_err *err, struct re_pos *end) { struct ast *ast; @@ -4435,6 +4353,7 @@ ZL0:; struct LX_STATE *lx; assert(f != NULL); + assert(end != NULL); ast = ast_new(); @@ -4458,11 +4377,6 @@ ZL0:; lex_state->buf.a = NULL; lex_state->buf.len = 0; -#if PCRE_DIALECT - lex_state->extended_mode_comment = 0; - lex_state->flags_ptr = &flags; -#endif /* PCRE_DIALECT */ - /* XXX: unneccessary since we're lexing from a string */ /* (except for pushing "[" and "]" around ::group-$dialect) */ lx->buf_opaque = &lex_state->buf; @@ -4493,6 +4407,9 @@ ZL0:; DIALECT_ENTRY(&flags, lex_state, act_state, err, &ast->expr); + /* we output this for reporting errors found through AST analysis */ + mark(end, &lx->end); + lx->free(lx->buf_opaque); if (err->e != RE_ESUCCESS) { @@ -4558,6 +4475,6 @@ ZL0:; return NULL; } -#line 4562 "src/libre/dialect/pcre/parser.c" +#line 4479 "src/libre/dialect/pcre/parser.c" /* END OF FILE */ diff --git a/src/libre/dialect/pcre/parser.h b/src/libre/dialect/pcre/parser.h index 20b24942d..84ef34223 100644 --- a/src/libre/dialect/pcre/parser.h +++ b/src/libre/dialect/pcre/parser.h @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 300 "src/libre/parser.act" +#line 281 "src/libre/parser.act" #include @@ -28,7 +28,7 @@ extern void p_re__pcre(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1247 "src/libre/parser.act" +#line 1207 "src/libre/parser.act" #line 35 "src/libre/dialect/pcre/parser.h" diff --git a/src/libre/dialect/pcre/parser.sid b/src/libre/dialect/pcre/parser.sid index 7d3e90c54..377060dbe 100644 --- a/src/libre/dialect/pcre/parser.sid +++ b/src/libre/dialect/pcre/parser.sid @@ -140,7 +140,6 @@ : (:pos, :pos) -> (); : (:pos, :pos) -> (); : (:pos, :pos) -> (); - : (:ast_expr, :pos, :pos) -> (); : () -> (:ast_class_id); : () -> (:ast_class_id); @@ -378,10 +377,7 @@ (start, end); ## ; - end = start; /* appease sid */ }; - - (tmp, start, end); }; type: () -> (node :ast_expr) = { @@ -389,8 +385,6 @@ node = ; (node, class); - - (node, start, end); }; flags: () -> (node :ast_expr) [ @@ -465,7 +459,7 @@ e = ; || END_NL; - e = ; + e = ; || ANY; a = ; @@ -644,11 +638,9 @@ }; re_pcre: () -> (node :ast_expr) = { - { - id = ; - e = expr; - node = (e, id); - }; + id = ; + e = expr; + node = (e, id); { EOF; diff --git a/src/libre/dialect/sql/lexer.c b/src/libre/dialect/sql/lexer.c index 6c35bf800..87459e786 100644 --- a/src/libre/dialect/sql/lexer.c +++ b/src/libre/dialect/sql/lexer.c @@ -12,11 +12,31 @@ static enum lx_sql_token z0(struct lx_sql_lx *lx); static enum lx_sql_token z1(struct lx_sql_lx *lx); static enum lx_sql_token z2(struct lx_sql_lx *lx); +static int +lx_sql_advance_end(struct lx_sql_lx *lx, int c) +{ + lx->end.byte++; + lx->end.col++; + if (c == '\n') { + lx->end.line++; + lx->end.saved_col = lx->end.col - 1; + lx->end.col = 1; + } + if (lx->push != NULL) { + if (-1 == lx->push(lx->buf_opaque, (char)c)) { + return 0; + } + } + return 1; +} + +/* This wrapper manages one character of lookahead/pushback + * and the line, column, and byte offsets. */ #if __STDC_VERSION__ >= 199901L inline #endif static int -lx_getc(struct lx_sql_lx *lx) +lx_sql_getc(struct lx_sql_lx *lx) { int c; @@ -32,18 +52,19 @@ lx_getc(struct lx_sql_lx *lx) } } - lx->end.byte++; - lx->end.col++; - - if (c == '\n') { - lx->end.line++; - lx->end.saved_col = lx->end.col - 1; - lx->end.col = 1; - } + if (!lx_sql_advance_end(lx, c)) { return EOF; } return c; } +/* This wrapper adapts calling lx_sql_getc to the interface + * in libfsm's generated code. */ +static int +fsm_getc(void *getc_opaque) +{ + return lx_sql_getc((struct lx_sql_lx *)getc_opaque); +} + #if __STDC_VERSION__ >= 199901L inline #endif @@ -52,10 +73,7 @@ lx_sql_ungetc(struct lx_sql_lx *lx, int c) { assert(lx != NULL); assert(lx->c == EOF); - lx->c = c; - - lx->end.byte--; lx->end.col--; @@ -107,6 +125,17 @@ lx_sql_dynpush(void *buf_opaque, char c) return 0; } +static void +lx_sql_dynpop(void *buf_opaque) +{ + struct lx_dynbuf *t = buf_opaque; + + assert(t != NULL); + + assert(t->p != t->a); + t->p--; +} + int lx_sql_dynclear(void *buf_opaque) { @@ -146,29 +175,28 @@ lx_sql_dynfree(void *buf_opaque) static enum lx_sql_token z0(struct lx_sql_lx *lx) { + int has_consumed_input = 0; int c; - enum { - S0, S1, S2, S3, NONE - } state; - assert(lx != NULL); if (lx->clear != NULL) { lx->clear(lx->buf_opaque); } - state = NONE; - lx->start = lx->end; - while (c = lx_getc(lx), c != EOF) { - if (state == NONE) { - state = S0; - } + void *getc_opaque = (void *)lx; + enum { + S0, S1, S2, S3 + } state; + + state = S0; + while (c = fsm_getc(getc_opaque), c != EOF) { + has_consumed_input = 1; switch (state) { - case S0: /* start */ + case S0: /* e.g. "" */ switch ((unsigned char) c) { case ',': state = S1; break; case '0': @@ -182,12 +210,14 @@ z0(struct lx_sql_lx *lx) case '8': case '9': state = S2; break; case '}': state = S3; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S1: /* e.g. "," */ - lx_sql_ungetc(lx, c); return TOK_SEP; + lx_sql_ungetc(lx, c); lx_sql_dynpop(lx->buf_opaque); return TOK_SEP; case S2: /* e.g. "0" */ switch ((unsigned char) c) { @@ -201,64 +231,70 @@ z0(struct lx_sql_lx *lx) case '7': case '8': case '9': break; - default: lx_sql_ungetc(lx, c); return TOK_COUNT; + default: lx_sql_ungetc(lx, c); lx_sql_dynpop(lx->buf_opaque); return TOK_COUNT; } break; case S3: /* e.g. "}" */ - lx_sql_ungetc(lx, c); return lx->z = z2, TOK_CLOSECOUNT; + lx_sql_ungetc(lx, c); lx_sql_dynpop(lx->buf_opaque); return lx->z = z2, TOK_CLOSECOUNT; default: ; /* unreached */ } + } + + /* end states */ + switch (state) { + case S1: return TOK_SEP; + case S2: return TOK_COUNT; + case S3: return lx->z = z2, TOK_CLOSECOUNT; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; + } if (lx->push != NULL) { if (-1 == lx->push(lx->buf_opaque, (char)c)) { return TOK_ERROR; } } - } lx->lgetc = NULL; - switch (state) { - case NONE: return TOK_EOF; - case S1: return TOK_SEP; - case S2: return TOK_COUNT; - case S3: return TOK_CLOSECOUNT; - default: errno = EINVAL; return TOK_ERROR; - } + if (!has_consumed_input) { + return TOK_EOF; + } + return TOK_ERROR; } static enum lx_sql_token z1(struct lx_sql_lx *lx) { + int has_consumed_input = 0; int c; - enum { - S0, S1, S2, S3, S4, S5, S6, S7, S8, S9, - S10, S11, S12, S13, S14, S15, S16, S17, S18, S19, - S20, S21, S22, S23, S24, S25, S26, S27, S28, S29, - S30, S31, S32, S33, S34, NONE - } state; - assert(lx != NULL); if (lx->clear != NULL) { lx->clear(lx->buf_opaque); } - state = NONE; - lx->start = lx->end; - while (c = lx_getc(lx), c != EOF) { - if (state == NONE) { - state = S0; - } + void *getc_opaque = (void *)lx; + enum { + S0, S1, S2, S3, S4, S5, S6, S7, S8, S9, + S10, S11, S12, S13, S14, S15, S16, S17, S18, S19, + S20, S21, S22, S23, S24, S25, S26, S27, S28, S29, + S30, S31, S32, S33, S34 + } state; + + state = S0; + while (c = fsm_getc(getc_opaque), c != EOF) { + has_consumed_input = 1; switch (state) { - case S0: /* start */ + case S0: /* e.g. "" */ switch ((unsigned char) c) { case '[': state = S1; break; case '-': state = S3; break; @@ -271,21 +307,21 @@ z1(struct lx_sql_lx *lx) case S1: /* e.g. "[" */ switch ((unsigned char) c) { case ':': state = S6; break; - default: lx_sql_ungetc(lx, c); return TOK_CHAR; + default: lx_sql_ungetc(lx, c); lx_sql_dynpop(lx->buf_opaque); return TOK_CHAR; } break; case S2: /* e.g. "\\x00" */ - lx_sql_ungetc(lx, c); return TOK_CHAR; + lx_sql_ungetc(lx, c); lx_sql_dynpop(lx->buf_opaque); return TOK_CHAR; case S3: /* e.g. "-" */ - lx_sql_ungetc(lx, c); return TOK_RANGE; + lx_sql_ungetc(lx, c); lx_sql_dynpop(lx->buf_opaque); return TOK_RANGE; case S4: /* e.g. "^" */ - lx_sql_ungetc(lx, c); return TOK_INVERT; + lx_sql_ungetc(lx, c); lx_sql_dynpop(lx->buf_opaque); return TOK_INVERT; case S5: /* e.g. "]" */ - lx_sql_ungetc(lx, c); return lx->z = z2, TOK_CLOSEGROUP; + lx_sql_ungetc(lx, c); lx_sql_dynpop(lx->buf_opaque); return lx->z = z2, TOK_CLOSEGROUP; case S6: /* e.g. "[:" */ switch ((unsigned char) c) { @@ -295,94 +331,120 @@ z1(struct lx_sql_lx *lx) case 'A': state = S10; break; case 'L': state = S11; break; case 'U': state = S12; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S7: /* e.g. "[:S" */ switch ((unsigned char) c) { case 'P': state = S32; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S8: /* e.g. "[:W" */ switch ((unsigned char) c) { case 'H': state = S28; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S9: /* e.g. "[:D" */ switch ((unsigned char) c) { case 'I': state = S25; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S10: /* e.g. "[:A" */ switch ((unsigned char) c) { case 'L': state = S20; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S11: /* e.g. "[:L" */ switch ((unsigned char) c) { case 'O': state = S19; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S12: /* e.g. "[:U" */ switch ((unsigned char) c) { case 'P': state = S13; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S13: /* e.g. "[:UP" */ switch ((unsigned char) c) { case 'P': state = S14; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S14: /* e.g. "[:LOW" */ switch ((unsigned char) c) { case 'E': state = S15; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S15: /* e.g. "[:LOWE" */ switch ((unsigned char) c) { case 'R': state = S16; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S16: /* e.g. "[:ALPHA" */ switch ((unsigned char) c) { case ':': state = S17; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S17: /* e.g. "[:ALPHA:" */ switch ((unsigned char) c) { case ']': state = S18; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S18: /* e.g. "[:ALPHA:]" */ - lx_sql_ungetc(lx, c); return TOK_NAMED__CLASS; + lx_sql_ungetc(lx, c); lx_sql_dynpop(lx->buf_opaque); return TOK_NAMED__CLASS; case S19: /* e.g. "[:LO" */ switch ((unsigned char) c) { case 'W': state = S14; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; @@ -390,160 +452,196 @@ z1(struct lx_sql_lx *lx) switch ((unsigned char) c) { case 'N': state = S21; break; case 'P': state = S22; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S21: /* e.g. "[:ALN" */ switch ((unsigned char) c) { case 'U': state = S24; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S22: /* e.g. "[:ALP" */ switch ((unsigned char) c) { case 'H': state = S23; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S23: /* e.g. "[:ALPH" */ switch ((unsigned char) c) { case 'A': state = S16; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S24: /* e.g. "[:ALNU" */ switch ((unsigned char) c) { case 'M': state = S16; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S25: /* e.g. "[:DI" */ switch ((unsigned char) c) { case 'G': state = S26; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S26: /* e.g. "[:DIG" */ switch ((unsigned char) c) { case 'I': state = S27; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S27: /* e.g. "[:DIGI" */ switch ((unsigned char) c) { case 'T': state = S16; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S28: /* e.g. "[:WH" */ switch ((unsigned char) c) { case 'I': state = S29; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S29: /* e.g. "[:WHI" */ switch ((unsigned char) c) { case 'T': state = S30; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S30: /* e.g. "[:WHIT" */ switch ((unsigned char) c) { case 'E': state = S31; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S31: /* e.g. "[:WHITE" */ switch ((unsigned char) c) { case 'S': state = S7; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S32: /* e.g. "[:SP" */ switch ((unsigned char) c) { case 'A': state = S33; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S33: /* e.g. "[:SPA" */ switch ((unsigned char) c) { case 'C': state = S34; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; case S34: /* e.g. "[:SPAC" */ switch ((unsigned char) c) { case 'E': state = S16; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; default: ; /* unreached */ } - - if (lx->push != NULL) { - if (-1 == lx->push(lx->buf_opaque, (char)c)) { - return TOK_ERROR; - } - } } - lx->lgetc = NULL; - + /* end states */ switch (state) { - case NONE: return TOK_EOF; case S1: return TOK_CHAR; case S2: return TOK_CHAR; case S3: return TOK_RANGE; case S4: return TOK_INVERT; - case S5: return TOK_CLOSEGROUP; + case S5: return lx->z = z2, TOK_CLOSEGROUP; case S18: return TOK_NAMED__CLASS; - default: errno = EINVAL; return TOK_ERROR; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } + + if (lx->push != NULL) { + if (-1 == lx->push(lx->buf_opaque, (char)c)) { + return TOK_ERROR; + } + } + + lx->lgetc = NULL; + + if (!has_consumed_input) { + return TOK_EOF; + } + return TOK_ERROR; } static enum lx_sql_token z2(struct lx_sql_lx *lx) { + int has_consumed_input = 0; int c; - enum { - S0, S1, S2, S3, S4, S5, S6, S7, S8, S9, - S10, S11, NONE - } state; - assert(lx != NULL); if (lx->clear != NULL) { lx->clear(lx->buf_opaque); } - state = NONE; - lx->start = lx->end; - while (c = lx_getc(lx), c != EOF) { - if (state == NONE) { - state = S0; - } + void *getc_opaque = (void *)lx; + enum { + S0, S1, S2, S3, S4, S5, S6, S7, S8, S9, + S10, S11 + } state; + state = S0; + + while (c = fsm_getc(getc_opaque), c != EOF) { + has_consumed_input = 1; switch (state) { - case S0: /* start */ + case S0: /* e.g. "" */ switch ((unsigned char) c) { case '{': state = S2; break; case '[': state = S3; break; @@ -560,56 +658,48 @@ z2(struct lx_sql_lx *lx) break; case S1: /* e.g. "\\x00" */ - lx_sql_ungetc(lx, c); return TOK_CHAR; + lx_sql_ungetc(lx, c); lx_sql_dynpop(lx->buf_opaque); return TOK_CHAR; case S2: /* e.g. "{" */ - lx_sql_ungetc(lx, c); return lx->z = z0, TOK_OPENCOUNT; + lx_sql_ungetc(lx, c); lx_sql_dynpop(lx->buf_opaque); return lx->z = z0, TOK_OPENCOUNT; case S3: /* e.g. "[" */ - lx_sql_ungetc(lx, c); return lx->z = z1, TOK_OPENGROUP; + lx_sql_ungetc(lx, c); lx_sql_dynpop(lx->buf_opaque); return lx->z = z1, TOK_OPENGROUP; case S4: /* e.g. "|" */ - lx_sql_ungetc(lx, c); return TOK_ALT; + lx_sql_ungetc(lx, c); lx_sql_dynpop(lx->buf_opaque); return TOK_ALT; case S5: /* e.g. "+" */ - lx_sql_ungetc(lx, c); return TOK_PLUS; + lx_sql_ungetc(lx, c); lx_sql_dynpop(lx->buf_opaque); return TOK_PLUS; case S6: /* e.g. "*" */ - lx_sql_ungetc(lx, c); return TOK_STAR; + lx_sql_ungetc(lx, c); lx_sql_dynpop(lx->buf_opaque); return TOK_STAR; - case S7: /* e.g. "?" */ - lx_sql_ungetc(lx, c); return TOK_OPT; + case S7: /* e.g. "\077" */ + lx_sql_ungetc(lx, c); lx_sql_dynpop(lx->buf_opaque); return TOK_OPT; case S8: /* e.g. ")" */ - lx_sql_ungetc(lx, c); return TOK_CLOSESUB; + lx_sql_ungetc(lx, c); lx_sql_dynpop(lx->buf_opaque); return TOK_CLOSESUB; case S9: /* e.g. "(" */ - lx_sql_ungetc(lx, c); return TOK_OPENSUB; + lx_sql_ungetc(lx, c); lx_sql_dynpop(lx->buf_opaque); return TOK_OPENSUB; case S10: /* e.g. "%" */ - lx_sql_ungetc(lx, c); return TOK_MANY; + lx_sql_ungetc(lx, c); lx_sql_dynpop(lx->buf_opaque); return TOK_MANY; case S11: /* e.g. "_" */ - lx_sql_ungetc(lx, c); return TOK_ANY; + lx_sql_ungetc(lx, c); lx_sql_dynpop(lx->buf_opaque); return TOK_ANY; default: ; /* unreached */ } - - if (lx->push != NULL) { - if (-1 == lx->push(lx->buf_opaque, (char)c)) { - return TOK_ERROR; - } - } } - lx->lgetc = NULL; - + /* end states */ switch (state) { - case NONE: return TOK_EOF; case S1: return TOK_CHAR; - case S2: return TOK_OPENCOUNT; - case S3: return TOK_OPENGROUP; + case S2: return lx->z = z0, TOK_OPENCOUNT; + case S3: return lx->z = z1, TOK_OPENGROUP; case S4: return TOK_ALT; case S5: return TOK_PLUS; case S6: return TOK_STAR; @@ -618,8 +708,23 @@ z2(struct lx_sql_lx *lx) case S9: return TOK_OPENSUB; case S10: return TOK_MANY; case S11: return TOK_ANY; - default: errno = EINVAL; return TOK_ERROR; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_sql_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } + + if (lx->push != NULL) { + if (-1 == lx->push(lx->buf_opaque, (char)c)) { + return TOK_ERROR; + } + } + + lx->lgetc = NULL; + + if (!has_consumed_input) { + return TOK_EOF; + } + return TOK_ERROR; } const char * @@ -747,6 +852,7 @@ lx_sql_init(struct lx_sql_lx *lx) lx->end.byte = 0; lx->end.line = 1; lx->end.col = 1; + (void)lx_sql_dynpop; } enum lx_sql_token diff --git a/src/libre/dialect/sql/parser.c b/src/libre/dialect/sql/parser.c index 993fcfc26..e7a4c2e75 100644 --- a/src/libre/dialect/sql/parser.c +++ b/src/libre/dialect/sql/parser.c @@ -68,14 +68,6 @@ #define TOK_CLASS__nspace TOK_CLASS_NSPACE #define TOK_CLASS__ndigit TOK_CLASS_NDIGIT - /* This is a hack to work around the AST files not being able to include lexer.h. */ - #define AST_POS_OF_LX_POS(AST_POS, LX_POS) \ - do { \ - AST_POS.line = LX_POS.line; \ - AST_POS.col = LX_POS.col; \ - AST_POS.byte = LX_POS.byte; \ - } while (0) - #include "parser.h" #include "lexer.h" @@ -132,92 +124,8 @@ /* TODO: use lx's generated conveniences for the pattern buffer */ char a[512]; char *p; -#if PCRE_DIALECT - int extended_mode_comment; - enum re_flags *flags_ptr; -#endif /* PCRE_DIALECT */ }; - #define ERROR_TERMINAL (TOK_ERROR) - - #if PCRE_DIALECT - static void mark(struct re_pos *, const struct lx_pos *); - - static void - pcre_advance_lexer(struct lex_state *lex_state, struct act_state *act_state) - { - mark(&act_state->synstart, &lex_state->lx.start); - mark(&act_state->synend, &lex_state->lx.end); - act_state->lex_tok = LX_NEXT(&lex_state->lx); - } - - static enum LX_TOKEN - pcre_current_token(struct lex_state *lex_state, struct act_state *act_state) - { - enum re_flags fl = (lex_state->flags_ptr != NULL) ? lex_state->flags_ptr[0] : 0; - enum LX_TOKEN tok; - - if ((fl & RE_EXTENDED) == 0) { - tok = act_state->lex_tok; - - switch (tok) { - case TOK_WHITESPACE: - case TOK_NEWLINE: - case TOK_MAYBE_COMMENT: - return TOK_CHAR; - - default: - return tok; - } - } - - restart: - tok = act_state->lex_tok; - if (lex_state->extended_mode_comment) { - switch (tok) { - case TOK_EOF: - case TOK_ERROR: - case TOK_UNKNOWN: - /* don't eat EOF or errors */ - return tok; - - case TOK_NEWLINE: - lex_state->extended_mode_comment = 0; - /* fall through */ - default: - pcre_advance_lexer(lex_state, act_state); - goto restart; - } - } else { - switch (tok) { - case TOK_MAYBE_COMMENT: - lex_state->extended_mode_comment = 1; - /*fall through */ - case TOK_WHITESPACE: - case TOK_NEWLINE: - pcre_advance_lexer(lex_state, act_state); - goto restart; - - default: - return tok; - } - } - } - - #define CURRENT_TERMINAL (pcre_current_token(lex_state, act_state)) - #define ADVANCE_LEXER do { pcre_advance_lexer(lex_state,act_state); } while(0) - #else /* !PCRE_DIALECT */ - #define CURRENT_TERMINAL (act_state->lex_tok) - #define ADVANCE_LEXER do { mark(&act_state->synstart, &lex_state->lx.start); \ - mark(&act_state->synend, &lex_state->lx.end); \ - act_state->lex_tok = LX_NEXT(&lex_state->lx); \ - } while (0) - #endif /* PCRE_DIALECT */ - - #define SAVE_LEXER(tok) do { act_state->lex_tok_save = act_state->lex_tok; \ - act_state->lex_tok = tok; } while (0) - #define RESTORE_LEXER do { act_state->lex_tok = act_state->lex_tok_save; } while (0) - static void mark(struct re_pos *r, const struct lx_pos *pos) { @@ -288,7 +196,80 @@ return s; } -#line 292 "src/libre/dialect/sql/parser.c" + static void + advance_lexer(struct lex_state *lex_state, struct act_state *act_state) + { + mark(&act_state->synstart, &lex_state->lx.start); + mark(&act_state->synend, &lex_state->lx.end); + act_state->lex_tok = LX_NEXT(&lex_state->lx); + } + + static enum LX_TOKEN + current_token(struct lex_state *lex_state, struct act_state *act_state, + enum re_flags *flags) + { +#ifndef PCRE_DIALECT + (void) lex_state; + (void) flags; + return act_state->lex_tok; +#else + if ((*flags & RE_EXTENDED) == 0) { + switch (act_state->lex_tok) { + case TOK_WHITESPACE: + case TOK_NEWLINE: + case TOK_MAYBE_COMMENT: + return TOK_CHAR; + + default: + return act_state->lex_tok; + } + } + + int extended_mode_comment = 0; + + restart: + + if (extended_mode_comment) { + switch (act_state->lex_tok) { + case TOK_EOF: + case TOK_ERROR: + case TOK_UNKNOWN: + /* don't eat EOF or errors */ + return act_state->lex_tok; + + case TOK_NEWLINE: + extended_mode_comment = 0; + /* fall through */ + default: + advance_lexer(lex_state, act_state); + goto restart; + } + } else { + switch (act_state->lex_tok) { + case TOK_MAYBE_COMMENT: + extended_mode_comment = 1; + /* fall through */ + case TOK_WHITESPACE: + case TOK_NEWLINE: + advance_lexer(lex_state, act_state); + goto restart; + + default: + return act_state->lex_tok; + } + } +#endif + } + + #define ADVANCE_LEXER (advance_lexer(lex_state, act_state)) + #define CURRENT_TERMINAL (current_token(lex_state, act_state, flags)) + #define ERROR_TERMINAL (TOK_ERROR) + + #define SAVE_LEXER(tok) do { act_state->lex_tok_save = act_state->lex_tok; \ + act_state->lex_tok = tok; } while (0) + #define RESTORE_LEXER do { act_state->lex_tok = act_state->lex_tok_save; } while (0) + +#line 273 "src/libre/dialect/sql/parser.c" #ifndef ERROR_TERMINAL @@ -306,9 +287,9 @@ static void p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags, lex_state, act_stat static void p_expr_C_Ccharacter_Hclass(flags, lex_state, act_state, err, t_ast__expr *); static void p_expr_C_Cpiece(flags, lex_state, act_state, err, t_ast__expr *); static void p_expr(flags, lex_state, act_state, err, t_ast__expr *); -static void p_206(flags, lex_state, act_state, err, t_ast__expr *); -static void p_210(flags, lex_state, act_state, err, t_char *, t_pos *, t_ast__expr *); -static void p_213(flags, lex_state, act_state, err, t_pos *, t_unsigned *, t_ast__count *); +static void p_204(flags, lex_state, act_state, err, t_ast__expr *); +static void p_208(flags, lex_state, act_state, err, t_char *, t_pos *, t_ast__expr *); +static void p_211(flags, lex_state, act_state, err, t_pos *, t_unsigned *, t_ast__count *); static void p_expr_C_Clist_Hof_Halts(flags, lex_state, act_state, err, t_ast__expr); static void p_expr_C_Cpiece_C_Ccount(flags, lex_state, act_state, err, t_ast__count *); static void p_expr_C_Cpiece_C_Catom(flags, lex_state, act_state, err, t_ast__expr *); @@ -326,21 +307,21 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hhead(flags flags, lex_state lex_state, act_ switch (CURRENT_TERMINAL) { case (TOK_INVERT): { - t_char ZI205; + t_char ZI203; /* BEGINNING OF EXTRACT: INVERT */ { -#line 321 "src/libre/parser.act" +#line 302 "src/libre/parser.act" - ZI205 = '^'; + ZI203 = '^'; - (void) ZI205; + (void) ZI203; -#line 340 "src/libre/dialect/sql/parser.c" +#line 321 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: INVERT */ ADVANCE_LEXER; - p_206 (flags, lex_state, act_state, err, ZIclass); + p_204 (flags, lex_state, act_state, err, ZIclass); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -350,47 +331,47 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hhead(flags flags, lex_state lex_state, act_ case (TOK_RANGE): { t_char ZIc; + t_pos ZI114; t_pos ZI115; - t_pos ZI116; t_ast__expr ZInode; /* BEGINNING OF EXTRACT: RANGE */ { -#line 327 "src/libre/parser.act" +#line 308 "src/libre/parser.act" ZIc = '-'; - ZI115 = lex_state->lx.start; - ZI116 = lex_state->lx.end; + ZI114 = lex_state->lx.start; + ZI115 = lex_state->lx.end; (void) ZIc; + (void) ZI114; (void) ZI115; - (void) ZI116; -#line 370 "src/libre/dialect/sql/parser.c" +#line 351 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 908 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode) == NULL) { goto ZL1; } -#line 383 "src/libre/dialect/sql/parser.c" +#line 364 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1084 "src/libre/parser.act" +#line 1046 "src/libre/parser.act" if (!ast_add_expr_alt((*ZIclass), (ZInode))) { goto ZL1; } -#line 394 "src/libre/dialect/sql/parser.c" +#line 375 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-add-alt */ } @@ -415,47 +396,65 @@ p_re__sql(flags flags, lex_state lex_state, act_state act_state, err err, t_ast_ return; } { - /* BEGINNING OF INLINE: 192 */ + t_group__id ZIid; + t_ast__expr ZIe; + + /* BEGINNING OF ACTION: make-group-id */ { - { - p_expr (flags, lex_state, act_state, err, &ZInode); - if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { - RESTORE_LEXER; - goto ZL1; - } - } +#line 881 "src/libre/parser.act" + + (ZIid) = act_state->group_id++; + +#line 409 "src/libre/dialect/sql/parser.c" + } + /* END OF ACTION: make-group-id */ + p_expr (flags, lex_state, act_state, err, &ZIe); + if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { + RESTORE_LEXER; + goto ZL1; + } + /* BEGINNING OF ACTION: ast-make-group */ + { +#line 911 "src/libre/parser.act" + + (ZInode) = ast_make_expr_group(act_state->poolp, *flags, (ZIe), (ZIid)); + if ((ZInode) == NULL) { + goto ZL1; + } + +#line 426 "src/libre/dialect/sql/parser.c" } - /* END OF INLINE: 192 */ - /* BEGINNING OF INLINE: 193 */ + /* END OF ACTION: ast-make-group */ + /* BEGINNING OF INLINE: 191 */ { { switch (CURRENT_TERMINAL) { case (TOK_EOF): break; default: - goto ZL4; + goto ZL3; } ADVANCE_LEXER; } - goto ZL3; - ZL4:; + goto ZL2; + ZL3:; { /* BEGINNING OF ACTION: err-expected-eof */ { -#line 767 "src/libre/parser.act" +#line 753 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXEOF; } goto ZL1; -#line 453 "src/libre/dialect/sql/parser.c" +#line 452 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-eof */ } - ZL3:; + ZL2:; } - /* END OF INLINE: 193 */ + /* END OF INLINE: 191 */ } goto ZL0; ZL1:; @@ -473,7 +472,7 @@ p_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms(flags flags, lex_state lex_ } ZL2_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms:; { - /* BEGINNING OF INLINE: 148 */ + /* BEGINNING OF INLINE: 147 */ { { t_ast__expr ZInode; @@ -485,13 +484,13 @@ ZL2_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms:; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1084 "src/libre/parser.act" +#line 1046 "src/libre/parser.act" if (!ast_add_expr_alt((ZIclass), (ZInode))) { goto ZL4; } -#line 495 "src/libre/dialect/sql/parser.c" +#line 494 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-add-alt */ } @@ -500,21 +499,21 @@ ZL2_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms:; { /* BEGINNING OF ACTION: err-expected-term */ { -#line 704 "src/libre/parser.act" +#line 690 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXTERM; } goto ZL1; -#line 511 "src/libre/dialect/sql/parser.c" +#line 510 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-term */ } ZL3:; } - /* END OF INLINE: 148 */ - /* BEGINNING OF INLINE: 149 */ + /* END OF INLINE: 147 */ + /* BEGINNING OF INLINE: 148 */ { switch (CURRENT_TERMINAL) { case (TOK_NAMED__CLASS): case (TOK_CHAR): @@ -528,7 +527,7 @@ ZL2_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms:; break; } } - /* END OF INLINE: 149 */ + /* END OF INLINE: 148 */ } return; ZL1:; @@ -561,16 +560,16 @@ ZL2_expr_C_Clist_Hof_Hpieces:; } /* BEGINNING OF ACTION: ast-add-concat */ { -#line 1078 "src/libre/parser.act" +#line 1040 "src/libre/parser.act" if (!ast_add_expr_concat((ZIcat), (ZIa))) { goto ZL1; } -#line 571 "src/libre/dialect/sql/parser.c" +#line 570 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-add-concat */ - /* BEGINNING OF INLINE: 184 */ + /* BEGINNING OF INLINE: 183 */ { switch (CURRENT_TERMINAL) { case (TOK_ANY): case (TOK_MANY): case (TOK_OPENSUB): case (TOK_OPENGROUP): @@ -585,7 +584,7 @@ ZL2_expr_C_Clist_Hof_Hpieces:; break; } } - /* END OF INLINE: 184 */ + /* END OF INLINE: 183 */ } return; ZL1:; @@ -601,30 +600,30 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hterm(flags flags, lex_state lex_state, act_ switch (CURRENT_TERMINAL) { case (TOK_CHAR): { - t_char ZI207; - t_pos ZI208; - t_pos ZI209; + t_char ZI205; + t_pos ZI206; + t_pos ZI207; /* BEGINNING OF EXTRACT: CHAR */ { -#line 589 "src/libre/parser.act" +#line 575 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); - ZI208 = lex_state->lx.start; - ZI209 = lex_state->lx.end; + ZI206 = lex_state->lx.start; + ZI207 = lex_state->lx.end; - (void) ZI208; - (void) ZI209; + (void) ZI206; + (void) ZI207; - ZI207 = lex_state->buf.a[0]; + ZI205 = lex_state->buf.a[0]; -#line 624 "src/libre/dialect/sql/parser.c" +#line 623 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; - p_210 (flags, lex_state, act_state, err, &ZI207, &ZI208, &ZInode); + p_208 (flags, lex_state, act_state, err, &ZI205, &ZI206, &ZInode); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -671,7 +670,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state case (TOK_OPENGROUP): /* BEGINNING OF EXTRACT: OPENGROUP */ { -#line 337 "src/libre/parser.act" +#line 318 "src/libre/parser.act" ZIopen__start = lex_state->lx.start; ZIopen__end = lex_state->lx.end; @@ -679,7 +678,7 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state (void) ZIopen__start; (void) ZIopen__end; -#line 683 "src/libre/dialect/sql/parser.c" +#line 682 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: OPENGROUP */ break; @@ -689,120 +688,100 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 901 "src/libre/parser.act" +#line 867 "src/libre/parser.act" (ZIclass) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZIclass) == NULL) { goto ZL1; } -#line 700 "src/libre/dialect/sql/parser.c" +#line 699 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-alt */ ZItmp = ZIclass; p_expr_C_Ccharacter_Hclass_C_Cclass_Hhead (flags, lex_state, act_state, err, &ZIclass); p_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms (flags, lex_state, act_state, err, ZItmp); p_expr_C_Ccharacter_Hclass_C_Cclass_Htail (flags, lex_state, act_state, err, ZItmp); - /* BEGINNING OF INLINE: 154 */ + /* BEGINNING OF INLINE: 153 */ { switch (CURRENT_TERMINAL) { case (TOK_CLOSEGROUP): { - t_char ZI155; + t_char ZI154; t_pos ZIclose__start; t_pos ZIclose__end; /* BEGINNING OF EXTRACT: CLOSEGROUP */ { -#line 369 "src/libre/parser.act" +#line 350 "src/libre/parser.act" - ZI155 = ']'; + ZI154 = ']'; ZIclose__start = lex_state->lx.start; ZIclose__end = lex_state->lx.end; - (void) ZI155; + (void) ZI154; (void) ZIclose__start; (void) ZIclose__end; -#line 728 "src/libre/dialect/sql/parser.c" +#line 727 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: CLOSEGROUP */ ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-group */ { -#line 781 "src/libre/parser.act" - - mark(&act_state->groupstart, &(ZIopen__start)); - mark(&act_state->groupend, &(ZIopen__end)); - -#line 739 "src/libre/dialect/sql/parser.c" - } - /* END OF ACTION: mark-group */ - /* BEGINNING OF ACTION: mark-expr */ - { -#line 796 "src/libre/parser.act" - - struct ast_pos ast_start, ast_end; - - AST_POS_OF_LX_POS(ast_start, (ZIopen__start)); - AST_POS_OF_LX_POS(ast_end, (ZIclose__end)); +#line 767 "src/libre/parser.act" mark(&act_state->groupstart, &(ZIopen__start)); mark(&act_state->groupend, &(ZIclose__end)); - -/* TODO: reinstate this, applies to an expr node in general - (ZItmp)->u.class.start = ast_start; - (ZItmp)->u.class.end = ast_end; -*/ -#line 759 "src/libre/dialect/sql/parser.c" +#line 738 "src/libre/dialect/sql/parser.c" } - /* END OF ACTION: mark-expr */ + /* END OF ACTION: mark-group */ ZInode = ZIclass; } break; case (TOK_INVERT): { - t_char ZI159; + t_char ZI158; t_ast__expr ZImask; t_ast__expr ZImask__tmp; /* BEGINNING OF EXTRACT: INVERT */ { -#line 321 "src/libre/parser.act" +#line 302 "src/libre/parser.act" - ZI159 = '^'; + ZI158 = '^'; - (void) ZI159; + (void) ZI158; -#line 779 "src/libre/dialect/sql/parser.c" +#line 758 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: INVERT */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-alt */ { -#line 901 "src/libre/parser.act" +#line 867 "src/libre/parser.act" (ZImask) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZImask) == NULL) { goto ZL3; } -#line 792 "src/libre/dialect/sql/parser.c" +#line 771 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-alt */ ZImask__tmp = ZImask; p_expr_C_Ccharacter_Hclass_C_Cclass_Hhead (flags, lex_state, act_state, err, &ZImask); p_expr_C_Ccharacter_Hclass_C_Clist_Hof_Hclass_Hterms (flags, lex_state, act_state, err, ZImask__tmp); p_expr_C_Ccharacter_Hclass_C_Cclass_Htail (flags, lex_state, act_state, err, ZImask__tmp); - /* BEGINNING OF INLINE: 163 */ + /* BEGINNING OF INLINE: 162 */ { if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL3; } { - t_char ZI164; + t_char ZI163; t_pos ZIclose__start; t_pos ZIclose__end; @@ -810,17 +789,17 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state case (TOK_CLOSEGROUP): /* BEGINNING OF EXTRACT: CLOSEGROUP */ { -#line 369 "src/libre/parser.act" +#line 350 "src/libre/parser.act" - ZI164 = ']'; + ZI163 = ']'; ZIclose__start = lex_state->lx.start; ZIclose__end = lex_state->lx.end; - (void) ZI164; + (void) ZI163; (void) ZIclose__start; (void) ZIclose__end; -#line 824 "src/libre/dialect/sql/parser.c" +#line 803 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: CLOSEGROUP */ break; @@ -830,84 +809,44 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-group */ { -#line 781 "src/libre/parser.act" - - mark(&act_state->groupstart, &(ZIclose__start)); - mark(&act_state->groupend, &(ZIclose__end)); - -#line 839 "src/libre/dialect/sql/parser.c" - } - /* END OF ACTION: mark-group */ - /* BEGINNING OF ACTION: mark-expr */ - { -#line 796 "src/libre/parser.act" - - struct ast_pos ast_start, ast_end; - - AST_POS_OF_LX_POS(ast_start, (ZIopen__start)); - AST_POS_OF_LX_POS(ast_end, (ZIclose__end)); - - mark(&act_state->groupstart, &(ZIopen__start)); - mark(&act_state->groupend, &(ZIclose__end)); - -/* TODO: reinstate this, applies to an expr node in general - (ZItmp)->u.class.start = ast_start; - (ZItmp)->u.class.end = ast_end; -*/ - -#line 859 "src/libre/dialect/sql/parser.c" - } - /* END OF ACTION: mark-expr */ - /* BEGINNING OF ACTION: mark-expr */ - { -#line 796 "src/libre/parser.act" - - struct ast_pos ast_start, ast_end; - - AST_POS_OF_LX_POS(ast_start, (ZIopen__start)); - AST_POS_OF_LX_POS(ast_end, (ZIclose__end)); +#line 767 "src/libre/parser.act" mark(&act_state->groupstart, &(ZIopen__start)); mark(&act_state->groupend, &(ZIclose__end)); - -/* TODO: reinstate this, applies to an expr node in general - (ZImask__tmp)->u.class.start = ast_start; - (ZImask__tmp)->u.class.end = ast_end; -*/ -#line 879 "src/libre/dialect/sql/parser.c" +#line 818 "src/libre/dialect/sql/parser.c" } - /* END OF ACTION: mark-expr */ + /* END OF ACTION: mark-group */ } goto ZL4; ZL5:; { /* BEGINNING OF ACTION: err-expected-closegroup */ { -#line 739 "src/libre/parser.act" +#line 725 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCLOSEGROUP; } goto ZL3; -#line 895 "src/libre/dialect/sql/parser.c" +#line 834 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-closegroup */ } ZL4:; } - /* END OF INLINE: 163 */ + /* END OF INLINE: 162 */ /* BEGINNING OF ACTION: ast-make-subtract */ { -#line 993 "src/libre/parser.act" +#line 959 "src/libre/parser.act" (ZInode) = ast_make_expr_subtract(act_state->poolp, *flags, (ZIclass), (ZImask)); if ((ZInode) == NULL) { goto ZL3; } -#line 911 "src/libre/dialect/sql/parser.c" +#line 850 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-subtract */ } @@ -923,32 +862,32 @@ p_expr_C_Ccharacter_Hclass(flags flags, lex_state lex_state, act_state act_state { /* BEGINNING OF ACTION: err-expected-closegroup */ { -#line 739 "src/libre/parser.act" +#line 725 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCLOSEGROUP; } goto ZL1; -#line 934 "src/libre/dialect/sql/parser.c" +#line 873 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-closegroup */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 887 "src/libre/parser.act" +#line 853 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 946 "src/libre/dialect/sql/parser.c" +#line 885 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-empty */ } ZL2:; } - /* END OF INLINE: 154 */ + /* END OF INLINE: 153 */ } goto ZL0; ZL1:; @@ -978,7 +917,7 @@ p_expr_C_Cpiece(flags flags, lex_state lex_state, act_state act_state, err err, } /* BEGINNING OF ACTION: ast-make-piece */ { -#line 931 "src/libre/parser.act" +#line 897 "src/libre/parser.act" if ((ZIc).min == 0 && (ZIc).max == 0) { (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); @@ -992,7 +931,7 @@ p_expr_C_Cpiece(flags flags, lex_state lex_state, act_state act_state, err err, goto ZL1; } -#line 996 "src/libre/dialect/sql/parser.c" +#line 935 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-piece */ } @@ -1015,14 +954,14 @@ p_expr(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__ex { /* BEGINNING OF ACTION: ast-make-alt */ { -#line 901 "src/libre/parser.act" +#line 867 "src/libre/parser.act" (ZInode) = ast_make_expr_alt(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 1026 "src/libre/dialect/sql/parser.c" +#line 965 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-alt */ p_expr_C_Clist_Hof_Halts (flags, lex_state, act_state, err, ZInode); @@ -1036,26 +975,26 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-alts */ { -#line 725 "src/libre/parser.act" +#line 711 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXALTS; } goto ZL2; -#line 1047 "src/libre/dialect/sql/parser.c" +#line 986 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-alts */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 887 "src/libre/parser.act" +#line 853 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL2; } -#line 1059 "src/libre/dialect/sql/parser.c" +#line 998 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -1068,58 +1007,58 @@ ZL0:; } static void -p_206(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__expr *ZIclass) +p_204(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__expr *ZIclass) { switch (CURRENT_TERMINAL) { case (TOK_RANGE): { t_char ZIc; + t_pos ZI117; t_pos ZI118; - t_pos ZI119; t_ast__expr ZInode; /* BEGINNING OF EXTRACT: RANGE */ { -#line 327 "src/libre/parser.act" +#line 308 "src/libre/parser.act" ZIc = '-'; - ZI118 = lex_state->lx.start; - ZI119 = lex_state->lx.end; + ZI117 = lex_state->lx.start; + ZI118 = lex_state->lx.end; (void) ZIc; + (void) ZI117; (void) ZI118; - (void) ZI119; -#line 1094 "src/libre/dialect/sql/parser.c" +#line 1033 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 908 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (ZIc)); if ((ZInode) == NULL) { goto ZL1; } -#line 1107 "src/libre/dialect/sql/parser.c" +#line 1046 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-literal */ /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1084 "src/libre/parser.act" +#line 1046 "src/libre/parser.act" if (!ast_add_expr_alt((*ZIclass), (ZInode))) { goto ZL1; } -#line 1118 "src/libre/dialect/sql/parser.c" +#line 1057 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-add-alt */ /* BEGINNING OF ACTION: ast-make-invert */ { -#line 1000 "src/libre/parser.act" +#line 966 "src/libre/parser.act" struct ast_expr *any; @@ -1157,7 +1096,7 @@ p_206(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp goto ZL1; } -#line 1161 "src/libre/dialect/sql/parser.c" +#line 1100 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-invert */ } @@ -1166,7 +1105,7 @@ p_206(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp { /* BEGINNING OF ACTION: ast-make-invert */ { -#line 1000 "src/libre/parser.act" +#line 966 "src/libre/parser.act" struct ast_expr *any; @@ -1204,7 +1143,7 @@ p_206(flags flags, lex_state lex_state, act_state act_state, err err, t_ast__exp goto ZL1; } -#line 1208 "src/libre/dialect/sql/parser.c" +#line 1147 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-invert */ } @@ -1219,7 +1158,7 @@ ZL1:; } static void -p_210(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI207, t_pos *ZI208, t_ast__expr *ZOnode) +p_208(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI205, t_pos *ZI206, t_ast__expr *ZOnode) { t_ast__expr ZInode; @@ -1228,14 +1167,14 @@ p_210(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI { /* BEGINNING OF ACTION: ast-make-literal */ { -#line 908 "src/libre/parser.act" +#line 874 "src/libre/parser.act" - (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (*ZI207)); + (ZInode) = ast_make_expr_literal(act_state->poolp, *flags, (*ZI205)); if ((ZInode) == NULL) { goto ZL1; } -#line 1239 "src/libre/dialect/sql/parser.c" +#line 1178 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -1243,37 +1182,37 @@ p_210(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI case (TOK_RANGE): { t_endpoint ZIa; - t_char ZI137; + t_char ZI136; + t_pos ZI137; t_pos ZI138; - t_pos ZI139; t_char ZIcz; - t_pos ZI141; + t_pos ZI140; t_pos ZIend; t_endpoint ZIz; /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 873 "src/libre/parser.act" +#line 839 "src/libre/parser.act" (ZIa).type = AST_ENDPOINT_LITERAL; - (ZIa).u.literal.c = (unsigned char)(*ZI207); + (ZIa).u.literal.c = (unsigned char) (*ZI205); -#line 1262 "src/libre/dialect/sql/parser.c" +#line 1201 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF EXTRACT: RANGE */ { -#line 327 "src/libre/parser.act" +#line 308 "src/libre/parser.act" - ZI137 = '-'; - ZI138 = lex_state->lx.start; - ZI139 = lex_state->lx.end; + ZI136 = '-'; + ZI137 = lex_state->lx.start; + ZI138 = lex_state->lx.end; + (void) ZI136; (void) ZI137; (void) ZI138; - (void) ZI139; -#line 1277 "src/libre/dialect/sql/parser.c" +#line 1216 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: RANGE */ ADVANCE_LEXER; @@ -1281,20 +1220,20 @@ p_210(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI case (TOK_CHAR): /* BEGINNING OF EXTRACT: CHAR */ { -#line 589 "src/libre/parser.act" +#line 575 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); - ZI141 = lex_state->lx.start; + ZI140 = lex_state->lx.start; ZIend = lex_state->lx.end; - (void) ZI141; + (void) ZI140; (void) ZIend; ZIcz = lex_state->buf.a[0]; -#line 1298 "src/libre/dialect/sql/parser.c" +#line 1237 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: CHAR */ break; @@ -1304,34 +1243,30 @@ p_210(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-range-endpoint-literal */ { -#line 873 "src/libre/parser.act" +#line 839 "src/libre/parser.act" (ZIz).type = AST_ENDPOINT_LITERAL; - (ZIz).u.literal.c = (unsigned char)(ZIcz); + (ZIz).u.literal.c = (unsigned char) (ZIcz); -#line 1313 "src/libre/dialect/sql/parser.c" +#line 1252 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-range-endpoint-literal */ /* BEGINNING OF ACTION: mark-range */ { -#line 786 "src/libre/parser.act" +#line 772 "src/libre/parser.act" - mark(&act_state->rangestart, &(*ZI208)); + mark(&act_state->rangestart, &(*ZI206)); mark(&act_state->rangeend, &(ZIend)); -#line 1323 "src/libre/dialect/sql/parser.c" +#line 1262 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: mark-range */ /* BEGINNING OF ACTION: ast-make-range */ { -#line 1038 "src/libre/parser.act" +#line 1004 "src/libre/parser.act" - struct ast_pos ast_start, ast_end; unsigned char lower, upper; - AST_POS_OF_LX_POS(ast_start, (*ZI208)); - AST_POS_OF_LX_POS(ast_end, (ZIend)); - if ((ZIa).type != AST_ENDPOINT_LITERAL || (ZIz).type != AST_ENDPOINT_LITERAL) { err->e = RE_EUNSUPPORTED; @@ -1352,12 +1287,12 @@ p_210(flags flags, lex_state lex_state, act_state act_state, err err, t_char *ZI goto ZL1; } - (ZInode) = ast_make_expr_range(act_state->poolp, *flags, &(ZIa), ast_start, &(ZIz), ast_end); + (ZInode) = ast_make_expr_range(act_state->poolp, *flags, &(ZIa), &(ZIz)); if ((ZInode) == NULL) { goto ZL1; } -#line 1361 "src/libre/dialect/sql/parser.c" +#line 1296 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-range */ } @@ -1374,63 +1309,58 @@ ZL0:; } static void -p_213(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI211, t_unsigned *ZIm, t_ast__count *ZOc) +p_211(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI209, t_unsigned *ZIm, t_ast__count *ZOc) { t_ast__count ZIc; switch (CURRENT_TERMINAL) { case (TOK_CLOSECOUNT): { - t_pos ZI177; + t_pos ZI176; t_pos ZIend; /* BEGINNING OF EXTRACT: CLOSECOUNT */ { -#line 397 "src/libre/parser.act" +#line 378 "src/libre/parser.act" - ZI177 = lex_state->lx.start; + ZI176 = lex_state->lx.start; ZIend = lex_state->lx.end; - (void) ZI177; + (void) ZI176; (void) ZIend; -#line 1398 "src/libre/dialect/sql/parser.c" +#line 1333 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: CLOSECOUNT */ ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-count */ { -#line 791 "src/libre/parser.act" +#line 777 "src/libre/parser.act" - mark(&act_state->countstart, &(*ZI211)); + mark(&act_state->countstart, &(*ZI209)); mark(&act_state->countend, &(ZIend)); -#line 1409 "src/libre/dialect/sql/parser.c" +#line 1344 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-range */ { -#line 853 "src/libre/parser.act" - - struct ast_pos ast_start, ast_end; +#line 824 "src/libre/parser.act" if ((*ZIm) < (*ZIm)) { err->e = RE_ENEGCOUNT; err->m = (*ZIm); err->n = (*ZIm); - mark(&act_state->countstart, &(*ZI211)); + mark(&act_state->countstart, &(*ZI209)); mark(&act_state->countend, &(ZIend)); goto ZL1; } - AST_POS_OF_LX_POS(ast_start, (*ZI211)); - AST_POS_OF_LX_POS(ast_end, (ZIend)); - - (ZIc) = ast_make_count((*ZIm), &ast_start, (*ZIm), &ast_end); + (ZIc) = ast_make_count((*ZIm), (*ZIm)); -#line 1434 "src/libre/dialect/sql/parser.c" +#line 1364 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-range */ } @@ -1438,7 +1368,7 @@ p_213(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 case (TOK_SEP): { t_unsigned ZIn; - t_pos ZI180; + t_pos ZI179; t_pos ZIend; ADVANCE_LEXER; @@ -1446,7 +1376,7 @@ p_213(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 case (TOK_COUNT): /* BEGINNING OF EXTRACT: COUNT */ { -#line 641 "src/libre/parser.act" +#line 627 "src/libre/parser.act" unsigned long u; char *e; @@ -1466,7 +1396,7 @@ p_213(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 ZIn = (unsigned int) u; -#line 1470 "src/libre/dialect/sql/parser.c" +#line 1400 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: COUNT */ break; @@ -1478,15 +1408,15 @@ p_213(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 case (TOK_CLOSECOUNT): /* BEGINNING OF EXTRACT: CLOSECOUNT */ { -#line 397 "src/libre/parser.act" +#line 378 "src/libre/parser.act" - ZI180 = lex_state->lx.start; + ZI179 = lex_state->lx.start; ZIend = lex_state->lx.end; - (void) ZI180; + (void) ZI179; (void) ZIend; -#line 1490 "src/libre/dialect/sql/parser.c" +#line 1420 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: CLOSECOUNT */ break; @@ -1496,37 +1426,32 @@ p_213(flags flags, lex_state lex_state, act_state act_state, err err, t_pos *ZI2 ADVANCE_LEXER; /* BEGINNING OF ACTION: mark-count */ { -#line 791 "src/libre/parser.act" +#line 777 "src/libre/parser.act" - mark(&act_state->countstart, &(*ZI211)); + mark(&act_state->countstart, &(*ZI209)); mark(&act_state->countend, &(ZIend)); -#line 1505 "src/libre/dialect/sql/parser.c" +#line 1435 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: mark-count */ /* BEGINNING OF ACTION: count-range */ { -#line 853 "src/libre/parser.act" - - struct ast_pos ast_start, ast_end; +#line 824 "src/libre/parser.act" if ((ZIn) < (*ZIm)) { err->e = RE_ENEGCOUNT; err->m = (*ZIm); err->n = (ZIn); - mark(&act_state->countstart, &(*ZI211)); + mark(&act_state->countstart, &(*ZI209)); mark(&act_state->countend, &(ZIend)); goto ZL1; } - AST_POS_OF_LX_POS(ast_start, (*ZI211)); - AST_POS_OF_LX_POS(ast_end, (ZIend)); - - (ZIc) = ast_make_count((*ZIm), &ast_start, (ZIn), &ast_end); + (ZIc) = ast_make_count((*ZIm), (ZIn)); -#line 1530 "src/libre/dialect/sql/parser.c" +#line 1455 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-range */ } @@ -1561,16 +1486,16 @@ ZL2_expr_C_Clist_Hof_Halts:; } /* BEGINNING OF ACTION: ast-add-alt */ { -#line 1084 "src/libre/parser.act" +#line 1046 "src/libre/parser.act" if (!ast_add_expr_alt((ZIalts), (ZIa))) { goto ZL1; } -#line 1571 "src/libre/dialect/sql/parser.c" +#line 1496 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-add-alt */ - /* BEGINNING OF INLINE: 190 */ + /* BEGINNING OF INLINE: 189 */ { switch (CURRENT_TERMINAL) { case (TOK_ALT): @@ -1585,21 +1510,21 @@ ZL2_expr_C_Clist_Hof_Halts:; break; } } - /* END OF INLINE: 190 */ + /* END OF INLINE: 189 */ } return; ZL1:; { /* BEGINNING OF ACTION: err-expected-alts */ { -#line 725 "src/libre/parser.act" +#line 711 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXALTS; } goto ZL4; -#line 1603 "src/libre/dialect/sql/parser.c" +#line 1528 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-alts */ } @@ -1618,21 +1543,21 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, switch (CURRENT_TERMINAL) { case (TOK_OPENCOUNT): { - t_pos ZI211; - t_pos ZI212; + t_pos ZI209; + t_pos ZI210; t_unsigned ZIm; /* BEGINNING OF EXTRACT: OPENCOUNT */ { -#line 389 "src/libre/parser.act" +#line 370 "src/libre/parser.act" - ZI211 = lex_state->lx.start; - ZI212 = lex_state->lx.end; + ZI209 = lex_state->lx.start; + ZI210 = lex_state->lx.end; - (void) ZI211; - (void) ZI212; + (void) ZI209; + (void) ZI210; -#line 1636 "src/libre/dialect/sql/parser.c" +#line 1561 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: OPENCOUNT */ ADVANCE_LEXER; @@ -1640,7 +1565,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, case (TOK_COUNT): /* BEGINNING OF EXTRACT: COUNT */ { -#line 641 "src/libre/parser.act" +#line 627 "src/libre/parser.act" unsigned long u; char *e; @@ -1660,7 +1585,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ZIm = (unsigned int) u; -#line 1664 "src/libre/dialect/sql/parser.c" +#line 1589 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: COUNT */ break; @@ -1668,7 +1593,7 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, goto ZL1; } ADVANCE_LEXER; - p_213 (flags, lex_state, act_state, err, &ZI211, &ZIm, &ZIc); + p_211 (flags, lex_state, act_state, err, &ZI209, &ZIm, &ZIc); if ((CURRENT_TERMINAL) == (ERROR_TERMINAL)) { RESTORE_LEXER; goto ZL1; @@ -1680,11 +1605,11 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-zero-or-one */ { -#line 845 "src/libre/parser.act" +#line 816 "src/libre/parser.act" - (ZIc) = ast_make_count(0, NULL, 1, NULL); + (ZIc) = ast_make_count(0, 1); -#line 1688 "src/libre/dialect/sql/parser.c" +#line 1613 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-zero-or-one */ } @@ -1694,11 +1619,11 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-one-or-more */ { -#line 841 "src/libre/parser.act" +#line 812 "src/libre/parser.act" - (ZIc) = ast_make_count(1, NULL, AST_COUNT_UNBOUNDED, NULL); + (ZIc) = ast_make_count(1, AST_COUNT_UNBOUNDED); -#line 1702 "src/libre/dialect/sql/parser.c" +#line 1627 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-one-or-more */ } @@ -1708,11 +1633,11 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, ADVANCE_LEXER; /* BEGINNING OF ACTION: count-zero-or-more */ { -#line 837 "src/libre/parser.act" +#line 808 "src/libre/parser.act" - (ZIc) = ast_make_count(0, NULL, AST_COUNT_UNBOUNDED, NULL); + (ZIc) = ast_make_count(0, AST_COUNT_UNBOUNDED); -#line 1716 "src/libre/dialect/sql/parser.c" +#line 1641 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-zero-or-more */ } @@ -1721,11 +1646,11 @@ p_expr_C_Cpiece_C_Ccount(flags flags, lex_state lex_state, act_state act_state, { /* BEGINNING OF ACTION: count-one */ { -#line 849 "src/libre/parser.act" +#line 820 "src/libre/parser.act" - (ZIc) = ast_make_count(1, NULL, 1, NULL); + (ZIc) = ast_make_count(1, 1); -#line 1729 "src/libre/dialect/sql/parser.c" +#line 1654 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-one */ } @@ -1738,23 +1663,23 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-count */ { -#line 711 "src/libre/parser.act" +#line 697 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXCOUNT; } goto ZL2; -#line 1749 "src/libre/dialect/sql/parser.c" +#line 1674 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-count */ /* BEGINNING OF ACTION: count-one */ { -#line 849 "src/libre/parser.act" +#line 820 "src/libre/parser.act" - (ZIc) = ast_make_count(1, NULL, 1, NULL); + (ZIc) = ast_make_count(1, 1); -#line 1758 "src/libre/dialect/sql/parser.c" +#line 1683 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-one */ } @@ -1779,24 +1704,24 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 811 "src/libre/parser.act" +#line 782 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; -#line 1788 "src/libre/dialect/sql/parser.c" +#line 1713 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1071 "src/libre/parser.act" +#line 1033 "src/libre/parser.act" (ZIe) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIe) == NULL) { goto ZL1; } -#line 1800 "src/libre/dialect/sql/parser.c" +#line 1725 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -1804,38 +1729,38 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e case (TOK_CHAR): { t_char ZIa; + t_pos ZI170; t_pos ZI171; - t_pos ZI172; /* BEGINNING OF EXTRACT: CHAR */ { -#line 589 "src/libre/parser.act" +#line 575 "src/libre/parser.act" /* the first byte may be '\x00' */ assert(lex_state->buf.a[1] == '\0'); - ZI171 = lex_state->lx.start; - ZI172 = lex_state->lx.end; + ZI170 = lex_state->lx.start; + ZI171 = lex_state->lx.end; + (void) ZI170; (void) ZI171; - (void) ZI172; ZIa = lex_state->buf.a[0]; -#line 1826 "src/libre/dialect/sql/parser.c" +#line 1751 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-literal */ { -#line 908 "src/libre/parser.act" +#line 874 "src/libre/parser.act" (ZIe) = ast_make_expr_literal(act_state->poolp, *flags, (ZIa)); if ((ZIe) == NULL) { goto ZL1; } -#line 1839 "src/libre/dialect/sql/parser.c" +#line 1764 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-literal */ } @@ -1849,38 +1774,38 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: class-any */ { -#line 811 "src/libre/parser.act" +#line 782 "src/libre/parser.act" /* TODO: or the unicode equivalent */ (ZIa) = (*flags & RE_SINGLE) ? &class_any : &class_notnl; -#line 1858 "src/libre/dialect/sql/parser.c" +#line 1783 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: class-any */ /* BEGINNING OF ACTION: ast-make-named */ { -#line 1071 "src/libre/parser.act" +#line 1033 "src/libre/parser.act" (ZIg) = ast_make_expr_named(act_state->poolp, *flags, (ZIa)); if ((ZIg) == NULL) { goto ZL1; } -#line 1870 "src/libre/dialect/sql/parser.c" +#line 1795 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-named */ /* BEGINNING OF ACTION: count-zero-or-more */ { -#line 837 "src/libre/parser.act" +#line 808 "src/libre/parser.act" - (ZIc) = ast_make_count(0, NULL, AST_COUNT_UNBOUNDED, NULL); + (ZIc) = ast_make_count(0, AST_COUNT_UNBOUNDED); -#line 1879 "src/libre/dialect/sql/parser.c" +#line 1804 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: count-zero-or-more */ /* BEGINNING OF ACTION: ast-make-piece */ { -#line 931 "src/libre/parser.act" +#line 897 "src/libre/parser.act" if ((ZIc).min == 0 && (ZIc).max == 0) { (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); @@ -1894,7 +1819,7 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e goto ZL1; } -#line 1898 "src/libre/dialect/sql/parser.c" +#line 1823 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-piece */ } @@ -1907,11 +1832,11 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e ADVANCE_LEXER; /* BEGINNING OF ACTION: make-group-id */ { -#line 915 "src/libre/parser.act" +#line 881 "src/libre/parser.act" (ZIid) = act_state->group_id++; -#line 1915 "src/libre/dialect/sql/parser.c" +#line 1840 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: make-group-id */ p_expr (flags, lex_state, act_state, err, &ZIg); @@ -1921,14 +1846,14 @@ p_expr_C_Cpiece_C_Catom(flags flags, lex_state lex_state, act_state act_state, e } /* BEGINNING OF ACTION: ast-make-group */ { -#line 945 "src/libre/parser.act" +#line 911 "src/libre/parser.act" (ZIe) = ast_make_expr_group(act_state->poolp, *flags, (ZIg), (ZIid)); if ((ZIe) == NULL) { goto ZL1; } -#line 1932 "src/libre/dialect/sql/parser.c" +#line 1857 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-group */ switch (CURRENT_TERMINAL) { @@ -1959,26 +1884,26 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-atom */ { -#line 718 "src/libre/parser.act" +#line 704 "src/libre/parser.act" if (err->e == RE_ESUCCESS) { err->e = RE_EXATOM; } goto ZL2; -#line 1970 "src/libre/dialect/sql/parser.c" +#line 1895 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: err-expected-atom */ /* BEGINNING OF ACTION: ast-make-empty */ { -#line 887 "src/libre/parser.act" +#line 853 "src/libre/parser.act" (ZIe) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZIe) == NULL) { goto ZL2; } -#line 1982 "src/libre/dialect/sql/parser.c" +#line 1907 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -2000,14 +1925,14 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hnamed(flags flags, lex_state lex_state, act } { t_ast__class__id ZIid; + t_pos ZI128; t_pos ZI129; - t_pos ZI130; switch (CURRENT_TERMINAL) { case (TOK_NAMED__CLASS): /* BEGINNING OF EXTRACT: NAMED_CLASS */ { -#line 661 "src/libre/parser.act" +#line 647 "src/libre/parser.act" ZIid = DIALECT_CLASS(lex_state->buf.a); if (ZIid == NULL) { @@ -2015,13 +1940,13 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hnamed(flags flags, lex_state lex_state, act goto ZL1; } - ZI129 = lex_state->lx.start; - ZI130 = lex_state->lx.end; + ZI128 = lex_state->lx.start; + ZI129 = lex_state->lx.end; + (void) ZI128; (void) ZI129; - (void) ZI130; -#line 2025 "src/libre/dialect/sql/parser.c" +#line 1950 "src/libre/dialect/sql/parser.c" } /* END OF EXTRACT: NAMED_CLASS */ break; @@ -2031,14 +1956,14 @@ p_expr_C_Ccharacter_Hclass_C_Cclass_Hnamed(flags flags, lex_state lex_state, act ADVANCE_LEXER; /* BEGINNING OF ACTION: ast-make-named */ { -#line 1071 "src/libre/parser.act" +#line 1033 "src/libre/parser.act" (ZInode) = ast_make_expr_named(act_state->poolp, *flags, (ZIid)); if ((ZInode) == NULL) { goto ZL1; } -#line 2042 "src/libre/dialect/sql/parser.c" +#line 1967 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-named */ } @@ -2061,14 +1986,14 @@ p_expr_C_Calt(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: ast-make-concat */ { -#line 894 "src/libre/parser.act" +#line 860 "src/libre/parser.act" (ZInode) = ast_make_expr_concat(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 2072 "src/libre/dialect/sql/parser.c" +#line 1997 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-concat */ p_expr_C_Clist_Hof_Hpieces (flags, lex_state, act_state, err, ZInode); @@ -2082,14 +2007,14 @@ p_expr_C_Calt(flags flags, lex_state lex_state, act_state act_state, err err, t_ { /* BEGINNING OF ACTION: ast-make-empty */ { -#line 887 "src/libre/parser.act" +#line 853 "src/libre/parser.act" (ZInode) = ast_make_expr_empty(act_state->poolp, *flags); if ((ZInode) == NULL) { goto ZL1; } -#line 2093 "src/libre/dialect/sql/parser.c" +#line 2018 "src/libre/dialect/sql/parser.c" } /* END OF ACTION: ast-make-empty */ } @@ -2107,7 +2032,7 @@ ZL0:; /* BEGINNING OF TRAILER */ -#line 1090 "src/libre/parser.act" +#line 1052 "src/libre/parser.act" static int @@ -2127,9 +2052,8 @@ ZL0:; struct ast * DIALECT_PARSE(re_getchar_fun *f, void *opaque, - const struct fsm_options *opt, enum re_flags flags, int overlap, - struct re_err *err) + struct re_err *err, struct re_pos *end) { struct ast *ast; @@ -2142,6 +2066,7 @@ ZL0:; struct LX_STATE *lx; assert(f != NULL); + assert(end != NULL); ast = ast_new(); @@ -2165,11 +2090,6 @@ ZL0:; lex_state->buf.a = NULL; lex_state->buf.len = 0; -#if PCRE_DIALECT - lex_state->extended_mode_comment = 0; - lex_state->flags_ptr = &flags; -#endif /* PCRE_DIALECT */ - /* XXX: unneccessary since we're lexing from a string */ /* (except for pushing "[" and "]" around ::group-$dialect) */ lx->buf_opaque = &lex_state->buf; @@ -2200,6 +2120,9 @@ ZL0:; DIALECT_ENTRY(&flags, lex_state, act_state, err, &ast->expr); + /* we output this for reporting errors found through AST analysis */ + mark(end, &lx->end); + lx->free(lx->buf_opaque); if (err->e != RE_ESUCCESS) { @@ -2265,6 +2188,6 @@ ZL0:; return NULL; } -#line 2269 "src/libre/dialect/sql/parser.c" +#line 2192 "src/libre/dialect/sql/parser.c" /* END OF FILE */ diff --git a/src/libre/dialect/sql/parser.h b/src/libre/dialect/sql/parser.h index 63ed8e773..7825ae3af 100644 --- a/src/libre/dialect/sql/parser.h +++ b/src/libre/dialect/sql/parser.h @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 300 "src/libre/parser.act" +#line 281 "src/libre/parser.act" #include @@ -28,7 +28,7 @@ extern void p_re__sql(flags, lex_state, act_state, err, t_ast__expr *); /* BEGINNING OF TRAILER */ -#line 1247 "src/libre/parser.act" +#line 1207 "src/libre/parser.act" #line 35 "src/libre/dialect/sql/parser.h" diff --git a/src/libre/dialect/sql/parser.sid b/src/libre/dialect/sql/parser.sid index c051c2a2c..f8609c6f5 100644 --- a/src/libre/dialect/sql/parser.sid +++ b/src/libre/dialect/sql/parser.sid @@ -139,7 +139,6 @@ : (:pos, :pos) -> (); : (:pos, :pos) -> (); : (:pos, :pos) -> (); - : (:ast_expr, :pos, :pos) -> (); : () -> (:ast_class_id); !: () -> (:ast_class_id); @@ -253,9 +252,7 @@ { (!, close_start, close_end) = CLOSEGROUP; - (open_start, open_end); - - (tmp, open_start, close_end); + (open_start, close_end); node = class; || @@ -287,11 +284,8 @@ { (!, close_start, close_end) = CLOSEGROUP; - (close_start, close_end); - /* TODO: would prefer to demark class and mask separately */ - (tmp, open_start, close_end); - (mask_tmp, open_start, close_end); + (open_start, close_end); ## ; }; @@ -412,9 +406,9 @@ }; re_sql: () -> (node :ast_expr) = { - { - node = expr; - }; + id = ; + e = expr; + node = (e, id); { EOF; diff --git a/src/libre/libre.syms b/src/libre/libre.syms index bc4e89c2a..b04833777 100644 --- a/src/libre/libre.syms +++ b/src/libre/libre.syms @@ -3,13 +3,17 @@ re_is_literal re_flags re_strerror re_perror +re_is_anchored +re_interpolate_groups -# XXX: for re(1) -ast_new_from_fsm +ast_print ast_print_dot ast_print_abnf ast_print_pcre ast_print_tree + +# XXX: for re(1) +ast_new_from_fsm ast_free re_parse re_strings diff --git a/src/libre/parser.act b/src/libre/parser.act index b836510ae..984036cb8 100644 --- a/src/libre/parser.act +++ b/src/libre/parser.act @@ -77,14 +77,6 @@ #define TOK_CLASS__nspace TOK_CLASS_NSPACE #define TOK_CLASS__ndigit TOK_CLASS_NDIGIT - /* This is a hack to work around the AST files not being able to include lexer.h. */ - #define AST_POS_OF_LX_POS(AST_POS, LX_POS) \ - do { \ - AST_POS.line = LX_POS.line; \ - AST_POS.col = LX_POS.col; \ - AST_POS.byte = LX_POS.byte; \ - } while (0) - #include "parser.h" #include "lexer.h" @@ -141,92 +133,8 @@ /* TODO: use lx's generated conveniences for the pattern buffer */ char a[512]; char *p; -#if PCRE_DIALECT - int extended_mode_comment; - enum re_flags *flags_ptr; -#endif /* PCRE_DIALECT */ }; - #define ERROR_TERMINAL @$ERROR - - #if PCRE_DIALECT - static void mark(struct re_pos *, const struct lx_pos *); - - static void - pcre_advance_lexer(struct lex_state *lex_state, struct act_state *act_state) - { - mark(&act_state->synstart, &lex_state->lx.start); - mark(&act_state->synend, &lex_state->lx.end); - act_state->lex_tok = LX_NEXT(&lex_state->lx); - } - - static enum LX_TOKEN - pcre_current_token(struct lex_state *lex_state, struct act_state *act_state) - { - enum re_flags fl = (lex_state->flags_ptr != NULL) ? lex_state->flags_ptr[0] : 0; - enum LX_TOKEN tok; - - if ((fl & RE_EXTENDED) == 0) { - tok = act_state->lex_tok; - - switch (tok) { - case TOK_WHITESPACE: - case TOK_NEWLINE: - case TOK_MAYBE_COMMENT: - return TOK_CHAR; - - default: - return tok; - } - } - - restart: - tok = act_state->lex_tok; - if (lex_state->extended_mode_comment) { - switch (tok) { - case TOK_EOF: - case TOK_ERROR: - case TOK_UNKNOWN: - /* don't eat EOF or errors */ - return tok; - - case TOK_NEWLINE: - lex_state->extended_mode_comment = 0; - /* fall through */ - default: - pcre_advance_lexer(lex_state, act_state); - goto restart; - } - } else { - switch (tok) { - case TOK_MAYBE_COMMENT: - lex_state->extended_mode_comment = 1; - /*fall through */ - case TOK_WHITESPACE: - case TOK_NEWLINE: - pcre_advance_lexer(lex_state, act_state); - goto restart; - - default: - return tok; - } - } - } - - #define CURRENT_TERMINAL (pcre_current_token(lex_state, act_state)) - #define ADVANCE_LEXER do { pcre_advance_lexer(lex_state,act_state); } while(0) - #else /* !PCRE_DIALECT */ - #define CURRENT_TERMINAL (act_state->lex_tok) - #define ADVANCE_LEXER do { mark(&act_state->synstart, &lex_state->lx.start); \ - mark(&act_state->synend, &lex_state->lx.end); \ - act_state->lex_tok = LX_NEXT(&lex_state->lx); \ - } while (0) - #endif /* PCRE_DIALECT */ - - #define SAVE_LEXER(tok) do { act_state->lex_tok_save = act_state->lex_tok; \ - act_state->lex_tok = tok; } while (0) - #define RESTORE_LEXER do { act_state->lex_tok = act_state->lex_tok_save; } while (0) - static void mark(struct re_pos *r, const struct lx_pos *pos) { @@ -297,6 +205,79 @@ return s; } + static void + advance_lexer(struct lex_state *lex_state, struct act_state *act_state) + { + mark(&act_state->synstart, &lex_state->lx.start); + mark(&act_state->synend, &lex_state->lx.end); + act_state->lex_tok = LX_NEXT(&lex_state->lx); + } + + static enum LX_TOKEN + current_token(struct lex_state *lex_state, struct act_state *act_state, + enum re_flags *flags) + { +#ifndef PCRE_DIALECT + (void) lex_state; + (void) flags; + return act_state->lex_tok; +#else + if ((*flags & RE_EXTENDED) == 0) { + switch (act_state->lex_tok) { + case TOK_WHITESPACE: + case TOK_NEWLINE: + case TOK_MAYBE_COMMENT: + return TOK_CHAR; + + default: + return act_state->lex_tok; + } + } + + int extended_mode_comment = 0; + + restart: + + if (extended_mode_comment) { + switch (act_state->lex_tok) { + case TOK_EOF: + case TOK_ERROR: + case TOK_UNKNOWN: + /* don't eat EOF or errors */ + return act_state->lex_tok; + + case TOK_NEWLINE: + extended_mode_comment = 0; + /* fall through */ + default: + advance_lexer(lex_state, act_state); + goto restart; + } + } else { + switch (act_state->lex_tok) { + case TOK_MAYBE_COMMENT: + extended_mode_comment = 1; + /* fall through */ + case TOK_WHITESPACE: + case TOK_NEWLINE: + advance_lexer(lex_state, act_state); + goto restart; + + default: + return act_state->lex_tok; + } + } +#endif + } + + #define ADVANCE_LEXER (advance_lexer(lex_state, act_state)) + #define CURRENT_TERMINAL (current_token(lex_state, act_state, flags)) + #define ERROR_TERMINAL @$ERROR + + #define SAVE_LEXER(tok) do { act_state->lex_tok_save = act_state->lex_tok; \ + act_state->lex_tok = tok; } while (0) + #define RESTORE_LEXER do { act_state->lex_tok = act_state->lex_tok_save; } while (0) + @}, @{ #include @@ -465,16 +446,21 @@ assert(lex_state->buf.a[3] == '\0'); @c = lex_state->buf.a[2]; - /* from http://www.pcre.org/current/doc/html/pcre2pattern.html#SEC5 + + /* + * From http://www.pcre.org/current/doc/html/pcre2pattern.html#SEC5 * - * The precise effect of \cx on ASCII characters is as follows: if x is a lower case letter, it is converted to - * upper case. Then bit 6 of the character (hex 40) is inverted. Thus \cA to \cZ become hex 01 to hex 1A (A is - * 41, Z is 5A), but \c{ becomes hex 3B ({ is 7B), and \c; becomes hex 7B (; is 3B). If the code unit following - * \c has a value less than 32 or greater than 126, a compile-time error occurs. + * The precise effect of \cx on ASCII characters is as follows: + * if x is a lower case letter, it is converted to upper case. + * Then bit 6 of the character (hex 40) is inverted. + * Thus \cA to \cZ become hex 01 to hex 1A (A is 41, Z is 5A), + * but \c{ becomes hex 3B ({ is 7B), and \c; becomes hex 7B (; is 3B). + * If the code unit following \c has a value less than 32 + * or greater than 126, a compile-time error occurs. */ { - unsigned char cc = (unsigned char)@c; + unsigned char cc = (unsigned char) @c; if (cc > 126 || cc < 32) { @!; } @@ -793,21 +779,6 @@ mark(&act_state->countend, &@end); @}; - : (node :ast_expr, start :pos, end :pos) -> () = @{ - struct ast_pos ast_start, ast_end; - - AST_POS_OF_LX_POS(ast_start, @start); - AST_POS_OF_LX_POS(ast_end, @end); - - mark(&act_state->groupstart, &@start); - mark(&act_state->groupend, &@end); - -/* TODO: reinstate this, applies to an expr node in general - @node->u.class.start = ast_start; - @node->u.class.end = ast_end; -*/ - @}; - : () -> (c :ast_class_id) = @{ /* TODO: or the unicode equivalent */ @c = (*flags & RE_SINGLE) ? &class_any : &class_notnl; @@ -835,24 +806,22 @@ @}; : () -> (c :ast_count) = @{ - @c = ast_make_count(0, NULL, AST_COUNT_UNBOUNDED, NULL); + @c = ast_make_count(0, AST_COUNT_UNBOUNDED); @}; : () -> (c :ast_count) = @{ - @c = ast_make_count(1, NULL, AST_COUNT_UNBOUNDED, NULL); + @c = ast_make_count(1, AST_COUNT_UNBOUNDED); @}; : () -> (c :ast_count) = @{ - @c = ast_make_count(0, NULL, 1, NULL); + @c = ast_make_count(0, 1); @}; : () -> (c :ast_count) = @{ - @c = ast_make_count(1, NULL, 1, NULL); + @c = ast_make_count(1, 1); @}; : (min :unsigned, start :pos, max :unsigned, end :pos) -> (c :ast_count) = @{ - struct ast_pos ast_start, ast_end; - if (@max < @min) { err->e = RE_ENEGCOUNT; err->m = @min; @@ -864,15 +833,12 @@ @!; } - AST_POS_OF_LX_POS(ast_start, @start); - AST_POS_OF_LX_POS(ast_end, @end); - - @c = ast_make_count(@min, &ast_start, @max, &ast_end); + @c = ast_make_count(@min, @max); @}; : (c :char) -> (e :endpoint) = @{ @e.type = AST_ENDPOINT_LITERAL; - @e.u.literal.c = (unsigned char)@c; + @e.u.literal.c = (unsigned char) @c; @}; : (id :ast_class_id) -> (e :endpoint) = @{ @@ -1036,12 +1002,8 @@ @}; : (from :endpoint, start :pos, to :endpoint, end :pos) -> (node :ast_expr) = @{ - struct ast_pos ast_start, ast_end; unsigned char lower, upper; - AST_POS_OF_LX_POS(ast_start, @start); - AST_POS_OF_LX_POS(ast_end, @end); - if (@from.type != AST_ENDPOINT_LITERAL || @to.type != AST_ENDPOINT_LITERAL) { err->e = RE_EUNSUPPORTED; @@ -1062,7 +1024,7 @@ @!; } - @node = ast_make_expr_range(act_state->poolp, *flags, &@from, ast_start, &@to, ast_end); + @node = ast_make_expr_range(act_state->poolp, *flags, &@from, &@to); if (@node == NULL) { @!; } @@ -1106,9 +1068,8 @@ struct ast * DIALECT_PARSE(re_getchar_fun *f, void *opaque, - const struct fsm_options *opt, enum re_flags flags, int overlap, - struct re_err *err) + struct re_err *err, struct re_pos *end) { struct ast *ast; @@ -1121,6 +1082,7 @@ struct LX_STATE *lx; assert(f != NULL); + assert(end != NULL); ast = ast_new(); @@ -1144,11 +1106,6 @@ lex_state->buf.a = NULL; lex_state->buf.len = 0; -#if PCRE_DIALECT - lex_state->extended_mode_comment = 0; - lex_state->flags_ptr = &flags; -#endif /* PCRE_DIALECT */ - /* XXX: unneccessary since we're lexing from a string */ /* (except for pushing "[" and "]" around ::group-$dialect) */ lx->buf_opaque = &lex_state->buf; @@ -1179,6 +1136,9 @@ DIALECT_ENTRY(&flags, lex_state, act_state, err, &ast->expr); + /* we output this for reporting errors found through AST analysis */ + mark(end, &lx->end); + lx->free(lx->buf_opaque); if (err->e != RE_ESUCCESS) { diff --git a/src/libre/print.c b/src/libre/print.c new file mode 100644 index 000000000..151545da5 --- /dev/null +++ b/src/libre/print.c @@ -0,0 +1,43 @@ +/* + * Copyright 2024 Katherine Flavel + * + * See LICENCE for the full copyright terms. + */ + +#include +#include +#include + +#include + +#include "print.h" + +int +ast_print(FILE *f, const struct ast *ast, const struct fsm_options *opt, + enum re_flags re_flags, enum ast_print_lang lang) +{ + ast_print_f *print; + + assert(ast != NULL); + + if (lang == AST_PRINT_NONE) { + return 0; + } + + switch (lang) { + case AST_PRINT_ABNF: print = ast_print_abnf; break; + case AST_PRINT_DOT: print = ast_print_dot; break; + case AST_PRINT_PCRE: print = ast_print_pcre; break; + case AST_PRINT_TREE: print = ast_print_tree; break; + + default: + errno = ENOTSUP; + return -1; + } + + // TODO: return int + print(f, opt, re_flags, ast); + + return 0; +} + diff --git a/src/libre/print.h b/src/libre/print.h index b7bc2dba8..2e70a4d51 100644 --- a/src/libre/print.h +++ b/src/libre/print.h @@ -1,5 +1,5 @@ /* - * Copyright 2018 Scott Vokes + * Copyright 2008-2024 Katherine Flavel * * See LICENCE for the full copyright terms. */ @@ -7,17 +7,27 @@ #ifndef RE_INTERNAL_PRINT_H #define RE_INTERNAL_PRINT_H -#include - struct ast; struct fsm_options; -typedef void (ast_print)(FILE *f, const struct fsm_options *opt, - enum re_flags re_flags, const struct ast *ast); +enum ast_print_lang { + AST_PRINT_NONE, + AST_PRINT_DOT, + AST_PRINT_ABNF, + AST_PRINT_PCRE, + AST_PRINT_TREE +}; + +// TODO: return int +typedef void ast_print_f(FILE *f, const struct fsm_options *opt, + enum re_flags re_flags, const struct ast *ast); + +ast_print_f ast_print_abnf; +ast_print_f ast_print_dot; +ast_print_f ast_print_pcre; +ast_print_f ast_print_tree; -ast_print ast_print_dot; -ast_print ast_print_abnf; -ast_print ast_print_pcre; -ast_print ast_print_tree; +int ast_print(FILE *f, const struct ast *ast, const struct fsm_options *opt, + enum re_flags re_flags, enum ast_print_lang lang); #endif diff --git a/src/libre/print/abnf.c b/src/libre/print/abnf.c index e8439aa28..a50771ca2 100644 --- a/src/libre/print/abnf.c +++ b/src/libre/print/abnf.c @@ -212,7 +212,6 @@ pp_iter(FILE *f, const struct fsm_options *opt, enum re_flags *re_flags, struct break; case AST_EXPR_SUBTRACT: - assert(!"unimplemented"); pp_atomic(f, opt, re_flags, n->u.subtract.a, n); fprintf(f, " - "); pp_atomic(f, opt, re_flags, n->u.subtract.b, n); diff --git a/src/libre/print/tree.c b/src/libre/print/tree.c index 5d2f78691..9422aafac 100644 --- a/src/libre/print/tree.c +++ b/src/libre/print/tree.c @@ -62,6 +62,9 @@ fprintf_flags(FILE *f, enum ast_flags flags) PR_FLAG(END_NL, "N"); PR_FLAG(CAN_CONSUME, "c"); PR_FLAG(ALWAYS_CONSUMES, "C"); + PR_FLAG(MATCHES_1NEWLINE, "n"); + PR_FLAG(CONSTRAINED_AT_START, "["); + PR_FLAG(CONSTRAINED_AT_END, "]"); #undef PR_FLAG @@ -147,7 +150,9 @@ pp_iter(FILE *f, const struct fsm_options *opt, size_t indent, enum re_flags re_ case AST_EXPR_ALT: { size_t i, count = n->u.alt.count; - fprintf(f, "ALT (%u):\n", (unsigned)count); + fprintf(f, "ALT (%u):%s\n", + (unsigned)count, + n->u.alt.contains_empty_groups ? " (contains_empty_groups)" : ""); for (i = 0; i < count; i++) { pp_iter(f, opt, indent + 1 * IND, re_flags, n->u.alt.n[i]); } @@ -155,7 +160,9 @@ pp_iter(FILE *f, const struct fsm_options *opt, size_t indent, enum re_flags re_ } case AST_EXPR_LITERAL: - fprintf(f, "LITERAL '%c'\n", n->u.literal.c); + fprintf(f, "LITERAL '"); + print_char_or_esc(f, n->u.literal.c); + fprintf(f, "'\n"); break; case AST_EXPR_CODEPOINT: @@ -167,18 +174,27 @@ pp_iter(FILE *f, const struct fsm_options *opt, size_t indent, enum re_flags re_ fprintf_count(f, n->u.repeat.min); fprintf(f, ","); fprintf_count(f, n->u.repeat.max); - fprintf(f, "}\n"); + fprintf(f, "}%s\n", n->u.repeat.contains_empty_groups ? " (contains_empty_groups)" : ""); pp_iter(f, opt, indent + 1 * IND, re_flags, n->u.repeat.e); break; case AST_EXPR_GROUP: - fprintf(f, "GROUP %p: %u\n", (void *) n, n->u.group.id); + fprintf(f, "GROUP: %u%s\n", n->u.group.id, + n->u.group.repeated ? " (repeated)" : ""); pp_iter(f, opt, indent + 1 * IND, re_flags, n->u.group.e); break; case AST_EXPR_ANCHOR: assert(n->u.anchor.type == AST_ANCHOR_START || n->u.anchor.type == AST_ANCHOR_END); - fprintf(f, "ANCHOR %s\n", n->u.anchor.type == AST_ANCHOR_START ? "^" : "$"); + if (n->u.anchor.type == AST_ANCHOR_START) { + fprintf(f, "ANCHOR ^\n"); + } else if (n->u.anchor.type == AST_ANCHOR_END) { + assert(n->u.anchor.type == AST_ANCHOR_START || n->u.anchor.type == AST_ANCHOR_END); + fprintf(f, "ANCHOR $%s\n", + n->u.anchor.is_end_nl ? " (with \\n)" : ""); + } else { + assert(!"unreachable"); + } break; case AST_EXPR_SUBTRACT: diff --git a/src/libre/re.c b/src/libre/re.c index 6c423dc36..f6c1d0c02 100644 --- a/src/libre/re.c +++ b/src/libre/re.c @@ -17,7 +17,6 @@ #include "ac.h" #include "class.h" -#include "print.h" #include "ast.h" #include "ast_analysis.h" #include "ast_compile.h" @@ -37,12 +36,12 @@ re_dialect(enum re_dialect dialect) size_t i; static const struct dialect a[] = { - { RE_LIKE, parse_re_like, 0, RE_SINGLE | RE_ANCHORED }, - { RE_LITERAL, parse_re_literal, 0, RE_SINGLE | RE_ANCHORED }, - { RE_GLOB, parse_re_glob, 0, RE_SINGLE | RE_ANCHORED }, + { RE_LIKE, parse_re_like, 0, RE_SINGLE | RE_ANCHORED | RE_NOCAPTURE }, + { RE_LITERAL, parse_re_literal, 0, RE_SINGLE | RE_ANCHORED | RE_NOCAPTURE }, + { RE_GLOB, parse_re_glob, 0, RE_SINGLE | RE_ANCHORED | RE_NOCAPTURE }, { RE_NATIVE, parse_re_native, 0, 0 }, { RE_PCRE, parse_re_pcre, 0, RE_END_NL }, - { RE_SQL, parse_re_sql, 1, RE_SINGLE | RE_ANCHORED } + { RE_SQL, parse_re_sql, 1, RE_SINGLE | RE_ANCHORED | RE_NOCAPTURE } }; for (i = 0; i < sizeof a / sizeof *a; i++) { @@ -92,12 +91,12 @@ re_flags(const char *s, enum re_flags *f) struct ast * re_parse(enum re_dialect dialect, int (*getc)(void *opaque), void *opaque, - const struct fsm_options *opt, enum re_flags flags, struct re_err *err, int *unsatisfiable) { const struct dialect *m; struct ast *ast = NULL; enum ast_analysis_res res; + struct re_pos end; assert(getc != NULL); @@ -109,7 +108,7 @@ re_parse(enum re_dialect dialect, int (*getc)(void *opaque), void *opaque, flags |= m->flags; - ast = m->parse(getc, opaque, opt, flags, m->overlap, err); + ast = m->parse(getc, opaque, flags, m->overlap, err, &end); if (ast == NULL) { return NULL; @@ -122,11 +121,15 @@ re_parse(enum re_dialect dialect, int (*getc)(void *opaque), void *opaque, /* Do a complete pass over the AST, filling in other details. */ res = ast_analysis(ast, flags); - if (res < 0) { - ast_free(ast); - if (err != NULL) { err->e = RE_EERRNO; } - return NULL; + if (err != NULL) { + if (res == AST_ANALYSIS_ERROR_UNSUPPORTED) { + err->e = RE_EUNSUPPORTED; + } else if (err->e == RE_ESUCCESS) { + err->e = RE_EERRNO; + } + } + goto error; } if (unsatisfiable != NULL) { @@ -134,11 +137,52 @@ re_parse(enum re_dialect dialect, int (*getc)(void *opaque), void *opaque, } return ast; + +error: + + ast_free(ast); + + if (err == NULL) { + return NULL; + } + + switch (res) { + case AST_ANALYSIS_ERROR_MEMORY: + /* This case comes up during fuzzing. */ + if (err->e == RE_ESUCCESS) { + err->e = RE_EERRNO; + errno = ENOMEM; + } + break; + + case AST_ANALYSIS_ERROR_UNSUPPORTED: + err->e = RE_EUNSUPPORTED; + + /* + * We can't tag AST nodes with re_pos, because it's + * also possible to construct an AST from an .fsm file. + * We detect RE_EUNSUPPORTED from annotations (e.g. nullable) + * on arbitary nodes. So at best we could tag an expr. + * But since in general we'd need to fabricate a pos anyway, + * I'm blaming the entire expression here. + */ + err->start.byte = 0; + err->end.byte = end.byte; + break; + + default: + if (err->e == RE_ESUCCESS) { + err->e = RE_EERRNO; + } + break; + } + + return NULL; } struct fsm * re_comp(enum re_dialect dialect, int (*getc)(void *opaque), void *opaque, - const struct fsm_options *opt, + const struct fsm_alloc *alloc, enum re_flags flags, struct re_err *err) { struct ast *ast; @@ -156,7 +200,7 @@ re_comp(enum re_dialect dialect, int (*getc)(void *opaque), void *opaque, flags |= m->flags; - ast = re_parse(dialect, getc, opaque, opt, flags, err, &unsatisfiable); + ast = re_parse(dialect, getc, opaque, flags, err, &unsatisfiable); if (ast == NULL) { return NULL; } @@ -174,7 +218,7 @@ re_comp(enum re_dialect dialect, int (*getc)(void *opaque), void *opaque, ast->expr = ast_expr_tombstone; } - new = ast_compile(ast, flags, opt, err); + new = ast_compile(ast, flags, alloc, err); ast_free(ast); @@ -204,7 +248,6 @@ re_comp(enum re_dialect dialect, int (*getc)(void *opaque), void *opaque, */ int re_is_literal(enum re_dialect dialect, int (*getc)(void *opaque), void *opaque, - const struct fsm_options *opt, enum re_flags flags, struct re_err *err, enum re_literal_category *category, char **s, size_t *n) { @@ -225,7 +268,7 @@ re_is_literal(enum re_dialect dialect, int (*getc)(void *opaque), void *opaque, flags |= m->flags; - ast = re_parse(dialect, getc, opaque, opt, flags, err, &unsatisfiable); + ast = re_parse(dialect, getc, opaque, flags, err, &unsatisfiable); if (ast == NULL) { return -1; } @@ -249,6 +292,8 @@ re_is_literal(enum re_dialect dialect, int (*getc)(void *opaque), void *opaque, /* * Literals have an enclosing group #0, and we skip it for our purposes. * Parsing a satisfiable expression is required to produce group #0. + * All dialects do this, regardless of whether their syntax provides + * for group capture of subexpressions. * If this doesn't exist, whatever we parsed, it's not a literal. * * I'm not doing this as an assertion because AST rewriting is free to @@ -297,3 +342,40 @@ re_is_literal(enum re_dialect dialect, int (*getc)(void *opaque), void *opaque, return -1; } +enum re_is_anchored_res +re_is_anchored(enum re_dialect dialect, re_getchar_fun *getc, void *opaque, + enum re_flags flags, struct re_err *err) +{ + /* FIXME: copy/pasted from above, factor out common code later. */ + + struct ast *ast; + const struct dialect *m; + int unsatisfiable; + + assert(getc != NULL); + + m = re_dialect(dialect); + if (m == NULL) { + if (err != NULL) { err->e = RE_EBADDIALECT; } + return RE_IS_ANCHORED_ERROR; + } + + flags |= m->flags; + + ast = re_parse(dialect, getc, opaque, flags, err, &unsatisfiable); + if (ast == NULL) { + return RE_IS_ANCHORED_ERROR; + } + + /* Copy anchoring flags, ending up with NONE, START, END, or BOTH. */ + enum re_is_anchored_res res = RE_IS_ANCHORED_NONE; + if (ast->expr->flags & AST_FLAG_ANCHORED_START) { + res |= RE_IS_ANCHORED_START; + } + if (ast->expr->flags & AST_FLAG_ANCHORED_END) { + res |= RE_IS_ANCHORED_END; + } + + ast_free(ast); + return res; +} diff --git a/src/libre/re_capvm_compile.c b/src/libre/re_capvm_compile.c new file mode 100644 index 000000000..be9e92c95 --- /dev/null +++ b/src/libre/re_capvm_compile.c @@ -0,0 +1,1568 @@ +/* + * Copyright 2022 Scott Vokes + * + * See LICENCE for the full copyright terms. + */ + +#include "re_capvm_compile.h" +#include "../libfsm/capture_vm.h" +#include "../libfsm/capture_vm_program.h" +#include "../libfsm/capture_vm_log.h" + +/* for EXPENSIVE_CHECKS */ +#include "adt/common.h" + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include + +#include + +#include "ast.h" + +#define DEF_OPCODE_CEIL 8 +#define DEF_CHARCLASS_BUCKETS 8 +#define DEF_CHARCLASS_CEIL 4 +#define DEF_REPEATED_ALT_BACKPATCH_CEIL 1 +#define NO_BUCKET_ID ((uint32_t)-1) +#define NO_CAPTURE_ID ((uint32_t)-1) + +#define LOG_REPETITION_CASES 0 + +/* Placeholder markers for pending offset values (which would + * otherwise temporarily be uninitialized memory), chosen so + * they stand out visually in a debugger. */ +enum pending_offset { + PENDING_OFFSET_REPEAT_OPTIONAL_NEW = 11111111, + PENDING_OFFSET_REPEAT_OPTIONAL_CONT = 22222222, + PENDING_OFFSET_ALT_BACKPATCH_JMP = 33333333, + PENDING_OFFSET_ALT_BACKPATCH_NEW = 44444444, + PENDING_OFFSET_ALT_BACKPATCH_AFTER_REPEAT_PLUS = 55555555, +}; + +struct capvm_compile_env { + const struct fsm_alloc *alloc; + enum re_flags re_flags; + struct capvm_program *program; + + uint32_t max_capture_seen; + + /* Hash table for interning character classes. + * Doubles and rehashes when half full. */ + struct charclass_htab { + uint32_t bucket_count; + uint32_t buckets_used; + uint32_t ids_used; + struct charclass_htab_bucket { + uint32_t id; /* or NO_BUCKET_ID for unused */ + struct capvm_char_class bitset; + } *buckets; + } charclass_htab; + +#define DEF_REPEATED_GROUPS_CEIL 8 + /* Linked list of nodes used at compile time to compile regexes + * such as '^(a((b*)*)*)$' as if they were '^(a(?:b*)(()))$'. + * Since the inner body of the repeated subexpression with the + * capture groups can be empty, it will always repeat after + * its body matches any input. We move the group captures to + * the end of the repeated subexpression to explicitly represent + * them always capturing afterward, because otherwise the + * infinite loop protection skips them. */ + struct repeated_group_info { + /* Ancestor node that should emit the SAVE opcodes; can + * be either a REPEAT or ALT. */ + const struct ast_expr *outermost_ancestor; + size_t ceil; + size_t count; + const struct ast_expr **groups; + /* linked list */ + struct repeated_group_info *prev; + } *repeated_groups; +}; + +static bool +ensure_program_capacity(const struct fsm_alloc *alloc, + struct capvm_program *p, uint32_t count) +{ +#define STRESS_GROWING (EXPENSIVE_CHECKS && 1) + + const uint32_t capacity = p->used + count; + + if (capacity > p->ceil) { +#if STRESS_GROWING + const uint32_t nceil = (p->ceil + 1 < capacity + ? capacity : p->ceil + 1); +#else + const uint32_t nceil = (p->ceil == 0 + ? DEF_OPCODE_CEIL + : 2*p->ceil); + /* This should always be enough for any capacity + * requested during compilation. */ + assert(nceil >= p->used + count); +#endif + LOG(3, "%s: growing %u -> %u (count %u)\n", + __func__, p->ceil, nceil, count); + struct capvm_opcode *nops = f_realloc(alloc, + p->ops, nceil * sizeof(p->ops[0])); + if (nops == NULL) { + return false; + } + +#if EXPENSIVE_CHECKS + for (size_t i = p->ceil; i < nceil; i++) { + /* out of range, will trigger asserts */ + nops[i].t = 'X'; + } +#endif + + p->ceil = nceil; + p->ops = nops; + } + return true; +} + +static void +check_program_for_invalid_labels(const struct capvm_program *p) +{ + for (uint32_t op_i = 0; op_i < p->used; op_i++) { + const struct capvm_opcode *op = &p->ops[op_i]; + switch (op->t) { + case CAPVM_OP_JMP: + assert(op->u.jmp != op_i); + break; + case CAPVM_OP_JMP_ONCE: + assert(op->u.jmp_once != op_i); + break; + case CAPVM_OP_SPLIT: + assert(op->u.split.greedy < p->used); + assert(op->u.split.greedy != op_i); + assert(op->u.split.nongreedy < p->used); + assert(op->u.split.nongreedy != op_i); + break; + + case CAPVM_OP_CHAR: + case CAPVM_OP_CHARCLASS: + case CAPVM_OP_MATCH: + case CAPVM_OP_SAVE: + case CAPVM_OP_ANCHOR: + break; + default: + assert(!"out of range"); + break; + } + } +} + +static uint32_t +get_program_offset(const struct capvm_program *p) +{ + assert(p->used < p->ceil); + +#if EXPENSIVE_CHECKS + struct capvm_opcode *op = &p->ops[p->used]; + op->t = 'X'; /* out of range */ +#endif + + return p->used; +} + +static uint32_t +reserve_program_opcode(struct capvm_program *p) +{ + assert(p->used < p->ceil); + const uint32_t res = p->used; + p->used++; + +#if EXPENSIVE_CHECKS + struct capvm_opcode *op = &p->ops[res]; + op->t = 'X'; /* out of range */ +#endif + + return res; +} + +static bool +grow_program_char_classes(const struct fsm_alloc *alloc, + struct capvm_program *p) +{ + const uint32_t nceil = (p->char_classes.ceil == 0 + ? DEF_CHARCLASS_CEIL + : 2*p->char_classes.ceil); + struct capvm_char_class *nsets = f_realloc(alloc, + p->char_classes.sets, nceil * sizeof(nsets[0])); + if (nsets == NULL) { + return false; + } + + p->char_classes.sets = nsets; + p->char_classes.ceil = nceil; + return true; +} + +static bool +intern_char_class(struct capvm_compile_env *env, + struct capvm_program *p, uint64_t chars[4], + uint32_t *id) +{ + LOG(5, "%s: used %u/%u\n", __func__, + env->charclass_htab.buckets_used, env->charclass_htab.bucket_count); + if (env->charclass_htab.buckets_used >= env->charclass_htab.bucket_count/2) { + const uint32_t ocount = env->charclass_htab.bucket_count; + const uint32_t ncount = (ocount == 0 + ? DEF_CHARCLASS_BUCKETS + : 2*env->charclass_htab.bucket_count); + LOG(3, "%s: growing from %u -> %u\n", __func__, ocount, ncount); + struct charclass_htab_bucket *nbuckets = + f_malloc(env->alloc, ncount * sizeof(nbuckets[0])); + if (nbuckets == NULL) { + return false; + } + for (uint32_t n_i = 0; n_i < ncount; n_i++) { + nbuckets[n_i].id = NO_BUCKET_ID; + } + + const uint32_t nmask = ncount - 1; + assert((ncount & nmask) == 0); + + struct charclass_htab_bucket *obuckets = env->charclass_htab.buckets; + for (uint32_t o_i = 0; o_i < ocount; o_i++) { + if (obuckets[o_i].id == NO_BUCKET_ID) { + continue; + } + const uint64_t h = hashrec(obuckets[o_i].bitset.octets, + sizeof(obuckets[o_i].bitset)); + + for (uint32_t n_i = 0; n_i < ncount; n_i++) { + const uint64_t b = (h + n_i) & nmask; + if (nbuckets[b].id == NO_BUCKET_ID) { + memcpy(&nbuckets[b].bitset, + &obuckets[o_i].bitset, + sizeof(obuckets[o_i].bitset)); + nbuckets[b].id = obuckets[o_i].id; + break; + } + } + } + f_free(env->alloc, obuckets); + env->charclass_htab.bucket_count = ncount; + env->charclass_htab.buckets = nbuckets; + } + + assert(env->charclass_htab.buckets_used < env->charclass_htab.bucket_count/2); + + const uint32_t count = env->charclass_htab.bucket_count; + const uint32_t mask = count - 1; + struct charclass_htab_bucket *buckets = env->charclass_htab.buckets; + + const uint64_t h = hashrec(chars, sizeof(buckets[0].bitset)); + + for (uint32_t i = 0; i < count; i++) { + const uint64_t b = (h + i) & mask; + LOG(5, "%s: buckets[%lu].id == %d\n", + __func__, b, buckets[b].id); + if (buckets[b].id == NO_BUCKET_ID) { + memcpy(&buckets[b].bitset, chars, sizeof(buckets[b].bitset)); + if (p->char_classes.count == p->char_classes.ceil) { + if (!grow_program_char_classes(env->alloc, p)) { + return false; + } + } + + memcpy(&p->char_classes.sets[p->char_classes.count], + chars, sizeof(buckets[b].bitset)); + p->char_classes.count++; + buckets[b].id = env->charclass_htab.ids_used; + env->charclass_htab.ids_used++; + env->charclass_htab.buckets_used++; + *id = buckets[b].id; + + return true; + } else if (0 == memcmp(chars, &buckets[b].bitset, sizeof(buckets[b].bitset))) { + *id = buckets[b].id; + return true; /* already present, reuse */ + } else { + /* collision */ + } + } + + assert(!"unreachable"); + return false; +} + +static void +dump_endpoint(const struct ast_endpoint *e) +{ + switch (e->type) { + case AST_ENDPOINT_LITERAL: + fprintf(stderr, "endpoint[LITERAL]: 0x%02x '%c'\n", + e->u.literal.c, + isprint(e->u.literal.c) ? e->u.literal.c : '.'); + break; + case AST_ENDPOINT_CODEPOINT: + fprintf(stderr, "endpoint[CODEPOINT]: 0x%x\n", + e->u.codepoint.u); + break; + case AST_ENDPOINT_NAMED: + assert(!"todo?"); + break; + } +} + +static bool +active_node(const struct ast_expr *n) +{ + assert(n != NULL); + + switch (n->type) { + case AST_EXPR_TOMBSTONE: + return false; + default: + return !(n->flags & AST_FLAG_UNSATISFIABLE); + } +} + +static bool +subtree_represents_character_class(const struct ast_expr *expr, uint64_t cc[4]) +{ + for (size_t i = 0; i < 4; i++) { + cc[i] = 0; + } + + switch (expr->type) { + case AST_EXPR_EMPTY: + /* empty set */ + return false; + + case AST_EXPR_LITERAL: + u64bitset_set(cc, (uint8_t)expr->u.literal.c); + return true; + + case AST_EXPR_RANGE: + { + const struct ast_endpoint *f = &expr->u.range.from; + const struct ast_endpoint *t = &expr->u.range.to; + if (f->type != AST_ENDPOINT_LITERAL + || t->type != AST_ENDPOINT_LITERAL) { + return false; + } + for (uint64_t c = (uint8_t)f->u.literal.c; c <= (uint8_t)t->u.literal.c; c++) { + u64bitset_set(cc, (uint8_t)c); + } + return true; + } + + case AST_EXPR_ALT: + { + /* union character classes from children */ + assert(expr->u.alt.count > 0); + for (size_t c_i = 0; c_i < expr->u.alt.count; c_i++) { + uint64_t child_cc[4]; + const struct ast_expr *child = expr->u.alt.n[c_i]; + if (subtree_represents_character_class(child, child_cc)) { + for (size_t cc_i = 0; cc_i < 4; cc_i++) { + cc[cc_i] |= child_cc[cc_i]; + } + } else { + return false; + } + } + return true; + } + + case AST_EXPR_SUBTRACT: + { + /* Only support AST_EXPR_SUBTRACT nodes where .a is a + * RANGE:0x00-0xff and .b is either a LITERAL, RANGE, EMPTY, + * or an ALT that itself represents a character class, */ + + const struct ast_expr *sub_a = expr->u.subtract.a; + if (sub_a->type != AST_EXPR_RANGE) { + return false; + } + + const struct ast_endpoint *f = &sub_a->u.range.from; + const struct ast_endpoint *t = &sub_a->u.range.to; + if (f->type != AST_ENDPOINT_LITERAL || t->type != AST_ENDPOINT_LITERAL) { + return false; + } + + for (uint64_t i = 0; i < 256; i++) { + if (i >= (uint8_t)f->u.literal.c && i <= (uint8_t)f->u.literal.c) { + u64bitset_set(cc, i); + } + } + + for (size_t i = 0; i < 4; i++) { + cc[i] = ~(uint64_t)0; + } + + uint64_t neg_cc[4]; + if (expr->u.subtract.b->type == AST_EXPR_EMPTY) { + for (size_t cc_i = 0; cc_i < 4; cc_i++) { + neg_cc[cc_i] = (uint64_t)0; + } + } else if (subtree_represents_character_class(expr->u.subtract.b, neg_cc)) { + for (size_t cc_i = 0; cc_i < 4; cc_i++) { + cc[cc_i] &=~ neg_cc[cc_i]; + } + } else { + return false; + } + return true; + } + + default: + return false; + } +} + +static void +make_charclass_case_insensitive(uint64_t *cc) +{ + for (size_t i = 0; i < 256; i++) { + if (isalpha(i) && u64bitset_get(cc, i)) { + const char c = (char)i; + const uint64_t cl = (uint64_t)tolower(c); + const uint64_t cu = (uint64_t)toupper(c); + u64bitset_set(cc, cl); + u64bitset_set(cc, cu); + } + } +} + +static bool +can_safely_skip_JMP_ONCE(const struct ast_expr *expr) +{ + /* There are potentially cases where it's safe to skip the + * JMP_ONCE special case, which would save memory by not + * expanding the path an extra bit per iteration, but the + * criteria are subtle enough that it can probably wait. */ + (void)expr; + return false; +} + +static bool +push_repeated_group_info(struct capvm_compile_env *env, const struct ast_expr *expr) +{ + LOG(3 - LOG_REPETITION_CASES, + "%s: setting env->repeated_groups.outermost_ancestor <- %p\n", + __func__, (void *)expr); + + assert(expr != NULL); + assert(expr->type == AST_EXPR_REPEAT || expr->type == AST_EXPR_ALT); + + struct repeated_group_info *rgi = f_calloc(env->alloc, 1, sizeof(*rgi)); + if (rgi == NULL) { + return false; + } + rgi->outermost_ancestor = expr; + rgi->prev = env->repeated_groups; + env->repeated_groups = rgi; + LOG(3 - LOG_REPETITION_CASES, + "%s: push rgi, allocated %p, prev %p\n", + __func__, (void *)rgi, (void *)rgi->prev); + return true; +} + +static void +pop_repeated_group_info(struct capvm_compile_env *env, const struct ast_expr *expr) +{ + assert(expr != NULL); + assert(expr->type == AST_EXPR_REPEAT || expr->type == AST_EXPR_ALT); + struct repeated_group_info *rgi = env->repeated_groups; + LOG(3 - LOG_REPETITION_CASES, + "%s: pop rgi, expecting %p, got %p\n", + __func__, (void *)expr, (void *)rgi->outermost_ancestor); + assert(rgi->outermost_ancestor == expr); + struct repeated_group_info *prev = rgi->prev; + LOG(3 - LOG_REPETITION_CASES, + "%s: pop rgi, freeing %p, prev %p\n", + __func__, (void *)rgi, (void *)prev); + + env->repeated_groups = prev; + if (rgi->groups != NULL) { + f_free(env->alloc, rgi->groups); + } + f_free(env->alloc, rgi); +} + +static bool +emit_repeated_groups(struct capvm_compile_env *env, struct capvm_program *p); + +static bool +capvm_compile_iter_save_groups_in_skipped_subtree(struct capvm_compile_env *env, + struct capvm_program *p, const struct ast_expr *expr); + +static bool +compile_kleene_star(struct capvm_compile_env *env, + struct capvm_program *p, const struct ast_expr *expr); + +static bool +capvm_compile_iter(struct capvm_compile_env *env, + struct capvm_program *p, const struct ast_expr *expr) +{ + LOG(4, "%s: expr %p, type %s, %u/%u used, re_flags 0x%02x\n", + __func__, (void *)expr, ast_node_type_name(expr->type), + p->used, p->ceil, expr->re_flags); + + switch (expr->type) { + case AST_EXPR_EMPTY: + case AST_EXPR_TOMBSTONE: + break; + case AST_EXPR_CONCAT: + for (size_t i = 0; i < expr->u.concat.count; i++) { + /* append instructions from each consecutive node */ + const struct ast_expr *n = expr->u.concat.n[i]; + if (!capvm_compile_iter(env, p, n)) { return false; } + } + break; + case AST_EXPR_ALT: + { + if (!ensure_program_capacity(env->alloc, p, 1)) { + return false; + } + assert(expr->u.alt.count > 0); + + if (expr->u.alt.contains_empty_groups) { + if (!push_repeated_group_info(env, expr)) { + return false; + } + } + + /* If this ALT node represents a character class (such as a + * rewritten . character's ALT[0x00 - 0x09, 0x0b - 0xff] or + * a rewritten [abc-ef]'s ... , then produce the corresponding + * character class literal. The direct representation of the + * subtree would take several instructions and introduce + * unnecessary splits, increasing memory usage at runtime. */ + uint64_t cc[4]; + if (subtree_represents_character_class(expr, cc)) { + const uint32_t pos = reserve_program_opcode(p); + struct capvm_opcode *op_cc = &p->ops[pos]; + op_cc->t = CAPVM_OP_CHARCLASS; + + if (expr->re_flags & RE_ICASE) { + make_charclass_case_insensitive(cc); + } + if (!intern_char_class(env, p, cc, &op_cc->u.charclass_id)) { + return false; + } + + if (expr->u.alt.contains_empty_groups) { + pop_repeated_group_info(env, expr); + } + break; + } + + uint32_t active_count = 0; + uint32_t last_active; + struct alt_flow_info { + bool is_active; + uint32_t backpatch; + }; + struct alt_flow_info *flow_info = f_calloc(env->alloc, + expr->u.alt.count, sizeof(flow_info[0])); + assert(flow_info != NULL); + + for (uint64_t i = 0; i < expr->u.alt.count; i++) { + const struct ast_expr *n = expr->u.alt.n[i]; + if (active_node(n)) { + last_active = i; + active_count++; + flow_info[i].is_active = true; + } + } + + /* If there are no children active this should terminate + * with an empty program. */ + LOG(3, "%s: active_count == %d\n", __func__, active_count); + if (active_count == 0) { + LOG(3, "%s: active_count == 0, skipping\n", __func__); + + for (uint64_t i = 0; i < expr->u.alt.count; i++) { + const struct ast_expr *n = expr->u.alt.n[i]; + capvm_compile_iter_save_groups_in_skipped_subtree(env, p, n); + if (n->flags & AST_FLAG_NULLABLE) { + break; + } + } + + f_free(env->alloc, flow_info); + if (expr->u.alt.contains_empty_groups) { + pop_repeated_group_info(env, expr); + } + + /* FIXME: May need distinct error case to not + * leak. There is currently no test reaching + * this and the fuzzer has not produced an input + * that reaches it -- unsatisfiability has probably + * already pruned subtrees that would get here. */ + return true; + } else if (active_count == 1) { + /* even if one of the later subtrees is active, an earlier + * subtree can still shadow it. */ + bool shadowed = false; + + for (uint64_t i = 0; i < expr->u.alt.count; i++) { + if (i != last_active) { /* evaluate for empty groups */ + const struct ast_expr *n = expr->u.alt.n[i]; + capvm_compile_iter_save_groups_in_skipped_subtree(env, p, n); + if (n->flags & AST_FLAG_NULLABLE) { + shadowed = true; + break; + } + } + } + + if (!shadowed) { + LOG(5, "narrowing to last_active %u\n", last_active); + assert(last_active < expr->u.alt.count); + const struct ast_expr *n = expr->u.alt.n[last_active]; + if (!capvm_compile_iter(env, p, n)) { + return false; + } + f_free(env->alloc, flow_info); + if (expr->u.alt.contains_empty_groups) { + pop_repeated_group_info(env, expr); + } + break; + } else { + f_free(env->alloc, flow_info); + if (expr->u.alt.contains_empty_groups) { + pop_repeated_group_info(env, expr); + } + return true; /* may need distinct error case to not leak */ + } + } + + LOG(3, "%s: compiling AST_EXPR_ALT with %u active nodes, last_active %u\n", + __func__, active_count, last_active); + + /* note: binarized split: for a|b|c, treat this like "a else (b else c)", + * leading to generated code like: + * + * // note: trying each case in order, earlier cases are more greedy + * - split_greedy j_try_a + * - split_nongreedy j_skip_a + * j_try_a: + * - + * - jmp pos_after_all // or split pos_after_all, PLUS_BACKPATCH, see below + * j_skip_a: + * - split_greedy j_try_b + * - split_nongreedy j_else_c + * j_try_b: + * - + * - jmp pos_after_all + * j_else_c: + * //// DO NOT EMIT split instructions here, treat like a final else + * - + * // fall through to pos_after_all + * pos_after_all: + * + * + * When an ALT case: + * - is nullable (can match the empty string) + * - is the first nullable case (shadowing cases after) + * - is in a subtree of a REPEAT{1,inf} (+) node whose entire subtree is nullable + * then that case's `jmp pos_after_all` should be replaced with + * `split pos_after_all pos_after_repeat_backjmp`, which will need a special + * label for batch-patching by the REPEAT later. This is necessary for cases + * like '^(?:($|x))+$', where the `jmp pos_after_all` would lead to code after + * the ALT that has already been executed at the current input position. + * */ + for (uint32_t c_i = 0; c_i < expr->u.alt.count; c_i++) { + const bool is_final_else_case = c_i == last_active; + LOG(3, "%s: %p c_i %u/%zu, is_final_else_case %d\n", + __func__, (void *)expr, c_i, expr->u.alt.count, is_final_else_case); + if (!flow_info[c_i].is_active) { continue; } + + if (is_final_else_case) { + /* Just add the case for the child node and + * then fall through to pos_after_all. */ + const struct ast_expr *n = expr->u.alt.n[c_i]; + LOG(3, "%s: %p recursing...\n", __func__, (void *)expr); + if (!capvm_compile_iter(env, p, n)) { + return false; + } + LOG(3, "%s: %p recursing...done (final-else-case)\n", __func__, (void *)expr); + + struct repeated_group_info *rgi = env->repeated_groups; + LOG(3 - LOG_REPETITION_CASES, + "%s: ALT %p: contains_empty_groups: %d, outermost_ancestor: %p == %p ? %d\n", + __func__, (void *)expr, + expr->u.alt.contains_empty_groups, + (void *)(rgi ? rgi->outermost_ancestor : NULL), + (void *)expr, + (rgi ? rgi->outermost_ancestor == expr : 0)); + if (expr->u.alt.contains_empty_groups) { + assert(rgi != NULL); + LOG(3 - LOG_REPETITION_CASES, + "%s: outermost_ancestor match, count %zu\n", __func__, rgi->count); + if (!emit_repeated_groups(env, p)) { + return false; + } + } + } else { + if (!ensure_program_capacity(env->alloc, p, 2)) { + return false; + } + const uint32_t pos_split_before_case = reserve_program_opcode(p); + struct capvm_opcode *op_split_before = &p->ops[pos_split_before_case]; + op_split_before->t = CAPVM_OP_SPLIT; + + /* greedier branch: trying the next case, in order */ + op_split_before->u.split.greedy = get_program_offset(p); + + /* less greedy branch: moving on to the next case. + * will backpatch .new to after this case's JMP later */ + op_split_before->u.split.nongreedy = PENDING_OFFSET_ALT_BACKPATCH_NEW; + + const struct ast_expr *n = expr->u.alt.n[c_i]; + LOG(3, "%s: %p recursing...\n", __func__, (void *)expr); + if (!capvm_compile_iter(env, p, n)) { + return false; + } + LOG(3, "%s: %p recursing...done (non-final)\n", __func__, (void *)expr); + + struct repeated_group_info *rgi = env->repeated_groups; + LOG(3 - LOG_REPETITION_CASES, + "%s: ALT %p: contains_empty_groups: %d, outermost_ancestor: %p == %p ? %d\n", + __func__, (void *)expr, expr->u.alt.contains_empty_groups, + (void *)(rgi ? rgi->outermost_ancestor : NULL), + (void *)expr, + (rgi ? rgi->outermost_ancestor == expr : 0)); + if (expr->u.alt.contains_empty_groups) { + assert(rgi != NULL); + LOG(3 - LOG_REPETITION_CASES, + "%s: outermost_ancestor match, count %zu\n", __func__, rgi->count); + if (!emit_repeated_groups(env, p)) { + return false; + } + } + + /* JMP or SPLIT, plus space after */ + if (!ensure_program_capacity(env->alloc, p, 2)) { + return false; + } + + /* Based on analysis, either emit a JMP or SPLIT. */ + if (n->type == AST_EXPR_ALT && n->u.alt.nullable_alt_inside_plus_repeat) { + const uint32_t pos_split_after = reserve_program_opcode(p); + flow_info[c_i].backpatch = pos_split_after; + struct capvm_opcode *op_split_after = &p->ops[pos_split_after]; + op_split_after->t = CAPVM_OP_SPLIT; + op_split_after->u.split.greedy = PENDING_OFFSET_ALT_BACKPATCH_JMP; + op_split_after->u.split.nongreedy = PENDING_OFFSET_ALT_BACKPATCH_AFTER_REPEAT_PLUS; + } else { + const uint32_t pos_jmp_after = reserve_program_opcode(p); + flow_info[c_i].backpatch = pos_jmp_after; + struct capvm_opcode *op_jmp = &p->ops[pos_jmp_after]; + op_jmp->t = CAPVM_OP_JMP; + op_jmp->u.jmp = PENDING_OFFSET_ALT_BACKPATCH_JMP; + } + + /* refresh pointer after possible realloc */ + op_split_before = &p->ops[pos_split_before_case]; + + /* and the original split jumps to after + * this case's JMP */ + op_split_before->u.split.nongreedy = get_program_offset(p); + } + } + + /* Ensure there's space for the next instruction, and then + * set every case's JMP suffix to it. */ + if (!ensure_program_capacity(env->alloc, p, 1)) { + return false; + } + const uint32_t pos_after_all = get_program_offset(p); + + // backpatch the jmps to pos_after_all + for (size_t i = 0; i < expr->u.alt.count - 1; i++) { + const bool is_final_else_case = i == last_active; + assert(flow_info[i].backpatch < p->used); + if (is_final_else_case || !flow_info[i].is_active) { + continue; + } + + struct capvm_opcode *op_patch = &p->ops[flow_info[i].backpatch]; + if (op_patch->t == CAPVM_OP_JMP) { + assert(op_patch->u.jmp == PENDING_OFFSET_ALT_BACKPATCH_JMP); + op_patch->u.jmp = pos_after_all; + } else if (op_patch->t == CAPVM_OP_SPLIT) { + assert(op_patch->u.split.greedy == PENDING_OFFSET_ALT_BACKPATCH_JMP); + op_patch->u.split.greedy = pos_after_all; + /* This will be patched by an ancestor repeat node after returning. */ + assert(op_patch->u.split.greedy == PENDING_OFFSET_ALT_BACKPATCH_AFTER_REPEAT_PLUS); + } else { + assert(!"type mismatch"); + } + } + + f_free(env->alloc, flow_info); + if (expr->u.alt.contains_empty_groups) { + pop_repeated_group_info(env, expr); + } + break; + } + case AST_EXPR_LITERAL: + { + if (!ensure_program_capacity(env->alloc, p, 1)) { + return false; + } + const uint32_t pos = reserve_program_opcode(p); + struct capvm_opcode *op = &p->ops[pos]; + + if (expr->re_flags & RE_ICASE) { + uint64_t cc[4] = { 0 }; + u64bitset_set(cc, (uint8_t)expr->u.literal.c); + + op->t = CAPVM_OP_CHARCLASS; + make_charclass_case_insensitive(cc); + if (!intern_char_class(env, p, cc, &op->u.charclass_id)) { + return false; + } + } else { + op->t = CAPVM_OP_CHAR; + op->u.chr = (uint8_t)expr->u.literal.c; + } + break; + } + case AST_EXPR_CODEPOINT: + assert(!"not implemented, unreachable"); + break; + case AST_EXPR_REPEAT: + { + const unsigned min = expr->u.repeat.min; + const unsigned max = expr->u.repeat.max; + const struct ast_expr *e = expr->u.repeat.e; + + /* collect groups to emit */ + if (expr->u.repeat.contains_empty_groups) { + if (!push_repeated_group_info(env, expr)) { + return false; + } + } + + if (min == 1 && max == 1) { /* {1,1} */ + /* if repeating exactly once, just defer to subtree, + * but still do the repeated_group_info cleanup below */ + if (!capvm_compile_iter(env, p, e)) { + return false; + } + } else if (min == 0 && max == 1) { /* ? */ + /* split l1, l2 + * l1: + * l2: */ + if (!ensure_program_capacity(env->alloc, p, 2)) { + return false; + } + + const uint32_t pos_split = reserve_program_opcode(p); + const uint32_t pos_l1 = get_program_offset(p); + + struct capvm_opcode *op_split = &p->ops[pos_split]; + op_split->t = CAPVM_OP_SPLIT; + op_split->u.split.greedy = pos_l1; + op_split->u.split.nongreedy = PENDING_OFFSET_REPEAT_OPTIONAL_NEW; + + if (!capvm_compile_iter(env, p, e)) { return false; } + + if (!ensure_program_capacity(env->alloc, p, 1)) { + return false; + } + op_split = &p->ops[pos_split]; /* refresh pointer */ + + const uint32_t after_expr = get_program_offset(p); + op_split->u.split.nongreedy = after_expr; + } else if (min == 0 && max == AST_COUNT_UNBOUNDED) { /* * */ + if (!compile_kleene_star(env, p, expr)) { + return false; + } + } else if (min == 1 && max == AST_COUNT_UNBOUNDED) { /* + */ + /* l1: + * split l1, l2 + * l2: */ + if (!ensure_program_capacity(env->alloc, p, 1)) { + return false; + } + const uint32_t pos_l1 = get_program_offset(p); + + if (!capvm_compile_iter(env, p, e)) { return false; } + + if (!ensure_program_capacity(env->alloc, p, 1)) { + return false; + } + + /* Only emit the backwards jump for repetition branching + * if the subtree added any instructions. */ + if (get_program_offset(p) != pos_l1) { + if (!ensure_program_capacity(env->alloc, p, 3)) { + return false; + } + const uint32_t pos_split = reserve_program_opcode(p); + const uint32_t pos_l2 = get_program_offset(p); + + struct capvm_opcode *op_split = &p->ops[pos_split]; + op_split->t = CAPVM_OP_SPLIT; + op_split->u.split.greedy = pos_l1; + op_split->u.split.nongreedy = pos_l2; + } + } else if (min == 0 && max == 0) { /* {0,0} */ + /* ignored, except any groups contained within that could match + * empty input still get emitted (unless unsatisfiable). */ + if (e->flags & AST_FLAG_UNSATISFIABLE) { + LOG(3, "%s: repeat{0,0} && UNSATISFIABILE -> skipping\n", __func__); + break; + } + + /* Unreachable group captures still need to be counted, otherwise + * subsequent ones would get shifted down. */ + if (!capvm_compile_iter_save_groups_in_skipped_subtree(env, p, e)) { return false; } + break; + } else { /* other bounded count */ + /* repeat the minimum number of times */ + for (size_t i = 0; i < min; i++) { + if (!capvm_compile_iter(env, p, e)) { return false; } + } + + if (max == AST_COUNT_UNBOUNDED) { + /* A repeat of {x,inf} should be treated like + * (?:subtree){x,x} (?:subtree)* , where any numbered + * capture groups inside have the same group ID in + * both copies of the subtree. */ + if (!compile_kleene_star(env, p, expr)) { + return false; + } + } else { + /* then repeat up to the max as ? + * + * split_cont l1 + * split_new l2 + * l1: + * l2: */ + for (size_t i = min; i < max; i++) { + if (!ensure_program_capacity(env->alloc, p, 3)) { + return false; + } + + const uint32_t pos_split = reserve_program_opcode(p); + const uint32_t pos_l1 = get_program_offset(p); + + struct capvm_opcode *op_split = &p->ops[pos_split]; + op_split->t = CAPVM_OP_SPLIT; + op_split->u.split.greedy = pos_l1; + op_split->u.split.nongreedy = PENDING_OFFSET_REPEAT_OPTIONAL_NEW; + + if (!capvm_compile_iter(env, p, e)) { return false; } + + if (!ensure_program_capacity(env->alloc, p, 1)) { + return false; + } + op_split = &p->ops[pos_split]; /* refresh pointer */ + + const uint32_t after_expr = get_program_offset(p); + op_split->u.split.nongreedy = after_expr; + } + } + } + + struct repeated_group_info *rgi = env->repeated_groups; + LOG(3 - LOG_REPETITION_CASES, + "%s: REPEAT %p: contains_empty_groups: %d, outermost_ancestor: %p == %p ? %d\n", + __func__, (void *)expr, expr->u.repeat.contains_empty_groups, + (void *)(rgi ? rgi->outermost_ancestor : NULL), + (void *)expr, + (rgi ? rgi->outermost_ancestor == expr : 0)); + if (expr->u.repeat.contains_empty_groups + && rgi != NULL + && rgi->outermost_ancestor == expr) { + LOG(3 - LOG_REPETITION_CASES, + "%s: outermost_ancestor match, count %zu\n", __func__, rgi->count); + if (!emit_repeated_groups(env, p)) { + return false; + } + pop_repeated_group_info(env, expr); + } + + break; + } + case AST_EXPR_GROUP: + { + const uint32_t id = expr->u.group.id; + const int is_repeated = expr->u.group.repeated; + + /* If the group is nullable and repeated, then move its save + * instructions to the end, since the final iteration matching + * nothing will always clobber any earlier saves. This is a + * workaround for cases that would otherwise incorrectly be + * halted by infinite loop prevention at runtime. */ + if (is_repeated && ((expr->flags & AST_FLAG_NULLABLE) + || !(expr->flags & AST_FLAG_CAN_CONSUME))) { + + struct repeated_group_info *rgi = env->repeated_groups; + + LOG(3 - LOG_REPETITION_CASES, + "%s: checking repeated group %u (capvm_compile_iter recurse), parent %p\n", + __func__, id, (void *)(rgi ? rgi->outermost_ancestor : NULL)); + if (!capvm_compile_iter(env, p, expr->u.group.e)) { return false; } + LOG(3 - LOG_REPETITION_CASES, + "%s: checking repeated group %u (capvm_compile_iter done), parent %p\n", + __func__, id, (void *)(rgi ? rgi->outermost_ancestor : NULL)); + + /* don't emit these here, parent repeat node will add them after. */ + if (rgi && rgi->outermost_ancestor != NULL) { + if (rgi->count == rgi->ceil) { + const size_t nceil = (rgi->ceil == 0 + ? DEF_REPEATED_GROUPS_CEIL + : 2*rgi->ceil); + const struct ast_expr **ngroups = f_realloc(env->alloc, + rgi->groups, + nceil * sizeof(ngroups[0])); + if (ngroups == NULL) { + return false; + } + rgi->groups = ngroups; + rgi->ceil = nceil; + } + + LOG(3 - LOG_REPETITION_CASES, + "%s: adding group %u (%p) to outermost_ancestor %p\n", + __func__, id, (void *)expr, + (void *)rgi->outermost_ancestor); + rgi->groups[rgi->count] = expr; + rgi->count++; + } + } else { + if (!ensure_program_capacity(env->alloc, p, 1)) { + return false; + } + const uint32_t pos_start = reserve_program_opcode(p); + struct capvm_opcode *op = &p->ops[pos_start]; + op->t = CAPVM_OP_SAVE; /* save capture start */ + op->u.save = 2*id; + + if (!capvm_compile_iter(env, p, expr->u.group.e)) { return false; } + + if (!ensure_program_capacity(env->alloc, p, 1)) { + return false; + } + const uint32_t pos_end = reserve_program_opcode(p); + op = &p->ops[pos_end]; + op->t = CAPVM_OP_SAVE; /* save capture end */ + op->u.save = 2*id + 1; + } + + if (id > env->max_capture_seen || env->max_capture_seen == NO_CAPTURE_ID) { + env->max_capture_seen = id; + } + + break; + } + + case AST_EXPR_ANCHOR: + { + if (!ensure_program_capacity(env->alloc, p, 1)) { + return false; + } + const uint32_t pos = reserve_program_opcode(p); + struct capvm_opcode *op = &p->ops[pos]; + op->t = CAPVM_OP_ANCHOR; + op->u.anchor = (expr->u.anchor.type == AST_ANCHOR_START + ? CAPVM_ANCHOR_START : CAPVM_ANCHOR_END); + break; + } + case AST_EXPR_SUBTRACT: + { + uint64_t cc[4]; + for (size_t i = 0; i < 4; i++) { + cc[i] = ~(uint64_t)0; + } + if (subtree_represents_character_class(expr, cc)) { + if (!ensure_program_capacity(env->alloc, p, 1)) { + return false; + } + const uint32_t pos = reserve_program_opcode(p); + struct capvm_opcode *op_cc = &p->ops[pos]; + op_cc->t = CAPVM_OP_CHARCLASS; + + if (expr->re_flags & RE_ICASE) { + make_charclass_case_insensitive(cc); + } + + if (!intern_char_class(env, p, cc, &op_cc->u.charclass_id)) { + return false; + } + } else { + /* FIXME: should return UNSUPPORTED */ + assert(!"unreachable"); + return false; + } + break; + } + case AST_EXPR_RANGE: + { + uint64_t cc[4] = { 0 }; + if (!subtree_represents_character_class(expr, cc)) { + dump_endpoint(&expr->u.range.from); + dump_endpoint(&expr->u.range.to); + assert(!"unreachable"); + return false; + } + + if (!ensure_program_capacity(env->alloc, p, 1)) { + return false; + } + const uint32_t pos = reserve_program_opcode(p); + struct capvm_opcode *op = &p->ops[pos]; + + op->t = CAPVM_OP_CHARCLASS; + if (expr->re_flags & RE_ICASE) { + make_charclass_case_insensitive(cc); + } + + if (!intern_char_class(env, p, cc, &op->u.charclass_id)) { + return false; + } + break; + } + default: + assert(!"matchfail"); + } + + return true; +} + +static bool +compile_kleene_star(struct capvm_compile_env *env, + struct capvm_program *p, const struct ast_expr *expr) +{ + /* Note: min count may be > 0 because this is also + * used for unbounded repetition with a lower count, + * as in `a{3,}`, but in that case the {min} + * repetitions have already been handled by the caller. */ + assert(expr && expr->type == AST_EXPR_REPEAT && + expr->u.repeat.max == AST_COUNT_UNBOUNDED); + + /* l1: split l2, l3 + * l2: + * jmp_once l1 OR jmp l1 + * l3: */ + if (!ensure_program_capacity(env->alloc, p, 2)) { + return false; + } + + const uint32_t pos_l1 = reserve_program_opcode(p); + const uint32_t pos_l2 = get_program_offset(p); + + struct capvm_opcode *op_split = &p->ops[pos_l1]; + op_split->t = CAPVM_OP_SPLIT; + op_split->u.split.greedy = PENDING_OFFSET_REPEAT_OPTIONAL_CONT; + op_split->u.split.nongreedy = PENDING_OFFSET_REPEAT_OPTIONAL_NEW; + + if (!capvm_compile_iter(env, p, expr->u.repeat.e)) { return false; } + + if (!ensure_program_capacity(env->alloc, p, 2)) { + return false; + } + + /* It's more expensive to always emit JMP_ONCE because it + * extends the path each iteration, so we could detect when + * it would be safe to use a JMP instead. */ + if (can_safely_skip_JMP_ONCE(expr)) { + const uint32_t pos_jmp = reserve_program_opcode(p); + struct capvm_opcode *op_jmp = &p->ops[pos_jmp]; + op_jmp->t = CAPVM_OP_JMP; + op_jmp->u.jmp = pos_l1; + } else { + const uint32_t pos_jmp_once = reserve_program_opcode(p); + struct capvm_opcode *op_jmp_once = &p->ops[pos_jmp_once]; + op_jmp_once->t = CAPVM_OP_JMP_ONCE; + op_jmp_once->u.jmp_once = pos_l1; + } + + const uint32_t pos_l3 = get_program_offset(p); + op_split = &p->ops[pos_l1]; /* refresh pointer */ + op_split->u.split.greedy = pos_l2; + op_split->u.split.nongreedy = pos_l3; + return true; +} + +static bool +emit_repeated_groups(struct capvm_compile_env *env, struct capvm_program *p) +{ + struct repeated_group_info *rgi = env->repeated_groups; + for (size_t i = 0; i < rgi->count; i++) { + const struct ast_expr *group = rgi->groups[i]; + assert(group->u.group.repeated); + const unsigned id = group->u.group.id; + LOG(3 - LOG_REPETITION_CASES, + "%s: checking %zu/%zu: group_id %u\n", + __func__, i, rgi->count, id); + + if (group->flags & (AST_FLAG_ANCHORED_START | AST_FLAG_ANCHORED_END)) { + /* if the otherwise empty group contains any anchors, + * then emit a subtree like (^)? so that its capture + * is only set when the anchors would match. */ + if (!ensure_program_capacity(env->alloc, p, 6)) { + return false; + } + + /* split l1, l2 + * l1: + * + * l2: save (start) + * save (end) + * */ + const uint32_t pos_split = reserve_program_opcode(p); + const uint32_t pos_l1 = get_program_offset(p); + + struct capvm_opcode *op_split = &p->ops[pos_split]; + op_split->t = CAPVM_OP_SPLIT; + op_split->u.split.greedy = pos_l1; + op_split->u.split.nongreedy = PENDING_OFFSET_REPEAT_OPTIONAL_NEW; + + if (group->flags & AST_FLAG_ANCHORED_START) { + const uint32_t pos_start = reserve_program_opcode(p); + struct capvm_opcode *op = &p->ops[pos_start]; + op->t = CAPVM_OP_ANCHOR; + op->u.anchor = CAPVM_ANCHOR_START; + } + + if (group->flags & AST_FLAG_ANCHORED_END) { + const uint32_t pos_end = reserve_program_opcode(p); + struct capvm_opcode *op = &p->ops[pos_end]; + op->t = CAPVM_OP_ANCHOR; + op->u.anchor = CAPVM_ANCHOR_END; + } + + const uint32_t pos_start = reserve_program_opcode(p); + struct capvm_opcode *op = &p->ops[pos_start]; + op->t = CAPVM_OP_SAVE; /* save capture start */ + op->u.save = 2*id; + + const uint32_t pos_end = reserve_program_opcode(p); + op = &p->ops[pos_end]; + op->t = CAPVM_OP_SAVE; /* save capture end */ + op->u.save = 2*group->u.group.id; + op->u.save = 2*id + 1; + + const uint32_t after_expr = get_program_offset(p); + op_split = &p->ops[pos_split]; /* refresh pointer */ + op_split->u.split.nongreedy = after_expr; + } else { + /* simple case, emit SAVE pair */ + if (!ensure_program_capacity(env->alloc, p, 2)) { + return false; + } + const uint32_t pos_start = reserve_program_opcode(p); + struct capvm_opcode *op = &p->ops[pos_start]; + op->t = CAPVM_OP_SAVE; /* save capture start */ + op->u.save = 2*id; + + const uint32_t pos_end = reserve_program_opcode(p); + op = &p->ops[pos_end]; + op->t = CAPVM_OP_SAVE; /* save capture end */ + op->u.save = 2*group->u.group.id; + op->u.save = 2*id + 1; + } + } + + /* clear, because an ALT's subtrees can have distinct repeated groups */ + rgi->count = 0; + + return true; +} + +static bool +capvm_compile_iter_save_groups_in_skipped_subtree(struct capvm_compile_env *env, + struct capvm_program *p, const struct ast_expr *expr) +{ + /* Follow the subtree as far as any expressions that could + * contain GROUPs. Emit any empty groups. This is necessary for + * regexes like /()*^/ and /(x|(x|))^/ whose subtrees are + * otherwise pruned but would still match the empty string + * before ^. */ + switch (expr->type) { + case AST_EXPR_EMPTY: + case AST_EXPR_LITERAL: + case AST_EXPR_CODEPOINT: + case AST_EXPR_ANCHOR: + case AST_EXPR_SUBTRACT: + case AST_EXPR_RANGE: + case AST_EXPR_TOMBSTONE: + /* none of these can contain groups */ + break; + + case AST_EXPR_CONCAT: + if (expr->flags & AST_FLAG_UNSATISFIABLE) { + return true; /* skip */ + } + for (size_t i = 0; i < expr->u.concat.count; i++) { + if (!capvm_compile_iter_save_groups_in_skipped_subtree(env, p, expr->u.concat.n[i])) { + return false; + } + } + break; + case AST_EXPR_ALT: + for (size_t i = 0; i < expr->u.alt.count; i++) { + if (!capvm_compile_iter_save_groups_in_skipped_subtree(env, p, expr->u.alt.n[i])) { + return false; + } + } + break; + + case AST_EXPR_REPEAT: + return capvm_compile_iter_save_groups_in_skipped_subtree(env, p, expr->u.repeat.e); + + case AST_EXPR_GROUP: + { + const uint32_t id = expr->u.group.id; + LOG(5, "%s: recording otherwise skipped group %u\n", __func__, id); + + if (!ensure_program_capacity(env->alloc, p, 2)) { + return false; + } + + if (id > env->max_capture_seen || env->max_capture_seen == NO_CAPTURE_ID) { + env->max_capture_seen = id; + } + + const uint32_t pos_start = reserve_program_opcode(p); + struct capvm_opcode *op = &p->ops[pos_start]; + op->t = CAPVM_OP_SAVE; /* save capture start */ + op->u.save = 2*id; + + const uint32_t pos_end = reserve_program_opcode(p); + op = &p->ops[pos_end]; + op->t = CAPVM_OP_SAVE; /* save capture end */ + op->u.save = 2*id + 1; + + if (!capvm_compile_iter_save_groups_in_skipped_subtree(env, p, expr->u.group.e)) { + return false; + } + + break; + } + default: + assert(!"match fail"); + + } + return true; +} + +static enum re_capvm_compile_ast_res +capvm_compile(struct capvm_compile_env *env, + const struct ast *ast) +{ + struct capvm_program *p = f_calloc(env->alloc, 1, sizeof(*p)); + if (p == NULL) { + return RE_CAPVM_COMPILE_AST_ERROR_ALLOC; + } + + LOG(3, "%s: has_unanchored: start? %d, end? %d\n", __func__, + ast->has_unanchored_start, + ast->has_unanchored_end); + + /* If the regex has an unanchored start, it gets a `.*` prefix, + * but with the labels swapped so that the unanchored start + * loop is NOT greedy. */ + if (ast->has_unanchored_start) { + if (!ensure_program_capacity(env->alloc, p, 4)) { + return RE_CAPVM_COMPILE_AST_ERROR_ALLOC; + } + + /* l1: split l3, l2 + * l2: . + * jmp l1 + * l3: */ + const uint32_t l1 = get_program_offset(p); + const uint32_t split_pos = reserve_program_opcode(p); + struct capvm_opcode *op_split = &p->ops[split_pos]; + + const uint32_t l2 = get_program_offset(p); + const uint32_t op_cc_pos = reserve_program_opcode(p); + struct capvm_opcode *op_cc = &p->ops[op_cc_pos]; + + const uint32_t op_jmp_pos = reserve_program_opcode(p); + struct capvm_opcode *op_jmp = &p->ops[op_jmp_pos]; + + const uint32_t l3 = get_program_offset(p); + + op_split->t = CAPVM_OP_SPLIT; + op_split->u.split.greedy = l3; + op_split->u.split.nongreedy = l2; + + op_cc->t = CAPVM_OP_CHARCLASS; + uint64_t any[4]; + for (size_t i = 0; i < 4; i++) { + any[i] = ~(uint64_t)0; + } + if (!intern_char_class(env, p, any, &op_cc->u.charclass_id)) { + goto cleanup; + } + + op_jmp->t = CAPVM_OP_JMP; + op_jmp->u.jmp = l1; + } + + /* Compile the regex AST, assuming match group 0 is + * explicitly represented. */ + if (!capvm_compile_iter(env, p, ast->expr)) { + goto cleanup; + } + + /* Add the unanchored end loop, outside of match group 0 */ + if (ast->has_unanchored_end) { + if (!ensure_program_capacity(env->alloc, p, 4)) { + return RE_CAPVM_COMPILE_AST_ERROR_ALLOC; + } + + /* l1: split l3, l2 + * l2: . + * jmp l1 + * l3: */ + const uint32_t l1 = reserve_program_opcode(p); + const uint32_t l2 = reserve_program_opcode(p); + const uint32_t l_jmp = reserve_program_opcode(p); + const uint32_t l3 = get_program_offset(p); + + struct capvm_opcode *op_split = &p->ops[l1]; + + struct capvm_opcode *op_any = &p->ops[l2]; + struct capvm_opcode *op_jmp = &p->ops[l_jmp]; + + op_split->t = CAPVM_OP_SPLIT; + op_split->u.split.greedy = l3; + op_split->u.split.nongreedy = l2; + + op_any->t = CAPVM_OP_CHARCLASS; + uint64_t any[4]; + for (size_t i = 0; i < 4; i++) { + any[i] = ~(uint64_t)0; + } + if (!intern_char_class(env, p, any, &op_any->u.charclass_id)) { + goto cleanup; + } + + op_jmp->t = CAPVM_OP_JMP; + op_jmp->u.jmp = l1; + } + + /* add MATCH opcode at end */ + if (!ensure_program_capacity(env->alloc, p, 1)) { + return RE_CAPVM_COMPILE_AST_ERROR_ALLOC; + } + const uint32_t pos_m = reserve_program_opcode(p); + struct capvm_opcode *op_m = &p->ops[pos_m]; + op_m->t = CAPVM_OP_MATCH; + + /* TODO: populate info about max threads, etc. in p, + * because it should be possible to calculate runtime + * memory limits at compile time. */ + env->program = p; + p->capture_count = (env->max_capture_seen == NO_CAPTURE_ID + ? 0 : env->max_capture_seen + 1); + + if (LOG_CAPVM > 2) { + LOG(0, "====\n"); + fsm_capvm_program_dump(stderr, p); + LOG(0, "====\n"); + } + + /* TODO: it may be worth exposing these static checks as + * something the caller can run at load-time */ + check_program_for_invalid_labels(p); + + return RE_CAPVM_COMPILE_AST_OK; + +cleanup: + fsm_capvm_program_free(env->alloc, p); + return RE_CAPVM_COMPILE_AST_ERROR_ALLOC; +} + +#define DUMP_AST 0 +#define DUMP_RESULT 0 /* should be 0 in production */ + +#if DUMP_AST || DUMP_RESULT +#include +#include "print.h" +static struct fsm_options opt = { .group_edges = 1 }; + +static unsigned +get_max_capture_id(const struct capvm_program *program) +{ + assert(program != NULL); + return (program->capture_count == 0 + ? 0 + : program->capture_base + program->capture_count - 1); +} + +#endif + +enum re_capvm_compile_ast_res +re_capvm_compile_ast(const struct fsm_alloc *alloc, + const struct ast *ast, + enum re_flags re_flags, + struct capvm_program **program) +{ +#if DUMP_AST + if (LOG_CAPVM > 2) { + ast_print_dot(stderr, &opt, re_flags, ast); + ast_print_tree(stderr, &opt, re_flags, ast); + } +#endif + + struct capvm_compile_env env = { + .alloc = alloc, + .re_flags = re_flags, + .max_capture_seen = NO_CAPTURE_ID, + }; + + enum re_capvm_compile_ast_res res; + res = capvm_compile(&env, ast); + + + struct repeated_group_info *rgi = env.repeated_groups; + while (rgi != NULL) { + struct repeated_group_info *prev = rgi->prev; + LOG(3 - LOG_REPETITION_CASES, + "%s: rgi cleanup, freeing %p, prev %p\n", + __func__, (void *)rgi, (void *)prev); + + if (rgi->groups != NULL) { + f_free(alloc, rgi->groups); + } + f_free(alloc, rgi); + rgi = prev; + } + + if (res == RE_CAPVM_COMPILE_AST_OK) { +#if DUMP_RESULT > 0 + if (DUMP_RESULT > 1 || getenv("DUMP")) { + ast_print_tree(stderr, &opt, re_flags, ast); + fsm_capvm_program_dump(stderr, env.program); + fprintf(stderr, "%s: max_capture_id %u\n", __func__, + get_max_capture_id(env.program)); + + } +#endif + + *program = env.program; + } + + free(env.charclass_htab.buckets); + + return res; +} diff --git a/src/libre/re_capvm_compile.h b/src/libre/re_capvm_compile.h new file mode 100644 index 000000000..b98ac9196 --- /dev/null +++ b/src/libre/re_capvm_compile.h @@ -0,0 +1,32 @@ +/* + * Copyright 2022 Scott Vokes + * + * See LICENCE for the full copyright terms. + */ + +#ifndef RE_CAPVM_COMPILE_H +#define RE_CAPVM_COMPILE_H + +/* The part of the capture VM interface that belongs in + * libre rather than libfsm, mostly related to compiling + * a libre AST into a capvm_program. */ + +#include + +#include "ast.h" +#include + +struct capvm_program; + +enum re_capvm_compile_ast_res { + RE_CAPVM_COMPILE_AST_OK, + RE_CAPVM_COMPILE_AST_ERROR_ALLOC = -1, +}; + +enum re_capvm_compile_ast_res +re_capvm_compile_ast(const struct fsm_alloc *alloc, + const struct ast *ast, + enum re_flags re_flags, + struct capvm_program **program); + +#endif diff --git a/src/libre/re_interpolate_groups.c b/src/libre/re_interpolate_groups.c new file mode 100644 index 000000000..52b61f116 --- /dev/null +++ b/src/libre/re_interpolate_groups.c @@ -0,0 +1,198 @@ +/* + * Copyright 2026 Katherine Flavel + * + * See LICENCE for the full copyright terms. + */ + +#include +#include +#include +#include +#include +#include + +#include +#include + +#define OUT_CHAR(c) \ + do if (outs != NULL) { \ + if (outn < 1) goto overflow; \ + *outs++ = (c); \ + outn--; \ + } while (0) + +#define OUT_GROUP(s) \ + do if (outs != NULL) { \ + size_t n = strlen((s)); \ + if (outn < n) goto overflow; \ + (void) memcpy(outs, s, n); \ + outs += n; \ + outn -= n; \ + } while (0) + +bool +re_interpolate_groups(const char *fmt, char esc, + const char *group0, unsigned groupc, const char *groupv[], const char *nonexistent, + char *outs, size_t outn, + struct re_pos *start, struct re_pos *end) +{ + unsigned group; // 0 meaning group0, 1 meaning groupv[0], etc + char *outs_orig; + const char *p; + + enum { + STATE_LIT, + STATE_ESC, + STATE_DIGIT + } state; + + assert(esc != '\0'); + assert(group0 != NULL || groupc == 0); + assert(groupc < UINT_MAX / 10 - 1); + assert(outs != NULL || outn == 0); + + state = STATE_LIT; + group = 0; + + outs_orig = outn > 0 ? outs : NULL; + + if (start != NULL) { + start->byte = 0; + } + + p = fmt; + + do { + switch (state) { + case STATE_LIT: + if (*p == '\0') { + break; + } + + if (*p == esc) { + if (start != NULL) { + start->byte = p - fmt; + } + + state = STATE_ESC; + continue; + } + + OUT_CHAR(*p); + continue; + + case STATE_ESC: + if (*p == '\0') { + goto error; + } + + if (*p == esc) { + OUT_CHAR(esc); + state = STATE_LIT; + continue; + } + + if (isdigit((unsigned char) *p)) { + group = *p - '0'; + state = STATE_DIGIT; + continue; + } + + goto error; + + case STATE_DIGIT: + if (isdigit((unsigned char) *p)) { + group *= 10; + group += *p - '0'; + + /* + * We need to handle numeric overflow somehow here, + * as we would with using strtol() or similar. But + * we don't need to distinguish this as a special + * error code, semantically it's the same as a group + * that doesn't exist. + * + * groupc + 1 is always out of bounds. So we cap to that, + * using it as a simple way to avoid needing to handle + * numeric overflow for subsequent digits. This assumes + * groupc *= 10 is <= UINT_MAX. + */ + if (group > groupc) { + group = groupc + 1; + } + continue; + } + + if (group == 0) { + OUT_GROUP(group0); + } else if (group <= groupc) { + assert(groupv[group - 1] != NULL); + OUT_GROUP(groupv[group - 1]); + } else if (nonexistent == NULL) { + /* + * We could indicate this independently from syntax errors, + * with some way to return different error codes. + * + * But there's no need, you can pre-check the fmt syntax + * by running ahead of time with groupc == 0 and pass + * nonexistent != NULL, because that eliminates the + * possibility for group-related errors. + */ + goto error; + } else { + OUT_GROUP(nonexistent); + } + + group = 0; + state = STATE_LIT; + + if (*p == '\0') { + break; + } + + if (*p == esc) { + if (start != NULL) { + start->byte = p - fmt; + } + + state = STATE_ESC; + continue; + } + + OUT_CHAR(*p); + continue; + + default: + assert(!"unreached"); + goto error; + } + } while (*p != '\0' && p++); + + if (state != STATE_LIT) { + goto error; + } + + OUT_CHAR('\0'); + + return true; + +overflow: + + /* we're blaming the entire fmt string for overflow */ + if (start != NULL) { + start->byte = 0; + } + +error: + + if (end != NULL) { + end->byte = p - fmt; + } + + if (outs_orig != NULL) { + *outs_orig = '\0'; + } + + return false; +} + diff --git a/src/libre/re_strings.c b/src/libre/re_strings.c index 06b7dc772..39a9b5c24 100644 --- a/src/libre/re_strings.c +++ b/src/libre/re_strings.c @@ -19,7 +19,8 @@ #include "ac.h" struct fsm * -re_strings(const struct fsm_options *opt, const char *a[], size_t n, +re_strings(const struct fsm_alloc *alloc, + const char *a[], size_t n, enum re_strings_flags flags) { struct re_strings *g; @@ -32,12 +33,12 @@ re_strings(const struct fsm_options *opt, const char *a[], size_t n, } for (i = 0; i < n; i++) { - if (!re_strings_add_str(g, a[i])) { + if (!re_strings_add_str(g, a[i], NULL)) { goto error; } } - fsm = re_strings_build(g, opt, flags); + fsm = re_strings_build(g, alloc, flags); re_strings_free(g); return fsm; @@ -64,25 +65,25 @@ re_strings_free(struct re_strings *g) } int -re_strings_add_raw(struct re_strings *g, const void *p, size_t n) +re_strings_add_raw(struct re_strings *g, const void *p, size_t n, const fsm_end_id_t *endid) { assert(p != NULL); - assert(n > 0); - return trie_add_word((struct trie_graph *) g, p, n) != NULL; + return trie_add_word((struct trie_graph *) g, p, n, endid) != NULL; } int -re_strings_add_str(struct re_strings *g, const char *s) +re_strings_add_str(struct re_strings *g, const char *s, const fsm_end_id_t *endid) { assert(s != NULL); - return re_strings_add_raw(g, s, strlen(s)); + return re_strings_add_raw(g, s, strlen(s), endid); } struct fsm * re_strings_build(struct re_strings *g, - const struct fsm_options *opt, enum re_strings_flags flags) + const struct fsm_alloc *alloc, + enum re_strings_flags flags) { struct fsm *fsm; fsm_state_t end; @@ -96,7 +97,8 @@ re_strings_build(struct re_strings *g, } } - fsm = fsm_new(opt); + /* TODO: count trie nodes and fsm_new_statealloc() */ + fsm = fsm_new(alloc); if (fsm == NULL) { goto error; } diff --git a/src/lx/Makefile b/src/lx/Makefile index 683341ab0..a8c293b88 100644 --- a/src/lx/Makefile +++ b/src/lx/Makefile @@ -3,6 +3,7 @@ SRC += src/lx/ast.c SRC += src/lx/lexer.c SRC += src/lx/parser.c +SRC += src/lx/print.c SRC += src/lx/main.c SRC += src/lx/tokens.c SRC += src/lx/var.c diff --git a/src/lx/ast.c b/src/lx/ast.c index 75754f58d..a3776b644 100644 --- a/src/lx/ast.c +++ b/src/lx/ast.c @@ -9,8 +9,10 @@ #include #include #include +#include #include +#include #include @@ -115,7 +117,7 @@ ast_addmapping(struct ast_zone *z, struct fsm *fsm, return NULL; } - m->fsm = fsm_new(fsm->opt); + m->fsm = fsm_new(fsm->alloc); if (m->fsm == NULL) { free(m); return NULL; @@ -123,7 +125,6 @@ ast_addmapping(struct ast_zone *z, struct fsm *fsm, m->token = token; m->to = to; - m->conflict = NULL; m->next = z->ml; z->ml = m; @@ -139,39 +140,6 @@ ast_addmapping(struct ast_zone *z, struct fsm *fsm, return m; } -/* TODO: merge with adt/set.c */ -struct mapping_set * -ast_addconflict(struct mapping_set **head, struct ast_mapping *m) -{ - struct mapping_set *new; - - assert(head != NULL); - assert(m != NULL); - - /* TODO: explain we find duplicate; return success */ - { - struct mapping_set *s; - - for (s = *head; s != NULL; s = s->next) { - if (s->m == m) { - return s; - } - } - } - - new = malloc(sizeof *new); - if (new == NULL) { - return NULL; - } - - new->m = m; - - new->next = *head; - *head = new; - - return new; -} - #define DEF_MAPPINGS 4 static fsm_end_id_t mapping_count = 0; static fsm_end_id_t mapping_ceil = DEF_MAPPINGS; @@ -233,50 +201,39 @@ ast_getendmappingbyendid(fsm_end_id_t id) struct ast_mapping * ast_getendmapping(const struct fsm *fsm, fsm_state_t s) { - #define ID_STACK_BUF_COUNT 4 - size_t id_count; - fsm_end_id_t *id_buf_dynamic = NULL; - fsm_end_id_t id_buf[ID_STACK_BUF_COUNT]; - enum fsm_getendids_res res; - size_t written; + fsm_end_id_t *ids; + size_t count; struct ast_mapping *m; + int res; - id_count = fsm_getendidcount(fsm, s); - if (id_count > ID_STACK_BUF_COUNT) { - id_buf_dynamic = malloc(id_count * sizeof(id_buf_dynamic[0])); - if (id_buf_dynamic == NULL) { - return NULL; - } + if (!fsm_isend(fsm, s)) { + errno = EINVAL; + return NULL; } - res = fsm_getendids(fsm, - s, id_count, - id_buf_dynamic == NULL ? id_buf : id_buf_dynamic, - &written); - if (res == FSM_GETENDIDS_NOT_FOUND) { - if (id_buf_dynamic != NULL) { - free(id_buf_dynamic); - } + count = fsm_endid_count(fsm, s); + if (count == 0) { + errno = EINVAL; return NULL; } - /* Should always have an appropriately sized buffer, - * or fail to allocate above */ - assert(res != FSM_GETENDIDS_ERROR_INSUFFICIENT_SPACE); + ids = malloc(count * sizeof *ids); + if (ids == NULL) { + return NULL; + } - assert(res == FSM_GETENDIDS_FOUND); - assert(written == id_count); - assert(written > 0); + res = fsm_endid_get(fsm, s, count, ids); + assert(res == 1); - m = ast_getendmappingbyendid(id_buf[0]); + m = ast_getendmappingbyendid(ids[0]); if (LOG()) { fprintf(stderr, "ast_getendmapping: got mapping %p mappings[%d]\n", - (void *)m, id_buf[0]); + (void *) m, ids[0]); } - if (id_buf_dynamic != NULL) { - free(id_buf_dynamic); - } + free(ids); + return m; } + diff --git a/src/lx/ast.h b/src/lx/ast.h index 097e43841..1f7fcf52f 100644 --- a/src/lx/ast.h +++ b/src/lx/ast.h @@ -14,7 +14,6 @@ struct ast_mapping { struct fsm *fsm; struct ast_token *token; struct ast_zone *to; - struct mapping_set *conflict; struct ast_mapping *next; }; @@ -34,11 +33,6 @@ struct ast_token { struct ast_token *next; }; -struct mapping_set { - struct ast_mapping *m; - struct mapping_set *next; -}; - struct ast { struct ast_zone *zl; struct ast_token *tl; @@ -59,9 +53,6 @@ struct ast_mapping * ast_addmapping(struct ast_zone *z, struct fsm *fsm, struct ast_token *token, struct ast_zone *to); -struct mapping_set * -ast_addconflict(struct mapping_set **head, struct ast_mapping *n); - int ast_setendmapping(struct fsm *fsm, struct ast_mapping *m); diff --git a/src/lx/lexer.c b/src/lx/lexer.c index 03bfbf463..ed7ca6d78 100644 --- a/src/lx/lexer.c +++ b/src/lx/lexer.c @@ -14,6 +14,26 @@ static enum lx_token z2(struct lx *lx); static enum lx_token z3(struct lx *lx); static enum lx_token z4(struct lx *lx); +static int +lx_advance_end(struct lx *lx, int c) +{ + lx->end.byte++; + lx->end.col++; + if (c == '\n') { + lx->end.line++; + lx->end.saved_col = lx->end.col - 1; + lx->end.col = 1; + } + if (lx->push != NULL) { + if (-1 == lx->push(lx->buf_opaque, (char)c)) { + return 0; + } + } + return 1; +} + +/* This wrapper manages one character of lookahead/pushback + * and the line, column, and byte offsets. */ #if __STDC_VERSION__ >= 199901L inline #endif @@ -34,18 +54,19 @@ lx_getc(struct lx *lx) } } - lx->end.byte++; - lx->end.col++; - - if (c == '\n') { - lx->end.line++; - lx->end.saved_col = lx->end.col - 1; - lx->end.col = 1; - } + if (!lx_advance_end(lx, c)) { return EOF; } return c; } +/* This wrapper adapts calling lx_getc to the interface + * in libfsm's generated code. */ +static int +fsm_getc(void *getc_opaque) +{ + return lx_getc((struct lx *)getc_opaque); +} + #if __STDC_VERSION__ >= 199901L inline #endif @@ -54,10 +75,7 @@ lx_ungetc(struct lx *lx, int c) { assert(lx != NULL); assert(lx->c == EOF); - lx->c = c; - - lx->end.byte--; lx->end.col--; @@ -67,13 +85,20 @@ lx_ungetc(struct lx *lx, int c) } } +/* Get a character from fgetc and push it to the buffer */ int lx_fgetc(struct lx *lx) { assert(lx != NULL); assert(lx->getc_opaque != NULL); - return fgetc(lx->getc_opaque); + const int c = fgetc(lx->getc_opaque); + if (c == EOF) { + lx->c = EOF; + return EOF; + } else { + return c; + } } int @@ -118,6 +143,17 @@ lx_dynpush(void *buf_opaque, char c) return 0; } +static void +lx_dynpop(void *buf_opaque) +{ + struct lx_dynbuf *t = buf_opaque; + + assert(t != NULL); + + assert(t->p != t->a); + t->p--; +} + int lx_dynclear(void *buf_opaque) { @@ -157,37 +193,36 @@ lx_dynfree(void *buf_opaque) static enum lx_token z0(struct lx *lx) { + int has_consumed_input = 0; int c; - enum { - S0, S1, S2, NONE - } state; - assert(lx != NULL); if (lx->clear != NULL) { lx->clear(lx->buf_opaque); } - state = NONE; - lx->start = lx->end; - while (c = lx_getc(lx), c != EOF) { - if (state == NONE) { - state = S0; - } + void *getc_opaque = (void *)lx; + enum { + S0, S1, S2 + } state; + + state = S0; + while (c = fsm_getc(getc_opaque), c != EOF) { + has_consumed_input = 1; switch (state) { - case S0: /* start */ + case S0: /* e.g. "" */ switch ((unsigned char) c) { case '/': state = S2; break; default: state = S1; break; } break; - case S1: /* e.g. "a" */ - lx_ungetc(lx, c); return TOK_CHAR; + case S1: /* e.g. "" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_CHAR; case S2: /* e.g. "\057" */ switch ((unsigned char) c) { @@ -243,79 +278,73 @@ z0(struct lx *lx) case 'x': case 'y': case 'z': break; - default: lx_ungetc(lx, c); return lx->z = z4, TOK_RE; + default: lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return lx->z = z4, TOK_RE; } break; default: ; /* unreached */ } + } + + /* end states */ + switch (state) { + case S1: return TOK_CHAR; + case S2: return lx->z = z4, TOK_RE; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; + } if (lx->push != NULL) { - if (-1 == lx->push(lx->buf_opaque, c)) { + if (-1 == lx->push(lx->buf_opaque, (char)c)) { return TOK_ERROR; } } - } lx->lgetc = NULL; - switch (state) { - case NONE: return TOK_EOF; - case S1: return TOK_CHAR; - case S2: return TOK_RE; - default: errno = EINVAL; return TOK_ERROR; - } + if (!has_consumed_input) { + return TOK_EOF; + } + return TOK_ERROR; } static enum lx_token z1(struct lx *lx) { + int has_consumed_input = 0; int c; - enum { - S0, S1, S2, S3, S4, S5, S6, S7, NONE - } state; - assert(lx != NULL); if (lx->clear != NULL) { lx->clear(lx->buf_opaque); } - state = NONE; - lx->start = lx->end; - while (c = lx_getc(lx), c != EOF) { - if (state == NONE) { - state = S0; - } + void *getc_opaque = (void *)lx; + enum { + S0, S1, S2, S3, S4, S5, S6, S7 + } state; + state = S0; + + while (c = fsm_getc(getc_opaque), c != EOF) { + has_consumed_input = 1; switch (state) { - case S0: /* start */ + case S0: /* e.g. "" */ switch ((unsigned char) c) { - case '"': state = S2; break; - case '\\': state = S3; break; - default: state = S1; break; + case '\\': state = S1; break; + case '"': state = S3; break; + default: state = S2; break; } break; - case S1: /* e.g. "a" */ - lx_ungetc(lx, c); return TOK_CHAR; - - case S2: /* e.g. "\"" */ - lx_ungetc(lx, c); return lx->z = z4, TOK_STR; - - case S3: /* e.g. "\\" */ + case S1: /* e.g. "\\" */ switch ((unsigned char) c) { - case '"': - case '\\': - case 'f': - case 'n': - case 'r': - case 't': - case 'v': state = S4; break; + case 'x': state = S4; break; case '0': case '1': case '2': @@ -324,29 +353,24 @@ z1(struct lx *lx) case '5': case '6': case '7': state = S5; break; - case 'x': state = S6; break; - default: lx_ungetc(lx, c); return TOK_CHAR; + case '"': + case '\\': + case 'f': + case 'n': + case 'r': + case 't': + case 'v': state = S6; break; + default: lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_CHAR; } break; - case S4: /* e.g. "\\f" */ - lx_ungetc(lx, c); return TOK_ESC; + case S2: /* e.g. "\\x00" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_CHAR; - case S5: /* e.g. "\\0" */ - switch ((unsigned char) c) { - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': break; - default: lx_ungetc(lx, c); return TOK_OCT; - } - break; + case S3: /* e.g. "\"" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return lx->z = z4, TOK_STR; - case S6: /* e.g. "\\x" */ + case S4: /* e.g. "\\x" */ switch ((unsigned char) c) { case '0': case '1': @@ -370,11 +394,30 @@ z1(struct lx *lx) case 'd': case 'e': case 'f': state = S7; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; + } + break; + + case S5: /* e.g. "\\0" */ + switch ((unsigned char) c) { + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': break; + default: lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_OCT; } break; - case S7: /* e.g. "\\xa" */ + case S6: /* e.g. "\\\"" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_ESC; + + case S7: /* e.g. "\\x0" */ switch ((unsigned char) c) { case '0': case '1': @@ -398,135 +441,157 @@ z1(struct lx *lx) case 'd': case 'e': case 'f': break; - default: lx_ungetc(lx, c); return TOK_HEX; + default: lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_HEX; } break; default: ; /* unreached */ } + } + + /* end states */ + switch (state) { + case S1: return TOK_CHAR; + case S2: return TOK_CHAR; + case S3: return lx->z = z4, TOK_STR; + case S5: return TOK_OCT; + case S6: return TOK_ESC; + case S7: return TOK_HEX; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; + } if (lx->push != NULL) { - if (-1 == lx->push(lx->buf_opaque, c)) { + if (-1 == lx->push(lx->buf_opaque, (char)c)) { return TOK_ERROR; } } - } lx->lgetc = NULL; - switch (state) { - case NONE: return TOK_EOF; - case S1: return TOK_CHAR; - case S2: return TOK_STR; - case S3: return TOK_CHAR; - case S4: return TOK_ESC; - case S5: return TOK_OCT; - case S7: return TOK_HEX; - default: errno = EINVAL; return TOK_ERROR; - } + if (!has_consumed_input) { + return TOK_EOF; + } + return TOK_ERROR; } static enum lx_token z2(struct lx *lx) { + int has_consumed_input = 0; int c; - enum { - S0, S1, S2, NONE - } state; - assert(lx != NULL); if (lx->clear != NULL) { lx->clear(lx->buf_opaque); } - state = NONE; - lx->start = lx->end; - while (c = lx_getc(lx), c != EOF) { - if (state == NONE) { - state = S0; - } + void *getc_opaque = (void *)lx; + enum { + S0, S1, S2 + } state; + + state = S0; + while (c = fsm_getc(getc_opaque), c != EOF) { + has_consumed_input = 1; switch (state) { - case S0: /* start */ + case S0: /* e.g. "" */ switch ((unsigned char) c) { case '\'': state = S2; break; default: state = S1; break; } break; - case S1: /* e.g. "a" */ - lx_ungetc(lx, c); return TOK_CHAR; + case S1: /* e.g. "" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_CHAR; case S2: /* e.g. "'" */ - lx_ungetc(lx, c); return lx->z = z4, TOK_STR; + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return lx->z = z4, TOK_STR; default: ; /* unreached */ } + } + + /* end states */ + switch (state) { + case S1: return TOK_CHAR; + case S2: return lx->z = z4, TOK_STR; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; + } if (lx->push != NULL) { - if (-1 == lx->push(lx->buf_opaque, c)) { + if (-1 == lx->push(lx->buf_opaque, (char)c)) { return TOK_ERROR; } } - } lx->lgetc = NULL; - switch (state) { - case NONE: return TOK_EOF; - case S1: return TOK_CHAR; - case S2: return TOK_STR; - default: errno = EINVAL; return TOK_ERROR; - } + if (!has_consumed_input) { + return TOK_EOF; + } + return TOK_ERROR; } static enum lx_token z3(struct lx *lx) { + int has_consumed_input = 0; int c; - enum { - S0, S1, S2, NONE - } state; - assert(lx != NULL); if (lx->clear != NULL) { lx->clear(lx->buf_opaque); } - state = NONE; - lx->start = lx->end; - while (c = lx_getc(lx), c != EOF) { - if (state == NONE) { - state = S0; - } + void *getc_opaque = (void *)lx; + enum { + S0, S1, S2 + } state; + state = S0; + + while (c = fsm_getc(getc_opaque), c != EOF) { + has_consumed_input = 1; switch (state) { - case S0: /* start */ + case S0: /* e.g. "" */ switch ((unsigned char) c) { case '\n': state = S2; break; default: state = S1; break; } break; - case S1: /* e.g. "a" */ - lx_ungetc(lx, c); return lx->z(lx); + case S1: /* e.g. "" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return lx->z(lx); case S2: /* e.g. "" */ - lx_ungetc(lx, c); return lx->z = z4, lx->z(lx); + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return lx->z = z4, lx->z(lx); default: ; /* unreached */ } + } + + /* end states */ + switch (state) { + case S1: return TOK_UNKNOWN; + case S2: return lx->z = z4, lx->z(lx); + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; + } switch (state) { case S0: @@ -536,75 +601,52 @@ z3(struct lx *lx) default: if (lx->push != NULL) { - if (-1 == lx->push(lx->buf_opaque, c)) { + if (-1 == lx->push(lx->buf_opaque, (char)c)) { return TOK_ERROR; } } break; } - } lx->lgetc = NULL; - switch (state) { - case NONE: return TOK_EOF; - case S1: return TOK_EOF; - case S2: return TOK_EOF; - default: errno = EINVAL; return TOK_ERROR; - } + if (!has_consumed_input) { + return TOK_EOF; + } + return TOK_ERROR; } static enum lx_token z4(struct lx *lx) { + int has_consumed_input = 0; int c; - enum { - S0, S1, S2, S3, S4, S5, S6, S7, S8, S9, - S10, S11, S12, S13, S14, S15, S16, S17, S18, S19, - S20, S21, S22, S23, S24, S25, S26, S27, S28, NONE - } state; - assert(lx != NULL); if (lx->clear != NULL) { lx->clear(lx->buf_opaque); } - state = NONE; - lx->start = lx->end; - while (c = lx_getc(lx), c != EOF) { - if (state == NONE) { - state = S0; - } + void *getc_opaque = (void *)lx; + enum { + S0, S1, S2, S3, S4, S5, S6, S7, S8, S9, + S10, S11, S12, S13, S14, S15, S16, S17, S18, S19, + S20, S21, S22, S23, S24, S25, S26, S27, S28 + } state; + state = S0; + + while (c = fsm_getc(getc_opaque), c != EOF) { + has_consumed_input = 1; switch (state) { - case S0: /* start */ + case S0: /* e.g. "" */ switch ((unsigned char) c) { - case '\t': - case '\n': - case '\r': - case ' ': state = S1; break; - case '!': state = S2; break; - case '"': state = S3; break; - case '#': state = S4; break; - case '$': state = S5; break; - case '&': state = S6; break; - case '\'': state = S7; break; - case '(': state = S8; break; - case ')': state = S9; break; - case '*': state = S10; break; - case '+': state = S11; break; - case ',': state = S12; break; - case '-': state = S13; break; - case '.': state = S14; break; - case '/': state = S15; break; - case ';': state = S16; break; - case '=': state = S17; break; - case '?': state = S18; break; + case ',': state = S1; break; + case '$': state = S2; break; case 'A': case 'B': case 'C': @@ -657,37 +699,42 @@ z4(struct lx *lx) case 'w': case 'x': case 'y': - case 'z': state = S19; break; - case '\\': state = S20; break; - case '^': state = S21; break; - case '{': state = S22; break; - case '|': state = S23; break; - case '}': state = S24; break; - case '~': state = S25; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; - } - break; - - case S1: /* e.g. "\\x09" */ - switch ((unsigned char) c) { + case 'z': state = S3; break; + case '&': state = S4; break; + case '|': state = S5; break; + case '.': state = S6; break; + case '-': state = S7; break; + case '\\': state = S8; break; + case '^': state = S9; break; + case '!': state = S10; break; + case '~': state = S11; break; + case '?': state = S12; break; + case '+': state = S13; break; + case '*': state = S14; break; + case ')': state = S15; break; + case '(': state = S16; break; + case '}': state = S17; break; + case '{': state = S18; break; + case ';': state = S19; break; + case '=': state = S20; break; + case '/': state = S21; break; + case '"': state = S22; break; + case '\'': state = S23; break; + case '#': state = S24; break; case '\t': case '\n': case '\r': - case ' ': break; - default: lx_ungetc(lx, c); return lx->z(lx); + case ' ': state = S25; break; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S2: /* e.g. "!" */ - lx_ungetc(lx, c); return TOK_BANG; - - case S3: /* e.g. "\"" */ - lx_ungetc(lx, c); return lx->z = z1, lx->z(lx); - - case S4: /* e.g. "#" */ - lx_ungetc(lx, c); return lx->z = z3, lx->z(lx); + case S1: /* e.g. "," */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_COMMA; - case S5: /* e.g. "$" */ + case S2: /* e.g. "$" */ switch ((unsigned char) c) { case 'A': case 'B': @@ -742,58 +789,13 @@ z4(struct lx *lx) case 'x': case 'y': case 'z': state = S28; break; - default: lx->lgetc = NULL; return TOK_UNKNOWN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; } break; - case S6: /* e.g. "&" */ - lx_ungetc(lx, c); return TOK_AND; - - case S7: /* e.g. "'" */ - lx_ungetc(lx, c); return lx->z = z2, lx->z(lx); - - case S8: /* e.g. "(" */ - lx_ungetc(lx, c); return TOK_LPAREN; - - case S9: /* e.g. ")" */ - lx_ungetc(lx, c); return TOK_RPAREN; - - case S10: /* e.g. "*" */ - lx_ungetc(lx, c); return TOK_STAR; - - case S11: /* e.g. "+" */ - lx_ungetc(lx, c); return TOK_CROSS; - - case S12: /* e.g. "," */ - lx_ungetc(lx, c); return TOK_COMMA; - - case S13: /* e.g. "-" */ - switch ((unsigned char) c) { - case '>': state = S27; break; - default: lx_ungetc(lx, c); return TOK_DASH; - } - break; - - case S14: /* e.g. "." */ - switch ((unsigned char) c) { - case '.': state = S26; break; - default: lx_ungetc(lx, c); return TOK_DOT; - } - break; - - case S15: /* e.g. "\057" */ - lx_ungetc(lx, c); return lx->z = z0, lx->z(lx); - - case S16: /* e.g. ";" */ - lx_ungetc(lx, c); return TOK_SEMI; - - case S17: /* e.g. "=" */ - lx_ungetc(lx, c); return TOK_BIND; - - case S18: /* e.g. "?" */ - lx_ungetc(lx, c); return TOK_QMARK; - - case S19: /* e.g. "a" */ + case S3: /* e.g. "A" */ switch ((unsigned char) c) { case '0': case '1': @@ -858,35 +860,98 @@ z4(struct lx *lx) case 'x': case 'y': case 'z': break; - default: lx_ungetc(lx, c); return TOK_IDENT; + default: lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_IDENT; } break; - case S20: /* e.g. "\\" */ - lx_ungetc(lx, c); return TOK_DASH; + case S4: /* e.g. "&" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_AND; + + case S5: /* e.g. "|" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_PIPE; + + case S6: /* e.g. "." */ + switch ((unsigned char) c) { + case '.': state = S27; break; + default: lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_DOT; + } + break; + + case S7: /* e.g. "-" */ + switch ((unsigned char) c) { + case '>': state = S26; break; + default: lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_DASH; + } + break; + + case S8: /* e.g. "\\" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_DASH; + + case S9: /* e.g. "^" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_HAT; - case S21: /* e.g. "^" */ - lx_ungetc(lx, c); return TOK_HAT; + case S10: /* e.g. "!" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_BANG; - case S22: /* e.g. "{" */ - lx_ungetc(lx, c); return TOK_OPEN; + case S11: /* e.g. "~" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_TILDE; - case S23: /* e.g. "|" */ - lx_ungetc(lx, c); return TOK_PIPE; + case S12: /* e.g. "\077" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_QMARK; - case S24: /* e.g. "}" */ - lx_ungetc(lx, c); return TOK_CLOSE; + case S13: /* e.g. "+" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_CROSS; - case S25: /* e.g. "~" */ - lx_ungetc(lx, c); return TOK_TILDE; + case S14: /* e.g. "*" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_STAR; - case S26: /* e.g. ".." */ - lx_ungetc(lx, c); return TOK_TO; + case S15: /* e.g. ")" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_RPAREN; - case S27: /* e.g. "->" */ - lx_ungetc(lx, c); return TOK_MAP; + case S16: /* e.g. "(" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_LPAREN; - case S28: /* e.g. "$a" */ + case S17: /* e.g. "}" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_CLOSE; + + case S18: /* e.g. "{" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_OPEN; + + case S19: /* e.g. ";" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_SEMI; + + case S20: /* e.g. "=" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_BIND; + + case S21: /* e.g. "\057" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return lx->z = z0, lx->z(lx); + + case S22: /* e.g. "\"" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return lx->z = z1, lx->z(lx); + + case S23: /* e.g. "'" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return lx->z = z2, lx->z(lx); + + case S24: /* e.g. "#" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return lx->z = z3, lx->z(lx); + + case S25: /* e.g. "\\x09" */ + switch ((unsigned char) c) { + case '\t': + case '\n': + case '\r': + case ' ': break; + default: lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return lx->z(lx); + } + break; + + case S26: /* e.g. "->" */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_MAP; + + case S27: /* e.g. ".." */ + lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_TO; + + case S28: /* e.g. "$A" */ switch ((unsigned char) c) { case '0': case '1': @@ -951,66 +1016,73 @@ z4(struct lx *lx) case 'x': case 'y': case 'z': break; - default: lx_ungetc(lx, c); return TOK_TOKEN; + default: lx_ungetc(lx, c); lx_dynpop(lx->buf_opaque); return TOK_TOKEN; } break; default: ; /* unreached */ } + } + + /* end states */ + switch (state) { + case S1: return TOK_COMMA; + case S3: return TOK_IDENT; + case S4: return TOK_AND; + case S5: return TOK_PIPE; + case S6: return TOK_DOT; + case S7: return TOK_DASH; + case S8: return TOK_DASH; + case S9: return TOK_HAT; + case S10: return TOK_BANG; + case S11: return TOK_TILDE; + case S12: return TOK_QMARK; + case S13: return TOK_CROSS; + case S14: return TOK_STAR; + case S15: return TOK_RPAREN; + case S16: return TOK_LPAREN; + case S17: return TOK_CLOSE; + case S18: return TOK_OPEN; + case S19: return TOK_SEMI; + case S20: return TOK_BIND; + case S21: return lx->z = z0, lx->z(lx); + case S22: return lx->z = z1, lx->z(lx); + case S23: return lx->z = z2, lx->z(lx); + case S24: return lx->z = z3, lx->z(lx); + case S25: return TOK_EOF; + case S26: return TOK_MAP; + case S27: return TOK_TO; + case S28: return TOK_TOKEN; + default: + if (!has_consumed_input) { return TOK_EOF; } + lx_ungetc(lx, c); lx->lgetc = NULL; return TOK_UNKNOWN; + } switch (state) { - case S1: - case S3: - case S4: - case S7: - case S15: + case S21: + case S22: + case S23: + case S24: + case S25: break; default: if (lx->push != NULL) { - if (-1 == lx->push(lx->buf_opaque, c)) { + if (-1 == lx->push(lx->buf_opaque, (char)c)) { return TOK_ERROR; } } break; } - } lx->lgetc = NULL; - switch (state) { - case NONE: return TOK_EOF; - case S1: return TOK_EOF; - case S2: return TOK_BANG; - case S3: return TOK_EOF; - case S4: return TOK_EOF; - case S6: return TOK_AND; - case S7: return TOK_EOF; - case S8: return TOK_LPAREN; - case S9: return TOK_RPAREN; - case S10: return TOK_STAR; - case S11: return TOK_CROSS; - case S12: return TOK_COMMA; - case S13: return TOK_DASH; - case S14: return TOK_DOT; - case S15: return TOK_EOF; - case S16: return TOK_SEMI; - case S17: return TOK_BIND; - case S18: return TOK_QMARK; - case S19: return TOK_IDENT; - case S20: return TOK_DASH; - case S21: return TOK_HAT; - case S22: return TOK_OPEN; - case S23: return TOK_PIPE; - case S24: return TOK_CLOSE; - case S25: return TOK_TILDE; - case S26: return TOK_TO; - case S27: return TOK_MAP; - case S28: return TOK_TOKEN; - default: errno = EINVAL; return TOK_ERROR; - } + if (!has_consumed_input) { + return TOK_EOF; + } + return TOK_ERROR; } const char * @@ -1238,6 +1310,7 @@ lx_init(struct lx *lx) lx->end.byte = 0; lx->end.line = 1; lx->end.col = 1; + (void)lx_dynpop; } enum lx_token diff --git a/src/lx/main.c b/src/lx/main.c index cbcfeea76..92098bfd9 100644 --- a/src/lx/main.c +++ b/src/lx/main.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -44,6 +45,8 @@ struct prefix prefix = { "" }; +/* TODO: use alloc hooks for -Q accounting */ +static struct fsm_alloc *alloc; struct fsm_options opt; int print_progress; @@ -128,28 +131,28 @@ io(const char *name) exit(EXIT_FAILURE); } -static lx_print * -print_name(const char *name) +static enum lx_print_lang +lang_name(const char *name) { size_t i; struct { const char *name; - lx_print *f; + enum lx_print_lang lang; } a[] = { - { "test", NULL }, - { "dot", lx_print_dot }, - { "dump", lx_print_dump }, - { "zdot", lx_print_zdot }, - { "c", lx_print_c }, - { "h", lx_print_h } + { "test", LX_PRINT_NONE }, + { "dot", LX_PRINT_DOT }, + { "dump", LX_PRINT_DUMP }, + { "zdot", LX_PRINT_ZDOT }, + { "c", LX_PRINT_C }, + { "h", LX_PRINT_H } }; assert(name != NULL); for (i = 0; i < sizeof a / sizeof *a; i++) { if (0 == strcmp(a[i].name, name)) { - return a[i].f; + return a[i].lang; } } @@ -306,15 +309,9 @@ zone_equal(const struct ast_zone *a, const struct ast_zone *b) return -1; } - { - /* opt.carryopaque = carryopaque; */ - - if (!fsm_determinise(q)) { - fsm_free(q); - return -1; - } - - /* opt.carryopaque = NULL; */ + if (!fsm_determinise(q)) { + fsm_free(q); + return -1; } { @@ -332,12 +329,6 @@ zone_equal(const struct ast_zone *a, const struct ast_zone *b) fprintf(stderr, "zone_equal: asserting ast_getendmapping(q, state %d) != NULL: %p\n", i, (void *)m); } assert(m != NULL); - - if (m->conflict != NULL) { - /* TODO: free conflict set */ - fsm_free(q); - return 0; - } } } @@ -373,7 +364,7 @@ zone_minimise(void *arg) } pthread_mutex_unlock(&zmtx); - z->fsm = fsm_new(&opt); + z->fsm = fsm_new(alloc); if (z->fsm == NULL) { pthread_mutex_lock(&zmtx); zerror = errno; @@ -530,19 +521,85 @@ run_threads(int concurrency, void *(*fn)(void *)) return 0; } +static int +conflict(FILE *f, const struct fsm_options *opt, + const struct fsm_state_metadata *state_metadata, const char *example, + void *hook_opaque) +{ + const struct ast *ast; + size_t i; + + assert(opt != NULL); + assert(opt->ambig == AMBIG_ERROR); + assert(state_metadata->end_id_count > 1); + + (void) f; + (void) opt; + (void) hook_opaque; + + ast = hook_opaque; + + /* TODO: // delimeters depend on dialect */ + /* TODO: would deal with dialect: prefix here, too */ + + fprintf(stderr, "ambiguous mappings to "); + + for (i = 0; i < state_metadata->end_id_count; i++) { + const struct ast_mapping *m; + + /* + * When the example is '', we have two patterns which match the empty string. + * Here we defer to the error about the start state accepting, + * and it seems redundant to also show an error about both patterns + * matching the same input, even if there's a non-empty part. + */ + if (example != NULL && strlen(example) == 0) { + continue; + } + + m = ast_getendmappingbyendid(state_metadata->end_ids[i]); + + if (m->token != NULL) { + fprintf(stderr, "$%s", m->token->s); + } else if (m->to == NULL) { + fprintf(stderr, "skip"); + } + if (m->token != NULL && m->to != NULL) { + fprintf(stderr, "/"); + } + if (m->to == ast->global) { + fprintf(stderr, "global zone"); + } else if (m->to != NULL) { + fprintf(stderr, "z%p", (void *) m->to); /* TODO: zindexof(n->to) */ + } + + if (i + 1 < state_metadata->end_id_count) { + fprintf(stderr, ", "); + } + } + + if (example != NULL) { + /* TODO: escape hex etc */ + fprintf(stderr, "; for example on input '%s'", example); + } + + fprintf(stderr, "\n"); + + return 0; +} + int main(int argc, char *argv[]) { struct ast *ast; - lx_print *print; + enum lx_print_lang lang; int concurrency; - print = lx_print_c; + lang = LX_PRINT_C; keep_nfa = 0; print_progress = 0; concurrency = 1; - /* TODO: populate options */ opt.anonymous_states = 1; opt.consolidate_edges = 1; opt.comments = 1; @@ -579,7 +636,7 @@ main(int argc, char *argv[]) case 'x': api_exclude |= lang_exclude(optarg); break; case 'l': - print = print_name(optarg); + lang = lang_name(optarg); break; case 'n': @@ -610,17 +667,17 @@ main(int argc, char *argv[]) } } - if (keep_nfa && print != lx_print_dot) { + if (keep_nfa && lang != LX_PRINT_DOT) { fprintf(stderr, "-n is for .dot output only\n"); return EXIT_FAILURE; } - if (api_tokbuf && (print != lx_print_c && print != lx_print_h && print != lx_print_dump)) { + if (api_tokbuf && (lang != LX_PRINT_C && lang != LX_PRINT_H && lang != LX_PRINT_DUMP)) { fprintf(stderr, "-b is for .c/.h output only\n"); return EXIT_FAILURE; } - if (api_getc && (print != lx_print_c && print != lx_print_h && print != lx_print_dump)) { + if (api_getc && (lang != LX_PRINT_C && lang != LX_PRINT_H && lang != LX_PRINT_DUMP)) { fprintf(stderr, "-g is for .c/.h output only\n"); return EXIT_FAILURE; } @@ -638,12 +695,16 @@ main(int argc, char *argv[]) opt.comments = 0; } + if (lang == LX_PRINT_C) { + opt.fragment = 1; + } + { if (print_progress) { fprintf(stderr, "-- parsing:"); } - ast = lx_parse(stdin, &opt); + ast = lx_parse(stdin, alloc); if (ast == NULL) { return EXIT_FAILURE; } @@ -668,7 +729,7 @@ main(int argc, char *argv[]) * of which end state is associated with each mapping. In other words, * without losing track of which regexp maps to which token. */ - if (print != lx_print_h) { + if (lang != LX_PRINT_H) { if (print_progress) { fprintf(stderr, "-- minimise AST:"); zn = 0; @@ -689,12 +750,10 @@ main(int argc, char *argv[]) zn = 0; } - /* opt.carryopaque = carryopaque; */ cur_zone = ast->zl; if (run_threads(concurrency, zone_determinise)) { return EXIT_FAILURE; } - /* opt.carryopaque = NULL; */ if (print_progress) { fprintf(stderr, "\n"); @@ -707,7 +766,7 @@ main(int argc, char *argv[]) * This converts the tree of zones to a DAG. * TODO: Fix */ - if (0 && print != lx_print_h) { + if (0 && lang != LX_PRINT_H) { struct ast_zone *z, **p; int changed; unsigned int zn, zd, zp; @@ -801,10 +860,9 @@ main(int argc, char *argv[]) /* * Semantic checks. */ - if (print != lx_print_h) { + if (lang != LX_PRINT_H) { struct ast_zone *z; unsigned int zn; - fsm_state_t i; int e; if (print_progress) { @@ -835,6 +893,18 @@ main(int argc, char *argv[]) zn++; } + switch (lang) { + case LX_PRINT_NONE: + case LX_PRINT_C: + case LX_PRINT_H: + opt.ambig = AMBIG_ERROR; + break; + + default: + opt.ambig = AMBIG_MULTIPLE; + continue; + } + (void) fsm_getstart(z->fsm, &start); if (fsm_isend(z->fsm, start)) { @@ -842,75 +912,33 @@ main(int argc, char *argv[]) e = 1; } - /* pick up conflicts flagged by carryopaque() */ - for (i = 0; i < z->fsm->statecount; i++) { - struct ast_mapping *m; - - if (!fsm_isend(z->fsm, i)) { - continue; - } - - m = ast_getendmapping(z->fsm, i); - if (LOG()) { - fprintf(stderr, "main: m <- ast_getendmapping(dst_fsm, i]: %d) = %p // pick up conflicts\n", i, (void *)m); - } - assert(m != NULL); - - if (m->conflict != NULL) { - struct mapping_set *p; - char buf[50]; /* 50 looks reasonable for an on-screen limit */ - int n; + { + static const struct fsm_hooks zero_hooks; + struct fsm_hooks hooks = zero_hooks; + int r; - n = fsm_example(z->fsm, i, buf, sizeof buf); - if (-1 == n) { - perror("fsm_example"); - return EXIT_FAILURE; - } - - /* - * When n == 0, we have two patterns which match the empty string. - * Here we defer to the error about the start state accepting, - * and it seems redundant to also show an error about both patterns - * matching the same input, even if there's a non-empty part. - */ - if (n > 0) { - fprintf(stderr, "ambiguous mappings to "); - - for (p = m->conflict; p != NULL; p = p->next) { - if (p->m->token != NULL) { - fprintf(stderr, "$%s", p->m->token->s); - } else if (p->m->to == NULL) { - fprintf(stderr, "skip"); - } - if (p->m->token != NULL && p->m->to != NULL) { - fprintf(stderr, "/"); - } - if (p->m->to == ast->global) { - fprintf(stderr, "global zone"); - } else if (p->m->to != NULL) { - fprintf(stderr, "z%p", (void *) p->m->to); /* TODO: zindexof(n->to) */ - } + hooks.conflict = conflict; + hooks.hook_opaque = ast; - if (p->next != NULL) { - fprintf(stderr, ", "); - } - } + r = fsm_print(stdout, z->fsm, &opt, &hooks, FSM_PRINT_NONE); - /* TODO: escape hex etc */ - fprintf(stderr, "; for example on input '%s%s'\n", buf, - n >= (int) sizeof buf - 1 ? "..." : ""); + if (r < 0) { + exit(EXIT_FAILURE); + } - e = 1; - } + if (r == 1) { + /* conflict */ + exit(EXIT_FAILURE); } } + } if (print_progress) { fprintf(stderr, "\n"); } - if (e && print != lx_print_dot) { + if (e && lang != LX_PRINT_DOT) { return EXIT_FAILURE; } } @@ -918,10 +946,10 @@ main(int argc, char *argv[]) /* XXX: can do this before semantic checks */ /* TODO: free ast */ /* TODO: free DFA ast_mappings, created in carryopaque, iff making a DFA. i.e. those which have non-NULL conflict sets */ - if (print == lx_print_h) { + if (lang == LX_PRINT_H) { /* TODO: special case to avoid overhead; free non-minimized NFA */ } - if (!keep_nfa && print != lx_print_h) { + if (!keep_nfa && lang != LX_PRINT_H) { struct ast_zone *z; struct ast_mapping *m; fsm_state_t i; @@ -938,9 +966,6 @@ main(int argc, char *argv[]) } if (m != NULL) { assert(m->fsm == NULL); - - /* TODO: free m->conflict, allocated in carryopaque */ - (void) m->conflict; } } } @@ -951,9 +976,7 @@ main(int argc, char *argv[]) fprintf(stderr, "-- output:"); } - if (print != NULL) { - print(stdout, ast); - } + lx_print(stdout, ast, &opt, lang); if (print_progress) { fprintf(stderr, "\n"); diff --git a/src/lx/parser.act b/src/lx/parser.act index 78254e123..f8f6b84a4 100644 --- a/src/lx/parser.act +++ b/src/lx/parser.act @@ -38,7 +38,6 @@ #include #include #include - #include #include @@ -68,7 +67,7 @@ enum lx_token lex_tok; enum lx_token lex_tok_save; unsigned int zn; - const struct fsm_options *opt; + const struct fsm_alloc *alloc; }; struct lex_state { @@ -135,7 +134,7 @@ typedef struct act_state * act_state; typedef struct ast * ast; - struct ast *lx_parse(FILE *f, const struct fsm_options *opt); + struct ast *lx_parse(FILE *f, const struct fsm_alloc *alloc); @}; @@ -223,18 +222,10 @@ TOKEN: () -> (s :string) = @{ /* TODO: submatch addressing */ @s = xstrdup(lex_state->buf.a + 1); /* +1 for '$' prefix */ - if (@s == NULL) { - perror("xstrdup"); - exit(EXIT_FAILURE); - } @}; IDENT: () -> (s :string) = @{ @s = xstrdup(lex_state->buf.a); - if (@s == NULL) { - perror("xstrdup"); - exit(EXIT_FAILURE); - } @}; RE: () -> (f :flags) = @{ @@ -317,7 +308,7 @@ assert(@t != NULL); assert(@t != NULL); - @r = fsm_new(act_state->opt); + @r = fsm_new(act_state->alloc); if (@r == NULL) { perror("fsm_new"); @!; @@ -373,7 +364,7 @@ assert(@a != NULL); assert(@a->p != NULL); - @r = re_comp(RE_LITERAL, act_agetc, @a, act_state->opt, 0, &err); + @r = re_comp(RE_LITERAL, act_agetc, @a, act_state->alloc, 0, &err); if (@r == NULL) { assert(err.e != RE_EBADDIALECT); /* TODO: pass filename for .lx source */ @@ -388,7 +379,7 @@ assert(@a != NULL); assert(@a->p != NULL); - @r = re_comp(RE_NATIVE, act_agetc, @a, act_state->opt, @f, &err); + @r = re_comp(RE_NATIVE, act_agetc, @a, act_state->alloc, @f | RE_NOCAPTURE, &err); if (@r == NULL) { assert(err.e != RE_EBADDIALECT); /* TODO: pass filename for .lx source */ @@ -401,7 +392,7 @@ : () -> (r :fsm) = @{ fsm_state_t start, end; - @r = fsm_new(act_state->opt); + @r = fsm_new(act_state->alloc); if (@r == NULL) { perror("fsm_new"); @!; @@ -826,7 +817,7 @@ %trailer% @{ - struct ast *lx_parse(FILE *f, const struct fsm_options *opt) { + struct ast *lx_parse(FILE *f, const struct fsm_alloc *alloc) { struct act_state act_state_s; struct act_state *act_state; struct lex_state lex_state_s; @@ -858,7 +849,7 @@ /* This is a workaround for ADVANCE_LEXER assuming a pointer */ act_state = &act_state_s; - act_state->opt = opt; + act_state->alloc = alloc; if (print_progress) { act_state->zn = 0; @@ -869,7 +860,19 @@ lx->free(lx->buf_opaque); - assert(ast != NULL); + /* + * This can happen when the first token is TOK_UNKNOWN. + * SID's generated parser bails out immediately. + * So we never reach , and never about it. + * + * Really I wanted this handled along with the usual syntax error + * case, from ## alts inside parser.sid, instead of reproducing + * the same error here. + */ + if (ast == NULL) { + err(lex_state, "Syntax error"); + exit(EXIT_FAILURE); + } return ast; } diff --git a/src/lx/parser.c b/src/lx/parser.c index cbc2067fd..5af0592d5 100644 --- a/src/lx/parser.c +++ b/src/lx/parser.c @@ -24,7 +24,6 @@ #include #include #include - #include #include @@ -54,7 +53,7 @@ enum lx_token lex_tok; enum lx_token lex_tok_save; unsigned int zn; - const struct fsm_options *opt; + const struct fsm_alloc *alloc; }; struct lex_state { @@ -111,7 +110,7 @@ return a->len--, (unsigned char) *a->p++; } -#line 115 "src/lx/parser.c" +#line 114 "src/lx/parser.c" #ifndef ERROR_TERMINAL @@ -183,11 +182,11 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-list */ { -#line 823 "src/lx/parser.act" +#line 814 "src/lx/parser.act" err_expected(lex_state, "list of mappings, bindings or zones"); -#line 191 "src/lx/parser.c" +#line 190 "src/lx/parser.c" } /* END OF ACTION: err-expected-list */ } @@ -205,21 +204,17 @@ p_pattern(lex_state lex_state, act_state act_state, zone ZIz, fsm *ZOr) /* BEGINNING OF EXTRACT: IDENT */ { -#line 232 "src/lx/parser.act" +#line 227 "src/lx/parser.act" ZIn = xstrdup(lex_state->buf.a); - if (ZIn == NULL) { - perror("xstrdup"); - exit(EXIT_FAILURE); - } -#line 217 "src/lx/parser.c" +#line 212 "src/lx/parser.c" } /* END OF EXTRACT: IDENT */ ADVANCE_LEXER; /* BEGINNING OF ACTION: deref-var */ { -#line 286 "src/lx/parser.act" +#line 277 "src/lx/parser.act" struct ast_zone *z; @@ -246,7 +241,7 @@ p_pattern(lex_state lex_state, act_state act_state, zone ZIz, fsm *ZOr) goto ZL1; } -#line 250 "src/lx/parser.c" +#line 245 "src/lx/parser.c" } /* END OF ACTION: deref-var */ } @@ -257,22 +252,18 @@ p_pattern(lex_state lex_state, act_state act_state, zone ZIz, fsm *ZOr) /* BEGINNING OF EXTRACT: TOKEN */ { -#line 223 "src/lx/parser.act" +#line 222 "src/lx/parser.act" /* TODO: submatch addressing */ ZIt = xstrdup(lex_state->buf.a + 1); /* +1 for '$' prefix */ - if (ZIt == NULL) { - perror("xstrdup"); - exit(EXIT_FAILURE); - } -#line 270 "src/lx/parser.c" +#line 261 "src/lx/parser.c" } /* END OF EXTRACT: TOKEN */ ADVANCE_LEXER; /* BEGINNING OF ACTION: deref-token */ { -#line 313 "src/lx/parser.act" +#line 304 "src/lx/parser.act" const struct ast_mapping *m; fsm_state_t start; @@ -280,7 +271,7 @@ p_pattern(lex_state lex_state, act_state act_state, zone ZIz, fsm *ZOr) assert((ZIt) != NULL); assert((ZIt) != NULL); - (ZIr) = fsm_new(act_state->opt); + (ZIr) = fsm_new(act_state->alloc); if ((ZIr) == NULL) { perror("fsm_new"); goto ZL1; @@ -329,7 +320,7 @@ p_pattern(lex_state lex_state, act_state act_state, zone ZIz, fsm *ZOr) fsm_setstart((ZIr), start); -#line 333 "src/lx/parser.c" +#line 324 "src/lx/parser.c" } /* END OF ACTION: deref-token */ } @@ -383,7 +374,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } /* BEGINNING OF ACTION: op-reverse */ { -#line 685 "src/lx/parser.act" +#line 676 "src/lx/parser.act" assert((ZI210) != NULL); @@ -392,7 +383,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z goto ZL1; } -#line 396 "src/lx/parser.c" +#line 387 "src/lx/parser.c" } /* END OF ACTION: op-reverse */ p_212 (lex_state, act_state, &ZIz, &ZI210, &ZI206); @@ -407,7 +398,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } /* BEGINNING OF ACTION: subtract-exit */ { -#line 744 "src/lx/parser.act" +#line 735 "src/lx/parser.act" assert((ZI231) != NULL); @@ -427,7 +418,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } } -#line 431 "src/lx/parser.c" +#line 422 "src/lx/parser.c" } /* END OF ACTION: subtract-exit */ p_251 (lex_state, act_state, &ZIa, &ZIz, &ZIexit, &ZI232, &ZI233); @@ -458,7 +449,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } /* BEGINNING OF ACTION: op-complete */ { -#line 676 "src/lx/parser.act" +#line 667 "src/lx/parser.act" assert((ZI210) != NULL); @@ -467,7 +458,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z goto ZL1; } -#line 471 "src/lx/parser.c" +#line 462 "src/lx/parser.c" } /* END OF ACTION: op-complete */ p_212 (lex_state, act_state, &ZIz, &ZI210, &ZI206); @@ -482,7 +473,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } /* BEGINNING OF ACTION: subtract-exit */ { -#line 744 "src/lx/parser.act" +#line 735 "src/lx/parser.act" assert((ZI239) != NULL); @@ -502,7 +493,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } } -#line 506 "src/lx/parser.c" +#line 497 "src/lx/parser.c" } /* END OF ACTION: subtract-exit */ p_251 (lex_state, act_state, &ZIa, &ZIz, &ZIexit, &ZI240, &ZI241); @@ -518,15 +509,11 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z /* BEGINNING OF EXTRACT: IDENT */ { -#line 232 "src/lx/parser.act" +#line 227 "src/lx/parser.act" ZIn = xstrdup(lex_state->buf.a); - if (ZIn == NULL) { - perror("xstrdup"); - exit(EXIT_FAILURE); - } -#line 530 "src/lx/parser.c" +#line 517 "src/lx/parser.c" } /* END OF EXTRACT: IDENT */ ADVANCE_LEXER; @@ -576,7 +563,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } /* BEGINNING OF ACTION: subtract-exit */ { -#line 744 "src/lx/parser.act" +#line 735 "src/lx/parser.act" assert((ZI248) != NULL); @@ -596,7 +583,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } } -#line 600 "src/lx/parser.c" +#line 587 "src/lx/parser.c" } /* END OF ACTION: subtract-exit */ p_251 (lex_state, act_state, &ZIa, &ZIz, &ZIexit, &ZI249, &ZI250); @@ -627,7 +614,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } /* BEGINNING OF ACTION: op-complement */ { -#line 667 "src/lx/parser.act" +#line 658 "src/lx/parser.act" assert((ZI210) != NULL); @@ -636,7 +623,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z goto ZL1; } -#line 640 "src/lx/parser.c" +#line 627 "src/lx/parser.c" } /* END OF ACTION: op-complement */ p_212 (lex_state, act_state, &ZIz, &ZI210, &ZI206); @@ -651,7 +638,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } /* BEGINNING OF ACTION: subtract-exit */ { -#line 744 "src/lx/parser.act" +#line 735 "src/lx/parser.act" assert((ZI223) != NULL); @@ -671,7 +658,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } } -#line 675 "src/lx/parser.c" +#line 662 "src/lx/parser.c" } /* END OF ACTION: subtract-exit */ p_251 (lex_state, act_state, &ZIa, &ZIz, &ZIexit, &ZI224, &ZI225); @@ -698,22 +685,18 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z /* BEGINNING OF EXTRACT: TOKEN */ { -#line 223 "src/lx/parser.act" +#line 222 "src/lx/parser.act" /* TODO: submatch addressing */ ZI253 = xstrdup(lex_state->buf.a + 1); /* +1 for '$' prefix */ - if (ZI253 == NULL) { - perror("xstrdup"); - exit(EXIT_FAILURE); - } -#line 711 "src/lx/parser.c" +#line 694 "src/lx/parser.c" } /* END OF EXTRACT: TOKEN */ ADVANCE_LEXER; /* BEGINNING OF ACTION: deref-token */ { -#line 313 "src/lx/parser.act" +#line 304 "src/lx/parser.act" const struct ast_mapping *m; fsm_state_t start; @@ -721,7 +704,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z assert((ZI253) != NULL); assert((ZI253) != NULL); - (ZI254) = fsm_new(act_state->opt); + (ZI254) = fsm_new(act_state->alloc); if ((ZI254) == NULL) { perror("fsm_new"); goto ZL1; @@ -770,7 +753,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z fsm_setstart((ZI254), start); -#line 774 "src/lx/parser.c" +#line 757 "src/lx/parser.c" } /* END OF ACTION: deref-token */ p_197 (lex_state, act_state, ZI254, &ZI210); @@ -786,7 +769,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } /* BEGINNING OF ACTION: subtract-exit */ { -#line 744 "src/lx/parser.act" +#line 735 "src/lx/parser.act" assert((ZI259) != NULL); @@ -806,7 +789,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } } -#line 810 "src/lx/parser.c" +#line 793 "src/lx/parser.c" } /* END OF ACTION: subtract-exit */ p_251 (lex_state, act_state, &ZIa, &ZIz, &ZIexit, &ZI260, &ZI261); @@ -846,7 +829,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } /* BEGINNING OF ACTION: subtract-exit */ { -#line 744 "src/lx/parser.act" +#line 735 "src/lx/parser.act" assert((ZI268) != NULL); @@ -866,7 +849,7 @@ p_list_Hof_Hthings_C_Cthing(lex_state lex_state, act_state act_state, ast ZIa, z } } -#line 870 "src/lx/parser.c" +#line 853 "src/lx/parser.c" } /* END OF ACTION: subtract-exit */ p_251 (lex_state, act_state, &ZIa, &ZIz, &ZIexit, &ZI269, &ZI270); @@ -886,11 +869,11 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-thing */ { -#line 819 "src/lx/parser.act" +#line 810 "src/lx/parser.act" err_expected(lex_state, "mapping, binding or zone"); -#line 894 "src/lx/parser.c" +#line 877 "src/lx/parser.c" } /* END OF ACTION: err-expected-thing */ } @@ -916,11 +899,11 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-open */ { -#line 807 "src/lx/parser.act" +#line 798 "src/lx/parser.act" err_expected(lex_state, "'{'"); -#line 924 "src/lx/parser.c" +#line 907 "src/lx/parser.c" } /* END OF ACTION: err-expected-open */ } @@ -946,11 +929,11 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-close */ { -#line 811 "src/lx/parser.act" +#line 802 "src/lx/parser.act" err_expected(lex_state, "'}'"); -#line 954 "src/lx/parser.c" +#line 937 "src/lx/parser.c" } /* END OF ACTION: err-expected-close */ } @@ -972,14 +955,14 @@ ZL2_pattern_C_Cbody:; { /* BEGINNING OF EXTRACT: CHAR */ { -#line 216 "src/lx/parser.act" +#line 215 "src/lx/parser.act" assert(lex_state->buf.a[0] != '\0'); assert(lex_state->buf.a[1] == '\0'); ZIc = lex_state->buf.a[0]; -#line 983 "src/lx/parser.c" +#line 966 "src/lx/parser.c" } /* END OF EXTRACT: CHAR */ ADVANCE_LEXER; @@ -989,7 +972,7 @@ ZL2_pattern_C_Cbody:; { /* BEGINNING OF EXTRACT: ESC */ { -#line 144 "src/lx/parser.act" +#line 143 "src/lx/parser.act" assert(lex_state->buf.a[0] == '\\'); assert(lex_state->buf.a[1] != '\0'); @@ -1007,7 +990,7 @@ ZL2_pattern_C_Cbody:; default: ZIc = '\0'; break; /* TODO: handle error */ } -#line 1011 "src/lx/parser.c" +#line 994 "src/lx/parser.c" } /* END OF EXTRACT: ESC */ ADVANCE_LEXER; @@ -1017,7 +1000,7 @@ ZL2_pattern_C_Cbody:; { /* BEGINNING OF EXTRACT: HEX */ { -#line 189 "src/lx/parser.act" +#line 188 "src/lx/parser.act" unsigned long u; char *e; @@ -1044,7 +1027,7 @@ ZL2_pattern_C_Cbody:; ZIc = (char) (unsigned char) u; -#line 1048 "src/lx/parser.c" +#line 1031 "src/lx/parser.c" } /* END OF EXTRACT: HEX */ ADVANCE_LEXER; @@ -1054,7 +1037,7 @@ ZL2_pattern_C_Cbody:; { /* BEGINNING OF EXTRACT: OCT */ { -#line 162 "src/lx/parser.act" +#line 161 "src/lx/parser.act" unsigned long u; char *e; @@ -1081,7 +1064,7 @@ ZL2_pattern_C_Cbody:; ZIc = (char) (unsigned char) u; -#line 1085 "src/lx/parser.c" +#line 1068 "src/lx/parser.c" } /* END OF EXTRACT: OCT */ ADVANCE_LEXER; @@ -1094,12 +1077,12 @@ ZL2_pattern_C_Cbody:; /* END OF INLINE: 84 */ /* BEGINNING OF ACTION: pattern-char */ { -#line 256 "src/lx/parser.act" +#line 247 "src/lx/parser.act" /* TODO */ *lex_state->p++ = (ZIc); -#line 1103 "src/lx/parser.c" +#line 1086 "src/lx/parser.c" } /* END OF ACTION: pattern-char */ /* BEGINNING OF INLINE: pattern::body */ @@ -1228,7 +1211,7 @@ ZL2_180:; } /* BEGINNING OF ACTION: op-alt */ { -#line 733 "src/lx/parser.act" +#line 724 "src/lx/parser.act" assert((ZI177) != NULL); assert((ZIb) != NULL); @@ -1239,7 +1222,7 @@ ZL2_180:; goto ZL1; } -#line 1243 "src/lx/parser.c" +#line 1226 "src/lx/parser.c" } /* END OF ACTION: op-alt */ /* BEGINNING OF INLINE: 180 */ @@ -1308,7 +1291,7 @@ p_expr_C_Cprefix_Hexpr(lex_state lex_state, act_state act_state, zone ZIz, fsm * } /* BEGINNING OF ACTION: op-reverse */ { -#line 685 "src/lx/parser.act" +#line 676 "src/lx/parser.act" assert((ZIq) != NULL); @@ -1317,7 +1300,7 @@ p_expr_C_Cprefix_Hexpr(lex_state lex_state, act_state act_state, zone ZIz, fsm * goto ZL1; } -#line 1321 "src/lx/parser.c" +#line 1304 "src/lx/parser.c" } /* END OF ACTION: op-reverse */ } @@ -1332,7 +1315,7 @@ p_expr_C_Cprefix_Hexpr(lex_state lex_state, act_state act_state, zone ZIz, fsm * } /* BEGINNING OF ACTION: op-complete */ { -#line 676 "src/lx/parser.act" +#line 667 "src/lx/parser.act" assert((ZIq) != NULL); @@ -1341,7 +1324,7 @@ p_expr_C_Cprefix_Hexpr(lex_state lex_state, act_state act_state, zone ZIz, fsm * goto ZL1; } -#line 1345 "src/lx/parser.c" +#line 1328 "src/lx/parser.c" } /* END OF ACTION: op-complete */ } @@ -1356,7 +1339,7 @@ p_expr_C_Cprefix_Hexpr(lex_state lex_state, act_state act_state, zone ZIz, fsm * } /* BEGINNING OF ACTION: op-complement */ { -#line 667 "src/lx/parser.act" +#line 658 "src/lx/parser.act" assert((ZIq) != NULL); @@ -1365,7 +1348,7 @@ p_expr_C_Cprefix_Hexpr(lex_state lex_state, act_state act_state, zone ZIz, fsm * goto ZL1; } -#line 1369 "src/lx/parser.c" +#line 1352 "src/lx/parser.c" } /* END OF ACTION: op-complement */ } @@ -1415,7 +1398,7 @@ ZL2_186:; } /* BEGINNING OF ACTION: op-intersect */ { -#line 722 "src/lx/parser.act" +#line 713 "src/lx/parser.act" assert((ZI183) != NULL); assert((ZIb) != NULL); @@ -1426,7 +1409,7 @@ ZL2_186:; goto ZL1; } -#line 1430 "src/lx/parser.c" +#line 1413 "src/lx/parser.c" } /* END OF ACTION: op-intersect */ /* BEGINNING OF INLINE: 186 */ @@ -1472,7 +1455,7 @@ ZL2_list_Hof_Hthings_C_Czone_Hthing_C_Clist_Hof_Hzone_Hto_Hmappings_C_Clist_Hof_ } /* BEGINNING OF ACTION: op-alt */ { -#line 733 "src/lx/parser.act" +#line 724 "src/lx/parser.act" assert((ZIold_Hexit) != NULL); assert((ZInew_Hexit) != NULL); @@ -1483,7 +1466,7 @@ ZL2_list_Hof_Hthings_C_Czone_Hthing_C_Clist_Hof_Hzone_Hto_Hmappings_C_Clist_Hof_ goto ZL1; } -#line 1487 "src/lx/parser.c" +#line 1470 "src/lx/parser.c" } /* END OF ACTION: op-alt */ /* BEGINNING OF INLINE: list-of-things::zone-thing::list-of-zone-to-mappings::list-of-zone-to-mappings-x */ @@ -1571,7 +1554,7 @@ ZL2_197:; ADVANCE_LEXER; /* BEGINNING OF ACTION: op-cross */ { -#line 577 "src/lx/parser.act" +#line 568 "src/lx/parser.act" fsm_state_t start, end; fsm_state_t old; @@ -1616,7 +1599,7 @@ ZL2_197:; fsm_setstart((ZIq), start); -#line 1620 "src/lx/parser.c" +#line 1603 "src/lx/parser.c" } /* END OF ACTION: op-cross */ /* BEGINNING OF INLINE: 197 */ @@ -1630,7 +1613,7 @@ ZL2_197:; ADVANCE_LEXER; /* BEGINNING OF ACTION: op-qmark */ { -#line 622 "src/lx/parser.act" +#line 613 "src/lx/parser.act" fsm_state_t start, end; fsm_state_t old; @@ -1675,7 +1658,7 @@ ZL2_197:; fsm_setstart((ZIq), start); -#line 1679 "src/lx/parser.c" +#line 1662 "src/lx/parser.c" } /* END OF ACTION: op-qmark */ /* BEGINNING OF INLINE: 197 */ @@ -1689,7 +1672,7 @@ ZL2_197:; ADVANCE_LEXER; /* BEGINNING OF ACTION: op-star */ { -#line 527 "src/lx/parser.act" +#line 518 "src/lx/parser.act" fsm_state_t start, end; fsm_state_t old; @@ -1739,7 +1722,7 @@ ZL2_197:; fsm_setstart((ZIq), start); -#line 1743 "src/lx/parser.c" +#line 1726 "src/lx/parser.c" } /* END OF ACTION: op-star */ /* BEGINNING OF INLINE: 197 */ @@ -1812,7 +1795,7 @@ p_204(lex_state lex_state, act_state act_state, zone *ZIz, fsm *ZI202, fsm *ZOq) } /* BEGINNING OF ACTION: op-subtract */ { -#line 711 "src/lx/parser.act" +#line 702 "src/lx/parser.act" assert((*ZI202) != NULL); assert((ZIb) != NULL); @@ -1823,7 +1806,7 @@ p_204(lex_state lex_state, act_state act_state, zone *ZIz, fsm *ZI202, fsm *ZOq) goto ZL1; } -#line 1827 "src/lx/parser.c" +#line 1810 "src/lx/parser.c" } /* END OF ACTION: op-subtract */ } @@ -1887,7 +1870,7 @@ p_208(lex_state lex_state, act_state act_state, zone *ZIz, fsm *ZI206, fsm *ZOq) } /* BEGINNING OF ACTION: op-concat */ { -#line 694 "src/lx/parser.act" +#line 685 "src/lx/parser.act" assert((*ZI206) != NULL); assert((ZIb) != NULL); @@ -1898,7 +1881,7 @@ p_208(lex_state lex_state, act_state act_state, zone *ZIz, fsm *ZI206, fsm *ZOq) goto ZL1; } -#line 1902 "src/lx/parser.c" +#line 1885 "src/lx/parser.c" } /* END OF ACTION: op-concat */ } @@ -1937,13 +1920,13 @@ p_212(lex_state lex_state, act_state act_state, zone *ZIz, fsm *ZI210, fsm *ZOq) } /* BEGINNING OF ACTION: op-product */ { -#line 705 "src/lx/parser.act" +#line 696 "src/lx/parser.act" fprintf(stderr, "unimplemented\n"); (ZIq) = NULL; goto ZL1; -#line 1947 "src/lx/parser.c" +#line 1930 "src/lx/parser.c" } /* END OF ACTION: op-product */ } @@ -1977,7 +1960,7 @@ p_215(lex_state lex_state, act_state act_state, fsm *ZOr) /* BEGINNING OF EXTRACT: RE */ { -#line 240 "src/lx/parser.act" +#line 231 "src/lx/parser.act" assert(lex_state->buf.a[0] == '/'); @@ -1991,13 +1974,13 @@ p_215(lex_state lex_state, act_state act_state, fsm *ZOr) ZIf |= RE_ANCHORED; /* regexps in lx are implicitly anchored */ ZIf |= RE_SINGLE; /* we can't assume we're lexing text */ -#line 1995 "src/lx/parser.c" +#line 1978 "src/lx/parser.c" } /* END OF EXTRACT: RE */ ADVANCE_LEXER; /* BEGINNING OF ACTION: pattern-buffer */ { -#line 261 "src/lx/parser.act" +#line 252 "src/lx/parser.act" size_t len; @@ -2022,19 +2005,19 @@ p_215(lex_state lex_state, act_state act_state, fsm *ZOr) lex_state->p = lex_state->a; -#line 2026 "src/lx/parser.c" +#line 2009 "src/lx/parser.c" } /* END OF ACTION: pattern-buffer */ /* BEGINNING OF ACTION: compile-regex */ { -#line 385 "src/lx/parser.act" +#line 376 "src/lx/parser.act" struct re_err err; assert((ZIa) != NULL); assert((ZIa)->p != NULL); - (ZIr) = re_comp(RE_NATIVE, act_agetc, (ZIa), act_state->opt, (ZIf), &err); + (ZIr) = re_comp(RE_NATIVE, act_agetc, (ZIa), act_state->alloc, (ZIf) | RE_NOCAPTURE, &err); if ((ZIr) == NULL) { assert(err.e != RE_EBADDIALECT); /* TODO: pass filename for .lx source */ @@ -2042,16 +2025,16 @@ p_215(lex_state lex_state, act_state act_state, fsm *ZOr) exit(EXIT_FAILURE); } -#line 2046 "src/lx/parser.c" +#line 2029 "src/lx/parser.c" } /* END OF ACTION: compile-regex */ /* BEGINNING OF ACTION: free-arr */ { -#line 774 "src/lx/parser.act" +#line 765 "src/lx/parser.act" free((ZIa)); -#line 2055 "src/lx/parser.c" +#line 2038 "src/lx/parser.c" } /* END OF ACTION: free-arr */ } @@ -2063,7 +2046,7 @@ p_215(lex_state lex_state, act_state act_state, fsm *ZOr) ADVANCE_LEXER; /* BEGINNING OF ACTION: pattern-buffer */ { -#line 261 "src/lx/parser.act" +#line 252 "src/lx/parser.act" size_t len; @@ -2088,19 +2071,19 @@ p_215(lex_state lex_state, act_state act_state, fsm *ZOr) lex_state->p = lex_state->a; -#line 2092 "src/lx/parser.c" +#line 2075 "src/lx/parser.c" } /* END OF ACTION: pattern-buffer */ /* BEGINNING OF ACTION: compile-literal */ { -#line 370 "src/lx/parser.act" +#line 361 "src/lx/parser.act" struct re_err err; assert((ZIa) != NULL); assert((ZIa)->p != NULL); - (ZIr) = re_comp(RE_LITERAL, act_agetc, (ZIa), act_state->opt, 0, &err); + (ZIr) = re_comp(RE_LITERAL, act_agetc, (ZIa), act_state->alloc, 0, &err); if ((ZIr) == NULL) { assert(err.e != RE_EBADDIALECT); /* TODO: pass filename for .lx source */ @@ -2108,16 +2091,16 @@ p_215(lex_state lex_state, act_state act_state, fsm *ZOr) exit(EXIT_FAILURE); } -#line 2112 "src/lx/parser.c" +#line 2095 "src/lx/parser.c" } /* END OF ACTION: compile-literal */ /* BEGINNING OF ACTION: free-arr */ { -#line 774 "src/lx/parser.act" +#line 765 "src/lx/parser.act" free((ZIa)); -#line 2121 "src/lx/parser.c" +#line 2104 "src/lx/parser.c" } /* END OF ACTION: free-arr */ } @@ -2181,7 +2164,7 @@ ZL2_list_Hof_Hthings_C_Czone_Hthing_C_Clist_Hof_Hzone_Hfrom_Hmappings_C_Clist_Ho } /* BEGINNING OF ACTION: subtract-exit */ { -#line 744 "src/lx/parser.act" +#line 735 "src/lx/parser.act" assert((ZIr) != NULL); @@ -2201,12 +2184,12 @@ ZL2_list_Hof_Hthings_C_Czone_Hthing_C_Clist_Hof_Hzone_Hfrom_Hmappings_C_Clist_Ho } } -#line 2205 "src/lx/parser.c" +#line 2188 "src/lx/parser.c" } /* END OF ACTION: subtract-exit */ /* BEGINNING OF ACTION: add-mapping */ { -#line 458 "src/lx/parser.act" +#line 449 "src/lx/parser.act" struct ast_token *t; struct ast_mapping *m; @@ -2239,7 +2222,7 @@ ZL2_list_Hof_Hthings_C_Czone_Hthing_C_Clist_Hof_Hzone_Hfrom_Hmappings_C_Clist_Ho nmappings++; } -#line 2243 "src/lx/parser.c" +#line 2226 "src/lx/parser.c" } /* END OF ACTION: add-mapping */ /* BEGINNING OF INLINE: list-of-things::zone-thing::list-of-zone-from-mappings::list-of-zone-from-mappings-x */ @@ -2273,16 +2256,16 @@ p_lx(lex_state lex_state, act_state act_state, ast *ZOa) /* BEGINNING OF ACTION: no-zone */ { -#line 523 "src/lx/parser.act" +#line 514 "src/lx/parser.act" (ZIparent) = NULL; -#line 2281 "src/lx/parser.c" +#line 2264 "src/lx/parser.c" } /* END OF ACTION: no-zone */ /* BEGINNING OF ACTION: make-ast */ { -#line 432 "src/lx/parser.act" +#line 423 "src/lx/parser.act" (ZIa) = ast_new(); if ((ZIa) == NULL) { @@ -2290,12 +2273,12 @@ p_lx(lex_state lex_state, act_state act_state, ast *ZOa) goto ZL1; } -#line 2294 "src/lx/parser.c" +#line 2277 "src/lx/parser.c" } /* END OF ACTION: make-ast */ /* BEGINNING OF ACTION: make-zone */ { -#line 440 "src/lx/parser.act" +#line 431 "src/lx/parser.act" assert((ZIa) != NULL); @@ -2313,28 +2296,28 @@ p_lx(lex_state lex_state, act_state act_state, ast *ZOa) goto ZL1; } -#line 2317 "src/lx/parser.c" +#line 2300 "src/lx/parser.c" } /* END OF ACTION: make-zone */ /* BEGINNING OF ACTION: no-exit */ { -#line 519 "src/lx/parser.act" +#line 510 "src/lx/parser.act" (ZIexit) = NULL; -#line 2326 "src/lx/parser.c" +#line 2309 "src/lx/parser.c" } /* END OF ACTION: no-exit */ /* BEGINNING OF ACTION: set-globalzone */ { -#line 508 "src/lx/parser.act" +#line 499 "src/lx/parser.act" assert((ZIa) != NULL); assert((ZIz) != NULL); (ZIa)->global = (ZIz); -#line 2338 "src/lx/parser.c" +#line 2321 "src/lx/parser.c" } /* END OF ACTION: set-globalzone */ p_list_Hof_Hthings (lex_state, act_state, ZIa, ZIz, ZIexit); @@ -2358,11 +2341,11 @@ p_lx(lex_state lex_state, act_state act_state, ast *ZOa) { /* BEGINNING OF ACTION: err-expected-eof */ { -#line 815 "src/lx/parser.act" +#line 806 "src/lx/parser.act" err_expected(lex_state, "EOF"); -#line 2366 "src/lx/parser.c" +#line 2349 "src/lx/parser.c" } /* END OF ACTION: err-expected-eof */ } @@ -2375,7 +2358,7 @@ ZL1:; { /* BEGINNING OF ACTION: make-ast */ { -#line 432 "src/lx/parser.act" +#line 423 "src/lx/parser.act" (ZIa) = ast_new(); if ((ZIa) == NULL) { @@ -2383,17 +2366,17 @@ ZL1:; goto ZL4; } -#line 2387 "src/lx/parser.c" +#line 2370 "src/lx/parser.c" } /* END OF ACTION: make-ast */ /* BEGINNING OF ACTION: err-syntax */ { -#line 782 "src/lx/parser.act" +#line 773 "src/lx/parser.act" err(lex_state, "Syntax error"); exit(EXIT_FAILURE); -#line 2397 "src/lx/parser.c" +#line 2380 "src/lx/parser.c" } /* END OF ACTION: err-syntax */ } @@ -2450,7 +2433,7 @@ p_list_Hof_Hthings_C_Czone_Hthing_C_Czone_Hto_Hmapping(lex_state lex_state, act_ } /* BEGINNING OF ACTION: add-mapping */ { -#line 458 "src/lx/parser.act" +#line 449 "src/lx/parser.act" struct ast_token *t; struct ast_mapping *m; @@ -2483,12 +2466,12 @@ p_list_Hof_Hthings_C_Czone_Hthing_C_Czone_Hto_Hmapping(lex_state lex_state, act_ nmappings++; } -#line 2487 "src/lx/parser.c" +#line 2470 "src/lx/parser.c" } /* END OF ACTION: add-mapping */ /* BEGINNING OF ACTION: clone */ { -#line 764 "src/lx/parser.act" +#line 755 "src/lx/parser.act" assert((ZIr) != NULL); @@ -2498,7 +2481,7 @@ p_list_Hof_Hthings_C_Czone_Hthing_C_Czone_Hto_Hmapping(lex_state lex_state, act_ goto ZL1; } -#line 2502 "src/lx/parser.c" +#line 2485 "src/lx/parser.c" } /* END OF ACTION: clone */ } @@ -2534,11 +2517,11 @@ p_112(lex_state lex_state, act_state act_state, string *ZOt) { /* BEGINNING OF ACTION: err-expected-map */ { -#line 791 "src/lx/parser.act" +#line 782 "src/lx/parser.act" err_expected(lex_state, "'->'"); -#line 2542 "src/lx/parser.c" +#line 2525 "src/lx/parser.c" } /* END OF ACTION: err-expected-map */ } @@ -2549,16 +2532,12 @@ p_112(lex_state lex_state, act_state act_state, string *ZOt) case (TOK_TOKEN): /* BEGINNING OF EXTRACT: TOKEN */ { -#line 223 "src/lx/parser.act" +#line 222 "src/lx/parser.act" /* TODO: submatch addressing */ ZIt = xstrdup(lex_state->buf.a + 1); /* +1 for '$' prefix */ - if (ZIt == NULL) { - perror("xstrdup"); - exit(EXIT_FAILURE); - } -#line 2562 "src/lx/parser.c" +#line 2541 "src/lx/parser.c" } /* END OF EXTRACT: TOKEN */ break; @@ -2572,11 +2551,11 @@ p_112(lex_state lex_state, act_state act_state, string *ZOt) { /* BEGINNING OF ACTION: no-token */ { -#line 515 "src/lx/parser.act" +#line 506 "src/lx/parser.act" (ZIt) = NULL; -#line 2580 "src/lx/parser.c" +#line 2559 "src/lx/parser.c" } /* END OF ACTION: no-token */ } @@ -2665,11 +2644,11 @@ ZL1:; { /* BEGINNING OF ACTION: err-expected-semi */ { -#line 799 "src/lx/parser.act" +#line 790 "src/lx/parser.act" err_expected(lex_state, "';'"); -#line 2673 "src/lx/parser.c" +#line 2652 "src/lx/parser.c" } /* END OF ACTION: err-expected-semi */ } @@ -2716,16 +2695,16 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit } /* BEGINNING OF ACTION: no-zone */ { -#line 523 "src/lx/parser.act" +#line 514 "src/lx/parser.act" (ZIto) = NULL; -#line 2724 "src/lx/parser.c" +#line 2703 "src/lx/parser.c" } /* END OF ACTION: no-zone */ /* BEGINNING OF ACTION: add-mapping */ { -#line 458 "src/lx/parser.act" +#line 449 "src/lx/parser.act" struct ast_token *t; struct ast_mapping *m; @@ -2758,7 +2737,7 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit nmappings++; } -#line 2762 "src/lx/parser.c" +#line 2741 "src/lx/parser.c" } /* END OF ACTION: add-mapping */ } @@ -2770,7 +2749,7 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit /* BEGINNING OF ACTION: make-zone */ { -#line 440 "src/lx/parser.act" +#line 431 "src/lx/parser.act" assert((*ZIa) != NULL); @@ -2788,12 +2767,12 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit goto ZL1; } -#line 2792 "src/lx/parser.c" +#line 2771 "src/lx/parser.c" } /* END OF ACTION: make-zone */ /* BEGINNING OF ACTION: add-mapping */ { -#line 458 "src/lx/parser.act" +#line 449 "src/lx/parser.act" struct ast_token *t; struct ast_mapping *m; @@ -2826,7 +2805,7 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit nmappings++; } -#line 2830 "src/lx/parser.c" +#line 2809 "src/lx/parser.c" } /* END OF ACTION: add-mapping */ p_list_Hof_Hthings_C_Czone_Hthing_C_Clist_Hof_Hzone_Hfrom_Hmappings_C_Clist_Hof_Hzone_Hfrom_Hmappings_Hx (lex_state, act_state, *ZIa, *ZIz, ZIchild, *ZIexit); @@ -2850,11 +2829,11 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit { /* BEGINNING OF ACTION: err-expected-to */ { -#line 803 "src/lx/parser.act" +#line 794 "src/lx/parser.act" err_expected(lex_state, "'..'"); -#line 2858 "src/lx/parser.c" +#line 2837 "src/lx/parser.c" } /* END OF ACTION: err-expected-to */ } @@ -2879,29 +2858,29 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit } /* BEGINNING OF ACTION: no-zone */ { -#line 523 "src/lx/parser.act" +#line 514 "src/lx/parser.act" (ZIx) = NULL; -#line 2887 "src/lx/parser.c" +#line 2866 "src/lx/parser.c" } /* END OF ACTION: no-zone */ /* BEGINNING OF ACTION: no-token */ { -#line 515 "src/lx/parser.act" +#line 506 "src/lx/parser.act" (ZIy) = NULL; -#line 2896 "src/lx/parser.c" +#line 2875 "src/lx/parser.c" } /* END OF ACTION: no-token */ /* BEGINNING OF ACTION: regex-any */ { -#line 401 "src/lx/parser.act" +#line 392 "src/lx/parser.act" fsm_state_t start, end; - (ZIw) = fsm_new(act_state->opt); + (ZIw) = fsm_new(act_state->alloc); if ((ZIw) == NULL) { perror("fsm_new"); goto ZL5; @@ -2928,12 +2907,12 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit fsm_setstart((ZIw), start); fsm_setend((ZIw), end, 1); -#line 2932 "src/lx/parser.c" +#line 2911 "src/lx/parser.c" } /* END OF ACTION: regex-any */ /* BEGINNING OF ACTION: subtract-exit */ { -#line 744 "src/lx/parser.act" +#line 735 "src/lx/parser.act" assert((ZIw) != NULL); @@ -2953,12 +2932,12 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit } } -#line 2957 "src/lx/parser.c" +#line 2936 "src/lx/parser.c" } /* END OF ACTION: subtract-exit */ /* BEGINNING OF ACTION: add-mapping */ { -#line 458 "src/lx/parser.act" +#line 449 "src/lx/parser.act" struct ast_token *t; struct ast_mapping *m; @@ -2991,7 +2970,7 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit nmappings++; } -#line 2995 "src/lx/parser.c" +#line 2974 "src/lx/parser.c" } /* END OF ACTION: add-mapping */ } @@ -3018,11 +2997,11 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit { /* BEGINNING OF ACTION: err-expected-list */ { -#line 823 "src/lx/parser.act" +#line 814 "src/lx/parser.act" err_expected(lex_state, "list of mappings, bindings or zones"); -#line 3026 "src/lx/parser.c" +#line 3005 "src/lx/parser.c" } /* END OF ACTION: err-expected-list */ } @@ -3038,16 +3017,16 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit /* BEGINNING OF ACTION: no-exit */ { -#line 519 "src/lx/parser.act" +#line 510 "src/lx/parser.act" (ZIr2) = NULL; -#line 3046 "src/lx/parser.c" +#line 3025 "src/lx/parser.c" } /* END OF ACTION: no-exit */ /* BEGINNING OF ACTION: make-zone */ { -#line 440 "src/lx/parser.act" +#line 431 "src/lx/parser.act" assert((*ZIa) != NULL); @@ -3065,12 +3044,12 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit goto ZL1; } -#line 3069 "src/lx/parser.c" +#line 3048 "src/lx/parser.c" } /* END OF ACTION: make-zone */ /* BEGINNING OF ACTION: add-mapping */ { -#line 458 "src/lx/parser.act" +#line 449 "src/lx/parser.act" struct ast_token *t; struct ast_mapping *m; @@ -3103,7 +3082,7 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit nmappings++; } -#line 3107 "src/lx/parser.c" +#line 3086 "src/lx/parser.c" } /* END OF ACTION: add-mapping */ /* BEGINNING OF INLINE: 131 */ @@ -3122,11 +3101,11 @@ p_251(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit { /* BEGINNING OF ACTION: err-expected-list */ { -#line 823 "src/lx/parser.act" +#line 814 "src/lx/parser.act" err_expected(lex_state, "list of mappings, bindings or zones"); -#line 3130 "src/lx/parser.c" +#line 3109 "src/lx/parser.c" } /* END OF ACTION: err-expected-list */ } @@ -3170,11 +3149,11 @@ p_252(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit { /* BEGINNING OF ACTION: err-expected-bind */ { -#line 795 "src/lx/parser.act" +#line 786 "src/lx/parser.act" err_expected(lex_state, "'='"); -#line 3178 "src/lx/parser.c" +#line 3157 "src/lx/parser.c" } /* END OF ACTION: err-expected-bind */ } @@ -3189,7 +3168,7 @@ p_252(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit } /* BEGINNING OF ACTION: add-binding */ { -#line 491 "src/lx/parser.act" +#line 482 "src/lx/parser.act" struct var *v; @@ -3206,7 +3185,7 @@ p_252(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit goto ZL1; } -#line 3210 "src/lx/parser.c" +#line 3189 "src/lx/parser.c" } /* END OF ACTION: add-binding */ } @@ -3232,7 +3211,7 @@ p_252(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit /* BEGINNING OF ACTION: deref-var */ { -#line 286 "src/lx/parser.act" +#line 277 "src/lx/parser.act" struct ast_zone *z; @@ -3259,7 +3238,7 @@ p_252(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit goto ZL1; } -#line 3263 "src/lx/parser.c" +#line 3242 "src/lx/parser.c" } /* END OF ACTION: deref-var */ p_197 (lex_state, act_state, ZI273, &ZI210); @@ -3275,7 +3254,7 @@ p_252(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit } /* BEGINNING OF ACTION: subtract-exit */ { -#line 744 "src/lx/parser.act" +#line 735 "src/lx/parser.act" assert((ZI278) != NULL); @@ -3295,7 +3274,7 @@ p_252(lex_state lex_state, act_state act_state, ast *ZIa, zone *ZIz, fsm *ZIexit } } -#line 3299 "src/lx/parser.c" +#line 3278 "src/lx/parser.c" } /* END OF ACTION: subtract-exit */ p_251 (lex_state, act_state, ZIa, ZIz, ZIexit, &ZI279, &ZI280); @@ -3318,10 +3297,10 @@ ZL1:; /* BEGINNING OF TRAILER */ -#line 827 "src/lx/parser.act" +#line 818 "src/lx/parser.act" - struct ast *lx_parse(FILE *f, const struct fsm_options *opt) { + struct ast *lx_parse(FILE *f, const struct fsm_alloc *alloc) { struct act_state act_state_s; struct act_state *act_state; struct lex_state lex_state_s; @@ -3353,7 +3332,7 @@ ZL1:; /* This is a workaround for ADVANCE_LEXER assuming a pointer */ act_state = &act_state_s; - act_state->opt = opt; + act_state->alloc = alloc; if (print_progress) { act_state->zn = 0; @@ -3364,11 +3343,23 @@ ZL1:; lx->free(lx->buf_opaque); - assert(ast != NULL); + /* + * This can happen when the first token is TOK_UNKNOWN. + * SID's generated parser bails out immediately. + * So we never reach , and never about it. + * + * Really I wanted this handled along with the usual syntax error + * case, from ## alts inside parser.sid, instead of reproducing + * the same error here. + */ + if (ast == NULL) { + err(lex_state, "Syntax error"); + exit(EXIT_FAILURE); + } return ast; } -#line 3373 "src/lx/parser.c" +#line 3364 "src/lx/parser.c" /* END OF FILE */ diff --git a/src/lx/parser.h b/src/lx/parser.h index fbe704fc4..947b194b5 100644 --- a/src/lx/parser.h +++ b/src/lx/parser.h @@ -9,7 +9,7 @@ /* BEGINNING OF HEADER */ -#line 128 "src/lx/parser.act" +#line 127 "src/lx/parser.act" #include @@ -20,7 +20,7 @@ typedef struct act_state * act_state; typedef struct ast * ast; - struct ast *lx_parse(FILE *f, const struct fsm_options *opt); + struct ast *lx_parse(FILE *f, const struct fsm_alloc *alloc); #line 26 "src/lx/parser.h" @@ -29,7 +29,7 @@ extern void p_lx(lex_state, act_state, ast *); /* BEGINNING OF TRAILER */ -#line 877 "src/lx/parser.act" +#line 880 "src/lx/parser.act" #line 36 "src/lx/parser.h" diff --git a/src/lx/print.c b/src/lx/print.c new file mode 100644 index 000000000..de71a54ef --- /dev/null +++ b/src/lx/print.c @@ -0,0 +1,46 @@ +/* + * Copyright 2008-2024 Katherine Flavel + * + * See LICENCE for the full copyright terms. + */ + +#include +#include +#include +#include + +#include +#include + +#include "print.h" + +int +lx_print(FILE *f, const struct ast *ast, const struct fsm_options *opt, + enum lx_print_lang lang) +{ + lx_print_f *print; + + assert(ast != NULL); + + if (lang == LX_PRINT_NONE) { + return 0; + } + + switch (lang) { + case LX_PRINT_C: print = lx_print_c; break; + case LX_PRINT_DOT: print = lx_print_dot; break; + case LX_PRINT_DUMP: print = lx_print_dump; break; + case LX_PRINT_H: print = lx_print_h; break; + case LX_PRINT_ZDOT: print = lx_print_zdot; break; + + default: + errno = ENOTSUP; + return -1; + } + + // TODO: return int + print(f, ast, opt); + + return 0; +} + diff --git a/src/lx/print.h b/src/lx/print.h index 118ec208c..771028e14 100644 --- a/src/lx/print.h +++ b/src/lx/print.h @@ -10,6 +10,7 @@ #include struct ast; +struct fsm_options; /* TODO: combine all variables here into an lx_outoptions struct */ struct prefix { @@ -42,19 +43,32 @@ enum api_exclude { API_BUF = 1 << 4 }; +enum lx_print_lang { + LX_PRINT_NONE, + LX_PRINT_C, + LX_PRINT_DOT, + LX_PRINT_DUMP, + LX_PRINT_H, + LX_PRINT_ZDOT +}; + extern enum api_tokbuf api_tokbuf; extern enum api_getc api_getc; extern enum api_exclude api_exclude; -extern struct fsm_options opt; +typedef void (lx_print_f)(FILE *f, + const struct ast *ast, const struct fsm_options *opt); -typedef void (lx_print)(FILE *f, const struct ast *ast); +lx_print_f lx_print_c; +lx_print_f lx_print_dot; +lx_print_f lx_print_dump; +lx_print_f lx_print_h; +lx_print_f lx_print_zdot; -lx_print lx_print_c; -lx_print lx_print_h; -lx_print lx_print_dot; -lx_print lx_print_dump; -lx_print lx_print_zdot; +int +lx_print(FILE *f, const struct ast *ast, + const struct fsm_options *opt, + enum lx_print_lang lang); #endif diff --git a/src/lx/print/c.c b/src/lx/print/c.c index 829f6ff21..3340f0d74 100644 --- a/src/lx/print/c.c +++ b/src/lx/print/c.c @@ -18,19 +18,17 @@ #include -#include "libfsm/internal.h" /* XXX */ -#include "libfsm/print/ir.h" /* XXX */ - #include "lx/lx.h" #include "lx/ast.h" #include "lx/print.h" -/* XXX: abstraction */ -int -fsm_print_cfrag(FILE *f, const struct ir *ir, const struct fsm_options *opt, - const char *cp, - int (*leaf)(FILE *, const struct fsm_end_ids *ids, const void *leaf_opaque), - const void *opaque); +struct lx_hook_env { + const struct ast *ast; + /* Name of variable for the current character of input in the + * current scope, which depends on the IO options. */ + const char *cur_char_var; +}; + static int skip(const struct fsm *fsm, fsm_state_t state) @@ -38,7 +36,7 @@ skip(const struct fsm *fsm, fsm_state_t state) struct ast_mapping *m; assert(fsm != NULL); - assert(state < fsm->statecount); + assert(state < fsm_countstates(fsm)); if (!fsm_isend(fsm, state)) { return 1; @@ -100,7 +98,8 @@ shortest_example(const struct fsm *fsm, const struct ast_token *token, (void) fsm_getstart(fsm, &goal); min = INT_MAX; - for (i = 0; i < fsm->statecount; i++) { + const size_t statecount = fsm_countstates(fsm); + for (i = 0; i < statecount; i++) { const struct ast_mapping *m; int n; @@ -137,57 +136,221 @@ shortest_example(const struct fsm *fsm, const struct ast_token *token, return 1; } +static const char * +buf_op_prefix(void) +{ + if (api_tokbuf & API_FIXEDBUF) { + return "fixed"; + } else if (api_tokbuf & API_DYNBUF) { + return "dyn"; + } else { + assert(!"buf is neither fixed nor dyn"); + return NULL; + } +} + +static void +unget_character(FILE *f, bool pop, const char *cur_char_var) +{ + fprintf(f, "%sungetc(lx, %s); ", prefix.api, cur_char_var); + if (pop && (~api_exclude & API_BUF)) { + fprintf(f, "%s%spop(lx->buf_opaque); ", + prefix.api, buf_op_prefix()); + } +} + +static bool +endid_represents_dead_end(fsm_end_id_t endid, const struct ast *ast) +{ + const struct ast_mapping *m = ast_getendmappingbyendid(endid); + if (m == NULL) { + return false; + } + + /* For each zone, check if this endid is associated with its z->ml zone. + * If so, that endid is the "dead end" for that zone. + * + * The total number of zones and end ids (each corresponding to mapping) + * should stay small enough that linear search is fine. If this becomes + * prohibitively expensive, then build a bitset of dead-end IDs upfront + * in one pass. */ + for (struct ast_zone *z = ast->zl; z != NULL; z = z->next) { + if (z->ml == m) { return true; } + } + + return false; +} + static int -leaf(FILE *f, const struct fsm_end_ids *ids, const void *leaf_opaque) +accept_c(FILE *f, const struct fsm_options *opt, + const struct fsm_state_metadata *state_metadata, + void *lang_opaque, void *hook_opaque) { const struct ast *ast; const struct ast_mapping *m; + struct lx_hook_env *env = hook_opaque; - ast = leaf_opaque; + assert(f != NULL); + assert(opt != NULL); + assert(state_metadata->end_ids != NULL); + assert(state_metadata->end_id_count > 0); + assert(lang_opaque == NULL); + assert(hook_opaque != NULL); - assert(ast != NULL); + ast = env->ast; + m = ast_getendmappingbyendid(state_metadata->end_ids[0]); + + /* re-sync before new call into zone */ + switch (opt->io) { + case FSM_IO_GETC: + break; + + case FSM_IO_STR: + case FSM_IO_PAIR: + fprintf(f, "lx->p = p; "); + break; + } - if (ids == NULL) { - m = NULL; + fprintf(f, "return "); + if (m->to == NULL) { + if (m->token == NULL) { + /* If accept-ing here doesn't actually map to a token or + * a different zone, then check whether the endid represents + * a dead end. In that case, it's stuck in the middle of a + * pattern pair like `'//' .. /\n/ -> $nl;` with an unexpected + * EOF, so tokenization should still fail (with TOK_UNKNOWN). + * + * An example where the endid doesn't represent a dead end is + * a zone ignoring trailing whitespace in a file, such as + * `/[\r\n\t ]+/;`. In that case, the EOF is valid, so still + * return TOK_EOF. */ + const fsm_end_id_t endid = state_metadata->end_ids[0]; + if (endid_represents_dead_end(endid, ast)) { + fprintf(f, "%sUNKNOWN", prefix.tok); + } else { + fprintf(f, "%sEOF", prefix.tok); + } + } else { + /* yield a token */ + fprintf(f, "%s", prefix.tok); + esctok(f, m->token->s); + } } else { - assert(ids->count > 0); - m = ast_getendmappingbyendid(ids->ids[0]); + if (m->token == NULL) { + /* update to a different zone, then call to it */ + fprintf(f, "lx->z = z%u, lx->z(lx)", zindexof(ast, m->to)); + } else { + /* update zone, then yield a token */ + fprintf(f, "lx->z = z%u, ", zindexof(ast, m->to)); + fprintf(f, "%s", prefix.tok); + esctok(f, m->token->s); + } } + fprintf(f, ";"); + return 0; +} - if (m == NULL) { - /* XXX: don't need this if complete */ - switch (opt.io) { +static int +reject_c(FILE *f, const struct fsm_options *opt, + const struct fsm_state_metadata *state_metadata, + void *lang_opaque, void *hook_opaque) +{ + assert(f != NULL); + assert(opt != NULL); + assert(lang_opaque == NULL); + assert(hook_opaque != NULL); + + (void) lang_opaque; + struct lx_hook_env *env = hook_opaque; + + const struct ast_mapping *m = state_metadata != NULL && state_metadata->end_id_count > 0 + ? ast_getendmappingbyendid(state_metadata->end_ids[0]) + : NULL; + + /* If there is an AST mapping associated with this end state, + * then unget the previous character (in most cases), and + * possibly emit its token type and/or new z state. */ + if (m != NULL) { + const bool has_endids = state_metadata && state_metadata->end_id_count > 0; + if (m->token == NULL && m->to == NULL && !has_endids) { + unget_character(f, true, env->cur_char_var); + } else if (m->token == NULL && m->to == NULL && has_endids) { + unget_character(f, true, env->cur_char_var); + } else if (m->token == NULL && m->to != NULL) { + unget_character(f, true, env->cur_char_var); + } else if (m->token != NULL && m->to == NULL) { + assert(has_endids); + unget_character(f, true, env->cur_char_var); + } else if (m->token != NULL && m->to != NULL) { + unget_character(f, true, env->cur_char_var); + } + + /* re-sync before new call into zone */ + switch (opt->io) { case FSM_IO_GETC: - fprintf(f, "lx->lgetc = NULL; "); break; case FSM_IO_STR: - fprintf(f, "lx->p = NULL; "); - break; - case FSM_IO_PAIR: - fprintf(f, "lx->p = NULL; "); + fprintf(f, "lx->p = p; "); break; } - fprintf(f, "return %sUNKNOWN;", prefix.tok); + fprintf(f, "return "); + if (m->to != NULL) { + fprintf(f, "lx->z = z%u, ", zindexof(env->ast, m->to)); + } + if (m->token != NULL) { + fprintf(f, "%s", prefix.tok); + esctok(f, m->token->s); + } else { + fprintf(f, "lx->z(lx)"); + } + fprintf(f, ";"); return 0; + } else { + fprintf(f, "\n\t\t\t\tif (!has_consumed_input) { return %sEOF; }\n", prefix.tok); + fprintf(f, "\t\t\t\t"); + unget_character(f, false, env->cur_char_var); } /* XXX: don't need this if complete */ - fprintf(f, "%sungetc(lx, c); ", prefix.api); - fprintf(f, "return "); - if (m->to != NULL) { - fprintf(f, "lx->z = z%u, ", zindexof(ast, m->to)); - } - if (m->token != NULL) { - fprintf(f, "%s", prefix.tok); - esctok(f, m->token->s); - } else { - fprintf(f, "lx->z(lx)"); + switch (opt->io) { + case FSM_IO_GETC: + fprintf(f, "lx->lgetc = NULL; "); + break; + + case FSM_IO_STR: + case FSM_IO_PAIR: + fprintf(f, "lx->p = NULL; "); + break; } - fprintf(f, ";"); + fprintf(f, "return %sUNKNOWN;", prefix.tok); + return 0; +} + +static int +advance_c(FILE *f, const struct fsm_options *opt, const char *cur_char_var, void *hook_opaque) +{ + (void)hook_opaque; + + fprintf(f, "\t\thas_consumed_input = 1;\n"); + + switch (opt->io) { + case FSM_IO_GETC: + break; + + case FSM_IO_STR: + case FSM_IO_PAIR: + /* When libfsm's generated code advances a character, update + * lx's token name buffer and position bookkeeping. */ + if (~api_exclude & API_POS) { + fprintf(f, "\t\tif (!%sadvance_end(lx, %s)) { return %sERROR; }\n", + prefix.api, cur_char_var, prefix.tok); + } + break; + } return 0; } @@ -203,20 +366,32 @@ print_proto(FILE *f, const struct ast *ast, const struct ast_zone *z) } static void -print_lgetc(FILE *f) +print_lgetc(FILE *f, const struct fsm_options *opt) { if (api_getc & API_FGETC) { if (print_progress) { fprintf(stderr, " fgetc"); } + if (opt->comments) { + fprintf(f, "/* Get a character from fgetc and push it to the buffer */\n"); + } fprintf(f, "int\n"); fprintf(f, "%sfgetc(struct %slx *lx)\n", prefix.api, prefix.lx); fprintf(f, "{\n"); fprintf(f, "\tassert(lx != NULL);\n"); fprintf(f, "\tassert(lx->getc_opaque != NULL);\n"); fprintf(f, "\n"); - fprintf(f, "\treturn fgetc(lx->getc_opaque);\n"); + + fprintf(f, "\tconst int c = fgetc(lx->getc_opaque);\n"); + fprintf(f, "\tif (c == EOF) {\n"); + fprintf(f, "\t\tlx->c = EOF;\n"); + fprintf(f, "\t\treturn EOF;\n"); + fprintf(f, "\t} else {\n"); + + fprintf(f, "\t\treturn c;\n"); + fprintf(f, "\t}\n"); + fprintf(f, "}\n"); fprintf(f, "\n"); } @@ -327,28 +502,73 @@ print_lgetc(FILE *f) } static void -print_io(FILE *f) +print_io(FILE *f, const struct fsm_options *opt) { if (print_progress) { fprintf(stderr, " io"); } - /* TODO: consider passing char *c, and return int 0/-1 for error */ - fprintf(f, "#if __STDC_VERSION__ >= 199901L\n"); - fprintf(f, "inline\n"); - fprintf(f, "#endif\n"); - fprintf(f, "static int\n"); - fprintf(f, "lx_getc(struct %slx *lx)\n", prefix.lx); - fprintf(f, "{\n"); - fprintf(f, "\tint c;\n"); - fprintf(f, "\n"); + if (opt->io == FSM_IO_GETC || (~api_exclude & API_POS)) { + fprintf(f, "static int\n"); + fprintf(f, "%sadvance_end(struct %slx *lx, int c)\n", prefix.api, prefix.lx); + fprintf(f, "{\n"); - fprintf(f, "\tassert(lx != NULL);\n"); + if (api_exclude & API_POS) { + fprintf(f, "\t(void)lx; (void)c;\n"); + } else { + fprintf(f, "\tlx->end.byte++;\n"); + fprintf(f, "\tlx->end.col++;\n"); + + fprintf(f, "\tif (c == '\\n') {\n"); + fprintf(f, "\t\tlx->end.line++;\n"); + fprintf(f, "\t\tlx->end.saved_col = lx->end.col - 1;\n"); + fprintf(f, "\t\tlx->end.col = 1;\n"); + + if (opt->io == FSM_IO_STR) { /* ignore terminating '\0' */ + fprintf(f, "\t} else if (c == '\\0') { /* don't count terminating '\\0' */\n"); + fprintf(f, "\t\tlx->end.byte--;\n"); + fprintf(f, "\t\tlx->end.col--;\n"); + fprintf(f, "\t}\n"); + } else { + fprintf(f, "\t}\n"); + } + } + + if (api_exclude & API_BUF) { + fprintf(f, "\t(void)lx; (void)c;\n"); + } else { + fprintf(f, "\tif (lx->push != NULL) {\n"); + fprintf(f, "\t\tif (-1 == lx->push(lx->buf_opaque, (char)c)) {\n"); + fprintf(f, "\t\t\treturn 0;\n"); + fprintf(f, "\t\t}\n"); + fprintf(f, "\t}\n"); + } + + fprintf(f, "\treturn 1;\n"); + fprintf(f, "}\n"); + fprintf(f, "\n"); + } + + if (opt->io == FSM_IO_GETC) { + /* TODO: consider passing char *c, and return int 0/-1 for error */ + if (opt->comments) { + fprintf(f, "/* This wrapper manages one character of lookahead/pushback\n"); + fprintf(f, " * and the line, column, and byte offsets. */\n"); + } + fprintf(f, "#if __STDC_VERSION__ >= 199901L\n"); + fprintf(f, "inline\n"); + fprintf(f, "#endif\n"); + fprintf(f, "static int\n"); + fprintf(f, "%sgetc(struct %slx *lx)\n", prefix.api, prefix.lx); + fprintf(f, "{\n"); + fprintf(f, "\tint c;\n"); + fprintf(f, "\n"); + + fprintf(f, "\tassert(lx != NULL);\n"); - switch (opt.io) { - case FSM_IO_GETC: fprintf(f, "\tassert(lx->lgetc != NULL);\n"); fprintf(f, "\n"); + fprintf(f, "\tif (lx->c != EOF) {\n"); fprintf(f, "\t\tc = lx->c, lx->c = EOF;\n"); fprintf(f, "\t} else {\n"); @@ -358,54 +578,28 @@ print_io(FILE *f) fprintf(f, "\t\t}\n"); fprintf(f, "\t}\n"); fprintf(f, "\n"); - break; - case FSM_IO_STR: - /* - * For FSM_IO_STR we treat '\0' as the end of input, - * and so there's no need to distinguish it from EOF. - * We return '\0' here to save the assignment. - */ - fprintf(f, "\tassert(lx->p != NULL);\n"); + /* FIXME: This should distinguish between alloc failure + * and EOF, but will require layers of interface changes. */ + fprintf(f, "\tif (!%sadvance_end(lx, c)) { return EOF; }\n", prefix.api); fprintf(f, "\n"); - fprintf(f, "\tc = *lx->p++;\n"); - fprintf(f, "\n"); - break; - case FSM_IO_PAIR: - fprintf(f, "\tassert(lx->p != NULL);\n"); - fprintf(f, "\n"); - fprintf(f, "\tif (lx->p == lx->e) {\n"); - fprintf(f, "\t\t\treturn EOF;\n"); - fprintf(f, "\t}\n"); - fprintf(f, "\n"); - fprintf(f, "\tc = *lx->p++;\n"); + fprintf(f, "\treturn c;\n"); + fprintf(f, "}\n"); fprintf(f, "\n"); - break; - } - if (~api_exclude & API_POS) { - fprintf(f, "\tlx->end.byte++;\n"); - fprintf(f, "\tlx->end.col++;\n"); - fprintf(f, "\n"); - fprintf(f, "\tif (c == '\\n') {\n"); - fprintf(f, "\t\tlx->end.line++;\n"); - fprintf(f, "\t\tlx->end.saved_col = lx->end.col - 1;\n"); - fprintf(f, "\t\tlx->end.col = 1;\n"); - - if (opt.io == FSM_IO_STR) { /* ignore terminating '\0' */ - fprintf(f, "\t} else if (c == '\\0') { /* don't count terminating '\\0' */\n"); - fprintf(f, "\t\tlx->end.byte--;\n"); - fprintf(f, "\t\tlx->end.col--;\n"); - fprintf(f, "\t}\n"); - } else { - fprintf(f, "\t}\n"); - } + /* Add an implementation of fsm_getc that calls back + * into lx_getc with the lx handle. */ + fprintf(f, "/* This wrapper adapts calling %sgetc to the interface\n", prefix.api); + fprintf(f, " * in libfsm's generated code. */\n"); + fprintf(f, "static int\n"); + fprintf(f, "fsm_getc(void *getc_opaque)\n"); + fprintf(f, "{\n"); + + fprintf(f, "\treturn %sgetc((struct %slx *)getc_opaque);\n", prefix.api, prefix.lx); + fprintf(f, "}\n"); fprintf(f, "\n"); } - fprintf(f, "\treturn c;\n"); - fprintf(f, "}\n"); - fprintf(f, "\n"); fprintf(f, "#if __STDC_VERSION__ >= 199901L\n"); fprintf(f, "inline\n"); @@ -415,33 +609,24 @@ print_io(FILE *f) fprintf(f, "{\n"); fprintf(f, "\tassert(lx != NULL);\n"); - switch (opt.io) { + switch (opt->io) { case FSM_IO_GETC: fprintf(f, "\tassert(lx->c == EOF);\n"); - fprintf(f, "\n"); fprintf(f, "\tlx->c = c;\n"); - fprintf(f, "\n"); break; case FSM_IO_STR: fprintf(f, "\tassert(lx->p != NULL);\n"); - fprintf(f, "\tassert(*(lx->p - 1) == c);\n"); - fprintf(f, "\n"); - fprintf(f, "\tlx->p--;\n"); - fprintf(f, "\n"); break; case FSM_IO_PAIR: fprintf(f, "\tassert(lx->p != NULL);\n"); - fprintf(f, "\tassert(*(lx->p - 1) == c);\n"); - fprintf(f, "\n"); - fprintf(f, "\tlx->p--;\n"); - fprintf(f, "\n"); break; } - if (~api_exclude & API_POS) { - fprintf(f, "\n"); + if (api_exclude & API_POS) { + fprintf(f, "\t(void)lx; (void)c;\n"); + } else { fprintf(f, "\tlx->end.byte--;\n"); fprintf(f, "\tlx->end.col--;\n"); fprintf(f, "\n"); @@ -455,7 +640,7 @@ print_io(FILE *f) } static void -print_buf(FILE *f) +print_buf(FILE *f, const struct fsm_options *opt) { if (api_tokbuf & API_DYNBUF) { if (print_progress) { @@ -505,7 +690,28 @@ print_buf(FILE *f) fprintf(f, "}\n"); fprintf(f, "\n"); + + if ((~api_exclude & API_BUF) && (api_tokbuf & API_DYNBUF)) { + fprintf(f, "static void\n"); + fprintf(f, "%sdynpop(void *buf_opaque)\n", prefix.api); + fprintf(f, "{\n"); + fprintf(f, "\tstruct lx_dynbuf *t = buf_opaque;\n"); + fprintf(f, "\n"); + fprintf(f, "\tassert(t != NULL);\n"); + fprintf(f, "\n"); + + if (opt->io == FSM_IO_GETC) { + fprintf(f, "\tassert(t->p != t->a);\n"); + } + + fprintf(f, "\tt->p--;\n"); + + fprintf(f, "}\n"); + fprintf(f, "\n"); + } + fprintf(f, "int\n"); + /* FIXME: handle error from dynclear */ fprintf(f, "%sdynclear(void *buf_opaque)\n", prefix.api); fprintf(f, "{\n"); fprintf(f, "\tstruct lx_dynbuf *t = buf_opaque;\n"); @@ -529,6 +735,7 @@ print_buf(FILE *f) fprintf(f, "\n"); fprintf(f, "\tt->p = t->a;\n"); fprintf(f, "\n"); + fprintf(f, "\treturn 0;\n"); fprintf(f, "}\n"); fprintf(f, "\n"); @@ -569,6 +776,24 @@ print_buf(FILE *f) fprintf(f, "}\n"); fprintf(f, "\n"); + if (~api_exclude & API_BUF && (api_tokbuf & API_FIXEDBUF)) { + fprintf(f, "static void\n"); + fprintf(f, "%sfixedpop(void *buf_opaque)\n", prefix.api); + fprintf(f, "{\n"); + fprintf(f, "\tstruct lx_fixedbuf *t = buf_opaque;\n"); + fprintf(f, "\n"); + fprintf(f, "\tassert(t != NULL);\n"); + fprintf(f, "\tassert(t->p != NULL);\n"); + fprintf(f, "\tassert(t->a != NULL);\n"); + + if (opt->io == FSM_IO_GETC) { + fprintf(f, "\tassert(t->p > t->a);\n"); + } + fprintf(f, "\tt->p--;\n"); + fprintf(f, "}\n"); + fprintf(f, "\n"); + } + fprintf(f, "int\n"); fprintf(f, "%sfixedclear(void *buf_opaque)\n", prefix.api); fprintf(f, "{\n"); @@ -586,47 +811,44 @@ print_buf(FILE *f) } } -static void -print_stateenum(FILE *f, const struct fsm *fsm) -{ - fsm_state_t i; - - fprintf(f, "\tenum {\n"); - fprintf(f, "\t\t"); - - for (i = 0; i < fsm->statecount; i++) { - fprintf(f, "S%u, ", i); - - if (i + 1 < fsm->statecount && (i + 1) % 10 == 0) { - fprintf(f, "\n"); - fprintf(f, "\t\t"); - } - } - - fprintf(f, "NONE"); - - fprintf(f, "\n"); - fprintf(f, "\t} state;\n"); -} - static int -print_zone(FILE *f, const struct ast *ast, const struct ast_zone *z) +print_zone(FILE *f, const struct ast *ast, const struct ast_zone *z, + const struct fsm_options *opt, const char *cur_char_var) { assert(f != NULL); assert(z != NULL); assert(z->fsm != NULL); assert(fsm_all(z->fsm, fsm_isdfa)); assert(ast != NULL); + assert(cur_char_var != NULL); - /* TODO: prerequisite that the FSM is a DFA */ + /* prerequisite that the FSM is a DFA */ + assert(fsm_all(z->fsm, fsm_isdfa)); fprintf(f, "static enum %stoken\n", prefix.api); fprintf(f, "z%u(struct %slx *lx)\n", zindexof(ast, z), prefix.lx); fprintf(f, "{\n"); - fprintf(f, "\tint c;\n"); - fprintf(f, "\n"); - print_stateenum(f, z->fsm); + /* This flag indicates whether the any of the input stream was + * consumed before getting EOF and skipping over the state and + * character logic expanded here. + * + * lx needs to track this for proper EOF handling. It previously + * generated the state enum itself, so that it could include an + * additional 'NONE' state. Inside the input loop, the default + * state of NONE would be updated to the start state, but if the + * input loop was skipped it would still be NONE. */ + fprintf(f, "\tint has_consumed_input = 0;\n"); + + switch (opt->io) { + case FSM_IO_GETC: + fprintf(f, "\tint c;\n"); + break; + case FSM_IO_STR: + case FSM_IO_PAIR: + break; + } + fprintf(f, "\n"); fprintf(f, "\tassert(lx != NULL);\n"); @@ -639,71 +861,45 @@ print_zone(FILE *f, const struct ast *ast, const struct ast_zone *z) fprintf(f, "\n"); } - fprintf(f, "\tstate = NONE;\n"); - fprintf(f, "\n"); - if (~api_exclude & API_POS) { fprintf(f, "\tlx->start = lx->end;\n"); fprintf(f, "\n"); } - switch (opt.io) { + switch (opt->io) { case FSM_IO_GETC: - fprintf(f, "\twhile (c = lx_getc(lx), c != EOF) {\n"); + fprintf(f, "\tvoid *getc_opaque = (void *)lx;\n"); + + /* This must be called with fragment sent, otherwise + * it will generate a nested function definition. */ + assert(opt->fragment); break; case FSM_IO_STR: - fprintf(f, "\twhile (c = lx_getc(lx), c != '\\0') {\n"); + fprintf(f, "const char *s = lx->p;\n"); + fprintf(f, "const char *p;\n"); break; case FSM_IO_PAIR: - fprintf(f, "\twhile (c = lx_getc(lx), c != EOF) {\n"); + fprintf(f, "\tconst char *p, *b = lx->p, *e = lx->e;\n"); break; } { - fsm_state_t start; - - if (!fsm_getstart(z->fsm, &start)) { - errno = EINVAL; - return -1; - } - - fprintf(f, "\t\tif (state == NONE) {\n"); - fprintf(f, "\t\t\tstate = S%u;\n", start); - fprintf(f, "\t\t}\n"); - fprintf(f, "\n"); - } - - { - const struct fsm_options *tmp; - static const struct fsm_options defaults; - struct fsm_options o = defaults; - struct ir *ir; - - tmp = z->fsm->opt; - - o.comments = z->fsm->opt->comments; - o.case_ranges = z->fsm->opt->case_ranges; - o.leaf = leaf; - o.leaf_opaque = (void *) ast; + static const struct fsm_hooks defaults; + struct fsm_hooks hooks = defaults; - z->fsm->opt = &o; + struct lx_hook_env hook_env = { + .ast = ast, + .cur_char_var = cur_char_var, + }; - assert(opt.cp != NULL); + hooks.accept = accept_c; + hooks.reject = reject_c; + hooks.advance = advance_c; + hooks.hook_opaque = &hook_env; - ir = make_ir(z->fsm); - if (ir == NULL) { - /* TODO */ - } - - /* XXX: abstraction */ - (void) fsm_print_cfrag(f, ir, &o, opt.cp, - z->fsm->opt->leaf != NULL ? z->fsm->opt->leaf : leaf, z->fsm->opt->leaf_opaque); - - free_ir(z->fsm, ir); - - z->fsm->opt = tmp; + fsm_print(f, z->fsm, opt, &hooks, FSM_PRINT_C); } if (~api_exclude & API_BUF) { @@ -712,7 +908,8 @@ print_zone(FILE *f, const struct ast *ast, const struct ast_zone *z) has_skips = 0; - for (i = 0; i < z->fsm->statecount; i++) { + const size_t statecount = fsm_countstates(z->fsm); + for (i = 0; i < statecount; i++) { int r; r = fsm_reachableall(z->fsm, i, skip); @@ -734,7 +931,7 @@ print_zone(FILE *f, const struct ast *ast, const struct ast_zone *z) fprintf(f, "\n"); fprintf(f, "\t\tswitch (state) {\n"); - for (i = 0; i < z->fsm->statecount; i++) { + for (i = 0; i < statecount; i++) { int r; r = fsm_reachableall(z->fsm, i, skip); @@ -754,7 +951,7 @@ print_zone(FILE *f, const struct ast *ast, const struct ast_zone *z) fprintf(f, "\t\tdefault:\n"); fprintf(f, "\t\t\tif (lx->push != NULL) {\n"); - fprintf(f, "\t\t\t\tif (-1 == lx->push(lx->buf_opaque, (char)%s)) {\n", opt.cp); + fprintf(f, "\t\t\t\tif (-1 == lx->push(lx->buf_opaque, (char)%s)) {\n", cur_char_var); fprintf(f, "\t\t\t\t\treturn %sERROR;\n", prefix.tok); fprintf(f, "\t\t\t\t}\n"); fprintf(f, "\t\t\t}\n"); @@ -765,21 +962,17 @@ print_zone(FILE *f, const struct ast *ast, const struct ast_zone *z) } else { fprintf(f, "\n"); fprintf(f, "\t\tif (lx->push != NULL) {\n"); - fprintf(f, "\t\t\tif (-1 == lx->push(lx->buf_opaque, (char)%s)) {\n", opt.cp); + fprintf(f, "\t\t\tif (-1 == lx->push(lx->buf_opaque, (char)%s)) {\n", cur_char_var); fprintf(f, "\t\t\t\treturn %sERROR;\n", prefix.tok); fprintf(f, "\t\t\t}\n"); fprintf(f, "\t\t}\n"); } } - fprintf(f, "\t}\n"); - fprintf(f, "\n"); { - fsm_state_t i; - - switch (opt.io) { + switch (opt->io) { case FSM_IO_GETC: fprintf(f, "\tlx->lgetc = NULL;\n"); fprintf(f, "\n"); @@ -796,41 +989,11 @@ print_zone(FILE *f, const struct ast *ast, const struct ast_zone *z) break; } - fprintf(f, "\tswitch (state) {\n"); - - fprintf(f, "\tcase NONE: return %sEOF;\n", prefix.tok); - - for (i = 0; i < z->fsm->statecount; i++) { - const struct ast_mapping *m; - - if (!fsm_isend(z->fsm, i)) { - continue; - } - - m = ast_getendmapping(z->fsm, i); - if (LOG()) { - fprintf(stderr, "print_zone: ast_getendmapping for state %d: %p (c)\n", - i, (void *)m); - } - assert(m != NULL); - - fprintf(f, "\tcase S%u: return ", (unsigned) i); - - /* note: no point in changing zone here, because getc is now NULL */ - - if (m->token == NULL) { - fprintf(f, "%sEOF;\n", prefix.tok); - } else { - /* TODO: maybe make a printf-like little language to simplify this */ - fprintf(f, "%s", prefix.tok); - esctok(f, m->token->s); - fprintf(f, ";\n"); - } - } - - fprintf(f, "\tdefault: errno = EINVAL; return %sERROR;\n", prefix.tok); - - fprintf(f, "\t}\n"); + fprintf(f, "\tif (!has_consumed_input) {\n"); + fprintf(f, "\t\treturn %sEOF;\n", prefix.tok); + fprintf(f, "\t} \n"); + fprintf(f, "\treturn %sERROR;", prefix.tok); + fprintf(f, "\n"); } fprintf(f, "}\n\n"); @@ -877,7 +1040,7 @@ print_name(FILE *f, const struct ast *ast) } static int -print_example(FILE *f, const struct ast *ast) +print_example(FILE *f, const struct ast *ast, const struct fsm_options *opt) { struct ast_token *t; struct ast_zone *z; @@ -924,7 +1087,7 @@ print_example(FILE *f, const struct ast *ast) fprintf(f, "\t\tcase %s", prefix.tok); esctok(f, t->s); fprintf(f, ": return \""); - escputs(f, z->fsm->opt, c_escputc_str, buf); + escputs(f, opt, c_escputc_str, buf); fprintf(f, "%s", n >= (int) sizeof buf - 1 ? "..." : ""); fprintf(f, "\";\n"); } @@ -949,18 +1112,20 @@ print_example(FILE *f, const struct ast *ast) } void -lx_print_c(FILE *f, const struct ast *ast) +lx_print_c(FILE *f, const struct ast *ast, const struct fsm_options *opt) { + const char *cp; const struct ast_zone *z; unsigned int zn; assert(f != NULL); assert(ast != NULL); + assert(opt != NULL); - switch (opt.io) { - case FSM_IO_GETC: opt.cp = "c"; break; - case FSM_IO_STR: opt.cp = "c"; break; - case FSM_IO_PAIR: opt.cp = "c"; break; + switch (opt->io) { + case FSM_IO_GETC: cp = "c"; break; + case FSM_IO_STR: cp = "*p"; break; + case FSM_IO_PAIR: cp = "*p"; break; } for (z = ast->zl; z != NULL; z = z->next) { @@ -997,10 +1162,10 @@ lx_print_c(FILE *f, const struct ast *ast) fprintf(f, "\n"); - print_io(f); - print_lgetc(f); + print_io(f, opt); + print_lgetc(f, opt); - print_buf(f); + print_buf(f, opt); if (print_progress) { zn = 0; @@ -1014,7 +1179,7 @@ lx_print_c(FILE *f, const struct ast *ast) zn++; } - if (-1 == print_zone(f, ast, z)) { + if (-1 == print_zone(f, ast, z, opt, cp)) { return; /* XXX: handle error */ } } @@ -1024,7 +1189,7 @@ lx_print_c(FILE *f, const struct ast *ast) } if (~api_exclude & API_EXAMPLE) { - if (-1 == print_example(f, ast)) { + if (-1 == print_example(f, ast, opt)) { return; } } @@ -1033,6 +1198,7 @@ lx_print_c(FILE *f, const struct ast *ast) fprintf(f, "void\n"); fprintf(f, "%sinit(struct %slx *lx)\n", prefix.api, prefix.lx); fprintf(f, "{\n"); + fprintf(f, "\tstatic const struct %slx lx_default;\n", prefix.lx); fprintf(f, "\n"); fprintf(f, "\tassert(lx != NULL);\n"); @@ -1040,16 +1206,8 @@ lx_print_c(FILE *f, const struct ast *ast) fprintf(f, "\t*lx = lx_default;\n"); fprintf(f, "\n"); - switch (opt.io) { - case FSM_IO_GETC: + if (opt->io == FSM_IO_GETC) { fprintf(f, "\tlx->c = EOF;\n"); - break; - - case FSM_IO_STR: - break; - - case FSM_IO_PAIR: - break; } fprintf(f, "\tlx->z = z%u;\n", zindexof(ast, ast->global)); @@ -1059,6 +1217,16 @@ lx_print_c(FILE *f, const struct ast *ast) fprintf(f, "\tlx->end.line = 1;\n"); fprintf(f, "\tlx->end.col = 1;\n"); } + + /* Suppress warning for possibly unused function */ + if (~api_exclude & API_BUF) { + if (api_tokbuf & API_FIXEDBUF) { + fprintf(f, "\t(void)%sfixedpop;\n", prefix.api); + } else if (api_tokbuf & API_DYNBUF) { + fprintf(f, "\t(void)%sdynpop;\n", prefix.api); + } + } + fprintf(f, "}\n"); fprintf(f, "\n"); } @@ -1074,15 +1242,12 @@ lx_print_c(FILE *f, const struct ast *ast) fprintf(f, "\tassert(lx->z != NULL);\n"); fprintf(f, "\n"); - switch (opt.io) { + switch (opt->io) { case FSM_IO_GETC: fprintf(f, "\tif (lx->lgetc == NULL) {\n"); break; case FSM_IO_STR: - fprintf(f, "\tif (lx->p == NULL) {\n"); - break; - case FSM_IO_PAIR: fprintf(f, "\tif (lx->p == NULL) {\n"); break; @@ -1112,14 +1277,16 @@ lx_print_c(FILE *f, const struct ast *ast) fprintf(f, "\n"); - if (opt.io == FSM_IO_STR) { + if (opt->io == FSM_IO_STR) { fprintf(f, "void\n"); fprintf(f, "%sinput_str(struct %slx *lx, const char *p)\n", prefix.api, prefix.lx); fprintf(f, "{\n"); fprintf(f, "\tassert(lx != NULL);\n"); fprintf(f, "\tassert(p != NULL);\n"); - fprintf(f, "\tlx->end.col = 1;\n"); + if (~api_exclude & API_POS) { + fprintf(f, "\tlx->end.col = 1;\n"); + } fprintf(f, "\tlx->p = p;\n"); fprintf(f, "}\n"); - } + } } diff --git a/src/lx/print/dot.c b/src/lx/print/dot.c index 4904c246c..09c14ce5b 100644 --- a/src/lx/print/dot.c +++ b/src/lx/print/dot.c @@ -65,46 +65,48 @@ mapping(FILE *f, const struct ast_mapping *m, const struct ast *ast) } static int -endleaf_dot(FILE *f, const struct fsm_end_ids *end_ids, - const void *endleaf_opaque) +accept_dot(FILE *f, const struct fsm_options *opt, + const struct fsm_state_metadata *state_metadata, + void *lang_opaque, void *hook_opaque) { const struct ast_mapping *m; const struct ast *ast; + fsm_state_t s; assert(f != NULL); - assert(endleaf_opaque != NULL); + assert(opt != NULL); + assert(state_metadata->end_ids != NULL); + assert(state_metadata->end_id_count > 0); + assert(lang_opaque != NULL); + assert(hook_opaque != NULL); - (void)end_ids; /* TODO */ - assert(end_ids->count > 0); - m = ast_getendmappingbyendid(end_ids->ids[0]); - - ast = endleaf_opaque; + ast = hook_opaque; + s = * (fsm_state_t *) lang_opaque; fprintf(f, "label = <"); -/* TODO: would need the fsm for indexing here */ -#if 0 if (!anonymous_states) { fprintf(f, "%u
", s); } -#endif - if (m->conflict != NULL) { - const struct mapping_set *p; + if (state_metadata->end_id_count == 1) { + m = ast_getendmappingbyendid(state_metadata->end_ids[0]); + mapping(f, m, ast); + } else { + size_t i; fprintf(f, ""); - for (p = m->conflict; p != NULL; p = p->next) { - mapping(f, p->m, ast); + for (i = 0; i < state_metadata->end_id_count; i++) { + m = ast_getendmappingbyendid(state_metadata->end_ids[i]); + mapping(f, m, ast); - if (p->next != NULL) { + if (i + 1 < state_metadata->end_id_count) { fprintf(f, "
"); } } fprintf(f, ""); - } else { - mapping(f, m, ast); } fprintf(f, ">"); @@ -134,35 +136,21 @@ singlestate(FILE *f, const struct fsm *fsm, const struct ast *ast, } assert(m != NULL); - if (m->conflict != NULL) { - const struct mapping_set *p; - - for (p = m->conflict; p != NULL; p = p->next) { - if (p->m->to != NULL) { - fsm_state_t start; - - (void) fsm_getstart(p->m->to->fsm, &start); - - fprintf(f, "\tz%uS%u -> z%uS%u [ color = red, style = dashed ];\n", - zindexof(ast, z), s, - zindexof(ast, p->m->to), start); - } - } - } else { - if (m->to != NULL) { - fsm_state_t start; + if (m->to != NULL) { + fsm_state_t start; - (void) fsm_getstart(m->to->fsm, &start); + (void) fsm_getstart(m->to->fsm, &start); - fprintf(f, "\tz%uS%u -> z%uS%u [ color = cornflowerblue, style = dashed ];\n", - zindexof(ast, z), s, - zindexof(ast, m->to), start); - } + fprintf(f, "\tz%uS%u -> z%uS%u [ color = %s, style = dashed ];\n", + zindexof(ast, z), s, + zindexof(ast, m->to), start, + fsm_endid_count(fsm, s) > 1 ? "red" : "cornflowerblue"); } } static void -print_zone(FILE *f, const struct ast *ast, const struct ast_zone *z) +print_zone(FILE *f, const struct ast *ast, const struct ast_zone *z, + const struct fsm_options *opt) { fsm_state_t start, i; @@ -170,6 +158,7 @@ print_zone(FILE *f, const struct ast *ast, const struct ast_zone *z) assert(ast != NULL); assert(z != NULL); assert(z->fsm != NULL); + assert(opt != NULL); (void) fsm_getstart(z->fsm, &start); @@ -186,28 +175,21 @@ print_zone(FILE *f, const struct ast *ast, const struct ast_zone *z) fprintf(f, "\t\n"); { - const struct fsm_options *tmp; - static const struct fsm_options defaults; - struct fsm_options opt = defaults; - char p[128]; + static const struct fsm_hooks defaults; + struct fsm_hooks hooks = defaults; + struct fsm_options o = *opt; - tmp = z->fsm->opt; + char p[128]; (void) sprintf(p, "z%u", zindexof(ast, z)); - opt.anonymous_states = anonymous_states; - opt.fragment = 1; /* XXX */ - opt.consolidate_edges = z->fsm->opt->consolidate_edges; - opt.comments = z->fsm->opt->comments; - opt.prefix = p; - opt.endleaf = endleaf_dot; - opt.endleaf_opaque = (void *) ast; - - z->fsm->opt = &opt; + o.prefix = p; + o.fragment = 1; - fsm_print_dot(f, z->fsm); + hooks.accept = accept_dot; + hooks.hook_opaque = (void *) ast; - z->fsm->opt = tmp; + fsm_print(f, z->fsm, &o, &hooks, FSM_PRINT_DOT); } for (i = 0; i < z->fsm->statecount; i++) { @@ -222,13 +204,14 @@ print_zone(FILE *f, const struct ast *ast, const struct ast_zone *z) } void -lx_print_dot(FILE *f, const struct ast *ast) +lx_print_dot(FILE *f, const struct ast *ast, const struct fsm_options *opt) { const struct ast_zone *z; unsigned int zn; fsm_state_t start; assert(f != NULL); + assert(opt != NULL); (void) fsm_getstart(ast->global->fsm, &start); @@ -293,7 +276,7 @@ lx_print_dot(FILE *f, const struct ast *ast) zn++; } - print_zone(f, ast, z); + print_zone(f, ast, z, opt); } fprintf(f, "}\n"); diff --git a/src/lx/print/dump.c b/src/lx/print/dump.c index 9b1b06b25..e693f9082 100644 --- a/src/lx/print/dump.c +++ b/src/lx/print/dump.c @@ -13,10 +13,14 @@ #include "lx/print.h" -static void -print_dump(FILE *f) +void +lx_print_dump(FILE *f, const struct ast *ast, const struct fsm_options *opt) { assert(f != NULL); + assert(ast != NULL); + assert(opt != NULL); + + (void) ast; if (api_getc == API_FDGETC) { fprintf(f, "#define _POSIX_SOURCE\n"); @@ -64,12 +68,12 @@ print_dump(FILE *f) fprintf(f, "main(int argc, char *argv[])\n"); fprintf(f, "{\n"); - fprintf(f, "\tenum lx_token t;\n"); - fprintf(f, "\tstruct lx lx = { 0 };\n"); + fprintf(f, "\tenum %stoken t;\n", prefix.api); + fprintf(f, "\tstruct %slx lx = { 0 };\n", prefix.lx); - switch (opt.io) { + switch (opt->io) { case FSM_IO_GETC: - fprintf(f, "\tint (*lgetc)(struct lx *lx);\n"); + fprintf(f, "\tint (*lgetc)(struct %slx *lx);\n", prefix.lx); fprintf(f, "\tvoid *getc_opaque;\n"); break; @@ -113,7 +117,7 @@ print_dump(FILE *f) fprintf(f, "\n"); - if (opt.io == FSM_IO_GETC && (api_getc != API_AGETC && api_getc != API_SGETC)) { + if (opt->io == FSM_IO_GETC && (api_getc != API_AGETC && api_getc != API_SGETC)) { fprintf(f, "\tif (argc != 1) {\n"); fprintf(f, "\t\tfprintf(stderr, \"usage: dump\\n\");\n"); fprintf(f, "\t\treturn 1;\n"); @@ -131,7 +135,7 @@ print_dump(FILE *f) switch (api_getc) { case API_FGETC: - fprintf(f, "\tlgetc = lx_fgetc;\n"); + fprintf(f, "\tlgetc = %sfgetc;\n", prefix.api); fprintf(f, "\tgetc_opaque = stdin;\n"); fprintf(f, "\n"); break; @@ -140,7 +144,7 @@ print_dump(FILE *f) fprintf(f, "\ts = argv[1];\n"); fprintf(f, "\n"); - fprintf(f, "\tlgetc = lx_sgetc;\n"); + fprintf(f, "\tlgetc = %ssgetc;\n", prefix.api); fprintf(f, "\tgetc_opaque = &s;\n"); fprintf(f, "\n"); break; @@ -150,7 +154,7 @@ print_dump(FILE *f) fprintf(f, "\tarr.len = strlen(arr.p);\n"); fprintf(f, "\n"); - fprintf(f, "\tlgetc = lx_agetc;\n"); + fprintf(f, "\tlgetc = %sagetc;\n", prefix.api); fprintf(f, "\tgetc_opaque = &arr;\n"); fprintf(f, "\n"); break; @@ -163,16 +167,16 @@ print_dump(FILE *f) fprintf(f, "\td.fd = fileno(stdin);\n"); fprintf(f, "\n"); - fprintf(f, "\tlgetc = lx_dgetc;\n"); + fprintf(f, "\tlgetc = %sdgetc;\n", prefix.api); fprintf(f, "\tgetc_opaque = &d;\n"); fprintf(f, "\n"); break; } - fprintf(f, "\tlx_init(&lx);\n"); + fprintf(f, "\t%sinit(&lx);\n", prefix.api); fprintf(f, "\n"); - switch (opt.io) { + switch (opt->io) { case FSM_IO_GETC: fprintf(f, "\tlx.lgetc = lgetc;\n"); fprintf(f, "\tlx.getc_opaque = getc_opaque;\n"); @@ -197,10 +201,12 @@ print_dump(FILE *f) fprintf(f, "\tbuf.len = 0;\n"); fprintf(f, "\n"); - fprintf(f, "\tlx.buf_opaque = &buf;\n"); - fprintf(f, "\tlx.push = lx_dynpush;\n"); - fprintf(f, "\tlx.clear = lx_dynclear;\n"); - fprintf(f, "\tlx.free = lx_dynfree;\n"); + if (~api_exclude & API_BUF) { + fprintf(f, "\tlx.buf_opaque = &buf;\n"); + fprintf(f, "\tlx.push = %sdynpush;\n", prefix.api); + fprintf(f, "\tlx.clear = %sdynclear;\n", prefix.api); + fprintf(f, "\tlx.free = %sdynfree;\n", prefix.api); + } fprintf(f, "\n"); break; @@ -210,10 +216,12 @@ print_dump(FILE *f) fprintf(f, "\tbuf.len = sizeof a;\n"); /* XXX: rename .len to .size */ fprintf(f, "\n"); - fprintf(f, "\tlx.buf_opaque = &buf;\n"); - fprintf(f, "\tlx.push = lx_fixedpush;\n"); - fprintf(f, "\tlx.clear = lx_fixedclear;\n"); - fprintf(f, "\tlx.free = NULL;\n"); + if (~api_exclude & API_BUF) { + fprintf(f, "\tlx.buf_opaque = &buf;\n"); + fprintf(f, "\tlx.push = %sfixedpush;\n", prefix.api); + fprintf(f, "\tlx.clear = %sfixedclear;\n", prefix.api); + fprintf(f, "\tlx.free = NULL;\n"); + } fprintf(f, "\n"); break; } @@ -223,7 +231,7 @@ print_dump(FILE *f) fprintf(f, "\t\tconst char *q;\n"); fprintf(f, "\n"); - fprintf(f, "\t\tt = lx_next(&lx);\n"); + fprintf(f, "\t\tt = %snext(&lx);\n", prefix.api); fprintf(f, "\n"); switch (api_tokbuf) { @@ -270,7 +278,7 @@ print_dump(FILE *f) fprintf(f, "\n"); fprintf(f, "\t\tcase TOK_ERROR:\n"); - fprintf(f, "\t\t\tperror(\"lx_next\");\n"); + fprintf(f, "\t\t\tperror(\"%snext\");\n", prefix.api); fprintf(f, "\t\t\tbreak;\n"); fprintf(f, "\n"); @@ -283,7 +291,7 @@ print_dump(FILE *f) fprintf(f, "\t\tdefault:\n"); if (~api_exclude & API_NAME) { - fprintf(f, "\t\t\tprintf(\"<%%s\", lx_name(t));\n"); + fprintf(f, "\t\t\tprintf(\"<%%s\", %sname(t));\n", prefix.api); fprintf(f, "\t\t\tdump_buf(q, l);\n"); fprintf(f, "\t\t\tprintf(\">\\n\");\n"); } else { @@ -299,18 +307,12 @@ print_dump(FILE *f) fprintf(f, "\t} while (t != TOK_ERROR && t != TOK_EOF && t != TOK_UNKNOWN);\n"); fprintf(f, "\n"); + if (api_tokbuf == API_DYNBUF && (~api_exclude & API_BUF)) { + fprintf(f, "\tlx.free(lx.buf_opaque);\n"); + fprintf(f, "\n"); + } + fprintf(f, "\treturn t == TOK_ERROR;\n"); fprintf(f, "}\n"); } -void -lx_print_dump(FILE *f, const struct ast *ast) -{ - assert(f != NULL); - assert(ast != NULL); - - (void) ast; - - print_dump(f); -} - diff --git a/src/lx/print/h.c b/src/lx/print/h.c index c41b89662..01a34c62e 100644 --- a/src/lx/print/h.c +++ b/src/lx/print/h.c @@ -41,10 +41,11 @@ print_tokens(FILE *f, const struct ast *ast) } void -lx_print_h(FILE *f, const struct ast *ast) +lx_print_h(FILE *f, const struct ast *ast, const struct fsm_options *opt) { assert(f != NULL); assert(ast != NULL); + assert(opt != NULL); fprintf(f, "/* Generated by lx */\n"); /* TODO: date, input etc */ fprintf(f, "\n"); @@ -73,7 +74,7 @@ lx_print_h(FILE *f, const struct ast *ast) fprintf(f, "struct %slx {\n", prefix.lx); - switch (opt.io) { + switch (opt->io) { case FSM_IO_GETC: fprintf(f, "\tint (*lgetc)(struct %slx *lx);\n", prefix.lx); fprintf(f, "\tvoid *getc_opaque;\n"); @@ -233,7 +234,7 @@ lx_print_h(FILE *f, const struct ast *ast) fprintf(f, "\n"); } - if (opt.io == FSM_IO_STR) { + if (opt->io == FSM_IO_STR) { fprintf(f, "void %sinput_str(struct %slx *lx, const char *p);\n", prefix.api, prefix.lx); fprintf(f, "\n"); } diff --git a/src/lx/print/zdot.c b/src/lx/print/zdot.c index 7fa9b8b07..3fcf72b56 100644 --- a/src/lx/print/zdot.c +++ b/src/lx/print/zdot.c @@ -74,13 +74,16 @@ print_zone(FILE *f, const struct ast *ast, const struct ast_zone *z) } void -lx_print_zdot(FILE *f, const struct ast *ast) +lx_print_zdot(FILE *f, const struct ast *ast, const struct fsm_options *opt) { const struct ast_zone *z; unsigned int zn; assert(f != NULL); assert(ast != NULL); + assert(opt != NULL); + + (void) opt; fprintf(f, "digraph %sG {\n", prefix.api); fprintf(f, "\trankdir = TB;\n"); diff --git a/src/lx/tokens.c b/src/lx/tokens.c index 3736ddf51..ac2fd52a9 100644 --- a/src/lx/tokens.c +++ b/src/lx/tokens.c @@ -6,6 +6,7 @@ #include #include +#include #include diff --git a/src/print/Makefile b/src/print/Makefile index 898028f5c..77b4446a0 100644 --- a/src/print/Makefile +++ b/src/print/Makefile @@ -6,6 +6,7 @@ SRC += src/print/dot.c SRC += src/print/abnf.c SRC += src/print/fsm.c SRC += src/print/json.c +SRC += src/print/llvm.c SRC += src/print/pcre.c SRC += src/print/rust.c SRC += src/print/tok.c diff --git a/src/print/llvm.c b/src/print/llvm.c new file mode 100644 index 000000000..ab0c69720 --- /dev/null +++ b/src/print/llvm.c @@ -0,0 +1,44 @@ +/* + * Copyright 2008-2017 Katherine Flavel + * + * See LICENCE for the full copyright terms. + */ + +#include +#include +#include +#include +#include + +#include +#include + +#include + +int +llvm_escputc_char(FILE *f, const struct fsm_options *opt, char c) +{ + assert(f != NULL); + assert(opt != NULL); + + if (opt->always_hex) { + return fprintf(f, "u%#02x", (unsigned char) c); + } + + return fprintf(f, "%u", (unsigned char) c); +} + +void +llvm_escputcharlit(FILE *f, const struct fsm_options *opt, char c) +{ + assert(f != NULL); + assert(opt != NULL); + + if (opt->always_hex || (unsigned char) c > SCHAR_MAX) { + fprintf(f, "u%#02x", (unsigned char) c); + return; + } + + llvm_escputc_char(f, opt, c); +} + diff --git a/src/re/main.c b/src/re/main.c index 77f147518..793ae6b62 100644 --- a/src/re/main.c +++ b/src/re/main.c @@ -18,6 +18,7 @@ #include #include +#include #include #include #include @@ -30,12 +31,13 @@ #include +#include + #include "libfsm/internal.h" /* XXX */ -#include "libre/print.h" /* XXX */ #include "libre/class.h" /* XXX */ #include "libre/ast.h" /* XXX */ #include "libre/ast_new_from_fsm.h" /* XXX */ - +#include "libre/print.h" /* XXX */ #define DEBUG_ESCAPES 0 #define DEBUG_VM_FSM 0 @@ -48,20 +50,25 @@ * TODO: flags; -r for RE_REVERSE, etc */ +static int +exec_with_captures(struct fsm *fsm, + int (*fsm_getc)(void *opaque), void *opaque, fsm_state_t *end); + struct match { fsm_end_id_t i; const char *s; struct match *next; }; -static struct fsm_options opt; +static char *const *matchv; +static size_t matchc; static void usage(void) { fprintf(stderr, "usage: re [-r ] [-nbiusfyz] [-x] ... [ | -- ... ]\n"); fprintf(stderr, " re [-r ] [-nbiusfyz] {-q } ...\n"); - fprintf(stderr, " re -p [-r ] [-nbiusfyz] [-l ] [-acwX] [-k ] [-e ] ...\n"); + fprintf(stderr, " re -p [-r ] [-nbiusfyz] [-l ] [-acwX] [-k ] [-E ] [-e ] ...\n"); fprintf(stderr, " re -m [-r ] [-nbiusfyz] ...\n"); fprintf(stderr, " re -G [-r ] [-biu] \n"); fprintf(stderr, " re -h\n"); @@ -100,60 +107,75 @@ io(const char *name) exit(EXIT_FAILURE); } +/* TODO: centralise */ static void -print_name(const char *name, - fsm_print **print_fsm, ast_print **print_ast) +lang_name(const char *name, enum fsm_print_lang *fsm_lang, enum ast_print_lang *ast_lang) { size_t i; - struct { + const struct { const char *name; - fsm_print *print_fsm; - ast_print *print_ast; + enum fsm_print_lang lang; } a[] = { - { "api", fsm_print_api, NULL }, - { "awk", fsm_print_awk, NULL }, - { "c", fsm_print_c, NULL }, - { "dot", fsm_print_dot, NULL }, - { "fsm", fsm_print_fsm, NULL }, - { "ir", fsm_print_ir, NULL }, - { "irjson", fsm_print_irjson, NULL }, - { "json", fsm_print_json, NULL }, - { "vmc", fsm_print_vmc, NULL }, - { "vmdot", fsm_print_vmdot, NULL }, - { "rust", fsm_print_rust, NULL }, - { "sh", fsm_print_sh, NULL }, - { "go", fsm_print_go, NULL }, - - { "vmops_c", fsm_print_vmops_c, NULL }, - { "vmops_h", fsm_print_vmops_h, NULL }, - { "vmops_main", fsm_print_vmops_main, NULL }, - - { "amd64", fsm_print_vmasm, NULL }, - { "amd64_att", fsm_print_vmasm_amd64_att, NULL }, - { "amd64_nasm", fsm_print_vmasm_amd64_nasm, NULL }, - { "amd64_go", fsm_print_vmasm_amd64_go, NULL }, - - { "tree", NULL, ast_print_tree }, - { "abnf", NULL, ast_print_abnf }, - { "ast", NULL, ast_print_dot }, - { "pcre", NULL, ast_print_pcre } + { "amd64", FSM_PRINT_AMD64_NASM }, + { "amd64_att", FSM_PRINT_AMD64_ATT }, + { "amd64_go", FSM_PRINT_AMD64_GO }, + { "amd64_nasm", FSM_PRINT_AMD64_NASM }, + + { "api", FSM_PRINT_API }, + { "awk", FSM_PRINT_AWK }, + { "c", FSM_PRINT_C }, + { "dot", FSM_PRINT_DOT }, + { "fsm", FSM_PRINT_FSM }, + { "go", FSM_PRINT_GO }, + { "ir", FSM_PRINT_IR }, + { "irjson", FSM_PRINT_IRJSON }, + { "json", FSM_PRINT_JSON }, + { "llvm", FSM_PRINT_LLVM }, + { "rust", FSM_PRINT_RUST }, + { "sh", FSM_PRINT_SH }, + { "vmc", FSM_PRINT_VMC }, + + { "vmdot", FSM_PRINT_VMDOT }, + { "vmops_c", FSM_PRINT_VMOPS_C }, + { "vmops_h", FSM_PRINT_VMOPS_H }, + { "vmops_main", FSM_PRINT_VMOPS_MAIN } + }; + + const struct { + const char *name; + enum ast_print_lang lang; + } b[] = { + { "abnf", AST_PRINT_ABNF }, + { "ast", AST_PRINT_DOT }, + { "pcre", AST_PRINT_PCRE }, + { "tree", AST_PRINT_TREE } }; assert(name != NULL); - assert(print_fsm != NULL); - assert(print_ast != NULL); for (i = 0; i < sizeof a / sizeof *a; i++) { if (0 == strcmp(a[i].name, name)) { - *print_fsm = a[i].print_fsm; - *print_ast = a[i].print_ast; + *fsm_lang = a[i].lang; + return; + } + } + + for (i = 0; i < sizeof b / sizeof *b; i++) { + if (0 == strcmp(b[i].name, name)) { + *ast_lang = b[i].lang; return; } } fprintf(stderr, "unrecognised output language; valid languages are: "); + for (i = 0; i < sizeof a / sizeof *a; i++) { + fprintf(stderr, "%s%s", + a[i].name, + ", "); + } + for (i = 0; i < sizeof a / sizeof *a; i++) { fprintf(stderr, "%s%s", a[i].name, @@ -261,140 +283,6 @@ xopen(const char *s) return f; } -struct matches_list { - struct matches_list *next; - struct match *head; -}; - -static struct matches_list *all_matches = NULL; - -static struct match * -find_match_with_id(fsm_end_id_t id) -{ - struct matches_list *res = all_matches; - while (res != NULL) { - struct match *m = res->head; - if (m->i == id) { - return m; - } - res = res->next; - } - return NULL; -} - -static struct match * -find_first_match_for_end_state(const struct fsm *dfa, fsm_state_t s) -{ -#define MAX_END_IDS 8 /* FIXME: what is reasonable here? */ - fsm_end_id_t end_id_buf[MAX_END_IDS]; - size_t end_ids_written; - enum fsm_getendids_res res; - - if (!fsm_isend(dfa, s)) { - return NULL; - } - - res = fsm_getendids(dfa, s, MAX_END_IDS, - end_id_buf, &end_ids_written); - if (res == FSM_GETENDIDS_ERROR_INSUFFICIENT_SPACE) { - fprintf(stderr, "Error: Multiple end IDs\n"); - return NULL; - } else if (res == FSM_GETENDIDS_NOT_FOUND) { - return NULL; - } else { - assert(res == FSM_GETENDIDS_FOUND); - /* continues below */ - } - - return find_match_with_id(end_id_buf[0]); -} - -static struct match * -addmatch(struct match **head, int i, const char *s) -{ - struct match *new; - - assert(head != NULL); - assert(s != NULL); - - if ((1U << i) > INT_MAX) { - fprintf(stderr, "Too many patterns for int bitmap\n"); - exit(EXIT_FAILURE); - } - - /* TODO: explain we find duplicate; return success */ - /* - * This is a purposefully shallow comparison (rather than calling strcmp) - * so that 're xyz xyz' will find the ambiguity. - */ - { - struct match *p; - - for (p = *head; p != NULL; p = p->next) { - if (p->s == s) { - return p; - } - } - } - - new = malloc(sizeof *new); - if (new == NULL) { - return NULL; - } - - new->i = i; - new->s = s; - - new->next = *head; - *head = new; - - return new; -} - -static void -add_matches_list(struct match *head) -{ - struct matches_list *new; - - if (head == NULL) { - return; - } - - new = malloc(sizeof *new); - if (new == NULL) { - perror("allocating matches list"); - abort(); - } - - new->next = all_matches; - new->head = head; - all_matches = new; -} - -static void -free_all_matches(void) -{ - struct matches_list *clst; - struct matches_list *nlst; - - for (clst = all_matches; clst != NULL; clst = nlst) { - struct match *curr; - struct match *next; - - nlst = clst->next; - - for (curr = clst->head; curr != NULL; curr = next) { - next = curr->next; - - free(curr); - } - - free(clst); - } - - all_matches = NULL; -} - static void printexample(FILE *f, const struct fsm *fsm, fsm_state_t state) { @@ -416,81 +304,68 @@ printexample(FILE *f, const struct fsm *fsm, fsm_state_t state) } static int -endleaf_c(FILE *f, const struct fsm_end_ids *end_ids, - const void *endleaf_opaque) +conflict(FILE *f, const struct fsm_options *opt, + const struct fsm_state_metadata *state_metadata, const char *example, + void *hook_opaque) { - const struct match *m; - int n; - size_t i, end_ids_count; + size_t i; - assert(endleaf_opaque == NULL); + assert(opt != NULL); + assert(opt->ambig == AMBIG_ERROR); + assert(state_metadata->end_id_count > 1); (void) f; - (void) endleaf_opaque; - - n = 0; + (void) opt; + (void) hook_opaque; - end_ids_count = (end_ids == NULL ? 0 : end_ids->count); + /* TODO: // delimeters depend on dialect */ + /* TODO: would deal with dialect: prefix here, too */ - for (i = 0; i < end_ids_count; i++) { - for (m = find_match_with_id(end_ids->ids[i]); m != NULL; m = m->next) { - n |= 1 << m->i; - } - } + fprintf(stderr, "ambiguous matches for "); - fprintf(f, "return %#x;", (unsigned) n); + for (i = 0; i < state_metadata->end_id_count; i++) { + assert(state_metadata->end_ids[i] < matchc); - fprintf(f, " /* "); + /* TODO: print nicely */ + fprintf(stderr, "/%s/", matchv[state_metadata->end_ids[i]]); - for (i = 0; i < end_ids_count; i++) { - for (m = find_match_with_id(end_ids->ids[i]); m != NULL; m = m->next) { - fprintf(f, "\"%s\"", m->s); /* XXX: escape string (and comment) */ - - if (m->next != NULL || i < end_ids_count - 1) { - fprintf(f, ", "); - } + if (i + 1 < state_metadata->end_id_count) { + fprintf(stderr, ", "); } } - fprintf(f, " */"); + if (example != NULL) { + /* TODO: escape hex etc. or perhaps make_ir() did that for us */ + fprintf(stderr, "; for example on input '%s'", example); + } + + fprintf(stderr, "\n"); return 0; } static int -endleaf_rust(FILE *f, const struct fsm_end_ids *end_ids, - const void *endleaf_opaque) +comment_c(FILE *f, const struct fsm_options *opt, + const struct fsm_state_metadata *state_metadata, + void *hook_opaque) { - const struct match *m; - int n; - size_t i, end_ids_count; - - assert(endleaf_opaque == NULL); - - (void) f; - (void) endleaf_opaque; - - n = 0; + size_t i; - end_ids_count = (end_ids == NULL ? 0 : end_ids->count); + assert(opt != NULL); + assert(hook_opaque == NULL); - for (i = 0; i < end_ids_count; i++) { - for (m = find_match_with_id(end_ids->ids[i]); m != NULL; m = m->next) { - n |= 1 << m->i; - } - } + (void) opt; + (void) hook_opaque; - fprintf(f, "return Some(%#x)", (unsigned) n); + fprintf(f, "/* "); - fprintf(f, " /* "); + for (i = 0; i < state_metadata->end_id_count; i++) { + assert(state_metadata->end_ids[i] < matchc); - for (i = 0; i < end_ids_count; i++) { - for (m = find_match_with_id(end_ids->ids[i]); m != NULL; m = m->next) { - fprintf(f, "\"%s\"", m->s); /* XXX: escape string (and comment) */ + fprintf(f, "\"%s\"", matchv[state_metadata->end_ids[i]]); /* XXX: escape string (and comment) */ - if (m->next != NULL || i < end_ids_count - 1) { - fprintf(f, ", "); - } + if (i + 1 < state_metadata->end_id_count) { + fprintf(f, ", "); } } @@ -500,85 +375,87 @@ endleaf_rust(FILE *f, const struct fsm_end_ids *end_ids, } static int -endleaf_dot(FILE *f, const struct fsm_end_ids *end_ids, - const void *endleaf_opaque) +comment_rust(FILE *f, const struct fsm_options *opt, + const struct fsm_state_metadata *state_metadata, + void *hook_opaque) { - const struct match *m; - size_t i, end_ids_count; + size_t i; - assert(f != NULL); - assert(endleaf_opaque == NULL); + assert(opt != NULL); + assert(hook_opaque == NULL); - (void) endleaf_opaque; + (void) opt; + (void) hook_opaque; - fprintf(f, "label = <"); + fprintf(f, "// "); - end_ids_count = (end_ids == NULL ? 0 : end_ids->count); + for (i = 0; i < state_metadata->end_id_count; i++) { + assert(state_metadata->end_ids[i] < matchc); - for (i = 0; i < end_ids_count; i++) { - for (m = find_match_with_id(end_ids->ids[i]); m != NULL; m = m->next) { - fprintf(f, "#%u", m->i); + fprintf(f, "\"%s\"", matchv[state_metadata->end_ids[i]]); /* XXX: escape string (and comment) */ - if (m->next != NULL || i < end_ids_count - 1) { - fprintf(f, ","); - } + if (i + 1 < state_metadata->end_id_count) { + fprintf(f, ", "); } } - fprintf(f, ">"); + return 0; +} + +static int +comment_llvm(FILE *f, const struct fsm_options *opt, + const struct fsm_state_metadata *state_metadata, + void *hook_opaque) +{ + size_t i; + + assert(opt != NULL); + assert(hook_opaque == NULL); - /* TODO: only if comments */ - /* TODO: centralise to libfsm/print/dot.c */ + (void) opt; + (void) hook_opaque; -#if 0 - fprintf(f, " # "); + fprintf(f, "; "); - for (i = 0; i < end_ids_count; i++) { - for (m = find_match_with_id(end_ids->ids[i]); m != NULL; m = m->next) { - fprintf(f, "\"%s\"", m->s); /* XXX: escape string (and comment) */ + for (i = 0; i < state_metadata->end_id_count; i++) { + assert(state_metadata->end_ids[i] < matchc); - if (m->next != NULL || i < end_ids_count - 1) { - fprintf(f, ", "); - } + fprintf(f, "\"%s\"", matchv[state_metadata->end_ids[i]]); /* XXX: escape string (and comment) */ + + if (i + 1 < state_metadata->end_id_count) { + fprintf(f, ", "); } } - fprintf(f, "\n"); -#endif - return 0; } static int -endleaf_json(FILE *f, const struct fsm_end_ids *end_ids, - const void *endleaf_opaque) +comment_dot(FILE *f, const struct fsm_options *opt, + const struct fsm_state_metadata *state_metadata, + void *hook_opaque) { - const struct match *m; - size_t i, end_ids_count; + size_t i; - assert(f != NULL); - assert(endleaf_opaque == NULL); + assert(opt != NULL); + assert(hook_opaque == NULL); - (void) endleaf_opaque; + (void) opt; + (void) hook_opaque; - fprintf(f, "[ "); + fprintf(f, "/* "); - end_ids_count = (end_ids == NULL ? 0 : end_ids->count); + for (i = 0; i < state_metadata->end_id_count; i++) { + assert(state_metadata->end_ids[i] < matchc); - for (i = 0; i < end_ids_count; i++) { - for (m = find_match_with_id(end_ids->ids[i]); m != NULL; m = m->next) { - fprintf(f, "%u", m->i); + fprintf(f, "\"%s\"", matchv[state_metadata->end_ids[i]]); /* XXX: escape string (and comment) */ - if (m->next != NULL) { - fprintf(f, ", "); - } + if (i + 1 < state_metadata->end_id_count) { + fprintf(f, ", "); } } - fprintf(f, " ]"); - - /* TODO: only if comments */ - /* TODO: centralise to libfsm/print/json.c */ + fprintf(f, " */"); return 0; } @@ -592,8 +469,6 @@ do_fsm_cleanup(void) fsm_free(fsm_to_cleanup); fsm_to_cleanup = NULL; } - - free_all_matches(); } static void @@ -601,6 +476,10 @@ parse_flags(const char *arg, enum re_flags *flags) { for (; *arg; arg++) { switch (*arg) { + case 'b': + *flags = *flags | RE_ANCHORED; + break; + case 'i': *flags = *flags | RE_ICASE; break; @@ -613,6 +492,10 @@ parse_flags(const char *arg, enum re_flags *flags) *flags = *flags | RE_EXTENDED; break; + case 'C': + *flags = *flags | RE_NOCAPTURE; + break; + /* others? */ default: @@ -626,32 +509,44 @@ parse_flags(const char *arg, enum re_flags *flags) int main(int argc, char *argv[]) { + static const struct fsm_options zero_options; + static const struct fsm_hooks zero_hooks; + + /* TODO: use alloc hooks for -Q accounting */ + struct fsm_alloc *alloc = NULL; + struct fsm_options opt; + struct fsm_hooks hooks; + + struct fsm *fsm; struct fsm *(*join)(struct fsm *, struct fsm *, struct fsm_combine_info *); int (*query)(const struct fsm *, const struct fsm *); - fsm_print *print_fsm; - ast_print *print_ast; + enum fsm_print_lang fsm_lang; + enum ast_print_lang ast_lang; enum re_dialect dialect; - struct fsm *fsm; enum re_flags flags; + size_t generate_bounds = 0; int xfiles, yfiles; int fsmfiles; int example; int isliteral; int keep_nfa; int patterns; - int ambig; int makevm; - size_t generate_bounds = 0; + int resolve_captures; struct fsm_dfavm *vm; atexit(do_fsm_cleanup); + opt = zero_options; + hooks = zero_hooks; + /* note these defaults are the opposite than for fsm(1) */ opt.anonymous_states = 1; opt.consolidate_edges = 1; + opt.ambig = AMBIG_ERROR; opt.comments = 1; opt.io = FSM_IO_GETC; @@ -663,10 +558,13 @@ main(int argc, char *argv[]) isliteral = 0; keep_nfa = 0; patterns = 0; - ambig = 0; makevm = 0; - print_fsm = NULL; - print_ast = NULL; + + resolve_captures = 0; + + fsm_lang = FSM_PRINT_NONE; + ast_lang = AST_PRINT_NONE; + query = NULL; join = fsm_union; dialect = RE_NATIVE; @@ -675,7 +573,7 @@ main(int argc, char *argv[]) { int c; - while (c = getopt(argc, argv, "h" "acCwXe:E:G:k:" "bi" "sq:r:l:F:" "upMmnftxyz"), c != -1) { + while (c = getopt(argc, argv, "h" "acCwXe:E:G:k:" "bi" "sq:r:l:F:" "upMmnRftxyz"), c != -1) { switch (c) { case 'a': opt.anonymous_states = 0; break; case 'c': opt.consolidate_edges = 0; break; @@ -694,18 +592,18 @@ main(int argc, char *argv[]) break; case 'l': - print_name(optarg, &print_fsm, &print_ast); + lang_name(optarg, &fsm_lang, &ast_lang); break; case 'F': parse_flags(optarg, &flags); break; - case 'p': print_fsm = fsm_print_fsm; break; + case 'p': fsm_lang = FSM_PRINT_FSM; break; case 'q': query = comparison(optarg); break; case 'r': dialect = dialect_name(optarg); break; + case 'u': opt.ambig = AMBIG_MULTIPLE; break; - case 'u': ambig = 1; break; case 'f': fsmfiles = 1; break; case 'x': xfiles = 1; break; case 'y': yfiles = 1; break; @@ -714,6 +612,7 @@ main(int argc, char *argv[]) case 't': isliteral = 1; break; case 'z': patterns = 1; break; case 'M': makevm = 1; break; + case 'R': resolve_captures = 1; break; case 'G': generate_bounds = strtoul(optarg, NULL, 10); @@ -743,12 +642,12 @@ main(int argc, char *argv[]) return EXIT_FAILURE; } - if (!!print_fsm + !!print_ast + example + isliteral + !!query > 1) { + if ((fsm_lang != FSM_PRINT_NONE) + (ast_lang != AST_PRINT_NONE) + example + isliteral + !!query > 1) { fprintf(stderr, "-m, -p, -q and -t are mutually exclusive\n"); return EXIT_FAILURE; } - if (!!print_fsm + !!print_ast + example + isliteral + !!query && xfiles) { + if ((fsm_lang != FSM_PRINT_NONE) + (ast_lang != AST_PRINT_NONE) + example + isliteral + !!query && xfiles) { fprintf(stderr, "-x applies only when executing\n"); return EXIT_FAILURE; } @@ -773,12 +672,12 @@ main(int argc, char *argv[]) return EXIT_FAILURE; } - if (print_fsm == NULL) { + if (fsm_lang == FSM_PRINT_NONE) { keep_nfa = 0; } if (keep_nfa) { - ambig = 1; + opt.ambig = AMBIG_MULTIPLE; } /* XXX: repetitive */ @@ -807,9 +706,8 @@ main(int argc, char *argv[]) literal_r = re_is_literal(dialect, fsm_fgetc, f, - &opt, flags, &err, + flags, &err, &literal_category, &literal_s, &literal_n); - fclose(f); } else { const char *s; @@ -818,7 +716,7 @@ main(int argc, char *argv[]) literal_r = re_is_literal(dialect, fsm_sgetc, &s, - &opt, flags, &err, + flags, &err, &literal_category, &literal_s, &literal_n); } @@ -866,7 +764,7 @@ main(int argc, char *argv[]) } /* XXX: repetitive */ - if (print_ast != NULL) { + if (ast_lang != AST_PRINT_NONE) { struct ast *ast; struct re_err err; @@ -881,11 +779,11 @@ main(int argc, char *argv[]) f = xopen(argv[0]); if (!fsmfiles) { - ast = re_parse(dialect, fsm_fgetc, f, &opt, flags, &err, NULL); + ast = re_parse(dialect, fsm_fgetc, f, flags, &err, NULL); } else { struct fsm *fsm; - fsm = fsm_parse(f, &opt); + fsm = fsm_parse(f, alloc); if (fsm == NULL) { perror("fsm_parse"); return EXIT_FAILURE; @@ -907,7 +805,7 @@ main(int argc, char *argv[]) s = argv[0]; - ast = re_parse(dialect, fsm_sgetc, &s, &opt, flags, &err, NULL); + ast = re_parse(dialect, fsm_sgetc, &s, flags, &err, NULL); } if (ast == NULL) { @@ -922,7 +820,7 @@ main(int argc, char *argv[]) return EXIT_FAILURE; } - print_ast(stdout, &opt, flags, ast); + ast_print(stdout, ast, &opt, flags, ast_lang); ast_free(ast); @@ -931,7 +829,7 @@ main(int argc, char *argv[]) flags |= RE_MULTI; - fsm = fsm_new(&opt); + fsm = fsm_new(alloc); if (fsm == NULL) { perror("fsm_new"); return EXIT_FAILURE; @@ -942,7 +840,7 @@ main(int argc, char *argv[]) { int i; - for (i = 0; i < argc - !(print_fsm || example || isliteral || !!query || argc <= 1); i++) { + for (i = 0; i < argc - !(fsm_lang != FSM_PRINT_NONE || example || isliteral || !!query || argc <= 1); i++) { struct re_err err; struct fsm *new, *q; @@ -961,9 +859,9 @@ main(int argc, char *argv[]) f = xopen(argv[i]); if (!fsmfiles) { - new = re_comp(dialect, fsm_fgetc, f, &opt, flags, &err); + new = re_comp(dialect, fsm_fgetc, f, alloc, flags, &err); } else { - new = fsm_parse(f, &opt); + new = fsm_parse(f, alloc); if (new == NULL) { perror("fsm_parse"); return EXIT_FAILURE; @@ -976,7 +874,7 @@ main(int argc, char *argv[]) s = argv[i]; - new = re_comp(dialect, fsm_sgetc, &s, &opt, flags, &err); + new = re_comp(dialect, fsm_sgetc, &s, alloc, flags, &err); } if (new == NULL) { @@ -1002,26 +900,13 @@ main(int argc, char *argv[]) } } - { - struct match *matches = NULL; - - /* - * Attach this mapping to each end state for this regexp. - * XXX: we should share the same matches struct for all end states - * in the same regexp, and keep an argc-sized array of pointers to free(). - */ - if (!fsm_setendid(new, i)) { - perror("fsm_setendid"); - return EXIT_FAILURE; - } - - if (!addmatch(&matches, i, argv[i])) { - perror("addmatch"); - return EXIT_FAILURE; - } - - add_matches_list(matches); - + /* + * Attach this mapping to each end state for this regexp. + * We pick this up from matchv[] indexed by the end id. + */ + if (!fsm_setendid(new, i)) { + perror("fsm_setendid"); + return EXIT_FAILURE; } /* TODO: implement concatenating patterns for -s in conjunction with -z. @@ -1060,6 +945,9 @@ main(int argc, char *argv[]) } } + matchv = argv; + matchc = i; + argc -= i; argv += i; } @@ -1068,14 +956,16 @@ main(int argc, char *argv[]) return EXIT_SUCCESS; } - if ((print_fsm || example) && argc > 0) { + if ((fsm_lang != FSM_PRINT_NONE || example) && argc > 0) { fprintf(stderr, "too many arguments\n"); return EXIT_FAILURE; } - if (!ambig) { + hooks.conflict = conflict; + + if ((opt.ambig | AMBIG_SINGLE)) { struct fsm *dfa; - size_t s; + int r; dfa = fsm_clone(fsm); if (dfa == NULL) { @@ -1083,60 +973,43 @@ main(int argc, char *argv[]) return EXIT_FAILURE; } - { - if (!fsm_determinise(dfa)) { - perror("fsm_determinise"); - return EXIT_FAILURE; - } + if (!fsm_determinise(dfa)) { + perror("fsm_determinise"); + return EXIT_FAILURE; } - for (s = 0; s < dfa->statecount; s++) { - const struct match *matches; - matches = find_first_match_for_end_state(dfa, s); - if (matches == NULL) { - continue; - } - - assert(matches != NULL); - if (matches->next != NULL) { - const struct match *m; - - /* TODO: // delimeters depend on dialect */ - /* TODO: would deal with dialect: prefix here, too */ - - fprintf(stderr, "ambiguous matches for "); - for (m = matches; m != NULL; m = m->next) { - /* TODO: print nicely */ - fprintf(stderr, "/%s/", m->s); - if (m->next != NULL) { - fprintf(stderr, ", "); - } - } + /* + * This is silly, we're doing this just to find conflicts, + * because otherwise we won't find them during print, + * because in re(1) we don't always print. + * + * I don't like that we have two places where we call fsm_print(). + */ + // XXX: stdout is a non-NULL placeholder, would prefer f=NULL because FSM_PRINT_NONE + r = fsm_print(stdout, dfa, &opt, &hooks, FSM_PRINT_NONE); - fprintf(stderr, "; for example on input '"); - printexample(stderr, dfa, s); - fprintf(stderr, "'\n"); + fsm_free(dfa); - /* TODO: consider different error codes */ - return EXIT_FAILURE; - } + if (r < 0) { + exit(EXIT_FAILURE); } - /* TODO: free opaques */ - - fsm_free(dfa); + if (r == 1) { + exit(EXIT_FAILURE); + } } if (!keep_nfa) { /* * Convert to a DFA, then minimise unless we need to keep the end * state information separated per regexp. */ +// TODO: redundant wrt above if (!fsm_determinise(fsm)) { perror("fsm_determinise"); return EXIT_FAILURE; } - if (!patterns && example && print_fsm != fsm_print_c) { + if (!patterns && example && fsm_lang != FSM_PRINT_C) { if (!fsm_minimise(fsm)) { perror("fsm_minimise"); return EXIT_FAILURE; @@ -1158,19 +1031,30 @@ main(int argc, char *argv[]) /* TODO: would deal with dialect: prefix here, too */ if (patterns) { - const struct match *m; - m = find_first_match_for_end_state(fsm, s); - - while (m != NULL) { - /* TODO: print nicely */ - printf("/%s/", m->s); - if (m->next != NULL) { - printf(", "); - } - m = m->next; + fsm_end_id_t *ids; + size_t count, i; + + count = fsm_endid_count(fsm, s); + if (count > 0) { + int res; + + ids = xmalloc(count * sizeof *ids); + + res = fsm_endid_get(fsm, s, count, ids); + assert(res == 1); + + for (i = 0; i < count; i++) { + /* TODO: print nicely */ + printf("/%s/", matchv[ids[i]]); + if (i + 1 < count) { + printf(", "); + } } - printf(": "); + free(ids); + } + + printf(": "); } printexample(stdout, fsm, s); @@ -1183,28 +1067,49 @@ main(int argc, char *argv[]) } if (generate_bounds > 0) { - return fsm_generate_matches(fsm, generate_bounds, fsm_generate_cb_printf_escaped, &opt); + if (!fsm_generate_matches(fsm, generate_bounds, 0, fsm_generate_cb_printf_escaped, &opt)) { + exit(EXIT_FAILURE); + } + + return 0; } - if (print_fsm != NULL) { - /* TODO: print examples in comments for end states; - * patterns in comments for the whole FSM */ - - if (print_fsm == fsm_print_c || print_fsm == fsm_print_vmc) { - opt.endleaf = endleaf_c; - } else if (print_fsm == fsm_print_rust) { - opt.endleaf = endleaf_rust; - } else if (print_fsm == fsm_print_dot || print_fsm == fsm_print_vmdot) { - opt.endleaf = patterns ? endleaf_dot : NULL; - } else if (print_fsm == fsm_print_json) { - opt.endleaf = patterns ? endleaf_json : NULL; + if (fsm_lang != FSM_PRINT_NONE) { + switch (fsm_lang) { + case FSM_PRINT_NONE: + break; + + case FSM_PRINT_C: + case FSM_PRINT_VMC: + hooks.comment = comment_c; + break; + + case FSM_PRINT_RUST: + case FSM_PRINT_GO: /* close enough */ + hooks.comment = comment_rust; + break; + + case FSM_PRINT_LLVM: + hooks.comment = comment_llvm; + break; + + case FSM_PRINT_DOT: + case FSM_PRINT_VMDOT: + hooks.comment = patterns ? comment_dot : NULL; + break; + + case FSM_PRINT_JSON: + break; + + default: + break; } - if (-1 == print_fsm(stdout, fsm)) { + if (-1 == fsm_print(stdout, fsm, &opt, &hooks, fsm_lang)) { if (errno == ENOTSUP) { fprintf(stderr, "unsupported IO API\n"); } else { - perror("print_fsm"); + perror("fsm_print"); } exit(EXIT_FAILURE); } @@ -1242,8 +1147,10 @@ main(int argc, char *argv[]) if (vm != NULL) { e = fsm_vm_match_file(vm, f); + } else if (resolve_captures) { + assert(!"todo"); } else { - e = fsm_exec(fsm, fsm_fgetc, f, &state, NULL); + e = fsm_exec(fsm, fsm_fgetc, f, &state); } fclose(f); @@ -1254,8 +1161,10 @@ main(int argc, char *argv[]) if (vm != NULL) { e = fsm_vm_match_buffer(vm, s, strlen(s)); + } else if (resolve_captures) { + e = exec_with_captures(fsm, fsm_sgetc, &s, &state); } else { - e = fsm_exec(fsm, fsm_sgetc, &s, &state, NULL); + e = fsm_exec(fsm, fsm_sgetc, &s, &state); } } @@ -1265,12 +1174,19 @@ main(int argc, char *argv[]) } if (patterns) { - const struct match *m; - m = find_first_match_for_end_state(fsm, state); - while (m != NULL) { + fsm_end_id_t ids[1]; + size_t count, i; + int res; + + count = fsm_endid_count(fsm, state); + assert(count == 1); + + res = fsm_endid_get(fsm, state, count, ids); + assert(res == 1); + + for (i = 0; i < count; i++) { /* TODO: print nicely */ - printf("match: /%s/\n", m->s); - m = m->next; + printf("match: /%s/\n", matchv[ids[i]]); } } } @@ -1288,3 +1204,56 @@ main(int argc, char *argv[]) return r; } } + +static int +exec_with_captures(struct fsm *fsm, + int (*fsm_getc)(void *opaque), void *opaque, fsm_state_t *end) +{ + int c; + size_t ceil = 16; + size_t used = 0; + unsigned char *buf = malloc(ceil); + size_t i; + size_t capture_ceil; + struct fsm_capture *captures; + int res; + + while (c = fsm_getc(opaque), c != EOF) { + if (used == ceil - 1) { + const size_t nceil = 2*ceil; + unsigned char *nbuf = realloc(buf, nceil); + if (nbuf == NULL) { + free(buf); + return -1; + } + ceil = nceil; + buf = nbuf; + } + buf[used] = c; + used++; + } + buf[used] = '\0'; + + capture_ceil = fsm_capture_ceiling(fsm); + + captures = malloc(capture_ceil * sizeof(captures[0])); + if (captures == NULL) { + free(buf); + return -1; + } + + res = fsm_exec_with_captures(fsm, buf, used, + end, captures, capture_ceil); + if (res == 1) { + for (i = 0; i < capture_ceil; i++) { + printf("-- %zu: %zd,%zd\n", + i, captures[i].pos[0], captures[i].pos[1]); + } + } else { + printf("-- no match\n"); + } + + free(buf); + free(captures); + return res; +} diff --git a/src/retest/cvtpcre.c b/src/retest/cvtpcre.c index 4bcd6e63a..d35b8459d 100644 --- a/src/retest/cvtpcre.c +++ b/src/retest/cvtpcre.c @@ -263,10 +263,6 @@ parse_modifiers(size_t linenum, const char *s, enum re_flags *mods_p) assert(s != NULL); buf = xstrdup(s); - if (buf == NULL) { - return 0; - } - pos = tok = buf; first = true; @@ -431,15 +427,13 @@ state_name(enum state st) return "unk"; } -static const struct fsm_options zero_options; - -int main(int argc, char **argv) +int +main(int argc, char **argv) { struct str l; enum state state; size_t count, nparsed, linenum, regexp_line; int re_ok = 0; - struct fsm_options opt; char **skip; size_t nskip; @@ -458,12 +452,6 @@ int main(int argc, char **argv) l = init_str; state = ST_DEFAULT; - opt = zero_options; - opt.comments = 0; - opt.anonymous_states = 1; - opt.consolidate_edges = 1; - opt.io = FSM_IO_GETC; - count = 0; nparsed = 0; linenum = 0; @@ -623,7 +611,7 @@ int main(int argc, char **argv) comp_err = err_zero; - fsm = re_comp(RE_PCRE, fsm_sgetc, &re, &opt, mods, &comp_err); + fsm = re_comp(RE_PCRE, fsm_sgetc, &re, NULL, mods, &comp_err); re_ok = (fsm != NULL); if (re_ok) { fsm_free(fsm); diff --git a/src/retest/main.c b/src/retest/main.c index b6b4c52f7..08850325b 100644 --- a/src/retest/main.c +++ b/src/retest/main.c @@ -34,7 +34,6 @@ #include #include "libfsm/internal.h" /* XXX */ -#include "libre/print.h" /* XXX */ #include "libre/class.h" /* XXX */ #include "libre/ast.h" /* XXX */ @@ -57,7 +56,6 @@ struct match { struct match *next; }; -static int tty_output = 0; static int do_timing = 0; static int do_watchdog = 0; @@ -179,6 +177,7 @@ usage(void) fprintf(stderr, " vmc compile as per fsm_print_vmc()\n"); fprintf(stderr, " vmops compile as per fsm_print_vmops_{c,h,main}()\n"); fprintf(stderr, " rust compile as per fsm_print_rust()\n"); + fprintf(stderr, " llvm compile as per fsm_print_llvm()\n"); fprintf(stderr, "\n"); fprintf(stderr, " -x \n"); @@ -393,7 +392,7 @@ parse_escapes(char *s, char **errpos, int *lenp) ndig++; } else { - s[j++] = ccode; + s[j++] = (char)ccode; st = ST_BARE; if (!hexcurly) { @@ -738,6 +737,7 @@ flagstring(enum re_flags flags, char buf[16]) */ static int process_test_file(const char *filename, + const struct fsm_alloc *alloc, enum re_dialect default_dialect, enum implementation impl, int max_errors, struct error_record *erec) { static const struct fsm_runner init_runner; @@ -968,16 +968,8 @@ process_test_file(const char *filename, flagstring(flags, &flagdesc[0]); - if (tty_output) { - char *re = dup_str_esc(regexp, NULL); - printf("[ ] line %d: working on %s regexp /%s/%s ...\r", - linenum, dialect_name, re, flagdesc); - fflush(stdout); - free(re); - } - re_str = regexp; - fsm = re_comp(dialect, fsm_sgetc, &re_str, &opt, flags, &err); + fsm = re_comp(dialect, fsm_sgetc, &re_str, alloc, flags, &err); if (fsm == NULL) { fprintf(stderr, "line %d: error with %s regexp /%s/%s: %s\n", linenum, dialect_name, regexp, flagdesc, re_strerror(err.e)); @@ -1048,11 +1040,11 @@ process_test_file(const char *filename, #if DEBUG_VM_FSM fprintf(stderr, "FSM:\n"); - fsm_print_fsm(stderr, fsm); + fsm_dump(stderr, fsm); fprintf(stderr, "---\n"); { FILE *f = fopen("dump.fsm", "w"); - fsm_print_fsm(f, fsm); + fsm_dump(f, fsm); fclose(f); } #endif /* DEBUG_VM_FSM */ @@ -1061,7 +1053,7 @@ process_test_file(const char *filename, fprintf(stderr, "REGEXP matching for /%s/%s\n", regexp, flagdesc); #endif /* DEBUG_TEST_REGEXP */ - ret = fsm_runner_initialize(fsm, &runner, impl, vm_opts); + ret = fsm_runner_initialize(fsm, &opt, &runner, impl, vm_opts); fsm_free(fsm); @@ -1181,7 +1173,7 @@ process_test_file(const char *filename, exit(EXIT_FAILURE); } - return num_errors; + return num_errors + num_re_errors; } static enum fsm_io @@ -1220,15 +1212,15 @@ io(const char *name) int main(int argc, char *argv[]) { + /* TODO: use alloc hooks for -Q accounting as well as watchdog timeouts */ + struct fsm_alloc *alloc = NULL; + enum re_dialect dialect; enum implementation impl; int max_test_errors; int optlevel = 1; - /* is output to a tty or not? */ - tty_output = isatty(fileno(stdout)); - /* note these defaults are the opposite than for fsm(1) */ opt.anonymous_states = 1; opt.consolidate_edges = 1; @@ -1266,22 +1258,24 @@ main(int argc, char *argv[]) break; case 'l': - if (strcmp(optarg, "vm") == 0) { - impl = IMPL_INTERPRET; + if (strcmp(optarg, "asm") == 0) { + impl = IMPL_VMASM; } else if (strcmp(optarg, "c") == 0) { impl = IMPL_C; - } else if (strcmp(optarg, "asm") == 0) { - impl = IMPL_VMASM; - } else if (strcmp(optarg, "vmc") == 0) { - impl = IMPL_VMC; - } else if (strcmp(optarg, "vmops") == 0) { - impl = IMPL_VMOPS; } else if (strcmp(optarg, "go") == 0) { impl = IMPL_GO; } else if (strcmp(optarg, "goasm") == 0) { impl = IMPL_GOASM; + } else if (strcmp(optarg, "llvm") == 0) { + impl = IMPL_LLVM; } else if (strcmp(optarg, "rust") == 0) { impl = IMPL_RUST; + } else if (strcmp(optarg, "vm") == 0) { + impl = IMPL_INTERPRET; + } else if (strcmp(optarg, "vmc") == 0) { + impl = IMPL_VMC; + } else if (strcmp(optarg, "vmops") == 0) { + impl = IMPL_VMOPS; } else { fprintf(stderr, "unknown argument to -l: %s\n", optarg); usage(); @@ -1342,7 +1336,7 @@ main(int argc, char *argv[]) } if (do_watchdog) { - opt.alloc = &watchdog_alloc; + alloc = &watchdog_alloc; } if (argc < 1) { @@ -1378,7 +1372,8 @@ main(int argc, char *argv[]) return EXIT_FAILURE; } - nerrs = process_test_file(argv[i], dialect, impl, max_test_errors, &erec); + nerrs = process_test_file(argv[i], alloc, + dialect, impl, max_test_errors, &erec); if (erec.len > 0) { error_record_print(stderr, &erec); diff --git a/src/retest/reperf.c b/src/retest/reperf.c index b87f16b3f..4536bcf5f 100644 --- a/src/retest/reperf.c +++ b/src/retest/reperf.c @@ -31,7 +31,6 @@ #include #include "libfsm/internal.h" /* XXX */ -#include "libre/print.h" /* XXX */ #include "libre/class.h" /* XXX */ #include "libre/ast.h" /* XXX */ @@ -652,6 +651,9 @@ static enum error_type perf_case_run(struct perf_case *c, enum halt halt, struct timing *t) { + /* TODO: use alloc hooks for -Q accounting */ + struct fsm_alloc *alloc = NULL; + struct fsm *fsm; struct fsm_runner runner; struct str contents; @@ -686,7 +688,7 @@ perf_case_run(struct perf_case *c, enum halt halt, re = c->regexp.data; - fsm = re_comp(c->dialect, fsm_sgetc, &re, &opt, flags, &comp_err); + fsm = re_comp(c->dialect, fsm_sgetc, &re, alloc, flags, &comp_err); if (fsm == NULL) { return ERROR_PARSING_REGEXP; } @@ -726,7 +728,7 @@ perf_case_run(struct perf_case *c, enum halt halt, goto done; } - ret = fsm_runner_initialize(fsm, &runner, c->impl, vm_opts); + ret = fsm_runner_initialize(fsm, &opt, &runner, c->impl, vm_opts); if (ret != ERROR_NONE) { fsm_free(fsm); return ret; @@ -734,11 +736,11 @@ perf_case_run(struct perf_case *c, enum halt halt, #if DEBUG_VM_FSM fprintf(stderr, "FSM:\n"); - fsm_print_fsm(stderr, fsm); + fsm_dump(stderr, fsm); fprintf(stderr, "---\n"); { FILE *f = fopen("dump.fsm", "w"); - fsm_print_fsm(f, fsm); + fsm_dump(f, fsm); fclose(f); } #endif /* DEBUG_VM_FSM */ @@ -1062,6 +1064,7 @@ usage(void) fprintf(stderr, " c compile as per fsm_print_c()\n"); fprintf(stderr, " vmc compile as per fsm_print_vmc()\n"); fprintf(stderr, " rust compile as per fsm_print_rust()\n"); + fprintf(stderr, " llvm compile as per fsm_print_llvm()\n"); fprintf(stderr, "\n"); fprintf(stderr, " -x \n"); @@ -1151,16 +1154,24 @@ main(int argc, char *argv[]) break; case 'l': - if (strcmp(optarg, "vm") == 0) { - impl = IMPL_INTERPRET; + if (strcmp(optarg, "asm") == 0) { + impl = IMPL_VMASM; } else if (strcmp(optarg, "c") == 0) { impl = IMPL_C; + } else if (strcmp(optarg, "go") == 0) { + impl = IMPL_GO; + } else if (strcmp(optarg, "goasm") == 0) { + impl = IMPL_GOASM; + } else if (strcmp(optarg, "llvm") == 0) { + impl = IMPL_LLVM; } else if (strcmp(optarg, "rust") == 0) { impl = IMPL_RUST; - } else if (strcmp(optarg, "asm") == 0) { - impl = IMPL_VMASM; + } else if (strcmp(optarg, "vm") == 0) { + impl = IMPL_INTERPRET; } else if (strcmp(optarg, "vmc") == 0) { impl = IMPL_VMC; + } else if (strcmp(optarg, "vmops") == 0) { + impl = IMPL_VMOPS; } else { fprintf(stderr, "unknown argument to -l: %s\n", optarg); usage(); diff --git a/src/retest/runner.c b/src/retest/runner.c index c5c2cde82..49cd611af 100644 --- a/src/retest/runner.c +++ b/src/retest/runner.c @@ -61,7 +61,10 @@ xmkstemps(char *s) } static int -print(const struct fsm *fsm, enum implementation impl, +print(const struct fsm *fsm, + const struct fsm_options *opt, + const struct fsm_hooks *hooks, + enum implementation impl, char *tmp_src) { int fd_src; @@ -76,7 +79,7 @@ print(const struct fsm *fsm, enum implementation impl, } /* the vmc codegen can emit memcmp() or strncmp() calls */ - if (impl == IMPL_VMC && (fsm_getoptions(fsm)->io == FSM_IO_PAIR || fsm_getoptions(fsm)->io == FSM_IO_STR)) { + if (impl == IMPL_VMC && (opt->io == FSM_IO_PAIR || opt->io == FSM_IO_STR)) { fprintf(f, "#include \n\n"); } @@ -84,17 +87,18 @@ print(const struct fsm *fsm, enum implementation impl, int e; switch (impl) { - case IMPL_C: e = fsm_print_c(f, fsm); break; - case IMPL_RUST: e = fsm_print_rust(f, fsm); break; - case IMPL_VMC: e = fsm_print_vmc(f, fsm); break; - case IMPL_GOASM: e = fsm_print_vmasm_amd64_go(f, fsm); break; - case IMPL_VMASM: e = fsm_print_vmasm_amd64_att(f, fsm); break; - case IMPL_GO: e = fsm_print_go(f, fsm); break; + case IMPL_C: e = fsm_print(f, fsm, opt, hooks, FSM_PRINT_C); break; + case IMPL_RUST: e = fsm_print(f, fsm, opt, hooks, FSM_PRINT_RUST); break; + case IMPL_LLVM: e = fsm_print(f, fsm, opt, hooks, FSM_PRINT_LLVM); break; + case IMPL_VMC: e = fsm_print(f, fsm, opt, hooks, FSM_PRINT_VMC); break; + case IMPL_GOASM: e = fsm_print(f, fsm, opt, hooks, FSM_PRINT_AMD64_GO); break; + case IMPL_VMASM: e = fsm_print(f, fsm, opt, hooks, FSM_PRINT_AMD64_ATT); break; + case IMPL_GO: e = fsm_print(f, fsm, opt, hooks, FSM_PRINT_GO); break; case IMPL_VMOPS: - e = fsm_print_vmops_h(f, fsm) - | fsm_print_vmops_c(f, fsm) - | fsm_print_vmops_main(f, fsm); + e = fsm_print(f, fsm, opt, hooks, FSM_PRINT_VMOPS_H) + | fsm_print(f, fsm, opt, hooks, FSM_PRINT_VMOPS_C) + | fsm_print(f, fsm, opt, hooks, FSM_PRINT_VMOPS_MAIN); break; case IMPL_INTERPRET: @@ -114,13 +118,12 @@ print(const struct fsm *fsm, enum implementation impl, fprintf(f, "use std::slice;\n"); fprintf(f, "\n"); - // XXX: fsm_main shouldn't return Option, Option or a bitfield type fprintf(f, "#[no_mangle]\n"); - fprintf(f, "pub extern \"C\" fn retest_trampoline(ptr: *const c_uchar, len: usize) -> i64 {\n"); + fprintf(f, "pub extern \"C\" fn retest_trampoline(ptr: *const c_uchar, len: usize) -> bool {\n"); fprintf(f, " let a: &[u8] = unsafe { slice::from_raw_parts(ptr, len as usize) };\n"); fprintf(f, " match fsm_main(a) {\n"); - fprintf(f, " Some(u) => u as i64,\n"); // XXX: we lose info for the i64 ABI - fprintf(f, " None => -1,\n"); + fprintf(f, " Some(()) => true,\n"); + fprintf(f, " None => false,\n"); fprintf(f, " }\n"); fprintf(f, "}\n"); } @@ -145,7 +148,7 @@ compile(enum implementation impl, case IMPL_C: case IMPL_VMC: case IMPL_VMOPS: - if (0 != systemf("%s %s -xc -shared -fPIC %s -o %s", + if (0 != systemf("%s %s -shared -fPIC %s -o %s", cc ? cc : "gcc", cflags ? cflags : "-std=c89 -pedantic -Wall -Werror -O3", tmp_src, tmp_so)) { @@ -155,7 +158,7 @@ compile(enum implementation impl, break; case IMPL_RUST: - if (0 != systemf("%s %s --crate-type dylib %s -o %s", + if (0 != systemf("%s %s -C opt-level=3 --crate-type dylib %s -o %s", "rustc", "--edition 2021", tmp_src, tmp_so)) { @@ -164,6 +167,16 @@ compile(enum implementation impl, break; + case IMPL_LLVM: + if (0 != systemf("clang %s -shared -fPIC -mllvm -opaque-pointers %s -o %s", + cflags ? cflags : "-pedantic -Wall -Werror -Wno-override-module -O3", + tmp_src, tmp_so)) + { + return 0; + } + + break; + case IMPL_GO: case IMPL_GOASM: { char tmp_o1[] = "/tmp/fsmcompile_o1-XXXXXX.o"; @@ -242,8 +255,12 @@ compile(enum implementation impl, return 0; } - if (0 != systemf("%s %s -shared %s -o %s", + if (0 != systemf("%s %s -shared %s %s -o %s", cc ? cc : "gcc", cflags ? cflags : "", + + // for "missing .note.GNU-stack section implies executable stack" + "-Wl,-z,noexecstack", + tmp_o, tmp_so)) { return 0; @@ -271,7 +288,9 @@ compile(enum implementation impl, } static enum error_type -runner_init_compiled(struct fsm *fsm, struct fsm_runner *r, enum implementation impl) +runner_init_compiled(struct fsm *fsm, + const struct fsm_options *opt, + struct fsm_runner *r, enum implementation impl) { void *h; @@ -283,6 +302,7 @@ runner_init_compiled(struct fsm *fsm, struct fsm_runner *r, enum implementation char tmp_src_go[] = "/tmp/fsmcompile_src-XXXXXX.go"; char tmp_src_c[] = "/tmp/fsmcompile_src-XXXXXX.c"; char tmp_src_rs[] = "/tmp/fsmcompile_src-XXXXXX.rs"; + char tmp_src_ll[] = "/tmp/fsmcompile_src-XXXXXX.ll"; char tmp_src_s[] = "/tmp/fsmcompile_src-XXXXXX.s"; char *tmp_src; @@ -291,6 +311,7 @@ runner_init_compiled(struct fsm *fsm, struct fsm_runner *r, enum implementation case IMPL_C: case IMPL_VMC: tmp_src = tmp_src_c; break; case IMPL_RUST: tmp_src = tmp_src_rs; break; + case IMPL_LLVM: tmp_src = tmp_src_ll; break; case IMPL_GOASM: case IMPL_VMASM: tmp_src = tmp_src_s; break; case IMPL_GO: tmp_src = tmp_src_go; break; @@ -300,7 +321,8 @@ runner_init_compiled(struct fsm *fsm, struct fsm_runner *r, enum implementation abort(); } - if (!print(fsm, r->impl, tmp_src)) { + /* we don't override the print hooks for retest */ + if (!print(fsm, opt, NULL, r->impl, tmp_src)) { return ERROR_FILE_IO; } @@ -352,16 +374,21 @@ runner_init_compiled(struct fsm *fsm, struct fsm_runner *r, enum implementation case IMPL_RUST: r->u.impl_rust.h = h; - r->u.impl_rust.func = (int64_t (*)(const unsigned char *, size_t)) (uintptr_t) dlsym(h, "retest_trampoline"); + r->u.impl_rust.func = (bool (*)(const unsigned char *, size_t)) (uintptr_t) dlsym(h, "retest_trampoline"); + break; + + case IMPL_LLVM: + r->u.impl_llvm.h = h; + r->u.impl_llvm.func = (bool (*)(const char *, const char *)) (uintptr_t) dlsym(h, "fsm.main"); break; case IMPL_GO: case IMPL_GOASM: r->u.impl_go.h = h; - r->u.impl_go.func = (int64_t (*)(const unsigned char *, int64_t)) (uintptr_t) dlsym(h, "retest_trampoline"); + r->u.impl_go.func = (bool (*)(const unsigned char *, int64_t)) (uintptr_t) dlsym(h, "retest_trampoline"); if (r->u.impl_go.func == NULL) { /* Sometime we need a leading underscore. */ - r->u.impl_go.func = (int64_t (*)(const unsigned char *, int64_t)) (uintptr_t) dlsym(h, "_retest_trampoline"); + r->u.impl_go.func = (bool (*)(const unsigned char *, int64_t)) (uintptr_t) dlsym(h, "_retest_trampoline"); } break; @@ -378,7 +405,8 @@ runner_init_compiled(struct fsm *fsm, struct fsm_runner *r, enum implementation } enum error_type -fsm_runner_initialize(struct fsm *fsm, struct fsm_runner *r, enum implementation impl, struct fsm_vm_compile_opts vm_opts) +fsm_runner_initialize(struct fsm *fsm, const struct fsm_options *opt, + struct fsm_runner *r, enum implementation impl, struct fsm_vm_compile_opts vm_opts) { static const struct fsm_runner zero; struct fsm_dfavm *vm; @@ -390,16 +418,17 @@ fsm_runner_initialize(struct fsm *fsm, struct fsm_runner *r, enum implementation switch (impl) { case IMPL_C: + case IMPL_LLVM: case IMPL_RUST: case IMPL_VMASM: case IMPL_VMC: case IMPL_VMOPS: case IMPL_GO: case IMPL_GOASM: - return runner_init_compiled(fsm, r, impl); + return runner_init_compiled(fsm, opt, r, impl); case IMPL_INTERPRET: - vm = fsm_vm_compile_with_options(fsm, vm_opts); + vm = fsm_vm_compile_with_options(fsm, opt, vm_opts); if (vm == NULL) { fsm_free(fsm); return ERROR_COMPILING_BYTECODE; @@ -432,6 +461,12 @@ fsm_runner_finalize(struct fsm_runner *r) } break; + case IMPL_LLVM: + if (r->u.impl_llvm.h != NULL) { + dlclose(r->u.impl_llvm.h); + } + break; + case IMPL_GO: case IMPL_GOASM: if (r->u.impl_go.h != NULL) { @@ -467,20 +502,24 @@ fsm_runner_run(const struct fsm_runner *r, const char *s, size_t n) case IMPL_VMC: case IMPL_VMOPS: assert(r->u.impl_c.func != NULL); - return r->u.impl_c.func(s, s+n) >= 0; + return r->u.impl_c.func(s, s + n); case IMPL_RUST: assert(r->u.impl_rust.func != NULL); - return r->u.impl_rust.func((const unsigned char *)s, n) >= 0; + return r->u.impl_rust.func((const unsigned char *) s, n); + + case IMPL_LLVM: + assert(r->u.impl_llvm.func != NULL); + return r->u.impl_llvm.func(s, s + n); case IMPL_GO: case IMPL_GOASM: assert(r->u.impl_go.func != NULL); - return r->u.impl_go.func((const unsigned char *)s, n) >= 0; + return r->u.impl_go.func((const unsigned char *) s, n); case IMPL_VMASM: assert(r->u.impl_asm.func != NULL); - return r->u.impl_asm.func((const unsigned char *)s, n) >= 0; + return r->u.impl_asm.func((const unsigned char *) s, n); case IMPL_INTERPRET: assert(r->u.impl_vm.vm != NULL); diff --git a/src/retest/runner.h b/src/retest/runner.h index 1df4d546f..40b269eb0 100644 --- a/src/retest/runner.h +++ b/src/retest/runner.h @@ -6,6 +6,7 @@ #include struct fsm; +struct fsm_options; struct fsm_dfavm; enum error_type { @@ -26,12 +27,13 @@ enum error_type { enum implementation { IMPL_C, - IMPL_RUST, IMPL_GO, IMPL_GOASM, - IMPL_VMC, - IMPL_VMASM, IMPL_INTERPRET, + IMPL_LLVM, + IMPL_RUST, + IMPL_VMASM, + IMPL_VMC, IMPL_VMOPS, }; @@ -47,12 +49,17 @@ struct fsm_runner { struct { /* pub extern "C" fn f(ptr: *const c_uchar, len: usize) -> usize */ void *h; - int64_t (*func)(const unsigned char *, size_t); + bool (*func)(const unsigned char *, size_t); } impl_rust; struct { void *h; - int64_t (*func)(const unsigned char *, int64_t); + bool (*func)(const char *, const char *); + } impl_llvm; + + struct { + void *h; + bool (*func)(const unsigned char *, int64_t); } impl_go; struct { @@ -67,7 +74,8 @@ struct fsm_runner { }; enum error_type -fsm_runner_initialize(struct fsm *fsm, struct fsm_runner *r, enum implementation impl, struct fsm_vm_compile_opts vm_opts); +fsm_runner_initialize(struct fsm *fsm, const struct fsm_options *opt, + struct fsm_runner *r, enum implementation impl, struct fsm_vm_compile_opts vm_opts); void fsm_runner_finalize(struct fsm_runner *r); diff --git a/src/rx/Makefile b/src/rx/Makefile new file mode 100644 index 000000000..2facbfd6c --- /dev/null +++ b/src/rx/Makefile @@ -0,0 +1,14 @@ +.include "../../share/mk/top.mk" + +SRC += src/rx/main.c + +PROG += rx + +.for lib in ${LIB:Mlibfsm} ${LIB:Mlibre} +${BUILD}/bin/rx: ${BUILD}/lib/${lib:R}.a +.endfor + +.for src in ${SRC:Msrc/rx/*.c} +${BUILD}/bin/rx: ${BUILD}/${src:R}.o +.endfor + diff --git a/src/rx/main.c b/src/rx/main.c new file mode 100644 index 000000000..4ac357bc8 --- /dev/null +++ b/src/rx/main.c @@ -0,0 +1,1418 @@ +/* + * Copyright 2024 Katherine Flavel + * + * See LICENCE for the full copyright terms. + */ + +#define _POSIX_C_SOURCE 200809L + +#ifdef __linux__ +#include +#endif + +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +/* placeholder for alloc hooks */ +static const struct fsm_alloc *alloc; + +enum category { + CATEGORY_EMPTY, + CATEGORY_LITERAL, + CATEGORY_GENERAL, + CATEGORY_ERROR, + CATEGORY_UNSUPPORTED, + CATEGORY_UNSATISFIABLE, + CATEGORY_NOT_PERMITTED +}; + +struct pattern { + fsm_end_id_t id; /* either per file or per pattern */ + enum re_dialect dialect; + const char *s; +}; + +struct literal { + fsm_end_id_t id; /* either per file or per literal */ + const void *p; + size_t n; +}; + +struct literal_set { + size_t count; + struct literal *a; +}; + +struct pattern_set { + size_t count; + struct pattern *a; +}; + +struct check_env { + struct literal_set (*literals)[4]; + const struct pattern_set *general; +}; + +// TODO: centralise, maybe adt/alloc.h +static FILE * +xopen(const char *s) +{ + FILE *f; + + assert(s != NULL); + + if (0 == strcmp(s, "-")) { + return stdin; + } + + f = fopen(s, "r"); + if (f == NULL) { + perror(s); + exit(EXIT_FAILURE); + } + + return f; +} + +// TODO: centralise +static char * +xgetline(FILE *f) +{ + char *buf; + size_t len; + ssize_t n; + + buf = NULL; + len = 0; + + n = getline(&buf, &len, f); + if (n == -1) { + free(buf); + + if (feof(f)) { + return NULL; + } + + perror("getline"); + exit(EXIT_FAILURE); + } + + /* n == 0 for an empty line */ + if (n > 0) { + assert(buf[n - 1] == '\n'); + buf[n - 1] = '\0'; + } + + return buf; +} + +static void +append_pattern(struct pattern_set *set, + fsm_end_id_t id, enum re_dialect dialect, const char *s) +{ + const size_t low = 16; /* must be power of 2 */ + const size_t factor = 2; /* must be even */ + + assert(set != NULL); + assert(s != NULL); + + if (set->count == 0) { + set->a = xmalloc(low * sizeof *set->a); + } else if (set->count >= low && (set->count & (set->count - 1)) == 0) { + size_t new = set->count * factor; + if (new < set->count) { + errno = E2BIG; + perror("realloc"); + exit(EXIT_FAILURE); + } + + set->a = xrealloc(set->a, new * sizeof *set->a); + } + + set->a[set->count].id = id; + set->a[set->count].dialect = dialect; + set->a[set->count].s = s; + set->count++; +} + +static void +append_literal(struct literal_set *set, + const char *p, size_t n, fsm_end_id_t id) +{ + const size_t low = 16; /* must be power of 2 */ + const size_t factor = 2; /* must be even */ + + assert(set != NULL); + assert(p != NULL); + + if (set->count == 0) { + set->a = xmalloc(low * sizeof *set->a); + } else if (set->count >= low && (set->count & (set->count - 1)) == 0) { + size_t new = set->count * factor; + if (new < set->count) { + errno = E2BIG; + perror("realloc"); + exit(EXIT_FAILURE); + } + + set->a = xrealloc(set->a, new * sizeof *set->a); + } + + set->a[set->count].id = id; + set->a[set->count].p = p; + set->a[set->count].n = n; + set->count++; +} + +static enum re_strings_flags +literal_flags(enum re_literal_category category) +{ + switch (category) { + case RE_LITERAL_UNANCHORED: + return 0; + + case RE_LITERAL_ANCHOR_START: + return RE_STRINGS_ANCHOR_LEFT; + + case RE_LITERAL_ANCHOR_END: + return RE_STRINGS_ANCHOR_RIGHT; + + case RE_LITERAL_ANCHOR_BOTH: + return RE_STRINGS_ANCHOR_LEFT | RE_STRINGS_ANCHOR_RIGHT; + + default: + assert(!"unreached"); + abort(); + } +} + +/* this interface doesn't allow us to have \0 in the character set */ +static struct fsm * +intersect_charset(bool show_stats, const char *charset, struct fsm *fsm) +{ + struct fsm *q; + + assert(charset != NULL); + assert(fsm_all(fsm, fsm_isdfa)); + + size_t pre = fsm_countstates(fsm); + if (show_stats) { + fprintf(stderr, "pre-intersection dfa: %zu states\n", pre); + } + + q = fsm_intersect_charset(fsm, strlen(charset), charset); + if (q == NULL) { + perror("fsm_intersect_charset"); + exit(EXIT_FAILURE); + } + + /* walking two DFA produces a DFA */ + assert(fsm_all(q, fsm_isdfa)); + + size_t post = fsm_countstates(q); + if (show_stats) { + fprintf(stderr, "post-intersection dfa: %zu states\n", post); + } + + /* intersecting with the charset FSM should never introduce new states */ + assert(post <= pre); + + return q; +} + +const char * +category_reason(enum category r) +{ + switch (r) { + case CATEGORY_EMPTY: return "empty string"; + case CATEGORY_LITERAL: return "literal"; + case CATEGORY_GENERAL: return "general"; + case CATEGORY_ERROR: return "parse error"; + case CATEGORY_UNSUPPORTED: return "unsupported"; + case CATEGORY_UNSATISFIABLE: return "unsatisfiable"; + case CATEGORY_NOT_PERMITTED: return "charset/reject"; + + default: + assert(!"unreached"); + abort(); + } +} + +/* return an error reason if a pattern was not successfully appended to a set */ +static enum category +categorize(enum re_dialect dialect, enum re_flags flags, + const char *charset, const char *reject, const char *s, + struct re_err *err, + const char **lit_s_out, size_t *lit_n_out, enum re_strings_flags *strings_flags) +{ + enum re_literal_category category; + char *lit_s; + size_t lit_n; + int r; + + assert(err != NULL); + assert(lit_s_out != NULL); + assert(lit_n_out != NULL); + + /* + * As an optimisation, we skip libre's parsing for RE_LITERAL and + * categorize empirically based on the dialect. We can't do this + * inside re_is_literal() because that uses the getc callback, + * and this shortcut depends on the storage for the input string s. + */ + if (dialect == RE_LITERAL) { + *lit_s_out = s; + *lit_n_out = strlen(s); + *strings_flags = RE_STRINGS_ANCHOR_LEFT | RE_STRINGS_ANCHOR_RIGHT; + return CATEGORY_LITERAL; + } + + /* + * -1: Error + * 1: Literal, *s may or may not be NULL (i.e. for an empty string), *n >= 0 + * and *category is set. + */ + const char *q = s; + r = re_is_literal(dialect, fsm_sgetc, &q, flags, err, &category, + &lit_s, &lit_n); + + /* unsupported */ + if (r == -1 && err->e == RE_EUNSUPPORTED) { + return CATEGORY_UNSUPPORTED; + } + + /* parse error */ + if (r == -1) { + return CATEGORY_ERROR; + } + + /* empty string */ + if (r == 1 && lit_n == 0) { + assert(lit_s == NULL); + return CATEGORY_EMPTY; + } + + /* unsatisfiable */ + if (r == 1 && category == RE_LITERAL_UNSATISFIABLE) { + assert(lit_s == NULL); + return CATEGORY_UNSATISFIABLE; + } + + /* is a literal */ + if (r == 1) { + assert(lit_s != NULL); + + if (category == RE_LITERAL_ANCHOR_END || category == RE_LITERAL_ANCHOR_BOTH) { + /* + * This newline is appended in ast_is_literal() as an attempt to + * account for the PCRE2_DOTALL behaviour. I'm not interested in + * newlines for this program (optional or legitimate), + * so I'm just cutting it off. + */ +// TODO: maybe only if it's not in the character set? + if (lit_n > 0 && lit_s[lit_n - 1] == '\n') { + lit_n--; + } + } + + /* + * The reject set does not apply for literals, + * it's intended only for regex operators. + * + * We're walking lit_s longhand here (rather than strcspn) + * because it isn't a \0-terminated string. + */ + if (charset != NULL) { + for (size_t i = 0; i < lit_n; i++) { + if (!strchr(charset, lit_s[i])) { + free(lit_s); + return CATEGORY_NOT_PERMITTED; + } + } + } + + *lit_s_out = lit_s; + *lit_n_out = lit_n; + *strings_flags = literal_flags(category); + + return CATEGORY_LITERAL; + } + + /* + * XXX: The charset accidentally includes regex operators, + * because we're cheesily looking at the string pre-parse, + * we need those in the charset too, in addition to the + * charset matching literals within the regex. + * + * So we shouldn't enforce positive charset pre-lexing here, + * this would need to be done at AST time. Until this is done, + * I'm just skipping the reject set. + */ + (void) reject; +#if 0 + /* + * Reject regexps that contain characters that may be expensive. + * + * If the regex contains a character that's not in the charset, + * we can also save some compilation time and decline it up front. + * + * This isn't perfect, for example consider hex encoded values. + * My design assumption here is that it's okay to pre-filter strings + * down to a character set. This means /./ doesn't match any character, + * it means any character within the charset. + * + * So we're relying on that here, and this is just a best-effort help. + */ + if (reject != NULL && s[strcspn(s, reject)]) { + return CATEGORY_NOT_PERMITTED; + } +#endif + + /* not a literal */ + return CATEGORY_GENERAL; +} + +/* + * This is re_strings() but for an array of struct literal + */ +static struct fsm * +literal_strings(const struct literal *a, size_t n, enum re_strings_flags flags) +{ + struct re_strings *g; + struct fsm *fsm; + + assert(a != NULL || n == 0); + + g = re_strings_new(); + if (g == NULL) { + return NULL; + perror("re_strings_new"); + exit(EXIT_FAILURE); + } + + for (size_t i = 0; i < n; i++) { + if (!re_strings_add_raw(g, a[i].p, a[i].n, & (fsm_end_id_t) { a[i].id })) { + perror("re_strings_add_raw"); + exit(EXIT_FAILURE); + } + } + + fsm = re_strings_build(g, alloc, flags); + re_strings_free(g); + + /* this is already a DFA, courtesy of re_strings() */ + assert(fsm_all(fsm, fsm_isdfa)); + + return fsm; +} + +static struct fsm * +build_literals_fsm(bool show_stats, + const char *charset, + struct literal_set *set, enum re_strings_flags flags) +{ + struct fsm *fsm; + + assert(set != NULL); + + fsm = literal_strings(set->a, set->count, flags); + + /* we do produce an empty fsm for 0 literals */ + if (set->count == 0) { + assert(fsm_empty(fsm)); + } else { + assert(!fsm_empty(fsm)); + } + + if (charset != NULL) { + fsm = intersect_charset(show_stats, charset, fsm); + if (fsm == NULL) { + perror("intersect_charset"); + exit(EXIT_FAILURE); + } + + if (fsm_countstates(fsm) == 0) { + fprintf(stderr, "literals produced empty intersection\n"); + exit(EXIT_FAILURE); + } + } + +#ifndef NDEBUG + /* + * We could test to see that the fsm isn't any different. + * For general regexps, lots will be different because of + * mapping down /./ edges to the character set. + * + * But literal strings should be no different after the mapping, + * except for the implicit .* at either end for anchoring. + * + * So in order to assert here, we'd depend on constructing + * those .* long-hand at union time. + */ +#endif + + /* We don't minimise here because this fsm has multiple endids, + * and the resulting FSM would be very similar to the current DFA */ + + return fsm; +} + +static struct fsm * +build_pattern_fsm(bool show_stats, + const char *charset, + enum re_dialect dialect, bool strict, + const char *s, enum re_flags flags, fsm_end_id_t id) +{ + struct fsm *fsm; + struct re_err err; + + assert(s != NULL); + +// TODO: want to compile these as if anchored, then do the unanchored parts during unioning +// then we can also construct the unanchored parts from the charset + + const char *q = s; +// TODO: RE_SINGLE here? i think it shouldn't have any effect when \n isn't present in the charset + fsm = re_comp(dialect, fsm_sgetc, &q, alloc, flags, &err); + + if (strict && fsm == NULL) { + re_perror(dialect, &err, NULL, s); + exit(EXIT_FAILURE); + } + + if (fsm == NULL && err.e == RE_EUNSUPPORTED) { + return NULL; + } + + /* + * An unsatisfiable regex compiles to an FSM that matches nothing, + * so that unioning it with other regexes will still work. + */ + if (strict && fsm_empty(fsm)) { + fprintf(stdout, "re_comp: /%s/ regex is unsatisfiable\n", s); + exit(EXIT_FAILURE); + } + + /* + * There's no need to minimise here, we only need a DFA in order + * to fsm_intersect_charset(). We'd want to minimise again anyway. + */ + if (!fsm_determinise(fsm)) { + perror("fsm_determinise"); + exit(EXIT_FAILURE); + } + + assert(!fsm_empty(fsm)); + + if (charset != NULL) { +// XXX: this is the wrong operation! +// XXX: i *think* intersect_charset must be doing the wrong thing. somehow + (void) charset; + (void) show_stats; +/* + fsm = intersect_charset(show_stats, charset, fsm); + if (fsm == NULL) { + perror("intersect_charset"); + exit(EXIT_FAILURE); + } +*/ + } + + /* + * This FSM has only one endid. fsm_minimise() does now maintain + * uniqueness for endids, but here we get the same FSM as if they + * didn't exist at all. + */ + if (!fsm_minimise(fsm)) { + perror("fsm_minimise"); + exit(EXIT_FAILURE); + } + + assert(!fsm_empty(fsm)); + + if (!fsm_setendid(fsm, id)) { + perror("fsm_setendid"); + exit(EXIT_FAILURE); + } + + return fsm; +} + +static int +conflict(FILE *f, const struct fsm_options *opt, + const struct fsm_state_metadata *state_metadata, + const char *example, + void *hook_opaque) +{ + const struct check_env *env = hook_opaque; + + assert(env != NULL); + assert(env->literals != NULL); + assert(env->general != NULL); + + /* in rx all end states have an end id */ + assert(state_metadata->end_id_count > 0); + + /* ...and we only conflict when there's more than one */ + assert(state_metadata->end_id_count > 1); + + (void) f; + (void) opt; + + fprintf(stderr, "error: ambiguous patterns"); + if (example != NULL) { + fprintf(stderr, ", for example on input '%s':", example); + } + fprintf(stderr, "\n"); + + for (fsm_end_id_t i = 0; i < state_metadata->end_id_count; i++) { + for (size_t k = 0; k < sizeof *env->literals / sizeof **env->literals; k++) { + const struct literal_set *literals = &(*env->literals)[k]; + for (size_t j = 0; j < literals->count; j++) { + if (literals->a[j].id == state_metadata->end_ids[i]) { + // TODO: escape, centralise with libre's re_perror() + fprintf(stderr, "#%u: /%s%.*s%s/\n", + state_metadata->end_ids[i], + (k & RE_STRINGS_ANCHOR_LEFT) ? "^" : "", + (int) literals->a[j].n, (const char *) literals->a[j].p, + (k & RE_STRINGS_ANCHOR_RIGHT) ? "$" : ""); + goto next; + } + } + } + + for (size_t j = 0; j < env->general->count; j++) { + // TODO: delimiters per dialect, centralise with libre's re_perror() + if (env->general->a[i].id == state_metadata->end_ids[i]) { + fprintf(stderr, "#%u: /%s/\n", + state_metadata->end_ids[i], + env->general->a[i].s); + goto next; + } + } + +next: ; + } + + return 0; +} + +static enum re_dialect +dialect_name(const char *name) +{ + size_t i; + + struct { + const char *name; + enum re_dialect dialect; + } a[] = { + { "like", RE_LIKE }, + { "literal", RE_LITERAL }, + { "glob", RE_GLOB }, + { "native", RE_NATIVE }, + { "pcre", RE_PCRE }, + { "sql", RE_SQL } + }; + + assert(name != NULL); + + for (i = 0; i < sizeof a / sizeof *a; i++) { + if (0 == strcmp(a[i].name, name)) { + return a[i].dialect; + } + } + + fprintf(stderr, "unrecognised regexp dialect \"%s\"; valid dialects are: ", name); + + for (i = 0; i < sizeof a / sizeof *a; i++) { + fprintf(stderr, "%s%s", + a[i].name, + i + 1 < sizeof a / sizeof *a ? ", " : "\n"); + } + + exit(EXIT_FAILURE); +} + +static enum fsm_io +io(const char *name) +{ + size_t i; + + struct { + const char *name; + enum fsm_io io; + } a[] = { + { "getc", FSM_IO_GETC }, + { "str", FSM_IO_STR }, + { "pair", FSM_IO_PAIR } + }; + + assert(name != NULL); + + for (i = 0; i < sizeof a / sizeof *a; i++) { + if (0 == strcmp(a[i].name, name)) { + return a[i].io; + } + } + + fprintf(stderr, "unrecognised IO API; valid IO APIs are: "); + + for (i = 0; i < sizeof a / sizeof *a; i++) { + fprintf(stderr, "%s%s", + a[i].name, + i + 1 < sizeof a / sizeof *a ? ", " : "\n"); + } + + exit(EXIT_FAILURE); +} + +static enum fsm_print_lang +print_name(const char *name, enum fsm_ambig ambig) +{ + size_t i; + bool failed_mask; + + struct { + const char *name; + enum fsm_print_lang lang; + enum fsm_ambig mask; + } a[] = { + { "c", FSM_PRINT_VMC, AMBIG_ANY }, + { "dot", FSM_PRINT_DOT, AMBIG_ANY }, + { "fsm", FSM_PRINT_FSM, AMBIG_ANY }, + { "rust", FSM_PRINT_RUST, AMBIG_ANY }, + { "go", FSM_PRINT_GO, AMBIG_ANY }, + { "ir", FSM_PRINT_IR, AMBIG_ANY }, + { "irjson", FSM_PRINT_IRJSON, AMBIG_ANY }, + { "llvm", FSM_PRINT_LLVM, AMBIG_ANY }, + { "vmdot", FSM_PRINT_VMDOT, AMBIG_ANY }, + { "vmops_c", FSM_PRINT_VMOPS_C, AMBIG_ANY }, + { "vmops_h", FSM_PRINT_VMOPS_H, AMBIG_ANY }, + { "vmops_main", FSM_PRINT_VMOPS_MAIN, AMBIG_ANY }, + { "amd64_att", FSM_PRINT_AMD64_ATT, AMBIG_ERROR }, + { "amd64_nasm", FSM_PRINT_AMD64_NASM, AMBIG_ERROR }, + { "amd64_go", FSM_PRINT_AMD64_GO, AMBIG_ERROR } + }; + + assert(name != NULL); + + failed_mask = false; + + for (i = 0; i < sizeof a / sizeof *a; i++) { + if (0 != strcmp(a[i].name, name)) { + continue; + } + + if ((a[i].mask & ambig)) { + return a[i].lang; + } + + failed_mask = true; + } + + if (failed_mask) { + fprintf(stderr, "unsupported ambig mode for output language\n"); + exit(EXIT_FAILURE); + } + + fprintf(stderr, "unrecognised output language; valid languages are: "); + + for (i = 0; i < sizeof a / sizeof *a; i++) { + fprintf(stderr, "%s%s", + a[i].name, + i + 1 < sizeof a / sizeof *a ? ", " : "\n"); + } + + exit(EXIT_FAILURE); +} + +/* TODO: centralize */ +static void +parse_flags(const char *arg, enum re_flags *flags) +{ + for (; *arg; arg++) { + switch (*arg) { + case 'b': + *flags = *flags | RE_ANCHORED; + break; + + case 'i': + *flags = *flags | RE_ICASE; + break; + + case 's': + *flags = *flags | RE_SINGLE; + break; + + case 'x': + *flags = *flags | RE_EXTENDED; + break; + + /* others? */ + + default: + printf("unknown flag '%c'\n", *arg); + exit(EXIT_FAILURE); + } + } +} + +static void +usage(const char *name) +{ + + if (name == NULL) { + name = ""; + } else { + const char *p = strrchr(name, '/'); + name = p != NULL ? p + 1 : name; + } + + printf("usage: %s: [-aciQqstuvwXx] [-C charset] [-k io] [-l ] [-r dialect] [-R reject] [-d declined-file] [-E ] [-e ] input-file...\n", name); + printf(" %s -h\n", name); +} + +int +main(int argc, char *argv[]) +{ + bool quiet = false; + bool strict = false; + bool verbose = false; + bool show_stats = false; + bool override_dialect = false; + bool unanchored_literals = false; + const char *lang = "c"; + enum fsm_print_lang fsm_lang = FSM_PRINT_VMC; + + size_t general_limit = 0; + + const char *charset = NULL; + const char *declined_file = NULL; + + enum re_dialect default_dialect = RE_PCRE; + + /* + * TODO: we need to provide a way to disable RE_END_NL + * TODO: RE_SINGLE. also be careful, flags disrupt re_is_literal() + */ + enum re_flags flags = 0; // TODO: notably not RE_END_NL + + /* regex syntax characters we decline at compile time + * because NFA->DFA is too costly */ + const char *reject = ""; + + struct fsm_options opt = { + .anonymous_states = true, + .consolidate_edges = true, + .comments = false, + .case_ranges = true, + .always_hex = false, + .group_edges = true, + .io = FSM_IO_PAIR, + .ambig = AMBIG_ERROR + }; + +// TODO: prepend comment with rx invocation to output +// TODO: cli option for memory limit to decline patterns (implement via allocation hooks) +// TODO: consider de-duplicating arrays. not sure there's any reason +// TODO: decide if i want dialect==RE_LITERAL unanchored or not, maybe a cli flag either way + + { + const char *name = argv[0]; + int c; + + while (c = getopt(argc, argv, "h" "aC:cd:E:e:ik:F:l:n:r:R:stQquvwXx"), c != -1) { + switch (c) { + case 'a': + opt.anonymous_states = false; + break; + + case 'C': + charset = optarg; + break; + + case 'c': + /* + * https://www.rfc-editor.org/rfc/rfc9110#name-tokens + * + * tchar = "!" / "#" / "$" / "%" / "&" / "'" / "*" + * / "+" / "-" / "." / "^" / "_" / "`" / "|" / "~" + * / DIGIT / ALPHA + * ; any VCHAR, except delimiters + * + * https://www.rfc-editor.org/rfc/rfc9110#appendix-A + * field-name = token + * token = 1*tchar + * + * So a valid header field-name has at least one character. + * But we don't enforce that here. + */ +// TODO: although we could, with the charset FSM as [a-z]+ rather than [a-z]* + charset = + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwzyx" + "0123456789" + "!#$%&'*+-.^_`|~"; + break; + + case 'd': + declined_file = optarg; + break; + + case 'E': + opt.package_prefix = optarg; + break; + + case 'e': + opt.prefix = optarg; + break; + + case 'h': + usage(name); + exit(EXIT_SUCCESS); + + case 'i': + strict = true; + break; + + case 'k': + opt.io = io(optarg); + break; + + case 'l': + lang = optarg; + break; + + case 'F': + parse_flags(optarg, &flags); + break; + + case 'n': + general_limit = atoi(optarg); + break; + + case 'R': + reject = optarg; + + /* XXX: see categorize() */ + fprintf(stderr, "reject charset not implemented\n"); + exit(EXIT_FAILURE); + break; + + case 'r': + default_dialect = dialect_name(optarg); + break; + + case 's': + /* override dialect by file extension */ + override_dialect = true; + break; + + case 'Q': + /* one-off stats on resource consumption */ + show_stats = true; + break; + + case 'q': + /* "quiet" applies to the generated code only */ + quiet = true; + break; + + case 't': + opt.ambig = AMBIG_EARLIEST; + break; + + case 'u': + opt.ambig = AMBIG_MULTIPLE; + break; + + case 'v': + /* "verbose" means showing information about every pattern and FSM, + * this is O(n) output to the length of the input file */ + verbose = true; + break; + + case 'w': + opt.fragment = true; + break; + + case 'X': + opt.always_hex = true; + break; + + case 'x': + unanchored_literals = true; + break; + + case '?': + default: + usage(name); + exit(EXIT_FAILURE); + } + } + + argc -= optind; + argv += optind; + + if (argc == 0) { + usage(name); + exit(EXIT_FAILURE); + } + } + + fsm_lang = print_name(lang, opt.ambig); + + if (quiet) { + fsm_lang = FSM_PRINT_NONE; + } + + if (charset != NULL && strchr(charset, '\n')) { + fprintf(stderr, "charset cannot contain \\n " + "(because regexes are delimited by newlines)\n"); + exit(EXIT_FAILURE); + } + + if (show_stats) { + fprintf(stderr, "charset: [%s]\n", + charset == NULL ? "(none)" : charset); + fprintf(stderr, "reject: [%s]\n", reject); + fprintf(stderr, "flags: %#x\n", (unsigned) flags); + } + + /* + * Phases: + * + * 1. per pattern, categorization + * 2. per set of literals/general regex to FSMs + * 3. union FSMs, codegen + */ + + if (general_limit == 0) { + general_limit = SIZE_MAX; + } + + /* + * We're keying unique id to regex pattern spellings. The index + * here is used as an fsm_end_id_t. + * + * We don't use patterns[] just for debugging, we use it for passing + * to re_comp(). But not to re_strings(), we pass lit_s there. + */ + + struct pattern_set declined; + struct pattern_set general; + struct literal_set literals[4]; /* unanchored, left, right, both */ + + declined.count = 0; + general.count = 0; + + for (size_t i = 0; i < sizeof literals / sizeof *literals; i++) { + literals[i].count = 0; + } + + /* + * On end id numbering: The UI must be one pattern per line, because + * it's the only format that doesn't introduce a syntax with escaping. + * So that means we need to track the line number associated with each + * regex. And that means the line number is necessarily the end id. + * So each pattern carries its associated end id. + * + * We have four whole FSMs from re_strings(), each potentially contains + * a set of end ids. And each regex in general.a[i] has a single end id. + */ + + /* + * Categorize patterns + */ + for (int arg = 0; arg < argc; arg++) { + fsm_end_id_t id = 0; + FILE *f; + char *s; + enum re_dialect dialect = default_dialect; + + if (override_dialect) { + const char *ext = strrchr(argv[arg], '.'); + if (ext != NULL) { + ext++; + fprintf(stderr, "overriding dialect by extension for %s: %s\n", + argv[arg], ext); + dialect = dialect_name(ext); + } + } + + f = xopen(argv[arg]); + + if (argc > 1) { + id = arg; + } + + while (s = xgetline(f), s != NULL) { + struct re_err err; + const char *lit_s; + size_t lit_n; + enum category r; + enum re_strings_flags strings_flags; + + /* + * This is so we can fit the endid value in an int for the + * return value of the generated code. + */ + if (id > INT_MAX) { + fprintf(stderr, "pattern count overflow\n"); + exit(EXIT_FAILURE); + } + + r = categorize(dialect, flags, + charset, reject, s, + &err, &lit_s, &lit_n, &strings_flags); + + /* + * For pattern sets, s storage belongs to the entry in the set + * (and is freed when the set is destroyed). For literals, we're + * done with the original syntax, and use lit_s/lit_n instead. + */ + if (dialect != RE_LITERAL && r == CATEGORY_LITERAL) { + free(s); + } + + if (dialect == RE_LITERAL && r == CATEGORY_LITERAL && unanchored_literals) { + strings_flags = 0; + } + + if (verbose) { + switch (r) { + case CATEGORY_LITERAL: + fprintf(stderr, "literal %s:#%u '%.*s'\n", + argv[arg], id, (int) lit_n, lit_s); + fprintf(stderr, "literal %s:#%u /%s%.*s%s/\n", + argv[arg], id, + (strings_flags & RE_STRINGS_ANCHOR_LEFT) ? "^" : "", + (int) lit_n, lit_s, + (strings_flags & RE_STRINGS_ANCHOR_RIGHT) ? "$" : ""); + break; + + case CATEGORY_EMPTY: + /* fallthrough */ + case CATEGORY_GENERAL: + fprintf(stderr, "general %s:#%u: /%s/\n", + argv[arg], id, s); + break; + + case CATEGORY_ERROR: + fprintf(stderr, "declined (%s) %s:#%u: /%s/", + category_reason(r), + argv[arg], id, s); + re_perror(dialect, &err, NULL, NULL); + break; + + default: + fprintf(stderr, "declined (%s) %s:#%u: /%s/\n", + category_reason(r), + argv[arg], id, s); + break; + } + } + + switch (r) { + case CATEGORY_LITERAL: + assert(strings_flags >= 0 && strings_flags <= 3); + append_literal(&literals[strings_flags], + lit_s, lit_n, + id); + break; + + case CATEGORY_EMPTY: + /* fallthrough */ + case CATEGORY_GENERAL: + append_pattern( + general.count < general_limit ? &general : &declined, + id, dialect, s); + break; + + default: + if (strict) { + exit(EXIT_FAILURE); + } + + append_pattern(&declined, id, dialect, s); + } + + if (argc == 1) { + id++; + } + } + + fclose(f); + } + + /* + * realloc down to size, note this leaves some .a arrays NULL + * when its count is 0. + */ + if (declined.count > 0) { + declined.a = xrealloc(declined.a, declined.count * sizeof *declined.a); + } + if (general.count > 0) { + general.a = xrealloc(general.a, general.count * sizeof *general.a); + } + for (size_t i = 0; i < sizeof literals / sizeof *literals; i++) { + if (literals[i].count > 0) { + literals[i].a = xrealloc(literals[i].a, literals[i].count * sizeof *literals[i].a); + } + } + + /* + * This is conservative allocation, the declined list may still grow yet. + * We track actual usage in fsm_count. + */ + struct fsm **fsms = xmalloc( + (general.count + sizeof literals / sizeof *literals) + * sizeof *fsms); + size_t fsm_count = 0; + + /* + * The order of processing literals and regexps here is arbitrary. + * If this were a threaded program, each set of literals and each + * individual general regex could be handled in parallel. + * + * Also, the order of FSMs in the fsms[] array is not significant, + * they could be populated in any order. They're just going to get + * unioned anyway. + * + * We'd probably have a CLI option for the size of the worker pool. + * However the bottleneck is fsm_determinise() after union, not here. + * So this doesn't help until fsm_determinise() also runs in parallel. + */ + +// TODO: also have strict mode error for empty strings and unsatisfiable expressions +// TODO: better to do this when looking at the DFA for both literals and for regexps + + /* sets of literals */ + { + enum re_strings_flags flags[sizeof literals / sizeof *literals] = { + 0, + RE_STRINGS_ANCHOR_LEFT, + RE_STRINGS_ANCHOR_RIGHT, + RE_STRINGS_ANCHOR_LEFT | RE_STRINGS_ANCHOR_RIGHT + }; + + for (size_t i = 0; i < sizeof literals / sizeof *literals; i++) { + if (show_stats) { + fprintf(stderr, "literals[%zu].count = %zu\n", i, literals[i].count); + } + + struct fsm *fsm = build_literals_fsm(show_stats, charset, + &literals[i], flags[i]); + assert(fsm != NULL); + + if (verbose) { + fprintf(stderr, "fsms[%zu] <- literal[%zu]\n", fsm_count, i); + } + + fsms[fsm_count] = fsm; + fsm_count++; + } + } + + /* individual regexps */ + { + for (size_t i = 0; i < general.count; i++) { + if (verbose) { + fprintf(stderr, "general[%zu]: #%u /%s/\n", + i, general.a[i].id, general.a[i].s); + } + + struct fsm *fsm = build_pattern_fsm(show_stats, charset, + general.a[i].dialect, strict, + general.a[i].s, flags, general.a[i].id); + if (fsm == NULL) { + append_pattern(&declined, i, general.a[i].dialect, general.a[i].s); + continue; + } + + if (verbose) { + fprintf(stderr, "fsms[%zu] <- general[%zu]\n", fsm_count, i); + } + + fsms[fsm_count] = fsm; + fsm_count++; + } + + if (general.count > general_limit) { + general.count = general_limit; + } + } + + fsms = xrealloc(fsms, fsm_count * sizeof *fsms); + + if (show_stats) { + fprintf(stderr, "literals (unanchored): %zu patterns, %u states\n", + literals[0].count, fsm_countstates(fsms[0])); + fprintf(stderr, "literals (^left): %zu patterns, %u states\n", + literals[1].count, fsm_countstates(fsms[1])); + fprintf(stderr, "literals (right$): %zu patterns, %u states\n", + literals[2].count, fsm_countstates(fsms[2])); + fprintf(stderr, "literals (^both$): %zu patterns, %u states\n", + literals[3].count, fsm_countstates(fsms[3])); + fprintf(stderr, "general: %zu patterns (limit %zu)\n", + general.count, general_limit); + fprintf(stderr, "declined: %zu patterns\n", + declined.count); + } + + if (verbose) { + for (size_t i = 0; i < declined.count; i++) { + fprintf(stderr, "declined[%zu]: #%u /%s/\n", + i, declined.a[i].id, declined.a[i].s); + } + } + + /* + * The declined patterns are intended to be handled by some other regex + * engine. The user is expected to disambiguate these by pattern spelling. + * This gives a way to map back to the per-line ID from the original list. + */ + if (declined_file != NULL) { + FILE *f; + + f = fopen(declined_file, "w"); + if (f == NULL) { + perror(declined_file); + exit(EXIT_FAILURE); + } + + for (size_t i = 0; i < declined.count; i++) { + fprintf(f, "%s\n", declined.a[i].s); + } + + fclose(f); + } + + for (size_t i = 0; i < declined.count; i++) { + free((void *) declined.a[i].s); + } + if (declined.count > 0) { + free(declined.a); + } + + if (show_stats) { + fprintf(stderr, "fsm_count = %zu FSMs prior to union\n", fsm_count); + } + + if (verbose) { + for (size_t i = 0; i < fsm_count; i++) { + fprintf(stderr, "fsm[%zu] = %u states\n", i, fsm_countstates(fsms[i])); + } + } + + /* + * Union down and codegen. + */ + { + struct fsm *fsm; + + struct fsm_hooks hooks = { + .conflict = conflict, + .hook_opaque = & (struct check_env) { &literals, &general } + }; + + /* + * fsm_union_array introduces epsilons. + * If we construct anchored DFA and deal with ^.* ourselves here, + * we can union union without introducing epislons. + * This should reduce pressure on fsm_determinise() below. + */ + + fsm = fsm_union_array(fsm_count, fsms, NULL); + if (fsm == NULL) { + perror("fsm_union_array"); + exit(EXIT_FAILURE); + } + + free(fsms); + + if (show_stats) { + fprintf(stderr, "nfa: %u states\n", fsm_countstates(fsm)); + } + + /* + * We determinise but do not minimise the single DFA, like in lx. + * This keeps end states (and therefore end ids) unique. But unlike lx, + * end ids don't overlap between patterns (because they represent lines + * in the input file). So there isn't an advantage to minimising here. + */ + if (!fsm_determinise(fsm)) { + perror("fsm_determinise"); + exit(EXIT_FAILURE); + } + + if (show_stats) { + fprintf(stderr, "dfa: %u states\n", fsm_countstates(fsm)); + } + + if (-1 == fsm_print(stdout, fsm, &opt, &hooks, fsm_lang)) { + exit(EXIT_FAILURE); + } + + fsm_free(fsm); + } + + for (size_t i = 0; i < sizeof literals / sizeof *literals; i++) { + for (size_t j = 0; j < literals[i].count; j++) { + free((void *) literals[i].a[j].p); /* allocated by re_is_literal() */ + } + if (literals[i].count > 0) { + free(literals[i].a); + } + } + + for (size_t i = 0; i < general.count; i++) { + free((void *) general.a[i].s); + } + if (general.count > 0) { + free(general.a); + } + +#ifdef __linux__ + if (show_stats) { + struct rusage ru; + + if (getrusage(RUSAGE_SELF, &ru) != 0) { + perror("getrusage"); + exit(EXIT_FAILURE); + } + + fprintf(stderr, "rusage.utime: %g\n", (double) ru.ru_utime.tv_sec + (double) ru.ru_utime.tv_usec / 1000000); + fprintf(stderr, "rusage.stime: %g\n", (double) ru.ru_stime.tv_sec + (double) ru.ru_stime.tv_usec / 1000000); + fprintf(stderr, "rusage.maxrss: %zu MiB\n", (size_t) ru.ru_maxrss / 1024); + } +#endif +} + diff --git a/tests/aho_corasick/Makefile b/tests/aho_corasick/Makefile index 5748ddd5c..6eef421bf 100644 --- a/tests/aho_corasick/Makefile +++ b/tests/aho_corasick/Makefile @@ -33,7 +33,7 @@ ${TEST_OUTDIR.tests/aho_corasick}/out${n}a.re: ${TEST_SRCDIR.tests/aho_corasick} < ${.ALLSRC:M*.txt} > $@ ${TEST_OUTDIR.tests/aho_corasick}/out${n}a.fsm: ${RE} ${TEST_OUTDIR.tests/aho_corasick}/out${n}a.re - ${RE} -r native -py ${.ALLSRC:M*.re} \ + ${RE} -FC -r native -py ${.ALLSRC:M*.re} \ > $@ ${TEST_OUTDIR.tests/aho_corasick}/got${n}a.fsm: ${AC_TEST} ${TEST_SRCDIR.tests/aho_corasick}/in${n}.txt @@ -48,7 +48,7 @@ ${TEST_OUTDIR.tests/aho_corasick}/out${n}l.re: ${TEST_SRCDIR.tests/aho_corasick} < ${.ALLSRC:M*.txt} > $@ ${TEST_OUTDIR.tests/aho_corasick}/out${n}l.fsm: ${RE} ${TEST_OUTDIR.tests/aho_corasick}/out${n}l.re - ${RE} -r native -py ${.ALLSRC:M*.re} \ + ${RE} -FC -r native -py ${.ALLSRC:M*.re} \ > $@ ${TEST_OUTDIR.tests/aho_corasick}/got${n}l.fsm: ${AC_TEST} ${TEST_SRCDIR.tests/aho_corasick}/in${n}.txt @@ -63,7 +63,7 @@ ${TEST_OUTDIR.tests/aho_corasick}/out${n}r.re: ${TEST_SRCDIR.tests/aho_corasick} < ${.ALLSRC:M*.txt} > $@ ${TEST_OUTDIR.tests/aho_corasick}/out${n}r.fsm: ${RE} ${TEST_OUTDIR.tests/aho_corasick}/out${n}r.re - ${RE} -r native -py ${.ALLSRC:M*.re} \ + ${RE} -FC -r native -py ${.ALLSRC:M*.re} \ > $@ ${TEST_OUTDIR.tests/aho_corasick}/got${n}r.fsm: ${AC_TEST} ${TEST_SRCDIR.tests/aho_corasick}/in${n}.txt @@ -79,7 +79,7 @@ ${TEST_OUTDIR.tests/aho_corasick}/out${n}u.re: ${TEST_SRCDIR.tests/aho_corasick} < ${.ALLSRC:M*.txt} > $@ ${TEST_OUTDIR.tests/aho_corasick}/out${n}u.fsm: ${RE} ${TEST_OUTDIR.tests/aho_corasick}/out${n}u.re - ${RE} -r native -py ${.ALLSRC:M*.re} \ + ${RE} -FC -r native -py ${.ALLSRC:M*.re} \ > $@ ${TEST_OUTDIR.tests/aho_corasick}/got${n}u.fsm: ${AC_TEST} ${TEST_SRCDIR.tests/aho_corasick}/in${n}.txt diff --git a/tests/aho_corasick/actest.c b/tests/aho_corasick/actest.c index facf8209b..40dcbdf4f 100644 --- a/tests/aho_corasick/actest.c +++ b/tests/aho_corasick/actest.c @@ -102,7 +102,7 @@ void wordlist_finalize(struct word_list *words) int main(int argc, char *argv[]) { struct fsm *fsm; char s[BUFSIZ]; - fsm_print *print; + enum fsm_print_lang lang; const char *pname = argv[0]; @@ -116,7 +116,7 @@ int main(int argc, char *argv[]) { opt.anonymous_states = 1; opt.consolidate_edges = 1; - print = fsm_print_fsm; + lang = FSM_PRINT_FSM; { int c; @@ -129,7 +129,7 @@ int main(int argc, char *argv[]) { case 'l': anchored_left = 1; break; case 'r': anchored_right = 1; break; - case 'd': print = fsm_print_dot; break; + case 'd': lang = FSM_PRINT_DOT; break; case 'h': case '?': @@ -164,7 +164,7 @@ int main(int argc, char *argv[]) { flags |= RE_STRINGS_ANCHOR_RIGHT; } - fsm = re_strings(&opt, (const char **)words.list, words.len, flags); + fsm = re_strings(NULL, (const char **)words.list, words.len, flags); wordlist_finalize(&words); if (fsm == NULL) { @@ -173,9 +173,7 @@ int main(int argc, char *argv[]) { exit(EXIT_FAILURE); } - if (print != NULL) { - print(stdout, fsm); - } + fsm_print(stdout, fsm, &opt, NULL, lang); fsm_free(fsm); diff --git a/tests/capture/Makefile b/tests/capture/Makefile index 53d63ff2b..bdee42df4 100644 --- a/tests/capture/Makefile +++ b/tests/capture/Makefile @@ -7,18 +7,20 @@ TEST_OUTDIR.tests/capture = ${BUILD}/tests/capture .for n in ${TEST.tests/capture:T:R:C/^capture//} test:: ${TEST_OUTDIR.tests/capture}/res${n} SRC += ${TEST_SRCDIR.tests/capture}/capture${n}.c -CFLAGS.${TEST_SRCDIR.tests/capture}/capture${n}.c = -UNDEBUG +CFLAGS.${TEST_SRCDIR.tests/capture}/capture${n}.c = -UNDEBUG -std=c99 ${TEST_OUTDIR.tests/capture}/run${n}: ${TEST_OUTDIR.tests/capture}/capture${n}.o ${TEST_OUTDIR.tests/capture}/captest.o - ${CC} ${CFLAGS} -o ${TEST_OUTDIR.tests/capture}/run${n} ${TEST_OUTDIR.tests/capture}/capture${n}.o ${TEST_OUTDIR.tests/capture}/captest.o ${BUILD}/lib/libfsm.a + ${CC} ${CFLAGS} -o ${TEST_OUTDIR.tests/capture}/run${n} ${TEST_OUTDIR.tests/capture}/capture${n}.o ${TEST_OUTDIR.tests/capture}/captest.o ${BUILD}/lib/libfsm.a ${BUILD}/lib/libre.a ${TEST_OUTDIR.tests/capture}/res${n}: ${TEST_OUTDIR.tests/capture}/run${n} ( ${TEST_OUTDIR.tests/capture}/run${n} 1>&2 && echo PASS || echo FAIL ) > ${TEST_OUTDIR.tests/capture}/res${n} -.for lib in ${LIB:Mlibfsm} +.for lib in ${LIB:Mlibfsm} ${LIB:Mlibre} ${TEST_OUTDIR.tests/capture}/run${n}: ${BUILD}/lib/${lib:R}.a .endfor .endfor +CFLAGS.${TEST_SRCDIR.tests/capture}/captest.c += -std=c99 + ${TEST_OUTDIR.tests/capture}/captest.o: tests/capture/captest.c ${CC} ${CFLAGS} -c -o ${TEST_OUTDIR.tests/capture}/captest.o tests/capture/captest.c diff --git a/tests/capture/captest.c b/tests/capture/captest.c index dc66f81d4..cfd0d8b05 100644 --- a/tests/capture/captest.c +++ b/tests/capture/captest.c @@ -1,210 +1,491 @@ #include "captest.h" -#include -#include -#include - -#if CAPTEST_RUN_SINGLE_LOG #include -#endif +#include -#define FAIL(MSG) \ - fprintf(stderr, "FAIL: %s:%d -- %s\n", \ - __FILE__, __LINE__, MSG); \ - exit(EXIT_FAILURE) +#include -int -captest_getc(void *opaque) -{ - struct captest_input *input = opaque; - int res = input->string[input->pos]; - input->pos++; - return res == 0 ? EOF : res; -} +/* for fsm_capvm_program_exec */ +#include "../../src/libfsm/capture_vm.h" -int -captest_run_single(const struct captest_single_fsm_test_info *info) -{ - size_t i; - struct captest_input input; - fsm_state_t end; - int exec_res; - struct fsm_capture got_captures[MAX_TEST_CAPTURES]; - struct fsm_capture exp_captures[MAX_TEST_CAPTURES]; - size_t capture_count = 0; - struct fsm *fsm = captest_fsm_of_string(info->string, 0); - - input.string = info->string; - input.pos = 0; +static struct fsm_options options = { + .group_edges = 1, +}; - if (fsm == NULL) { - FAIL("fsm_of_string"); +#define MAX_INPUT_WITH_NL_LENGTH 1000 +static char +input_with_nl[MAX_INPUT_WITH_NL_LENGTH]; + +enum captest_run_case_res +captest_run_case(const struct captest_case_single *testcase, + int verbosity, bool trailing_newline, FILE *prog_output) +{ + bool dump_captures = false; + enum captest_run_case_res res = CAPTEST_RUN_CASE_ERROR; + struct re_err err; + + if (verbosity == DUMP_PROGRAMS_VERBOSITY) { + assert(prog_output != NULL); + } else { + assert(prog_output == NULL); } - for (i = 0; i < MAX_TEST_CAPTURES; i++) { - exp_captures[i].pos[0] = FSM_CAPTURE_NO_POS; - exp_captures[i].pos[1] = FSM_CAPTURE_NO_POS; + if (verbosity > 0) { + printf("/%s/ <- \"%s%s\": ", + testcase->regex, testcase->input, + trailing_newline ? "\\n" : ""); + fflush(stdout); } - for (i = 0; i < MAX_SINGLE_FSM_TEST_PATHS; i++) { - const struct captest_single_fsm_test_path *path = - &info->paths[i]; - if (path->start == 0 && path->end == 0 && i > 0) { - break; /* end of list */ - } + /* build regex */ + const enum re_flags flags = 0; + + const char *regex = testcase->regex; - /* no zero-width captures */ - assert(path->end > path->start); + struct fsm *fsm = re_comp(RE_PCRE, + fsm_sgetc, ®ex, + NULL, flags, &err); - if (!fsm_capture_set_path(fsm, i, - path->start, path->end)) { - fprintf(stderr, - "failed to set capture path %lu\n", i); - FAIL("fsm_capture_set_path"); + if (testcase->match == SHOULD_REJECT_AS_UNSUPPORTED) { + if (fsm != NULL) { + fsm_free(fsm); + return CAPTEST_RUN_CASE_FAIL; } + return CAPTEST_RUN_CASE_PASS; + } - exp_captures[i].pos[0] = path->start; - exp_captures[i].pos[1] = path->end; + if (fsm == NULL) { + re_perror(RE_PCRE, &err, NULL, testcase->regex); + return CAPTEST_RUN_CASE_ERROR; + } - capture_count = i + 1; + if (!fsm_determinise(fsm)) { + return CAPTEST_RUN_CASE_ERROR; } - { - const unsigned count = fsm_countcaptures(fsm); - const unsigned expected = capture_count; - if (count != expected) { - fprintf(stderr, "expected %u, got %u\n", - expected, count); - FAIL("countcaptures"); - } + if (!fsm_minimise(fsm)) { + return CAPTEST_RUN_CASE_ERROR; } -#if CAPTEST_RUN_SINGLE_LOG - fsm_print_fsm(stderr, fsm); - fsm_capture_dump(stderr, "fsm", fsm); -#endif + if (verbosity > 3) { + fsm_print(stdout, fsm, &options, NULL, FSM_PRINT_FSM); + } + + if (trailing_newline) { + const size_t length = strlen(testcase->input); + assert(length + 1 < MAX_INPUT_WITH_NL_LENGTH); + memcpy(input_with_nl, testcase->input, + length); + input_with_nl[length] = '\n'; + input_with_nl[length + 1] = '\0'; + } - exec_res = fsm_exec(fsm, captest_getc, &input, &end, got_captures); - if (exec_res != 1) { FAIL("exec_res"); } - if (end != strlen(info->string)) { FAIL("exec end pos"); } + const char *input = trailing_newline + ? input_with_nl + : testcase->input; + assert(input != NULL); + const size_t length = strlen(input); + + fsm_state_t end; /* unused but required by API */ + struct fsm_capture capture_buf[MAX_CAPTEST_SINGLE_CAPTURE_PAIRS]; + const size_t capture_buf_length = MAX_CAPTEST_SINGLE_CAPTURE_PAIRS; + + /* Initialize with values that are distinct from FSM_CAPTURE_NO_POS + * and will stand out visually. Should never see these. */ + for (size_t i = 0; i < MAX_CAPTEST_SINGLE_CAPTURE_PAIRS; i++) { + capture_buf[i].pos[0] = 88888888; + capture_buf[i].pos[1] = 99999999; + } - { - fsm_end_id_t id_buf[1] = { ~0 }; - enum fsm_getendids_res gres; - size_t written; - if (1 != fsm_getendidcount(fsm, end)) { - FAIL("did not have exactly one end ID"); + /* If verbosity is exactly DUMP_PROGRAMS_VERBOSITY, then print out capture info and pass. */ + if (verbosity == DUMP_PROGRAMS_VERBOSITY) { + assert(prog_output != NULL); + if (!trailing_newline) { + const char *match_str = testcase->match == SHOULD_MATCH ? "SHOULD_MATCH" + : testcase->match == SHOULD_NOT_MATCH ? "SHOULD_NOT_MATCH" + : testcase->match == SHOULD_REJECT_AS_UNSUPPORTED ? "SHOULD_REJECT_AS_UNSUPPORTED" + : "ERROR"; + fprintf(prog_output, "regex \"%s\", input \"%s\", match %s, no_nl %d, count %zu:", + testcase->regex, testcase->input, match_str, testcase->no_nl, + testcase->count); + for (size_t i = 0; i < testcase->count; i++) { + fprintf(prog_output, " %zu:[%zd, %zd]", + i, testcase->expected[i].pos[0], testcase->expected[i].pos[1]); + } + fprintf(prog_output, "\n"); + fsm_capture_dump(prog_output, "capture_info", fsm); } + fsm_free(fsm); + return CAPTEST_RUN_CASE_PASS; + } - gres = fsm_getendids(fsm, end, 1, id_buf, &written); - if (gres != FSM_GETENDIDS_FOUND) { - FAIL("failed to get end IDs"); + /* first, execute with a capture buffer that is one cell too small and check for an error */ + const size_t capture_ceil = fsm_capture_ceiling(fsm); + assert(capture_ceil > 0); + const size_t insufficient_capture_buf_length = capture_ceil - 1; + errno = 0; + int exec_res = fsm_exec_with_captures(fsm, + (const unsigned char *)input, length, &end, capture_buf, insufficient_capture_buf_length); + assert(exec_res == -1); + assert(errno == EINVAL); + errno = 0; + + /* then, execute and check result & captures */ + exec_res = fsm_exec_with_captures(fsm, + (const unsigned char *)input, length, &end, capture_buf, capture_buf_length); + if (exec_res == -1) { + perror("fsm_exec_with_captures"); + return CAPTEST_RUN_CASE_ERROR; + } + + if (testcase->match == SHOULD_NOT_MATCH) { /* expect match failure */ + res = (exec_res == 0 + ? CAPTEST_RUN_CASE_PASS + : CAPTEST_RUN_CASE_FAIL); + } else if (exec_res == 0) { + res = CAPTEST_RUN_CASE_FAIL; /* didn't match, should have */ + } else { + res = CAPTEST_RUN_CASE_PASS; + if (verbosity > 1) { + dump_captures = true; } - if (0 != id_buf[0]) { - FAIL("failed to get end ID of 0"); + /* check captures against expected */ + for (size_t i = 0; i < testcase->count; i++) { + if (testcase->expected[i].pos[0] != capture_buf[i].pos[0] || + testcase->expected[i].pos[1] != capture_buf[i].pos[1]) { + res = CAPTEST_RUN_CASE_FAIL; + dump_captures = true; + } } } - for (i = 0; i < capture_count; i++) { -#if CAPTEST_RUN_SINGLE_LOG - fprintf(stderr, "captest: capture %lu: exp (%ld, %ld), got (%ld, %ld)\n", - i, exp_captures[i].pos[0], exp_captures[i].pos[1], - got_captures[i].pos[0], got_captures[i].pos[1]); -#endif - if (got_captures[i].pos[0] != exp_captures[i].pos[0]) { - fprintf(stderr, "capture[%lu].pos[0]: exp %lu, got %lu\n", - i, exp_captures[i].pos[0], - got_captures[i].pos[0]); - FAIL("capture mismatch"); + switch (res) { + case CAPTEST_RUN_CASE_PASS: + if (verbosity > 0) { + printf("pass\n"); + } + break; + case CAPTEST_RUN_CASE_FAIL: + if (verbosity == 0) { + printf("/%s/ <- \"%s%s\": FAIL\n", + testcase->regex, testcase->input, + trailing_newline ? "\\n" : ""); } - if (got_captures[i].pos[1] != exp_captures[i].pos[1]) { - fprintf(stderr, "capture[%lu].pos[1]: exp %lu, got %lu\n", - i, exp_captures[i].pos[1], - got_captures[i].pos[1]); - FAIL("capture mismatch"); + if (verbosity > 0) { + printf("FAIL\n"); + } + break; + case CAPTEST_RUN_CASE_ERROR: + printf("ERROR\n"); + break; + } + + if (dump_captures) { + for (size_t i = 0; i < testcase->count; i++) { + printf("exp %zd, %zd, got %zd, %zd%s\n", + testcase->expected[i].pos[0], testcase->expected[i].pos[1], + capture_buf[i].pos[0], capture_buf[i].pos[1], + (testcase->expected[i].pos[0] != capture_buf[i].pos[0] || + testcase->expected[i].pos[1] != capture_buf[i].pos[1]) + ? " *" : ""); } } fsm_free(fsm); - return 0; + return res; } -struct fsm * -captest_fsm_of_string(const char *string, unsigned end_id) +enum captest_run_case_res +captest_run_case_multi(const struct captest_case_multi *testcase, + int verbosity, bool trailing_newline, FILE *prog_output, + struct captest_case_multi_result *result) { - struct fsm *fsm = captest_fsm_with_options(); - const size_t length = strlen(string); - size_t i; + struct re_err err; + const enum re_flags flags = 0; - if (fsm == NULL) { - return NULL; + struct captest_case_multi_result ignored_result; + if (result == NULL) { + result = &ignored_result; } + memset(result, 0x00, sizeof(*result)); - if (!fsm_addstate_bulk(fsm, length + 1)) { - goto cleanup; + if (verbosity == DUMP_PROGRAMS_VERBOSITY) { + assert(prog_output != NULL); + } else { + assert(prog_output == NULL); } - fsm_setstart(fsm, 0); - for (i = 0; i < length; i++) { - if (!fsm_addedge_literal(fsm, i, i + 1, string[i])) { + /* build each regex, combining them and keeping track of capture offsets */ + struct fsm *fsms[testcase->regex_count]; + struct fsm_combined_base_pair bases[testcase->regex_count]; + struct fsm *combined_fsm = NULL; + + for (size_t i = 0; i < testcase->regex_count; i++) { + fsms[i] = NULL; + } + + /* compile each individually */ + for (size_t i = 0; i < testcase->regex_count; i++) { + const char *regex = testcase->regexes[i]; + + if (verbosity > 1) { + fprintf(stderr, "%s: compiling \"%s\"\n", + __func__, regex); + } + + struct fsm *fsm = re_comp(RE_PCRE, + fsm_sgetc, ®ex, + NULL, flags, &err); + assert(fsm != NULL); + + if (!fsm_determinise(fsm)) { + goto cleanup; + } + + if (!fsm_minimise(fsm)) { goto cleanup; } + + if (verbosity > 3) { + char tag_buf[16] = { 0 }; + snprintf(tag_buf, sizeof(tag_buf), "fsm[%zu]", i); + + fprintf(stderr, "==== fsm[%zu]\n", i); + fsm_print(stderr, fsm, &options, NULL, FSM_PRINT_FSM); + fsm_capture_dump(stderr, tag_buf, fsm); + } + + fsms[i] = fsm; + } + + combined_fsm = fsm_union_array(testcase->regex_count, fsms, bases); + assert(combined_fsm != NULL); + if (verbosity > 0) { + fprintf(stderr, "%s: combined_fsm: %d states after fsm_union_array\n", + __func__, fsm_countstates(combined_fsm)); + } + if (verbosity > 1) { + for (size_t i = 0; i < testcase->regex_count; i++) { + fprintf(stderr, "%s: base[%zu]: state %d, capture %u\n", + __func__, i, bases[i].state, bases[i].capture); + } } - fsm_setend(fsm, length, 1); - if (!fsm_setendid(fsm, end_id)) { + if (!fsm_determinise(combined_fsm)) { goto cleanup; } + if (verbosity > 0) { + fprintf(stderr, "%s: combined_fsm: %d states after determinise\n", + __func__, fsm_countstates(combined_fsm)); + } - return fsm; + if (!fsm_minimise(combined_fsm)) { + goto cleanup; + } + if (verbosity > 0) { + fprintf(stderr, "%s: combined_fsm: %d states after minimise\n", + __func__, fsm_countstates(combined_fsm)); + } + + /* If verbosity is exactly 9, then print out capture info and pass. */ + if (verbosity == DUMP_PROGRAMS_VERBOSITY) { + fsm_capture_dump(prog_output, "capture_info", combined_fsm); + fsm_free(combined_fsm); + return CAPTEST_RUN_CASE_PASS; + } + + if (verbosity > 3) { + fprintf(stderr, "==== combined\n"); + fsm_print(stderr, combined_fsm, &options, NULL, FSM_PRINT_FSM); + fsm_capture_dump(stderr, "combined", combined_fsm); + } + + /* for each input, execute and check result */ + const struct multi_case_input_info *info; + for (info = &testcase->inputs[0]; info->input != NULL; info++) { + if (trailing_newline) { + const size_t length = strlen(info->input); + assert(length + 1 < MAX_INPUT_WITH_NL_LENGTH); + memcpy(input_with_nl, info->input, + length); + input_with_nl[length] = '\n'; + input_with_nl[length + 1] = '\0'; + } + + const char *input = trailing_newline + ? input_with_nl + : info->input; + assert(input != NULL); + const size_t length = strlen(input); + + if (verbosity > 1) { + fprintf(stderr, "%s: input: %s\n", __func__, input); + } + + fsm_state_t end; /* unused but required by API */ + struct fsm_capture capture_buf[MAX_CAPTEST_MULTI_CAPTURE_PAIRS]; + const size_t capture_buf_length = MAX_CAPTEST_MULTI_CAPTURE_PAIRS; + for (size_t i = 0; i < capture_buf_length; i++) { + capture_buf[i].pos[0] = (size_t)-2; + capture_buf[i].pos[1] = (size_t)-3; + } + + /* execute and check result & captures */ + int exec_res = fsm_exec_with_captures(combined_fsm, + (const unsigned char *)input, length, &end, capture_buf, capture_buf_length); + if (exec_res == -1) { + perror("fsm_exec_with_captures"); + return CAPTEST_RUN_CASE_ERROR; + } + + /* The .regex field should be in ascending order so we know + * when we've reached the all-0 suffix of expected[]. */ + uint8_t prev_regex = 0; + for (const struct case_multi_expected *exp = &info->expected[0]; + exp->regex >= prev_regex; exp++) { + prev_regex = exp->regex; + bool match = true; + const unsigned capture_base = bases[exp->regex].capture; + const unsigned capture_id = capture_base + exp->capture; + assert(capture_id < MAX_CAPTEST_MULTI_CAPTURE_PAIRS); + const size_t exp_s = exp->pos[0]; + const size_t exp_e = exp->pos[1]; + const size_t got_s = capture_buf[capture_id].pos[0]; + const size_t got_e = capture_buf[capture_id].pos[1]; + if (exp_s == got_s && exp_e == got_e) { + result->pass++; + } else { + match = false; + result->fail++; + } + + if (!match || verbosity > 2) { + fprintf(stderr, "%s: regex %u, capture %u (%u + base %u), exp (%zd, %zd), got (%zd, %zd)%s\n", + __func__, exp->regex, + capture_id, exp->capture, capture_base, + exp_s, exp_e, got_s, got_e, + match ? "" : " *** mismatch ***"); + } + } + } + + fsm_free(combined_fsm); + + /* this could populate a result struct so it can pass/fail multiple inputs */ + + return result->fail == 0 + ? CAPTEST_RUN_CASE_PASS + : CAPTEST_RUN_CASE_FAIL; cleanup: - fsm_free(fsm); - return NULL; -} + if (combined_fsm != NULL) { + fsm_free(combined_fsm); + } else { + for (size_t i = 0; i < testcase->regex_count; i++) { + if (fsms[i] != NULL) { + fsm_free(fsms[i]); + } + } + } -static struct fsm_options options; + return CAPTEST_RUN_CASE_ERROR; +} -struct fsm * -captest_fsm_with_options(void) +static struct capvm_program * +get_program_copy(const struct captest_case_program *testcase) { - struct fsm *fsm = NULL; + static struct capvm_program prog; + static struct capvm_opcode ops[MAX_PROGRAM_OPS + 1] = { 0 }; + static struct capvm_char_class cc_sets[MAX_PROGRAM_CHAR_CLASSES] = { 0 }; + + memset(&prog, 0x00, sizeof(prog)); + + memcpy(ops, testcase->ops, + MAX_PROGRAM_OPS * sizeof(testcase->ops[0])); + memcpy(cc_sets, testcase->char_class, + MAX_PROGRAM_CHAR_CLASSES * sizeof(testcase->char_class[0])); + + assert(testcase->expected.count < MAX_CAPTEST_PROGRAM_CAPTURE_PAIRS); + prog.capture_count = testcase->expected.count; + prog.capture_base = testcase->expected.base; + + uint32_t max_cc_used = (uint32_t)-1; + + prog.used = MAX_PROGRAM_OPS; + for (size_t i = 0; i < MAX_PROGRAM_OPS; i++) { + const struct capvm_opcode *op = &testcase->ops[i]; + if (op->t == CAPVM_OP_CHAR && op->u.chr == 0x00) { + prog.used = i; + break; + } else if (op->t == CAPVM_OP_CHARCLASS) { + if (max_cc_used == (uint32_t)-1 || op->u.charclass_id > max_cc_used) { + assert(op->u.charclass_id < MAX_PROGRAM_CHAR_CLASSES); + max_cc_used = op->u.charclass_id; + } + } + } + + prog.ceil = MAX_PROGRAM_OPS; + prog.ops = ops; - /* We currently don't need to set anything custom on this. */ - fsm = fsm_new(&options); - return fsm; + prog.char_classes.sets = cc_sets; + prog.char_classes.count = max_cc_used == (uint32_t)-1 ? 0 : max_cc_used + 1; + prog.char_classes.ceil = MAX_PROGRAM_CHAR_CLASSES; + + return &prog; } -int -captest_check_single_end_id(const struct fsm *fsm, fsm_state_t end_state, - unsigned expected_end_id, const char **msg) +enum captest_run_case_res +captest_run_case_program(const struct captest_case_program *testcase, + int verbosity) { - fsm_end_id_t id_buf[1] = { ~0 }; - enum fsm_getendids_res gres; - size_t written; - const char *unused; + (void)verbosity; + + /* copy program */ + const size_t input_length = strlen(testcase->input); + struct fsm_capture capture_buf[MAX_CAPTEST_PROGRAM_CAPTURE_PAIRS]; + const size_t capture_buf_length = MAX_CAPTEST_PROGRAM_CAPTURE_PAIRS; - if (msg == NULL) { - msg = &unused; + /* Initialize with FSM_CAPTURE_NO_POS, as the caller would */ + for (size_t i = 0; i < capture_buf_length; i++) { + capture_buf[i].pos[0] = FSM_CAPTURE_NO_POS; + capture_buf[i].pos[1] = FSM_CAPTURE_NO_POS; } - if (1 != fsm_getendidcount(fsm, end_state)) { - *msg = "did not have exactly one end ID"; - return 0; + struct capvm_program *program = get_program_copy(testcase); + + if (verbosity > 2) { + fsm_capvm_program_dump(stderr, program); } - gres = fsm_getendids(fsm, end_state, 1, id_buf, &written); - if (gres != FSM_GETENDIDS_FOUND) { - *msg = "failed to get end IDs"; - return 0; + fsm_capvm_program_exec(program, (const uint8_t *)testcase->input, input_length, + capture_buf, capture_buf_length); + + bool dump_captures = false; + enum captest_run_case_res res = CAPTEST_RUN_CASE_PASS; + + /* check captures against expected */ + for (size_t i = 0; i < testcase->expected.count; i++) { + if (testcase->expected.captures[i].pos[0] != capture_buf[i].pos[0] || + testcase->expected.captures[i].pos[1] != capture_buf[i].pos[1]) { + res = CAPTEST_RUN_CASE_FAIL; + dump_captures = true; + } } - if (expected_end_id != id_buf[0]) { - *msg = "failed to get expected end ID"; - return 0; + if (dump_captures) { + for (size_t i = 0; i < testcase->expected.count; i++) { + printf("exp %zd, %zd, got %zd, %zd%s\n", + testcase->expected.captures[i].pos[0], + testcase->expected.captures[i].pos[1], + capture_buf[i].pos[0], capture_buf[i].pos[1], + (testcase->expected.captures[i].pos[0] != capture_buf[i].pos[0] || + testcase->expected.captures[i].pos[1] != capture_buf[i].pos[1]) + ? " *" : ""); + } } - return 1; + return res; } diff --git a/tests/capture/captest.h b/tests/capture/captest.h index a9debff6c..cdfcfa008 100644 --- a/tests/capture/captest.h +++ b/tests/capture/captest.h @@ -1,56 +1,122 @@ /* - * Copyright 2020 Scott Vokes + * Copyright 2022 Scott Vokes * * See LICENCE for the full copyright terms. */ + #ifndef CAPTEST_H #define CAPTEST_H #include #include #include +#include +#include #include #include #include #include +#include -#define MAX_SINGLE_FSM_TEST_PATHS 8 -#define MAX_TEST_CAPTURES 8 +#include -#define CAPTEST_RUN_SINGLE_LOG 0 +/* for captest_run_case_program, to evaluate handwritten programs */ +#include "../../src/libfsm/capture_vm_program.h" +#include "../../src/libfsm/capture_vm.h" -#ifndef LOG_INTERMEDIATE_FSMS -#define LOG_INTERMEDIATE_FSMS 0 -#endif +#define MAX_CAPTEST_SINGLE_CAPTURE_PAIRS 16 +#define MAX_CAPTEST_MULTI_CAPTURE_PAIRS 16 +#define MAX_CAPTEST_PROGRAM_CAPTURE_PAIRS 16 + +/* position representing no match */ +#define POS_NONE { (size_t)-1, (size_t)-1 } + +/* If verbosity is set to this (with -vvvvvvvvv) then dump all the + * compiled programs to 'prog_output'. */ +#define DUMP_PROGRAMS_VERBOSITY 9 + +enum captest_match { + SHOULD_MATCH = 0, /* implied, set by designated initializer */ + SHOULD_NOT_MATCH = 1, + SHOULD_REJECT_AS_UNSUPPORTED = 2, + SHOULD_SKIP = 3, +}; + +struct captest_case_single { + const char *regex; + const char *input; + enum captest_match match; + bool no_nl; /* do not retry with trailing newline */ -struct captest_single_fsm_test_info { - const char *string; - struct captest_single_fsm_test_path { - fsm_state_t start; - fsm_state_t end; - } paths[MAX_SINGLE_FSM_TEST_PATHS]; + size_t count; + struct fsm_capture expected[MAX_CAPTEST_SINGLE_CAPTURE_PAIRS]; }; -struct captest_input { - const char *string; - size_t pos; +/* Same as captest_case_single, but these expect multiple (possibly overlapping) + * regexes to be combined before checking the match/capture behavior. */ +#define MAX_REGEXES 4 +#define MAX_INPUTS 8 +#define MAX_CAPTEST_MULTI_EXPECTED 8 +struct captest_case_multi { + uint8_t regex_count; + const char *regexes[MAX_REGEXES]; + enum captest_match match; + bool no_nl; + + struct multi_case_input_info { + const char *input; /* first NULL input = end of list */ + struct case_multi_expected { + uint8_t regex; /* expected: ascending order */ + uint8_t capture; /* 0 is default */ + size_t pos[2]; + } expected[MAX_CAPTEST_MULTI_EXPECTED]; + } inputs[MAX_INPUTS]; +}; + +struct captest_case_multi_result { + size_t pass; + size_t fail; }; -int -captest_run_single(const struct captest_single_fsm_test_info *info); +#define MAX_PROGRAM_CHAR_CLASSES 4 +#define MAX_PROGRAM_OPS 32 +struct captest_case_program { + const char *input; + + struct capvm_char_class char_class[MAX_PROGRAM_CHAR_CLASSES]; -int -captest_getc(void *opaque); + struct { + uint32_t count; + uint32_t base; + struct fsm_capture captures[MAX_CAPTEST_PROGRAM_CAPTURE_PAIRS]; + } expected; -struct fsm * -captest_fsm_with_options(void); + /* termined by 0'd record, { .t == CAPVM_OP_CHAR, .u.chr = 0x00 } */ + struct capvm_opcode ops[MAX_PROGRAM_OPS]; +}; + +enum captest_run_case_res { + CAPTEST_RUN_CASE_PASS, + CAPTEST_RUN_CASE_FAIL, + CAPTEST_RUN_CASE_ERROR, +}; +enum captest_run_case_res +captest_run_case(const struct captest_case_single *testcase, + int verbosity, bool trailing_newline, FILE *prog_output); -struct fsm * -captest_fsm_of_string(const char *string, unsigned end_id); +enum captest_run_case_res +captest_run_case_multi(const struct captest_case_multi *testcase, + int verbosity, bool trailing_newline, FILE *prog_output, + struct captest_case_multi_result *result); -int -captest_check_single_end_id(const struct fsm *fsm, fsm_state_t end_state, - unsigned expected_end_id, const char **msg); +/* This should probably only be used for evaluating specific + * hand-written programs for development, because we only care + * about supporting the kinds of programs that could be produced + * by compiling from valid regexes. In other words, this is not + * a stable public interface. */ +enum captest_run_case_res +captest_run_case_program(const struct captest_case_program *testcase, + int verbosity); #endif diff --git a/tests/capture/capture0.c b/tests/capture/capture0.c deleted file mode 100644 index 4e7d0e3fa..000000000 --- a/tests/capture/capture0.c +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright 2020 Scott Vokes - * - * See LICENCE for the full copyright terms. - */ - -#include -#include -#include -#include - -#include -#include - -#include "captest.h" - -/* /a(bcd)e/ */ - -int main(void) { - struct captest_single_fsm_test_info test_info = { - "abcde", - { - { 1, 4 }, - } - }; - return captest_run_single(&test_info); -} diff --git a/tests/capture/capture1.c b/tests/capture/capture1.c deleted file mode 100644 index e9fe53ab9..000000000 --- a/tests/capture/capture1.c +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright 2020 Scott Vokes - * - * See LICENCE for the full copyright terms. - */ - -#include -#include -#include -#include - -#include -#include - -#include "captest.h" -/* (a(b(c))) */ - -int main(void) { - struct captest_single_fsm_test_info test_info = { - "abc", - { - { 0, 3 }, - { 1, 3 }, - { 2, 3 }, - } - }; - return captest_run_single(&test_info); -} diff --git a/tests/capture/capture2.c b/tests/capture/capture2.c deleted file mode 100644 index 20a1c1bac..000000000 --- a/tests/capture/capture2.c +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright 2020 Scott Vokes - * - * See LICENCE for the full copyright terms. - */ - -#include -#include -#include -#include - -#include -#include - -#include "captest.h" - -/* (a(b((c))(d))) */ - -int main(void) { - struct captest_single_fsm_test_info test_info = { - "abcd", - { - { 0, 4 }, - { 1, 4 }, - { 2, 3 }, - { 2, 3 }, - { 3, 4 }, - } - }; - return captest_run_single(&test_info); -} diff --git a/tests/capture/capture3.c b/tests/capture/capture3.c deleted file mode 100644 index 9d4d284ab..000000000 --- a/tests/capture/capture3.c +++ /dev/null @@ -1,221 +0,0 @@ -/* - * Copyright 2020 Scott Vokes - * - * See LICENCE for the full copyright terms. - */ - -#include -#include -#include -#include - -#include -#include -#include -#include - -#include "captest.h" - -/* Combine 3 fully disjoint FSMs: - * - * - 0: "(a(b))" - * - 1: "(cd(e))" - * - 2: "(fgh(i))" - * - * Shift the captures for 1 and 2 forward and use/combine - * opaques on them to track which one(s) matched. - * - * This tracking of which DFA matched should be more directly - * supported by the API later. */ - -static void -check(const struct fsm *fsm, const char *string, - unsigned end_id, unsigned capture_base); - -static void -det_and_min(const char *tag, struct fsm *fsm); - -int main(void) { - struct fsm *f_ab = captest_fsm_of_string("ab", 0); - struct fsm *f_cde = captest_fsm_of_string("cde", 1); - struct fsm *f_fghi = captest_fsm_of_string("fghi", 2); - struct fsm *f_all = NULL; - unsigned captures; - - struct fsm_combined_base_pair bases[3]; - struct fsm *fsms[3]; - - assert(f_ab); - assert(f_cde); - assert(f_fghi); - - /* set captures */ -#define SET_CAPTURE(FSM, STATE, CAPTURE, TYPE) \ - if (!fsm_set_capture_action(FSM, STATE, CAPTURE, TYPE)) { \ - fprintf(stderr, "failed to set capture on line %d\n", \ - __LINE__); \ - exit(EXIT_FAILURE); \ - } - - /* (a(b)) */ - if (!fsm_capture_set_path(f_ab, 0, 0, 2)) { - exit(EXIT_FAILURE); - } - if (!fsm_capture_set_path(f_ab, 1, 1, 2)) { - exit(EXIT_FAILURE); - } - - /* (cd(e)) */ - if (!fsm_capture_set_path(f_cde, 0, 0, 3)) { - exit(EXIT_FAILURE); - } - if (!fsm_capture_set_path(f_cde, 1, 2, 3)) { - exit(EXIT_FAILURE); - } - - /* (fgh(i)) */ - if (!fsm_capture_set_path(f_fghi, 0, 0, 4)) { - exit(EXIT_FAILURE); - } - if (!fsm_capture_set_path(f_fghi, 1, 3, 4)) { - exit(EXIT_FAILURE); - } - -#if LOG_INTERMEDIATE_FSMS - fprintf(stderr, "\n=== f_ab...\n"); - fsm_print_fsm(stderr, f_ab); - fsm_capture_dump(stderr, "#### f_ab", f_ab); - - fprintf(stderr, "\n=== f_cde...\n"); - fsm_print_fsm(stderr, f_cde); - fsm_capture_dump(stderr, "#### f_cde", f_cde); - - fprintf(stderr, "\n=== f_fghi...\n"); - fsm_print_fsm(stderr, f_fghi); - fsm_capture_dump(stderr, "#### f_fghi", f_fghi); -#endif - - /* determinise and minimise each before unioning */ - det_and_min("ab", f_ab); - det_and_min("cde", f_cde); - det_and_min("fghi", f_fghi); - - /* union them */ - fsms[0] = f_ab; - fsms[1] = f_cde; - fsms[2] = f_fghi; - - f_all = fsm_union_array(3, fsms, bases); - assert(f_all != NULL); - -#if LOG_INTERMEDIATE_FSMS - fprintf(stderr, "=== unioned f_ab with f_cde... (CB ab: %u, cde: %u)\n", - bases[0].capture, bases[1].capture); - fsm_print_fsm(stderr, f_all); - fsm_capture_dump(stderr, "#### f_all", f_all); -#endif - -#if LOG_INTERMEDIATE_FSMS - fprintf(stderr, "=== unioned f_all with f_fghi... (CB fghi: %u), %u captures\n", - bases[2].capture, fsm_countcaptures(f_all)); - fsm_print_fsm(stderr, f_all); - fsm_capture_dump(stderr, "#### f_all #2", f_all); -#endif - - if (!fsm_determinise(f_all)) { - fprintf(stderr, "NOPE %d\n", __LINE__); - exit(EXIT_FAILURE); - } - -#if LOG_INTERMEDIATE_FSMS - fprintf(stderr, "==== after determinise\n"); - fsm_print_fsm(stderr, f_all); - fsm_capture_dump(stderr, "#### f_all", f_all); -#endif - - captures = fsm_countcaptures(f_all); - if (captures != 6) { - fprintf(stderr, "expected 6 captures, got %u\n", captures); - exit(EXIT_FAILURE); - } - - check(f_all, "ab", 0, bases[0].capture); - check(f_all, "cde", 1, bases[1].capture); - check(f_all, "fghi", 2, bases[2].capture); - - - fsm_free(f_all); - - return 0; -} - -static void -det_and_min(const char *tag, struct fsm *fsm) -{ - if (!fsm_determinise(fsm)) { - fprintf(stderr, "Failed to determise '%s'\n", tag); - exit(EXIT_FAILURE); - } - - if (!fsm_minimise(fsm)) { - fprintf(stderr, "Failed to minimise '%s'\n", tag); - exit(EXIT_FAILURE); - } - -#if LOG_INTERMEDIATE_FSMS - fprintf(stderr, "==== after det_and_min: '%s'\n", tag); - fsm_print_fsm(stderr, fsm); - fsm_capture_dump(stderr, tag, fsm); -#endif - -} - -static void -check(const struct fsm *fsm, const char *string, - unsigned end_id, unsigned capture_base) -{ - int exec_res; - size_t i; - struct captest_input input; - fsm_state_t end; - struct fsm_capture captures[MAX_TEST_CAPTURES]; - const size_t length = strlen(string); - const unsigned cb = capture_base; /* alias */ - - input.string = string; - input.pos = 0; - - for (i = 0; i < MAX_TEST_CAPTURES; i++) { - captures[i].pos[0] = FSM_CAPTURE_NO_POS; - captures[i].pos[1] = FSM_CAPTURE_NO_POS; - } - - exec_res = fsm_exec(fsm, captest_getc, &input, &end, captures); - if (exec_res != 1) { - fprintf(stderr, "fsm_exec: %d for '%s', expected 1\n", - exec_res, string); - exit(EXIT_FAILURE); - } - - /* check end ID */ - { - const char *msg; - if (!captest_check_single_end_id(fsm, end, end_id, &msg)) { - fprintf(stderr, "%s\n", msg); - exit(EXIT_FAILURE); - } - } - - /* check captures */ - if (0) { - fprintf(stderr, "captures for '%s' (cb %u): [%ld, %ld], [%ld, %ld]\n", - string, capture_base, - captures[0 + cb].pos[0], captures[0 + cb].pos[1], - captures[1 + cb].pos[0], captures[1 + cb].pos[1]); - } - - assert(captures[0 + cb].pos[0] == 0); - assert(captures[0 + cb].pos[1] == length); - assert(captures[1 + cb].pos[0] == length - 1); - assert(captures[1 + cb].pos[1] == length); -} diff --git a/tests/capture/capture4.c b/tests/capture/capture4.c deleted file mode 100644 index 170cbe8b0..000000000 --- a/tests/capture/capture4.c +++ /dev/null @@ -1,242 +0,0 @@ -/* - * Copyright 2020 Scott Vokes - * - * See LICENCE for the full copyright terms. - */ - -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -#include "captest.h" - -/* Combine 2 mostly overlapping FSMs: - * - 0: "(abc)" - * - 1: "(ab*c)" - * and check for false positives in the match. - */ - -static struct fsm * -build_and_combine(unsigned *cb_a, unsigned *cb_b); - -static void -det_and_min(const char *tag, struct fsm *fsm); - -static struct fsm * -build_ab_c(void); - -static void -check(const struct fsm *fsm, const char *string, - unsigned expected_ends, - unsigned cb_a, size_t pa_0, size_t pa_1, - unsigned cb_b, size_t pb_0, size_t pb_1); - -int main(void) { - unsigned cb_abc, cb_ab_c; - struct fsm *f_all = build_and_combine(&cb_abc, &cb_ab_c); - unsigned captures; - const unsigned exp_0 = 1U << 0; - const unsigned exp_1 = 1U << 1; - - captures = fsm_countcaptures(f_all); - if (captures != 2) { - fprintf(stderr, "expected 2 captures, got %u\n", captures); - exit(EXIT_FAILURE); - } - - #define NO_POS FSM_CAPTURE_NO_POS - check(f_all, "abc", /* captures 0 and 1 */ - exp_0 | exp_1, - cb_abc, 0, 3, - cb_ab_c, 0, 3); - check(f_all, "ac", /* only capture 1 */ - exp_1, - cb_abc, NO_POS, NO_POS, - cb_ab_c, 0, 2); - check(f_all, "abbc", /* only capture 1 */ - exp_1, - cb_abc, NO_POS, NO_POS, - cb_ab_c, 0, 4); - - fsm_free(f_all); - - return 0; -} - -static struct fsm * -build_and_combine(unsigned *cb_a, unsigned *cb_b) -{ - struct fsm *f_abc = captest_fsm_of_string("abc", 0); - struct fsm *f_ab_c = build_ab_c(); - struct fsm *f_all; - struct fsm_combine_info ci; - - assert(f_abc); - assert(f_ab_c); - - if (!fsm_capture_set_path(f_abc, 0, 0, 3)) { - exit(EXIT_FAILURE); - } - if (!fsm_capture_set_path(f_ab_c, 0, 0, 3)) { - exit(EXIT_FAILURE); - } - -#if LOG_INTERMEDIATE_FSMS - fprintf(stderr, "==================== abc \n"); - fsm_print_fsm(stderr, f_abc); - fsm_capture_dump(stderr, "abc", f_abc); - - fprintf(stderr, "==================== ab*c \n"); - fsm_print_fsm(stderr, f_ab_c); - fsm_capture_dump(stderr, "ab*c", f_ab_c); -#endif - - det_and_min("abc", f_abc); - det_and_min("ab*c", f_ab_c); - - /* union them */ - f_all = fsm_union(f_abc, f_ab_c, &ci); - assert(f_all != NULL); - - *cb_a = ci.capture_base_a; - *cb_b = ci.capture_base_b; - -#if LOG_INTERMEDIATE_FSMS - fprintf(stderr, "==================== post-union \n"); - fsm_print_fsm(stderr, f_all); - fsm_capture_dump(stderr, "capture_actions", f_all); - fprintf(stderr, "====================\n"); -#endif - - if (!fsm_determinise(f_all)) { - exit(EXIT_FAILURE); - } - -#if LOG_INTERMEDIATE_FSMS - fprintf(stderr, "==================== post-det \n"); - fsm_print_fsm(stderr, f_all); - fsm_capture_dump(stderr, "capture_actions", f_all); - fprintf(stderr, "====================\n"); -#endif - - return f_all; -} - -static void -det_and_min(const char *tag, struct fsm *fsm) -{ - if (!fsm_determinise(fsm)) { - fprintf(stderr, "Failed to determise '%s'\n", tag); - exit(EXIT_FAILURE); - } - - if (!fsm_minimise(fsm)) { - fprintf(stderr, "Failed to minimise '%s'\n", tag); - exit(EXIT_FAILURE); - } - -#if LOG_INTERMEDIATE_FSMS - fprintf(stderr, "==== after det_and_min: '%s'\n", tag); - fsm_print_fsm(stderr, fsm); - fsm_capture_dump(stderr, tag, fsm); -#endif - -} - -static struct fsm * -build_ab_c(void) -{ - struct fsm *fsm = captest_fsm_with_options(); - assert(fsm != NULL); - - if (!fsm_addstate_bulk(fsm, 4)) { goto fail; } - - fsm_setstart(fsm, 0); - if (!fsm_addedge_literal(fsm, 0, 1, 'a')) { goto fail; } - - if (!fsm_addedge_literal(fsm, 1, 2, 'b')) { goto fail; } - if (!fsm_addedge_literal(fsm, 1, 3, 'c')) { goto fail; } - - if (!fsm_addedge_literal(fsm, 2, 2, 'b')) { goto fail; } - if (!fsm_addedge_literal(fsm, 2, 3, 'c')) { goto fail; } - - fsm_setend(fsm, 3, 1); - if (!fsm_setendid(fsm, 1)) { - goto fail; - } - - return fsm; - -fail: - exit(EXIT_FAILURE); -} - -static void -check(const struct fsm *fsm, const char *string, - unsigned expected_ends, - unsigned cb_a, size_t pa_0, size_t pa_1, - unsigned cb_b, size_t pb_0, size_t pb_1) -{ - int exec_res; - size_t i; - struct captest_input input; - fsm_state_t end; - struct fsm_capture captures[MAX_TEST_CAPTURES]; - - fprintf(stderr, "#### check '%s', exp: ends 0x%u, c%u: (%ld, %ld), c%u: %ld, %ld)\n", - string, expected_ends, - cb_a, pa_0, pa_1, - cb_b, pb_0, pb_1); - - input.string = string; - input.pos = 0; - - for (i = 0; i < MAX_TEST_CAPTURES; i++) { - captures[i].pos[0] = FSM_CAPTURE_NO_POS; - captures[i].pos[1] = FSM_CAPTURE_NO_POS; - } - - exec_res = fsm_exec(fsm, captest_getc, &input, &end, captures); - if (exec_res != 1) { - fprintf(stderr, "fsm_exec: %d\n", exec_res); - exit(EXIT_FAILURE); - } - - /* check captures */ - fprintf(stderr, "captures for '%s': [%ld, %ld], [%ld, %ld]\n", - string, - captures[0].pos[0], captures[0].pos[1], - captures[1].pos[0], captures[1].pos[1]); - assert(captures[cb_a].pos[0] == pa_0); - assert(captures[cb_a].pos[1] == pa_1); - assert(captures[cb_b].pos[0] == pb_0); - assert(captures[cb_b].pos[1] == pb_1); - - { - enum fsm_getendids_res gres; - fsm_end_id_t id_buf[2]; - size_t written; - gres = fsm_getendids(fsm, end, 2, id_buf, &written); - if (gres != FSM_GETENDIDS_FOUND) { - assert(!"fsm_getendids failed"); - } - - if (expected_ends == 0x2) { - assert(written == 1); - assert(id_buf[0] == 1); - } else if (expected_ends == 0x3) { - assert(written == 2); - assert(id_buf[0] == 0); - assert(id_buf[1] == 1); - } else { - assert(!"test not handled"); - } - } -} diff --git a/tests/capture/capture5.c b/tests/capture/capture5.c deleted file mode 100644 index b3a4be3ee..000000000 --- a/tests/capture/capture5.c +++ /dev/null @@ -1,150 +0,0 @@ -/* - * Copyright 2020 Scott Vokes - * - * See LICENCE for the full copyright terms. - */ - -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -#define LOG_INTERMEDIATE_FSMS 0 -#include "captest.h" - -/* Check that self edges are handled properly in the - * capture action analysis. - * - * The DFA corresponds to /a(b*)(c)/. */ - -static struct fsm * -build(void); - -static void -check(struct fsm *f, const char *input, - unsigned pa_0, unsigned pa_1, - unsigned pb_0, unsigned pb_1); - -int main(void) { - struct fsm *f = build(); - unsigned captures; - assert(f != NULL); - - captures = fsm_countcaptures(f); - if (captures != 2) { - fprintf(stderr, "expected 2 captures, got %u\n", captures); - exit(EXIT_FAILURE); - } - - check(f, "ac", - 1, 1, - 1, 2); - check(f, "abc", - 1, 2, - 2, 3); - check(f, "abbc", - 1, 3, - 3, 4); - - fsm_free(f); - - return 0; -} - -static struct fsm * -build(void) -{ - struct fsm *fsm = captest_fsm_with_options(); - - if (!fsm_addstate_bulk(fsm, 4)) { goto fail; } - - fsm_setstart(fsm, 0); - if (!fsm_addedge_literal(fsm, 0, 1, 'a')) { goto fail; } - - if (!fsm_addedge_literal(fsm, 1, 1, 'b')) { goto fail; } - if (!fsm_addedge_literal(fsm, 1, 2, 'c')) { goto fail; } - - fsm_setend(fsm, 2, 1); - - if (!fsm_capture_set_path(fsm, 0, 1, 1)) { goto fail; } - if (!fsm_capture_set_path(fsm, 1, 1, 2)) { goto fail; } - -#if LOG_INTERMEDIATE_FSMS - fprintf(stderr, "==== built\n"); - fsm_print_fsm(stderr, fsm); - fsm_capture_dump(stderr, "built", fsm); -#endif - - if (!fsm_determinise(fsm)) { - fprintf(stderr, "Failed to determise\n"); - exit(EXIT_FAILURE); - } - -#if LOG_INTERMEDIATE_FSMS - fprintf(stderr, "==== after det\n"); - fsm_print_fsm(stderr, fsm); - fsm_capture_dump(stderr, "after det", fsm); -#endif - - if (!fsm_minimise(fsm)) { - fprintf(stderr, "Failed to minimise\n"); - exit(EXIT_FAILURE); - } - -#if LOG_INTERMEDIATE_FSMS - fprintf(stderr, "==== after min\n"); - fsm_print_fsm(stderr, fsm); - fsm_capture_dump(stderr, "after min", fsm); -#endif - return fsm; - -fail: - exit(EXIT_FAILURE); -} - -static void -check(struct fsm *fsm, const char *string, - unsigned pa_0, unsigned pa_1, - unsigned pb_0, unsigned pb_1) -{ - int exec_res; - size_t i; - struct captest_input input; - fsm_state_t end; - struct fsm_capture captures[MAX_TEST_CAPTURES]; - - fprintf(stderr, "#### check '%s', exp: c%u: (%u, %u), c%u: %u, %u)\n", - string, - 0, pa_0, pa_1, - 1, pb_0, pb_1); - - input.string = string; - input.pos = 0; - - for (i = 0; i < MAX_TEST_CAPTURES; i++) { - captures[i].pos[0] = FSM_CAPTURE_NO_POS; - captures[i].pos[1] = FSM_CAPTURE_NO_POS; - } - - exec_res = fsm_exec(fsm, captest_getc, &input, &end, captures); - if (exec_res != 1) { - fprintf(stderr, "fsm_exec: %d\n", exec_res); - exit(EXIT_FAILURE); - } - - /* check captures */ - fprintf(stderr, "captures for '%s': [%ld, %ld], [%ld, %ld]\n", - string, - captures[0].pos[0], captures[0].pos[1], - captures[1].pos[0], captures[1].pos[1]); - assert(captures[0].pos[0] == pa_0); - assert(captures[0].pos[1] == pa_1); - assert(captures[1].pos[0] == pb_0); - assert(captures[1].pos[1] == pb_1); -} diff --git a/tests/capture/capture_concat1.c b/tests/capture/capture_concat1.c deleted file mode 100644 index ee9c8aaab..000000000 --- a/tests/capture/capture_concat1.c +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Copyright 2020 Scott Vokes - * - * See LICENCE for the full copyright terms. - */ - -#include -#include -#include -#include - -#include -#include -#include - -#include "captest.h" - -/* concat /(ab)/ and /(cde)/ */ - -static struct fsm * -build(unsigned *cb_a, unsigned *cb_b); - -static void -check(const struct fsm *fsm, const char *input, unsigned end_id, - unsigned cb_ab, size_t exp_start_ab, size_t exp_end_ab, - unsigned cb_cde, size_t exp_start_cde, size_t exp_end_cde); - -int main(void) { - unsigned cb_ab, cb_cde; /* capture base */ - struct fsm *abcde = build(&cb_ab, &cb_cde); - - check(abcde, "abcde", 1, - cb_ab, 0, 2, - cb_cde, 2, 5); - - fsm_free(abcde); - - return EXIT_SUCCESS; -} - -static struct fsm * -build(unsigned *cb_a, unsigned *cb_b) -{ - struct fsm *ab = captest_fsm_of_string("ab", 0); - struct fsm *cde = captest_fsm_of_string("cde", 1); - struct fsm *abcde; - struct fsm_combine_info ci; - size_t cc_ab, cc_cde, cc_abcde; - - assert(ab); - assert(cde); - - if (!fsm_capture_set_path(ab, 0, 0, 2)) { - assert(!"path 0"); - } - if (!fsm_capture_set_path(cde, 0, 0, 3)) { - assert(!"path 1"); - } - - cc_ab = fsm_countcaptures(ab); - assert(cc_ab == 1); - - cc_cde = fsm_countcaptures(cde); - assert(cc_cde == 1); - - abcde = fsm_concat(ab, cde, &ci); - assert(abcde); - *cb_a = ci.capture_base_a; - *cb_b = ci.capture_base_b; - - cc_abcde = fsm_countcaptures(abcde); - assert(cc_abcde == cc_ab + cc_cde); - -#if LOG_INTERMEDIATE_FSMS - fprintf(stderr, "==== after concat: cb_ab %u, cb_cde %u\n", - *cb_a, *cb_b); - fsm_print_fsm(stderr, abcde); - - fsm_capture_dump(stderr, "#### after concat", abcde); - - fprintf(stderr, "==== determinise\n"); -#endif - - if (!fsm_determinise(abcde)) { - assert(!"determinise"); - } - -#if LOG_INTERMEDIATE_FSMS - fprintf(stderr, "==== after determinise\n"); - fsm_print_fsm(stderr, abcde); - - assert(fsm_countcaptures(abcde) == cc_abcde); - - fsm_capture_dump(stderr, "#### after det", abcde); -#endif - - assert(fsm_countcaptures(abcde) == cc_abcde); - return abcde; -} - -static void -check(const struct fsm *fsm, const char *input, unsigned end_id, - unsigned cb_ab, size_t exp_start_ab, size_t exp_end_ab, - unsigned cb_cde, size_t exp_start_cde, size_t exp_end_cde) -{ - struct captest_input ci; - fsm_state_t end; - int exec_res; - struct fsm_capture captures[MAX_TEST_CAPTURES]; - - ci.string = input; - ci.pos = 0; - - exec_res = fsm_exec(fsm, captest_getc, &ci, &end, captures); - if (exec_res != 1) { - fprintf(stderr, "exec_res: %d\n", exec_res); - exit(EXIT_FAILURE); - } - - { - const char *msg; - if (!captest_check_single_end_id(fsm, end, end_id, &msg)) { - fprintf(stderr, "%s\n", msg); - exit(EXIT_FAILURE); - } - } - - assert(captures[cb_ab].pos[0] == exp_start_ab); - assert(captures[cb_ab].pos[1] == exp_end_ab); - - assert(captures[cb_cde].pos[0] == exp_start_cde); - assert(captures[cb_cde].pos[1] == exp_end_cde); -} diff --git a/tests/capture/capture_concat2.c b/tests/capture/capture_concat2.c deleted file mode 100644 index a8f070c7e..000000000 --- a/tests/capture/capture_concat2.c +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Copyright 2020 Scott Vokes - * - * See LICENCE for the full copyright terms. - */ - -#include -#include -#include -#include - -#include -#include -#include - -#include "captest.h" - -/* concat /(abc)/ and /(de)/ */ - -static struct fsm * -build(unsigned *cb_a, unsigned *cb_b); - -static void -check(const struct fsm *fsm, const char *input, unsigned end_id, - unsigned cb_ab, size_t exp_start_ab, size_t exp_end_ab, - unsigned cb_cde, size_t exp_start_cde, size_t exp_end_cde); - -int main(void) { - unsigned cb_abc, cb_de; /* capture base */ - struct fsm *abcde = build(&cb_abc, &cb_de); - - check(abcde, "abcde", 1, - cb_abc, 0, 3, - cb_de, 3, 5); - - fsm_free(abcde); - - return EXIT_SUCCESS; -} - -static struct fsm * -build(unsigned *cb_a, unsigned *cb_b) -{ - struct fsm *abc = captest_fsm_of_string("abc", 0); - struct fsm *de = captest_fsm_of_string("de", 1); - struct fsm *abcde; - struct fsm_combine_info ci; - size_t cc_abc, cc_de, cc_abcde; - - assert(abc); - assert(de); - - if (!fsm_capture_set_path(abc, 0, 0, 3)) { - assert(!"path 0"); - } - if (!fsm_capture_set_path(de, 0, 0, 2)) { - assert(!"path 1"); - } - - cc_abc = fsm_countcaptures(abc); - assert(cc_abc == 1); - - cc_de = fsm_countcaptures(de); - assert(cc_de == 1); - - abcde = fsm_concat(abc, de, &ci); - assert(abcde); - *cb_a = ci.capture_base_a; - *cb_b = ci.capture_base_b; - - cc_abcde = fsm_countcaptures(abcde); - assert(cc_abcde == cc_abc + cc_de); - -#if LOG_INTERMEDIATE_FSMS - fprintf(stderr, "==== after concat: cb_abc %u, cb_de %u\n", - *cb_a, *cb_b); - fsm_print_fsm(stderr, abcde); - - fsm_capture_dump(stderr, "#### after concat", abcde); - - fprintf(stderr, "==== determinise\n"); -#endif - - if (!fsm_determinise(abcde)) { - assert(!"determinise"); - } - -#if LOG_INTERMEDIATE_FSMS - fprintf(stderr, "==== after determinise\n"); - fsm_print_fsm(stderr, abcde); - - assert(fsm_countcaptures(abcde) == cc_abcde); - - fsm_capture_dump(stderr, "#### after det", abcde); -#endif - - assert(fsm_countcaptures(abcde) == cc_abcde); - return abcde; -} - -static void -check(const struct fsm *fsm, const char *input, unsigned end_id, - unsigned cb_abc, size_t exp_start_abc, size_t exp_end_abc, - unsigned cb_de, size_t exp_start_de, size_t exp_end_de) -{ - struct captest_input ci; - fsm_state_t end; - int exec_res; - struct fsm_capture captures[MAX_TEST_CAPTURES]; - - ci.string = input; - ci.pos = 0; - - exec_res = fsm_exec(fsm, captest_getc, &ci, &end, captures); - if (exec_res != 1) { - fprintf(stderr, "exec_res: %d\n", exec_res); - exit(EXIT_FAILURE); - } - - { - const char *msg; - if (!captest_check_single_end_id(fsm, end, end_id, &msg)) { - fprintf(stderr, "%s\n", msg); - exit(EXIT_FAILURE); - } - } - - assert(captures[cb_abc].pos[0] == exp_start_abc); - assert(captures[cb_abc].pos[1] == exp_end_abc); - - assert(captures[cb_de].pos[0] == exp_start_de); - assert(captures[cb_de].pos[1] == exp_end_de); -} diff --git a/tests/capture/capture_long_trail.c b/tests/capture/capture_long_trail.c deleted file mode 100644 index 349717b0f..000000000 --- a/tests/capture/capture_long_trail.c +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright 2020 Scott Vokes - * - * See LICENCE for the full copyright terms. - */ - -#include -#include -#include -#include - -#include -#include - -#include "captest.h" -/* a(bcdefghijklmnopqrstuvwxy)z - * This is long enough to exercise growing the trail for - * capture action analysis. */ - -int main(void) { - struct captest_single_fsm_test_info test_info = { - "abcdefghijklmnopqrstuvwxyz", - { - { 1, 25 }, - } - }; - return captest_run_single(&test_info); -} diff --git a/tests/capture/capture_test_case_list.c b/tests/capture/capture_test_case_list.c new file mode 100644 index 000000000..6b72d1018 --- /dev/null +++ b/tests/capture/capture_test_case_list.c @@ -0,0 +1,1944 @@ +#include "captest.h" + +#include + +#define NO_POS FSM_CAPTURE_NO_POS + +const struct captest_case_single single_cases[] = { + { + .regex = "^", + .input = "", + .count = 1, .expected = { + { .pos = {0, 0}, }, + }, + }, + { + .regex = "$", + .input = "", + .count = 1, .expected = { + { .pos = {0, 0}, }, + }, + }, + { + .regex = "$^", + .input = "", + .count = 1, .expected = { + { .pos = {0, 0}, }, + }, + }, + { + .regex = "$^", .input = "x", + .match = SHOULD_NOT_MATCH, + }, + { + .regex = "()*", + .input = "", + .count = 2, .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "()*", + .input = "x", + .count = 2, .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "^$", + .input = "", + .count = 1, .expected = { + { .pos = {0, 0}, }, + }, + }, + { + .regex = "^($|($)|(($))|((($))))", + .input = "", + .count = 2, .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "^(((($)))|(($))|($)|$)", + .input = "", + .count = 5, .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "^((((a$)))|((b$))|(c$)|d$)", + .input = "a", + .count = 5, .expected = { + { .pos = {0, 1}, }, + { .pos = {0, 1}, }, + { .pos = {0, 1}, }, + { .pos = {0, 1}, }, + { .pos = {0, 1}, }, + }, + }, + { + .regex = "^((((a$)))|((b$))|(c$)|d$)", + .input = "b", + .count = 7, .expected = { + { .pos = {0, 1}, }, + { .pos = {0, 1}, }, + { .pos = {-1, -1}, }, + { .pos = {-1, -1}, }, + { .pos = {-1, -1}, }, + { .pos = {0, 1}, }, + { .pos = {0, 1}, }, + }, + }, + { + .regex = "^((((b$)))|((b$))|(c$)|d$)", + .input = "b", + .count = 5, .expected = { + { .pos = {0, 1}, }, + { .pos = {0, 1}, }, + { .pos = {0, 1}, }, + { .pos = {0, 1}, }, + { .pos = {0, 1}, }, + }, + }, + { + .regex = "^((x?))*$", + .input = "x", + .count = 3, .expected = { + { .pos = {0, 1}, }, + { .pos = {1, 1}, }, + { .pos = {1, 1}, }, + }, + }, + { + .regex = "^((x?)*)*$", + .input = "", + .count = 3, .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "^((x?)*)*$", + .input = "xxxxx", + .count = 3, .expected = { + { .pos = {0, 5}, }, + { .pos = {5, 5}, }, + { .pos = {5, 5}, }, + }, + }, + { + .regex = "xx*x", + .input = "xx", + .count = 1, .expected = { + { .pos = {0, 2}, }, + }, + }, + { + .regex = "^(x?)*$", + .input = "xx", + .count = 2, .expected = { + { .pos = {0, 2}, }, + { .pos = {2, 2}, }, + }, + }, + { + .regex = "^(x?)*$", + .input = "xxx", + .count = 2, .expected = { + { .pos = {0, 3}, }, + { .pos = {3, 3}, }, + }, + }, + { + .regex = "^(x?)+$", + .input = "xx", + .count = 2, .expected = { + { .pos = {0, 2}, }, + { .pos = {2, 2}, }, + }, + }, + { + .regex = "^(x?)+$", + .input = "xxx", + .count = 2, .expected = { + { .pos = {0, 3}, }, + { .pos = {3, 3}, }, + }, + }, + { + .regex = "^x(z?)*y$", + .input = "xy", + .count = 2, .expected = { + { .pos = {0, 2}, }, + { .pos = {1, 1}, }, + }, + }, + { + .regex = "()|x", + .input = "", + .count = 2, .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "()|x", + .input = "x", + .count = 2, .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "x|()", + .input = "x", + .count = 1, .expected = { + { .pos = {0, 1}, }, + }, + }, + { + .regex = "x|()", + .input = "", + .count = 2, .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "$|", + .input = "x", + .count = 1, .expected = { + { .pos = {0, 0}, }, + }, + }, + { + .regex = ".|$^", + .input = "", + .count = 1, .expected = { + { .pos = {0, 0}, }, + }, + }, + { + .regex = ".|$^", + .input = "x", + .count = 1, .expected = { + { .pos = {0, 1}, }, + }, + }, + { + .regex = "$^|.", + .input = "", + .count = 1, .expected = { + { .pos = {0, 0}, }, + }, + }, + { + .regex = "$^|.", + .input = "x", + .count = 1, .expected = { + { .pos = {0, 1}, }, + }, + }, + { + .regex = "$$$^|...", + .input = "xxx", + .count = 1, .expected = { + { .pos = {0, 3}, }, + }, + }, + { + .regex = "x?$x?^x?|x?$x?^x?", + .input = "", + .count = 1, .expected = { + { .pos = {0, 0}, }, + }, + }, + { + .regex = "[^x]", .input = "", + .no_nl = true, + .match = SHOULD_NOT_MATCH, + }, + { + .regex = "[^x]", + .input = "\n", + .count = 1, .expected = { + { .pos = {0, 1}, }, + }, + }, + { + .regex = ".$()", + .input = "x", + .count = 2, .expected = { + { .pos = {0, 1}, }, + { .pos = {1, 1}, }, + }, + }, + { + .regex = ".$()", .input = "", + .match = SHOULD_NOT_MATCH, + }, + { + .regex = "^.$()", + .input = "x", + .count = 2, .expected = { + { .pos = {0, 1}, }, + { .pos = {1, 1}, }, + }, + }, + { + .regex = "^.$()", .input = "", + .match = SHOULD_NOT_MATCH, + }, + { + .regex = "$(x?)(y?)(z?)", + .input = "a", + .count = 4, .expected = { + { .pos = {1, 1}, }, + { .pos = {1, 1}, }, + { .pos = {1, 1}, }, + { .pos = {1, 1}, }, + }, + }, + { + .regex = ".$(x?)(y?)(z?)", + .input = "a", + .count = 4, .expected = { + { .pos = {0, 1}, }, + { .pos = {1, 1}, }, + { .pos = {1, 1}, }, + { .pos = {1, 1}, }, + }, + }, + { + .regex = "[^y]", + .input = "xx", + .count = 1, .expected = { + { .pos = {0, 1}, }, + }, + }, + { + .regex = ".", + .input = "xx", + .count = 1, .expected = { + { .pos = {0, 1}, }, + }, + }, + { + .regex = "(x)+", + .input = "xxx", + .count = 2, .expected = { + { .pos = {0, 3}, }, + { .pos = {2, 3}, }, + }, + }, + { + .regex = "^(x)*.", + .input = "xx", + .count = 2, .expected = { + { .pos = {0, 2}, }, + { .pos = {0, 1}, }, + }, + }, + { + .regex = "^(x)*.", + .input = "xy", + .count = 2, .expected = { + { .pos = {0, 2}, }, + { .pos = {0, 1}, }, + }, + }, + { + .regex = "a.b(c)*", + .input = "axbc", + .count = 2, .expected = { + { .pos = {0, 4}, }, + { .pos = {3, 4}, }, + }, + }, + { + .regex = "^x?^", + .input = "", + .count = 1, .expected = { + { .pos = {0, 0}, }, + }, + }, + { + .regex = "^x?^", + .input = "x", + .count = 1, .expected = { + { .pos = {0, 0}, }, + }, + }, + { + .regex = "$(^)", + .input = "", + .count = 2, .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "($)", + .input = "", + .count = 2, .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "($$$)", + .input = "", + .count = 2, .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "$x?^", .input = "x", + .match = SHOULD_NOT_MATCH, + }, + { + .regex = "$(^)*", + .input = "", + .count = 2, .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "$(^)*", + .input = "x", + .count = 1, .expected = { + { .pos = {1, 1}, }, + }, + }, + { + .regex = "$()*", + .input = "", + .count = 2, .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "$()*", + .input = "x", + .count = 2, .expected = { + { .pos = {1, 1}, }, + { .pos = {1, 1}, }, + }, + }, + { + .regex = "^$^", .input = "x", + .match = SHOULD_NOT_MATCH, + }, + { + .regex = "$^$", .input = "x", + .match = SHOULD_NOT_MATCH, + }, + { + .regex = "$y?^x*", .input = "x", + .match = SHOULD_NOT_MATCH, + }, + { + .regex = "x|$^", + .input = "x", + .count = 1, .expected = { + { .pos = {0, 1}, }, + }, + }, + { + .regex = "x|$^", .input = "y", + .match = SHOULD_NOT_MATCH, + }, + { + .regex = "x|$^$^", + .input = "x", + .count = 1, .expected = { + { .pos = {0, 1}, }, + }, + }, + { + .regex = "x|$^$^", .input = "y", + .match = SHOULD_NOT_MATCH, + }, + { + .regex = "$^|x", + .input = "x", + .count = 1, .expected = { + { .pos = {0, 1}, }, + }, + }, + { + .regex = "$^|x", .input = "y", + .match = SHOULD_NOT_MATCH, + }, + { + .regex = "$^$^|x", + .input = "x", + .count = 1, .expected = { + { .pos = {0, 1}, }, + }, + }, + { + .regex = "$^$^|x", .input = "y", + .match = SHOULD_NOT_MATCH, + }, + { + .regex = "^$|.", + .input = "x", + .count = 1, .expected = { + { .pos = {0, 1}, }, + }, + }, + { + .regex = "x|^$^$", .input = "y", + .match = SHOULD_NOT_MATCH, + }, + { + .regex = "^$^$|x", .input = "y", + .match = SHOULD_NOT_MATCH, + }, + { + .regex = "$|^|a$", + .input = "x", + .count = 1, .expected = { + { .pos = {0, 0}, }, + }, + }, + { + .regex = "[^a]x", .input = "x", + .match = SHOULD_NOT_MATCH, + }, + { + .regex = "[^a]x", + .input = "xx", + .count = 1, .expected = { + { .pos = {0, 2}, }, + }, + }, + { + .regex = "a(b|c$)d", .input = "ac", + .match = SHOULD_NOT_MATCH, + }, + { + .regex = "a(^b|c)d", .input = "bd", + .match = SHOULD_NOT_MATCH, + }, + { + .regex = "(a|b|)*", + .input = "", + .count = 2, .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "xx*y$", + .input = "x_xxy", + .count = 1, .expected = { + { .pos = {2, 5}, }, + }, + }, + { + .regex = "(|.$)*", + .input = "x", + .count = 2, .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "(.$)*x", .input = "y", + .match = SHOULD_NOT_MATCH, + }, + { + .regex = "(.$)*", + .input = "x", + .count = 2, .expected = { + { .pos = {0, 1}, }, + { .pos = {0, 1}, }, + }, + }, + { + .regex = "^(|.$)*", + .input = "x", + .count = 2, .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "^(|.$)*$", + .input = "x", + .count = 2, .expected = { + { .pos = {0, 1}, }, + { .pos = {1, 1}, }, + }, + }, + { + .regex = "x|y(^)", .input = "", + .match = SHOULD_NOT_MATCH, + }, + { + .regex = "(?:x*.|^$).", .input = "", + .match = SHOULD_NOT_MATCH, + }, + { + .regex = "(?:x|^$)x", .input = "", + .match = SHOULD_NOT_MATCH, + }, + { + .regex = "()+x", + .input = "x", + .count = 2, .expected = { + { .pos = {0, 1}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "($$)^", + .input = "", + .count = 2, .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "$($|$a)", + .input = "", + .count = 2, .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "^(?i)abc$", + .input = "AbC", + .count = 1, .expected = { + { .pos = {0, 3}, }, + }, + }, + { + .regex = "^(?i)ab(?-i)c$", .input = "AbC", + .match = SHOULD_NOT_MATCH, + }, + { + .regex = "^(?i)ab(?-i)c$", + .input = "Abc", + .count = 1, .expected = { + { .pos = {0, 3}, }, + }, + }, + { + .regex = "^(?i)a[b]c$", + .input = "ABC", + .count = 1, .expected = { + { .pos = {0, 3}, }, + }, + }, + { + .regex = "^(?i)a[^b]c$", .input = "ABC", + .match = SHOULD_NOT_MATCH, + }, + { + .regex = "^(?i)a[bx]c$", + .input = "ABC", + .count = 1, .expected = { + { .pos = {0, 3}, }, + }, + }, + { + .regex = "^(?i)a[b-c]c$", + .input = "ABC", + .count = 1, .expected = { + { .pos = {0, 3}, }, + }, + }, + { + .regex = "(a()b)+a", + .input = "a!aba", + .count = 3, .expected = { + { .pos = {2, 5}, }, + { .pos = {2, 4}, }, + { .pos = {3, 3}, }, + }, + }, + { + .regex = "^^[^]]", + .input = "\n", + .count = 1, .expected = { + { .pos = {0, 1}, }, + }, + }, + { + .regex = "x(x()y)*", + .input = "xxy", + .count = 3, .expected = { + { .pos = {0, 3}, }, + { .pos = {1, 3}, }, + { .pos = {2, 2}, }, + }, + }, + { + .regex = "x(()x)*", + .input = "xx", + .count = 3, .expected = { + { .pos = {0, 2}, }, + { .pos = {1, 2}, }, + { .pos = {1, 1}, }, + }, + }, + { + .regex = "b(x*x*a()*y)*(a)a*", + .input = "ba", + .count = 4, .expected = { + { .pos = {0, 2}, }, + { .pos = {-1, -1}, }, + { .pos = {-1, -1}, }, + { .pos = {1, 2}, }, + }, + }, + { + .regex = "a(().x)*ab", + .input = "a.a.aaxab", + .count = 3, .expected = { + { .pos = {4, 9}, }, + { .pos = {5, 7}, }, + { .pos = {5, 5}, }, + }, + }, + { + .regex = "ab(b()*()*)*()*z", + .input = "a!abz", + .count = 5, .expected = { + { .pos = {2, 5}, }, + { .pos = {-1, -1}, }, + { .pos = {-1, -1}, }, + { .pos = {-1, -1}, }, + { .pos = {4, 4}, }, + }, + }, + { + .regex = "^x(y?z*)*$", + .input = "x", + .count = 2, .expected = { + { .pos = {0, 1}, }, + { .pos = {1, 1}, }, + }, + }, + { + .regex = "^(y?z*)*$", + .input = "", + .count = 2, .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "^(x|$x?)*$", + .input = "", + .count = 2, .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "^(^|$x)*$", + .input = "", + .count = 2, .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "((x?)*(x?)*(x?)*(x?)*(x?)*(x?)*(x?)*(x?)*(x?)*(x?)*(x?)*(x?)*(x?)*)*y$", + .input = "xxxxxxxxxxy", + .count = 15, .expected = { + { .pos = {0, 11}, }, + { .pos = {10, 10}, }, + { .pos = {10, 10}, }, + { .pos = {10, 10}, }, + { .pos = {10, 10}, }, + { .pos = {10, 10}, }, + { .pos = {10, 10}, }, + { .pos = {10, 10}, }, + { .pos = {10, 10}, }, + { .pos = {10, 10}, }, + { .pos = {10, 10}, }, + { .pos = {10, 10}, }, + { .pos = {10, 10}, }, + { .pos = {10, 10}, }, + { .pos = {10, 10}, }, + }, + }, + { + .regex = "^a$", + .input = "a", + .count = 1, .expected = { + { .pos = {0, 1}, }, + }, + }, + { + .regex = "^a(bcd)e$", + .input = "abcde", + .count = 2, .expected = { + { .pos = {0, 5}, }, + { .pos = {1, 4}, }, + }, + }, + { + .regex = "^(a(b((c))(d)))$", + .input = "abcd", + .count = 6, .expected = { + { .pos = {0, 4}, }, + { .pos = {0, 4}, }, + { .pos = {1, 4}, }, + { .pos = {2, 3}, }, + { .pos = {2, 3}, }, + { .pos = {3, 4}, }, + }, + }, + { + .regex = "^(a(b(c)))$", + .input = "abc", + .count = 4, .expected = { + { .pos = {0, 3}, }, + { .pos = {0, 3}, }, + { .pos = {1, 3}, }, + { .pos = {2, 3}, }, + }, + }, + { + .regex = "^a(b*)(c)$", + .input = "ac", + .count = 3, .expected = { + { .pos = {0, 2}, }, + { .pos = {1, 1}, }, + { .pos = {1, 2}, }, + }, + }, + { + .regex = "^a(b*)(c)$", + .input = "abc", + .count = 3, .expected = { + { .pos = {0, 3}, }, + { .pos = {1, 2}, }, + { .pos = {2, 3}, }, + }, + }, + { + .regex = "^a(b*)(c)$", + .input = "abbc", + .count = 3, .expected = { + { .pos = {0, 4}, }, + { .pos = {1, 3}, }, + { .pos = {3, 4}, }, + }, + }, + { + .regex = "^(ab*c)$", + .input = "ac", + .count = 2, .expected = { + { .pos = {0, 2}, }, + { .pos = {0, 2}, }, + }, + }, + { + .regex = "^(ab*c)$", + .input = "abc", + .count = 2, .expected = { + { .pos = {0, 3}, }, + { .pos = {0, 3}, }, + }, + }, + { + .regex = "^(ab*c)$", + .input = "abbc", + .count = 2, .expected = { + { .pos = {0, 4}, }, + { .pos = {0, 4}, }, + }, + }, + { + .regex = "^(a*)", + .input = "", + .count = 2, .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "^(a*)", + .input = "x", + .count = 2, .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "^(a*)", + .input = "a", + .count = 2, .expected = { + { .pos = {0, 1}, }, + { .pos = {0, 1}, }, + }, + }, + { + .regex = "^(a*)", + .input = "ax", + .count = 2, .expected = { + { .pos = {0, 1}, }, + { .pos = {0, 1}, }, + }, + }, + { + .regex = "^a*", + .input = "", + .count = 1, .expected = { + { .pos = {0, 0}, }, + }, + }, + { + .regex = "^a*", + .input = "a", + .count = 1, .expected = { + { .pos = {0, 1}, }, + }, + }, + { + .regex = "^a*", + .input = "ax", + .count = 1, .expected = { + { .pos = {0, 1}, }, + }, + }, + { + .regex = ".|", + .input = "", + .count = 1, .expected = { + { .pos = {0, 0}, }, + }, + }, + { + .regex = "()*^", + .input = "", + .count = 2, .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "(((())))*^", + .input = "", + .count = 5, .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + + { + .regex = "(x|(x|))^", + .input = "", + .count = 3, .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = ".*(x|())^", + .input = "", + .count = 3, .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "(()|(()|x)^|x)^", + .input = "", + .count = 3, + .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + + { + .regex = "x^()()|()", + .input = "", + .count = 4, + .expected = { + { .pos = {0, 0}, }, + { .pos = {NO_POS, NO_POS}, }, + { .pos = {NO_POS, NO_POS}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "y^()|()^x", + .input = "x", + .count = 3, + .expected = { + { .pos = {0, 1}, }, + { .pos = {NO_POS, NO_POS}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "()$a|()", + .input = "", + .count = 3, + .expected = { + { .pos = {0, 0}, }, + { .pos = {NO_POS, NO_POS}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "()$z|(x)$", + .input = "x", + .count = 3, + .expected = { + { .pos = {0, 1}, }, + { .pos = {NO_POS, NO_POS}, }, + { .pos = {0, 1}, }, + }, + }, + + { + /* long enough to exercise the USE_COLLAPSED_ZERO_PREFIX optimization */ + .regex = "a*(ba*)c$", + .input = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaac", + .count = 2, + .expected = { + { .pos = {101, 303}, }, + { .pos = {201, 302}, }, + }, + }, + + /* regression: losing the first character on the transition from + * the unanchored start loop to the capture */ + { + .regex = "aa+b$", + .input = "aXaXaaab", + .count = 1, + .expected = { + { .pos = {4, 8}, }, + }, + }, + { + .regex = "aa*b$", + .input = "aXaXaaab", + .count = 1, + .expected = { + { .pos = {4, 8}, }, + }, + }, + { + .regex = "!!!+$", + .input = "!\"!\"!\"!!!!", + .count = 1, + .expected = { + { .pos = {6, 10}, }, + }, + }, + + /* new fuzzer regressions */ + { + /* PCRE does not set the first capture, which is unsatisfiable */ + .regex = "^(.^)*^(a*)", + .input = "", + .count = 3, + .expected = { + { .pos = {0, 0}, }, + { .pos = {NO_POS, NO_POS}, }, + { .pos = {0, 0}, }, + }, + }, + { + /* similar to the previous case, but with different anchoring */ + .regex = "(a)*(^)*^", + .input = "", + .count = 3, + .expected = { + { .pos = {0, 0}, }, + { .pos = {NO_POS, NO_POS}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "^(.a)*^(.a)", + .input = "!a", + .count = 3, + .expected = { + { .pos = {0, 2}, }, + { .pos = {NO_POS, NO_POS}, }, + { .pos = {0, 2}, }, + }, + }, + { + .regex = "(A)*^()*^", + .input = "", + .count = 3, + .expected = { + { .pos = {0, 0}, }, + { .pos = {NO_POS, NO_POS}, }, + { .pos = {0, 0}, }, + }, + }, + + { + .regex = "(a(b*)*|)*bc", + .input = "b!bc", + .count = 3, + .expected = { + { .pos = {2, 4}, }, + { .pos = {2, 2}, }, + { .pos = {NO_POS, NO_POS}, }, + }, + }, + { + .regex = "^(a(b*)*|)*bc$", + .input = "bc", + .count = 3, + .expected = { + { .pos = {0, 2}, }, + { .pos = {0, 0}, }, + { .pos = {NO_POS, NO_POS}, }, + }, + }, + { + .regex = "(|a((b*)*b*))*", + .input = "", + .count = 4, + .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + { .pos = {NO_POS, NO_POS}, }, + { .pos = {NO_POS, NO_POS}, }, + }, + }, + { + /* simplified version of the above */ + .regex = "^(|a(b*)*)*$", + .input = "", + .count = 3, + .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + { .pos = {NO_POS, NO_POS}, }, + }, + }, + { + /* zero repetitions should not set the capture */ + .regex = "^(a)*$", + .input = "", + .count = 2, + .expected = { + { .pos = {0, 0}, }, + { .pos = {NO_POS, NO_POS}, }, + }, + }, + { + .regex = "^(a)*(^)$", + .input = "", + .count = 3, + .expected = { + { .pos = {0, 0}, }, + { .pos = {NO_POS, NO_POS}, }, + { .pos = {0, 0}, }, + }, + }, + { + /* raw fuzzer output */ + .regex = "()((()(^|$|$^|^|$|$^^|$|$^|^|$|$^^^^|^|(|)($)|)+|^^|^|(|)($)|)+|)($)()+", + .input = "", + .count = 12, + .match = SHOULD_REJECT_AS_UNSUPPORTED, + .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + { .pos = {NO_POS, NO_POS}, }, + { .pos = {NO_POS, NO_POS}, }, + { .pos = {NO_POS, NO_POS}, }, + { .pos = {NO_POS, NO_POS}, }, + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "(^|())+()", + .input = "", + .count = 4, + .match = SHOULD_REJECT_AS_UNSUPPORTED, + .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + { .pos = {NO_POS, NO_POS}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "(?:(^|^$)+|)+", + .input = "", + .match = SHOULD_REJECT_AS_UNSUPPORTED, + .count = 2, + .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "^((|)($)|)+a$", + .input = "a", + .match = SHOULD_REJECT_AS_UNSUPPORTED, + .count = 4, + .expected = { + { .pos = {0, 1}, }, + { .pos = {0, 0}, }, + { .pos = {NO_POS, NO_POS}, }, + { .pos = {NO_POS, NO_POS}, }, + }, + }, + { + .regex = "^(($)|)+a$", + .input = "a", + .match = SHOULD_REJECT_AS_UNSUPPORTED, + .count = 3, + .expected = { + { .pos = {0, 1}, }, + { .pos = {0, 0}, }, + { .pos = {NO_POS, NO_POS}, }, + }, + }, + { + .regex = "^(|(|x))*$", + .input = "x", + .count = 3, + .expected = { + { .pos = {0, 1}, }, + { .pos = {1, 1}, }, + { .pos = {0, 1}, }, + }, + }, + { + /* same as the previous but without outer capture */ + .regex = "^(?:|(|x))*$", + .input = "x", + .count = 2, + .expected = { + { .pos = {0, 1}, }, + { .pos = {0, 1}, }, + }, + }, + { + .regex = "(((($)|)+|)a|)+", + .input = "", + .match = SHOULD_REJECT_AS_UNSUPPORTED, + .count = 5, + .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + { .pos = {NO_POS, NO_POS}, }, + { .pos = {NO_POS, NO_POS}, }, + { .pos = {NO_POS, NO_POS}, }, + }, + }, + + { + .regex = "^(|(|(|x)))*$", + .input = "x", + .count = 4, + .expected = { + { .pos = {0, 1}, }, + { .pos = {1, 1}, }, + { .pos = {0, 1}, }, + { .pos = {0, 1}, }, + }, + }, + + + { + .regex = "^(?:(?:(x?)^)y?)+$", + .input = "", + .count = 2, + .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "^(?:^())+$", + .input = "", + .count = 2, + .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "^(?:($|x))+$", + .input = "x", + + .match = SHOULD_REJECT_AS_UNSUPPORTED, + + .count = 2, + .expected = { + { .pos = {0, 1}, }, + { .pos = {1, 1}, }, + }, + }, + { + .regex = "^(($)|x)+$", + .input = "x", + .match = SHOULD_REJECT_AS_UNSUPPORTED, + .count = 3, + .expected = { + { .pos = {0, 1}, }, + { .pos = {1, 1}, }, + { .pos = {1, 1}, }, + }, + }, + { + .regex = "^(?:()?^()?)+$", + .input = "", + .count = 3, + .expected = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + }, + }, + { + .regex = "^(?:($|x)())+$", + .input = "x", + .match = SHOULD_REJECT_AS_UNSUPPORTED, + .count = 3, + .expected = { + { .pos = {0, 1}, }, + { .pos = {1, 1}, }, + { .pos = {1, 1}, }, + }, + }, + + { + .regex = "()~((|)($)|%)+", + .input = "~%", + .match = SHOULD_REJECT_AS_UNSUPPORTED, + .count = 5, + .expected = { + { .pos = {0, 2}, }, + { .pos = {0, 0}, }, + { .pos = {2, 2}, }, + { .pos = {2, 2}, }, + { .pos = {2, 2}, }, + }, + }, + + { + /* (slightly) reduced version of the previous */ + .regex = "^(()($)|x)+$", + .input = "x", + .match = SHOULD_REJECT_AS_UNSUPPORTED, + .count = 4, + .expected = { + { .pos = {0, 1}, }, + { .pos = {1, 1}, }, + { .pos = {1, 1}, }, + { .pos = {1, 1}, }, + }, + }, + + { + .regex = "a|_$[^b]", + .input = "a", + .count = 1, + .expected = { + { .pos = {0, 1}, }, + }, + }, + + { + .regex = "\\z", + .input = "", + .count = 1, + .match = SHOULD_REJECT_AS_UNSUPPORTED, + }, +}; + +const struct captest_case_multi multi_cases[] = { + { + .regex_count = 4, + .regexes = { + "^aa$", /* exactly two 'a's */ + "^a*", /* zero or more 'a's followed by anything */ + "^ab?$", /* 'a' and optionally 'b' */ + "a*$", /* anything ending in zero or more 'a's */ + }, + .inputs = { + { + .input = "", + .expected = { + { .regex = 0, .pos = POS_NONE }, + { .regex = 1, .pos = { 0, 0 } }, + { .regex = 2, .pos = POS_NONE }, + { .regex = 3, .pos = { 0, 0 } }, + }, + }, + + { + .input = "a", + .expected = { + { .regex = 0, .pos = POS_NONE }, + { .regex = 1, .pos = { 0, 1 } }, + { .regex = 2, .pos = { 0, 1 } }, + { .regex = 3, .pos = { 0, 1 } }, + }, + }, + + { + .input = "aa", + .expected = { + { .regex = 0, .pos = { 0, 2 } }, + { .regex = 1, .pos = { 0, 2 } }, + { .regex = 2, .pos = POS_NONE }, + { .regex = 3, .pos = { 0, 2 } }, + }, + }, + + { + .input = "aaa", + .expected = { + { .regex = 0, .pos = POS_NONE }, + { .regex = 1, .pos = { 0, 3 } }, + { .regex = 2, .pos = POS_NONE }, + { .regex = 3, .pos = { 0, 3 } }, + }, + }, + + { + .input = "ba", + .expected = { + { .regex = 0, .pos = POS_NONE }, + { .regex = 1, .pos = { 0, 0 } }, + { .regex = 2, .pos = POS_NONE }, + { .regex = 3, .pos = { 1, 2 } }, + }, + }, + + { + .input = "ab", + .expected = { + { .regex = 0, .pos = POS_NONE }, + { .regex = 1, .pos = { 0, 1 } }, + { .regex = 2, .pos = { 0, 2 } }, + { .regex = 3, .pos = { 2, 2 } }, + }, + }, + + { + .input = NULL, + }, + }, + }, + + { + .regex_count = 3, + .regexes = { + "a(b?)*c", + "(ab)(c)", + "ab+(c)", + }, + .inputs = { + { + .input = "", + .expected = { + { .regex = 0, .capture = 0, .pos = POS_NONE }, + { .regex = 0, .capture = 1, .pos = POS_NONE }, + { .regex = 1, .capture = 0, .pos = POS_NONE }, + { .regex = 1, .capture = 1, .pos = POS_NONE }, + { .regex = 1, .capture = 2, .pos = POS_NONE }, + { .regex = 2, .capture = 0, .pos = POS_NONE }, + { .regex = 2, .capture = 1, .pos = POS_NONE }, + }, + }, + { + .input = "abc", + .expected = { + { .regex = 0, .capture = 0, .pos = {0, 3} }, + { .regex = 0, .capture = 1, .pos = {2, 2} }, + { .regex = 1, .capture = 0, .pos = {0, 3} }, + { .regex = 1, .capture = 1, .pos = {0, 2} }, + { .regex = 1, .capture = 2, .pos = {2, 3} }, + { .regex = 2, .capture = 0, .pos = {0, 3} }, + { .regex = 2, .capture = 1, .pos = {2, 3} }, + }, + }, + }, + }, + { + /* fuzzer regression: This led to an execution path in fsm_union_array, + * fsm_union, fsm_merge, merge that did not init or otherwise set the + * `struct fsm_combine_info`, leading to an out of range offset for + * the capture base. */ + .regex_count = 3, + .regexes = { + ".", + ".^", + "^^_", + }, + .inputs = { + { + .input = "", + .expected = { + { .regex = 0, .pos = POS_NONE }, + { .regex = 1, .pos = POS_NONE }, + { .regex = 2, .pos = POS_NONE }, + }, + }, + { + .input = "_", + .expected = { + { .regex = 0, .pos = { 0, 1 } }, + { .regex = 1, .pos = { 0, 1 } }, + { .regex = 2, .pos = { 0, 1 } }, + }, + }, + }, + }, + + { + /* This checks that minimisation doesn't incorrectly + * merge these and lead to capture false positives. */ + .regex_count = 2, + .regexes = { + "^a(b)c$", /* exactly one 'b' */ + "^a(b*)c$", /* any number of 'b's */ + }, + .inputs = { + { + .input = "", + .expected = { + { .regex = 0, .capture = 0, .pos = POS_NONE }, + { .regex = 0, .capture = 1, .pos = POS_NONE }, + { .regex = 1, .capture = 0, .pos = POS_NONE }, + { .regex = 1, .capture = 1, .pos = POS_NONE }, + }, + }, + { + .input = "a", + .expected = { + { .regex = 0, .capture = 0, .pos = POS_NONE }, + { .regex = 0, .capture = 1, .pos = POS_NONE }, + { .regex = 1, .capture = 0, .pos = POS_NONE }, + { .regex = 1, .capture = 1, .pos = POS_NONE }, + }, + }, + { + .input = "ab", + .expected = { + { .regex = 0, .capture = 0, .pos = POS_NONE }, + { .regex = 0, .capture = 1, .pos = POS_NONE }, + { .regex = 1, .capture = 0, .pos = POS_NONE }, + { .regex = 1, .capture = 1, .pos = POS_NONE }, + }, + }, + { + .input = "ac", + .expected = { + { .regex = 0, .capture = 0, .pos = POS_NONE }, + { .regex = 0, .capture = 1, .pos = POS_NONE }, + { .regex = 1, .capture = 0, .pos = { 0, 2 } }, + { .regex = 1, .capture = 1, .pos = { 1, 1 } }, + }, + }, + { + .input = "abc", + .expected = { + { .regex = 0, .capture = 0, .pos = {0, 3 } }, + { .regex = 0, .capture = 1, .pos = {1, 2 } }, + { .regex = 1, .capture = 0, .pos = { 0, 3 } }, + { .regex = 1, .capture = 1, .pos = { 1, 2 } }, + }, + }, + { + .input = "abbc", + .expected = { + { .regex = 0, .capture = 0, .pos = POS_NONE }, + { .regex = 0, .capture = 1, .pos = POS_NONE }, + { .regex = 1, .capture = 0, .pos = { 0, 4 } }, + { .regex = 1, .capture = 1, .pos = { 1, 3 } }, + }, + }, + + { + .input = NULL, + }, + }, + } +}; + + +static struct captest_case_program program_cases[] = { + { + .input = "", + .char_class = { + { .octets = { ~0, ~0, ~0, ~0 }}, /* 0x00 <= x <= 0xff */ + }, + .expected = { + .count = 4, + .captures = { + { .pos = {0, 0}, }, + { .pos = {0, 0}, }, + { .pos = {NO_POS, NO_POS}, }, + { .pos = {0, 0}, }, + }, + }, + + .ops = { + { .t = CAPVM_OP_SPLIT, .u.split = { .greedy = 3, .nongreedy = 1 }}, + { .t = CAPVM_OP_CHARCLASS, .u.charclass_id = 0 }, + { .t = CAPVM_OP_JMP, .u.jmp = 0 }, + { .t = CAPVM_OP_SAVE, .u.save = 0 }, + { .t = CAPVM_OP_SPLIT, .u.split = { .greedy = 5, .nongreedy = 7 }}, + { .t = CAPVM_OP_ANCHOR, .u.anchor = CAPVM_ANCHOR_START }, + + { .t = CAPVM_OP_JMP, .u.jmp = 9 }, /* jump after |() */ + { .t = CAPVM_OP_SAVE, .u.save = 4 }, + { .t = CAPVM_OP_SAVE, .u.save = 5 }, + + { .t = CAPVM_OP_SPLIT, .u.split = { .greedy = 4, .nongreedy = 10 }}, + + { .t = CAPVM_OP_SAVE, .u.save = 2 }, + { .t = CAPVM_OP_SAVE, .u.save = 3 }, + { .t = CAPVM_OP_SAVE, .u.save = 6 }, + { .t = CAPVM_OP_SAVE, .u.save = 7 }, + { .t = CAPVM_OP_SAVE, .u.save = 1 }, + { .t = CAPVM_OP_SPLIT, .u.split = { .greedy = 18, .nongreedy = 16 }}, + { .t = CAPVM_OP_CHARCLASS, .u.charclass_id = 0 }, + { .t = CAPVM_OP_JMP, .u.jmp = 15 }, + { .t = CAPVM_OP_MATCH }, + }, + }, + + + { + /* correcting compilation of '^(?:($|x))+$' */ + .input = "x", + .expected = { + .count = 2, + .captures = { + { .pos = {0, 1}, }, + { .pos = {1, 1}, }, + }, + }, + + .ops = { + [0] = { .t = CAPVM_OP_SAVE, .u.save = 0 }, + [1] = { .t = CAPVM_OP_ANCHOR, .u.anchor = CAPVM_ANCHOR_START }, + [2] = { .t = CAPVM_OP_SAVE, .u.save = 2 }, + [3] = { .t = CAPVM_OP_SPLIT, .u.split = { .greedy = 4, .nongreedy = 6 }}, + [4] = { .t = CAPVM_OP_ANCHOR, .u.anchor = CAPVM_ANCHOR_END }, + + /* [5] = { .t = CAPVM_OP_JMP, .u.jmp = 7 }, */ + [5] = { .t = CAPVM_OP_SPLIT, .u.split = { .greedy = 7, .nongreedy = 9 }}, + + [6] = { .t = CAPVM_OP_CHAR, .u.chr = 'x' }, + [7] = { .t = CAPVM_OP_SAVE, .u.save = 3 }, + [8] = { .t = CAPVM_OP_SPLIT, .u.split = { .greedy = 2, .nongreedy = 9 }}, + [9] = { .t = CAPVM_OP_ANCHOR, .u.anchor = CAPVM_ANCHOR_END }, + [10] = { .t = CAPVM_OP_SAVE, .u.save = 1 }, + [11] = { .t = CAPVM_OP_MATCH }, + }, + }, +}; + +#define NO_FILTER ((size_t)-1) +struct options { + size_t filter; + int verbosity; + bool track_timing; + FILE *prog_output; + enum groups { + GROUP_SINGLE = 0x01, + GROUP_MULTI = 0x02, + GROUP_PROGRAMS = 0x04, + GROUP_ALL = 0xff, + } group; +}; + +static void +print_usage(FILE *f, const char *progname) +{ + fprintf(f, "%s: [-h] [-v] [-s | -m | -p] [-f ] [-t]\n", progname); + fprintf(f, " -h: print this usage info\n"); + fprintf(f, " -v: increase verbosity (can repeat: -vvv)\n"); + fprintf(f, " -f : just run a specific test, by numeric ID\n"); + fprintf(f, " -s: only single casse\n"); + fprintf(f, " -m: only multi cases\n"); + fprintf(f, " -p: only program cases\n"); + fprintf(f, " -t: print timing info\n"); +} + +static void +get_options(struct options *opt, int argc, char **argv) +{ + const char *progname = argv[0]; + int c; + while (c = getopt(argc, argv, "hf:mpstv"), c != -1) { + switch (c) { + case 'h': + print_usage(stdout, progname); + exit(EXIT_SUCCESS); + break; + case 'v': + opt->verbosity++; + break; + case 'f': + opt->filter = atol(optarg); + break; + case 't': + opt->track_timing = true; + break; + case 'p': + opt->group = GROUP_PROGRAMS; + break; + case 's': + opt->group = GROUP_SINGLE; + break; + case 'm': + opt->group = GROUP_MULTI; + break; + case '?': + default: + print_usage(stderr, progname); + exit(EXIT_FAILURE); + } + } +} + +int main(int argc, char **argv) { + size_t pass = 0; + size_t fail = 0; + size_t skip = 0; + size_t nth = 0; + + struct options options = { + .filter = NO_FILTER, + .verbosity = 0, + .group = GROUP_ALL, + }; + get_options(&options, argc, argv); + + if (options.verbosity == DUMP_PROGRAMS_VERBOSITY) { + options.prog_output = fopen("prog_output", "w"); + assert(options.prog_output != NULL); + } + + /* avoid an extra layer of indentation here */ + if (!(options.group & GROUP_SINGLE)) { goto after_single; } + + printf("-- single cases without trailing newline\n"); + const size_t single_case_count = sizeof(single_cases)/sizeof(single_cases[0]); + for (size_t c_i = 0; c_i < single_case_count; c_i++) { + const size_t cur = nth++; + if (options.filter != NO_FILTER && options.filter != cur) { + continue; + } + + if (options.verbosity > 0) { + printf("%zu: ", cur); + if (options.verbosity > 2) { + fflush(stdout); + } + } + + if (options.verbosity == DUMP_PROGRAMS_VERBOSITY) { + fprintf(options.prog_output, "\n\n==== test_case %zu\n", c_i); + } + + const struct captest_case_single *t = &single_cases[c_i]; + + if (t->match == SHOULD_SKIP) { + printf("%zd: SKIP (regex \"%s\", input \"%s\")\n", + cur, t->regex, t->input); + skip++; + continue; + } + + enum captest_run_case_res res = captest_run_case(t, options.verbosity, false, options.prog_output); + + switch (res) { + case CAPTEST_RUN_CASE_PASS: + pass++; + break; + case CAPTEST_RUN_CASE_FAIL: + if (options.verbosity == 0) { + printf("-- test case %zd (regex \"%s\", input \"%s\")\n", cur, t->regex, t->input); + } + fail++; + break; + case CAPTEST_RUN_CASE_ERROR: + assert(!"error"); + return EXIT_FAILURE; + } + } + + /* second pass, adding a trailing newline to input */ + printf("-- single cases with trailing newline\n"); + for (size_t c_i = 0; c_i < single_case_count; c_i++) { + const size_t cur = nth++; + if (options.filter != NO_FILTER && options.filter != cur) { + continue; + } + + const struct captest_case_single *t = &single_cases[c_i]; + if (t->no_nl) { continue; } + if (t->match == SHOULD_SKIP) { + printf("%zd: SKIP (regex \"%s\", input \"%s\\n\")\n", + cur, t->regex, t->input); + skip++; + continue; + } + + if (options.verbosity > 0) { + printf("%zu: ", cur); + if (options.verbosity > 2) { + fflush(stdout); + } + } + + enum captest_run_case_res res = captest_run_case(t, options.verbosity, true, options.prog_output); + + switch (res) { + case CAPTEST_RUN_CASE_PASS: + pass++; + break; + case CAPTEST_RUN_CASE_FAIL: + if (options.verbosity == 0) { + printf("-- test case %zd (regex \"%s\", input \"%s\\n\")\n", cur, t->regex, t->input); + } + fail++; + break; + case CAPTEST_RUN_CASE_ERROR: + assert(!"error"); + return EXIT_FAILURE; + } + } +after_single: + + /* multi-regex tests */ + if (!(options.group & GROUP_MULTI)) { goto after_multi; } + + printf("-- multi-regex cases\n"); + const size_t multi_case_count = sizeof(multi_cases)/sizeof(multi_cases[0]); + for (size_t c_i = 0; c_i < multi_case_count; c_i++) { + const size_t cur = nth++; + if ((options.filter != NO_FILTER && options.filter != cur)) { + continue; + } + + const struct captest_case_multi *t = &multi_cases[c_i]; + if (t->match == SHOULD_SKIP) { + printf("%zu: SKIP (multi)\n", c_i); + skip++; + continue; + } + + if (options.verbosity > 0) { + printf("%zu: ", cur); + } + + struct captest_case_multi_result result; + enum captest_run_case_res res = captest_run_case_multi(t, + options.verbosity, false, options.prog_output, &result); + + pass += result.pass; + fail += result.fail; + + switch (res) { + case CAPTEST_RUN_CASE_PASS: + if (options.verbosity > 0) { + printf("pass\n"); + } + break; + case CAPTEST_RUN_CASE_FAIL: + if (options.verbosity > 0) { + printf("FAIL\n"); + } else { + printf("-- test case %zd\n", cur); + } + break; + case CAPTEST_RUN_CASE_ERROR: + assert(!"error"); + return EXIT_FAILURE; + } + } +after_multi: + + /* hardcoded programs */ + if (!(options.group & GROUP_PROGRAMS)) { goto after_programs; } + + const size_t prog_case_count = sizeof(program_cases)/sizeof(program_cases[0]); + for (size_t c_i = 0; c_i < prog_case_count; c_i++) { + const size_t cur = nth++; + if ((options.filter != NO_FILTER && options.filter != cur)) { + continue; + } + + const struct captest_case_program *t = &program_cases[c_i]; + + if (options.verbosity > 0) { + printf("%zu: ", cur); + } + + enum captest_run_case_res res = captest_run_case_program(t, + options.verbosity); + + switch (res) { + case CAPTEST_RUN_CASE_PASS: + if (options.verbosity > 0) { + printf("pass\n"); + } + pass++; + break; + case CAPTEST_RUN_CASE_FAIL: + fail++; + if (options.verbosity > 0) { + printf("FAIL\n"); + } else if (options.verbosity == 0) { + printf("-- test case %zd\n", cur); + } + break; + case CAPTEST_RUN_CASE_ERROR: + assert(!"error"); + return EXIT_FAILURE; + } + } +after_programs: + + printf("-- pass %zu, fail %zu, skip %zu\n", pass, fail, skip); + + return fail > 0 + ? EXIT_FAILURE + : EXIT_SUCCESS; +} diff --git a/tests/capture/capture_union1.c b/tests/capture/capture_union1.c deleted file mode 100644 index 5d9bd2920..000000000 --- a/tests/capture/capture_union1.c +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Copyright 2020 Scott Vokes - * - * See LICENCE for the full copyright terms. - */ - -#include -#include -#include -#include - -#include -#include -#include -#include - -#include "captest.h" - -/* union /(ab)/ and /(cde)/ */ - -static struct fsm * -build(unsigned *cb_a, unsigned *cb_b); - -static void -check(const struct fsm *fsm, const char *input, - unsigned end_id, unsigned exp_capture_id, - size_t exp_start, size_t exp_end); - -int main(void) { - unsigned cb_ab, cb_cde; /* capture base */ - struct fsm *abcde = build(&cb_ab, &cb_cde); - - check(abcde, "ab", 0, cb_ab, 0, 2); - check(abcde, "cde", 1, cb_cde, 0, 3); - - fsm_free(abcde); - - return EXIT_SUCCESS; -} - -static struct fsm * -build(unsigned *cb_a, unsigned *cb_b) -{ - struct fsm *ab = captest_fsm_of_string("ab", 0); - struct fsm *cde = captest_fsm_of_string("cde", 1); - struct fsm *abcde; - struct fsm_combine_info ci; - size_t cc_ab, cc_cde, cc_abcde; - - assert(ab); - assert(cde); - - if (!fsm_capture_set_path(ab, 0, 0, 2)) { - assert(!"path 0"); - } - if (!fsm_capture_set_path(cde, 0, 0, 3)) { - assert(!"path 1"); - } - - cc_ab = fsm_countcaptures(ab); - assert(cc_ab == 1); - - cc_cde = fsm_countcaptures(cde); - assert(cc_cde == 1); - - abcde = fsm_union(ab, cde, &ci); - assert(abcde); - *cb_a = ci.capture_base_a; - *cb_b = ci.capture_base_b; - - cc_abcde = fsm_countcaptures(abcde); - assert(cc_abcde == cc_ab + cc_cde); - -#if LOG_INTERMEDIATE_FSMS - fprintf(stderr, "==== after union: cb_ab %u, cb_cde %u\n", - *cb_a, *cb_b); - fsm_print_fsm(stderr, abcde); - - fsm_capture_dump(stderr, "#### after union", abcde); - - fprintf(stderr, "==== determinise\n"); -#endif - - if (!fsm_determinise(abcde)) { - assert(!"determinise"); - } - -#if LOG_INTERMEDIATE_FSMS - fprintf(stderr, "==== after determinise\n"); - fsm_print_fsm(stderr, abcde); - - assert(fsm_countcaptures(abcde) == cc_abcde); - - fsm_capture_dump(stderr, "#### after det", abcde); -#endif - - assert(fsm_countcaptures(abcde) == cc_abcde); - return abcde; -} - -static void -check(const struct fsm *fsm, const char *input, - unsigned end_id, unsigned exp_capture_id, - size_t exp_start, size_t exp_end) -{ - struct captest_input ci; - fsm_state_t end; - int exec_res; - struct fsm_capture got_captures[MAX_TEST_CAPTURES]; - - ci.string = input; - ci.pos = 0; - - exec_res = fsm_exec(fsm, captest_getc, &ci, &end, got_captures); - if (exec_res != 1) { - fprintf(stderr, "exec_res: %d\n", exec_res); - exit(EXIT_FAILURE); - } - - { - const char *msg; - if (!captest_check_single_end_id(fsm, end, end_id, &msg)) { - fprintf(stderr, "%s\n", msg); - exit(EXIT_FAILURE); - } - } - - if (got_captures[exp_capture_id].pos[0] != exp_start) { - fprintf(stderr, "capture[%u].pos[0]: exp %lu, got %lu\n", - exp_capture_id, exp_start, - got_captures[exp_capture_id].pos[0]); - exit(EXIT_FAILURE); - } - if (got_captures[exp_capture_id].pos[1] != exp_end) { - fprintf(stderr, "capture[%u].pos[1]: exp %lu, got %lu\n", - exp_capture_id, exp_end, - got_captures[exp_capture_id].pos[1]); - exit(EXIT_FAILURE); - } -} diff --git a/tests/capture/capture_union2.c b/tests/capture/capture_union2.c deleted file mode 100644 index 7fab2f18d..000000000 --- a/tests/capture/capture_union2.c +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Copyright 2020 Scott Vokes - * - * See LICENCE for the full copyright terms. - */ - -#include -#include -#include -#include - -#include -#include -#include -#include - -#include "captest.h" - -/* union /(abcd)/ and /(abed)/ */ - -static struct fsm * -build(unsigned *cb_a, unsigned *cb_b); - -static void -check(const struct fsm *fsm, const char *input, - unsigned end_id, unsigned exp_capture_id, - size_t exp_start, size_t exp_end); - -int main(void) { - unsigned cb_abcd, cb_abed; - struct fsm *fsm = build(&cb_abcd, &cb_abed); - - check(fsm, "abcd", 0, cb_abcd, 0, 4); - check(fsm, "abed", 1, cb_abed, 0, 4); - - fsm_free(fsm); - - return EXIT_SUCCESS; -} - -static struct fsm * -build(unsigned *cb_a, unsigned *cb_b) -{ - struct fsm *abcd = captest_fsm_of_string("abcd", 0); - struct fsm *abed = captest_fsm_of_string("abed", 1); - struct fsm *res; - - assert(abcd); - assert(abed); - - if (!fsm_capture_set_path(abcd, 0, 0, 4)) { - assert(!"path 0"); - } - if (!fsm_capture_set_path(abed, 0, 0, 4)) { - assert(!"path 1"); - } - - { - struct fsm *fsms[2]; - struct fsm_combined_base_pair bases[2]; - fsms[0] = abcd; - fsms[1] = abed; - res = fsm_union_array(2, fsms, bases); - assert(res); - *cb_a = bases[0].capture; - *cb_b = bases[1].capture; - } - - if (!fsm_determinise(res)) { - assert(!"determinise"); - } - - assert(fsm_countcaptures(res) == 2); - - return res; -} - -static void -check(const struct fsm *fsm, const char *input, - unsigned end_id, unsigned exp_capture_id, - size_t exp_start, size_t exp_end) -{ - struct captest_input ci; - fsm_state_t end; - int exec_res; - struct fsm_capture got_captures[MAX_TEST_CAPTURES]; - - ci.string = input; - ci.pos = 0; - - exec_res = fsm_exec(fsm, captest_getc, &ci, &end, got_captures); - if (exec_res != 1) { - fprintf(stderr, "exec_res: %d\n", exec_res); - exit(EXIT_FAILURE); - } - - { - const char *msg; - if (!captest_check_single_end_id(fsm, end, end_id, &msg)) { - fprintf(stderr, "%s\n", msg); - exit(EXIT_FAILURE); - } - } - - if (got_captures[exp_capture_id].pos[0] != exp_start) { - fprintf(stderr, "capture[%u].pos[0]: exp %lu, got %lu\n", - exp_capture_id, exp_start, - got_captures[exp_capture_id].pos[0]); - exit(EXIT_FAILURE); - } - if (got_captures[exp_capture_id].pos[1] != exp_end) { - fprintf(stderr, "capture[%u].pos[1]: exp %lu, got %lu\n", - exp_capture_id, exp_end, - got_captures[exp_capture_id].pos[1]); - exit(EXIT_FAILURE); - } -} diff --git a/tests/detect_required/Makefile b/tests/detect_required/Makefile new file mode 100644 index 000000000..06d76277e --- /dev/null +++ b/tests/detect_required/Makefile @@ -0,0 +1,43 @@ +.include "../../share/mk/top.mk" + +TEST.tests/detect_required != ls -1 tests/detect_required/detect_required*.c +TEST.tests/detect_required != ls -1 tests/detect_required/out*.txt +TEST_SRCDIR.tests/detect_required = tests/detect_required +TEST_OUTDIR.tests/detect_required = ${BUILD}/tests/detect_required + +FSM=${BUILD}/bin/fsm +RE=${BUILD}/bin/re + +${TEST_OUTDIR.tests/detect_required}/testutil.o: tests/detect_required/testutil.c + ${CC} ${CFLAGS} -c -o ${TEST_OUTDIR.tests/detect_required}/testutil.o tests/detect_required/testutil.c + +.for n in ${TEST.tests/detect_required:T:Mout*.txt:R:C/^out//} + +${TEST_OUTDIR.tests/detect_required}/got${n}.txt: ${TEST_SRCDIR.tests/detect_required}/in${n}.re + ( ${RE} -r pcre -l fsm -p -y ${.ALLSRC:M*.re} | ${FSM} -q chars ) \ + > $@ + +${TEST_OUTDIR.tests/detect_required}/res${n}: \ + ${TEST_SRCDIR.tests/detect_required}/out${n}.txt \ + ${TEST_OUTDIR.tests/detect_required}/got${n}.txt + +TXTTEST_RESULT += ${TEST_OUTDIR.tests/detect_required}/res${n} +.endfor + + +test:: ${TEST_OUTDIR.tests/detect_required}/res_step_limit +SRC += ${TEST_SRCDIR.tests/detect_required}/detect_required_step_limit.c +CFLAGS.${TEST_SRCDIR.tests/detect_required}/detect_required_step_limit.c = -UNDEBUG + +${TEST_OUTDIR.tests/detect_required}/run_step_limit: ${TEST_OUTDIR.tests/detect_required}/detect_required_step_limit.o ${TEST_OUTDIR.tests/detect_required}/testutil.o + ${CC} ${CFLAGS} -o ${TEST_OUTDIR.tests/detect_required}/run_step_limit ${TEST_OUTDIR.tests/detect_required}/detect_required_step_limit.o ${TEST_OUTDIR.tests/detect_required}/testutil.o ${BUILD}/lib/libfsm.a ${BUILD}/lib/libre.a + +${TEST_OUTDIR.tests/detect_required}/detect_required_step_limit.o: tests/detect_required/testutil.h + +${TEST_OUTDIR.tests/detect_required}/res_step_limit: ${TEST_OUTDIR.tests/detect_required}/run_step_limit + ( ${TEST_OUTDIR.tests/detect_required}/run_step_limit 1>&2 && echo PASS || echo FAIL ) > ${TEST_OUTDIR.tests/detect_required}/res_step_limit + +.for lib in ${LIB:Mlibfsm} ${LIB:Mlibre} +${TEST_OUTDIR.tests/detect_required}/run_step_limit: ${BUILD}/lib/${lib:R}.a +.endfor + diff --git a/tests/detect_required/detect_required_step_limit.c b/tests/detect_required/detect_required_step_limit.c new file mode 100644 index 000000000..743173f90 --- /dev/null +++ b/tests/detect_required/detect_required_step_limit.c @@ -0,0 +1,56 @@ +#include "testutil.h" + +#include +#include +#include +#include +#include + +int main() +{ + enum re_flags flags = 0; + struct re_err err; + const char *regex = "^abcde$"; + + struct fsm *fsm = re_comp(RE_PCRE, fsm_sgetc, ®ex, NULL, flags, &err); + assert(fsm != NULL); + + if (!fsm_determinise(fsm)) { + assert(!"determinise"); + return EXIT_FAILURE; + } + if (!fsm_minimise(fsm)) { + assert(!"minimise"); + return EXIT_FAILURE; + } + + uint64_t charmap[4]; + + /* keep decreasing the step limit until it's hit, and check that + * the bitmap is cleared. */ + bool hit_step_limit = false; + size_t step_limit = 25; + while (!hit_step_limit) { + assert(step_limit > 0); + + const enum fsm_detect_required_characters_res res = fsm_detect_required_characters(fsm, step_limit, charmap, NULL); + if (res == FSM_DETECT_REQUIRED_CHARACTERS_STEP_LIMIT_REACHED) { + hit_step_limit = true; + + /* this should not contain any partially complete information */ + for (size_t i = 0; i < 4; i++) { + if (charmap[i] != 0) { + fprintf(stderr, "-- Test failure: partial information set when step limit reached\n"); + return EXIT_FAILURE; + } + } + } + + step_limit--; + } + assert(hit_step_limit); + printf("-- successfully hit step limit at %zd\n", step_limit); + + fsm_free(fsm); + return EXIT_SUCCESS; +} diff --git a/tests/detect_required/in0.re b/tests/detect_required/in0.re new file mode 100644 index 000000000..5d762054a --- /dev/null +++ b/tests/detect_required/in0.re @@ -0,0 +1 @@ +^$ \ No newline at end of file diff --git a/tests/detect_required/in1.re b/tests/detect_required/in1.re new file mode 100644 index 000000000..25081a78c --- /dev/null +++ b/tests/detect_required/in1.re @@ -0,0 +1 @@ +^a$ \ No newline at end of file diff --git a/tests/detect_required/in2.re b/tests/detect_required/in2.re new file mode 100644 index 000000000..fe1ec125a --- /dev/null +++ b/tests/detect_required/in2.re @@ -0,0 +1 @@ +^abcde$ \ No newline at end of file diff --git a/tests/detect_required/in3.re b/tests/detect_required/in3.re new file mode 100644 index 000000000..bed2c040e --- /dev/null +++ b/tests/detect_required/in3.re @@ -0,0 +1 @@ +^(ab|cd)$ \ No newline at end of file diff --git a/tests/detect_required/in4.re b/tests/detect_required/in4.re new file mode 100644 index 000000000..72c6c30d0 --- /dev/null +++ b/tests/detect_required/in4.re @@ -0,0 +1 @@ +^(ab|cd|ef)$ \ No newline at end of file diff --git a/tests/detect_required/in5.re b/tests/detect_required/in5.re new file mode 100644 index 000000000..454fb8261 --- /dev/null +++ b/tests/detect_required/in5.re @@ -0,0 +1 @@ +^(abc|def)$ \ No newline at end of file diff --git a/tests/detect_required/in6.re b/tests/detect_required/in6.re new file mode 100644 index 000000000..bdbaa3c85 --- /dev/null +++ b/tests/detect_required/in6.re @@ -0,0 +1 @@ +^(abc|dbf)$ \ No newline at end of file diff --git a/tests/detect_required/in7.re b/tests/detect_required/in7.re new file mode 100644 index 000000000..1957f292a --- /dev/null +++ b/tests/detect_required/in7.re @@ -0,0 +1 @@ +^abc(def)*ghi$ \ No newline at end of file diff --git a/tests/detect_required/in8.re b/tests/detect_required/in8.re new file mode 100644 index 000000000..9ec28ce00 --- /dev/null +++ b/tests/detect_required/in8.re @@ -0,0 +1 @@ +^abc(def)+ghi$ \ No newline at end of file diff --git a/tests/detect_required/in9.re b/tests/detect_required/in9.re new file mode 100644 index 000000000..e3f42fa55 --- /dev/null +++ b/tests/detect_required/in9.re @@ -0,0 +1 @@ +^ghi(def)abc$ \ No newline at end of file diff --git a/tests/detect_required/out0.txt b/tests/detect_required/out0.txt new file mode 100644 index 000000000..ff95acde3 --- /dev/null +++ b/tests/detect_required/out0.txt @@ -0,0 +1 @@ +0 diff --git a/tests/detect_required/out1.txt b/tests/detect_required/out1.txt new file mode 100644 index 000000000..8ba4a7a8d --- /dev/null +++ b/tests/detect_required/out1.txt @@ -0,0 +1 @@ +1 a diff --git a/tests/detect_required/out2.txt b/tests/detect_required/out2.txt new file mode 100644 index 000000000..7a88aba43 --- /dev/null +++ b/tests/detect_required/out2.txt @@ -0,0 +1 @@ +5 abcde diff --git a/tests/detect_required/out3.txt b/tests/detect_required/out3.txt new file mode 100644 index 000000000..ff95acde3 --- /dev/null +++ b/tests/detect_required/out3.txt @@ -0,0 +1 @@ +0 diff --git a/tests/detect_required/out4.txt b/tests/detect_required/out4.txt new file mode 100644 index 000000000..ff95acde3 --- /dev/null +++ b/tests/detect_required/out4.txt @@ -0,0 +1 @@ +0 diff --git a/tests/detect_required/out5.txt b/tests/detect_required/out5.txt new file mode 100644 index 000000000..ff95acde3 --- /dev/null +++ b/tests/detect_required/out5.txt @@ -0,0 +1 @@ +0 diff --git a/tests/detect_required/out6.txt b/tests/detect_required/out6.txt new file mode 100644 index 000000000..bacfc4097 --- /dev/null +++ b/tests/detect_required/out6.txt @@ -0,0 +1 @@ +1 b diff --git a/tests/detect_required/out7.txt b/tests/detect_required/out7.txt new file mode 100644 index 000000000..099f9b056 --- /dev/null +++ b/tests/detect_required/out7.txt @@ -0,0 +1 @@ +6 abcghi diff --git a/tests/detect_required/out8.txt b/tests/detect_required/out8.txt new file mode 100644 index 000000000..19fa3c37b --- /dev/null +++ b/tests/detect_required/out8.txt @@ -0,0 +1 @@ +9 abcdefghi diff --git a/tests/detect_required/out9.txt b/tests/detect_required/out9.txt new file mode 100644 index 000000000..19fa3c37b --- /dev/null +++ b/tests/detect_required/out9.txt @@ -0,0 +1 @@ +9 abcdefghi diff --git a/tests/detect_required/testutil.c b/tests/detect_required/testutil.c new file mode 100644 index 000000000..a6ec8c296 --- /dev/null +++ b/tests/detect_required/testutil.c @@ -0,0 +1,85 @@ +#include "testutil.h" + +#include + +#include +#include +#include +#include +#include +#include + +#include + +bool +run_test(const struct testcase *tc) +{ + bool test_res = false; + + enum re_flags flags = 0; + struct re_err err; + char *regex = (char *)tc->regex; + const char *required = tc->required ? tc->required : ""; + const size_t step_limit = tc->step_limit ? tc->step_limit : DEF_STEP_LIMIT; + + fprintf(stderr, "-- test: regex '%s', required '%s'\n", tc->regex, required); + + struct fsm *fsm = re_comp(RE_PCRE, fsm_sgetc, ®ex, NULL, flags, &err); + if (fsm == NULL) { + return false; + } + + if (!fsm_determinise(fsm)) { + assert(!"determinise"); + return false; + } + if (!fsm_minimise(fsm)) { + assert(!"minimise"); + return false; + } + + uint64_t charmap[4]; + + { + const size_t statecount = fsm_countstates(fsm); + size_t ends = 0; + for (size_t i = 0; i < statecount; i++) { + if (fsm_isend(fsm, i)) { + ends++; + } + } + fprintf(stderr, "-- statecount %zu, %zu ends\n", statecount, ends); + } + + + const enum fsm_detect_required_characters_res res = fsm_detect_required_characters(fsm, step_limit, charmap, NULL); + if (res == FSM_DETECT_REQUIRED_CHARACTERS_STEP_LIMIT_REACHED) { + fprintf(stderr, "-- step limit reached, halting\n"); + goto cleanup; + } + assert(res == FSM_DETECT_REQUIRED_CHARACTERS_WRITTEN); + + char buf[257] = {0}; + size_t used = 0; + assert(!u64bitset_get(charmap, 0)); /* does not contain 0x00 */ + + for (size_t i = 0; i < 256; i++) { + if (u64bitset_get(charmap, i)) { + buf[used++] = (char)i; + } + } + + if (0 != strcmp(required, buf)) { + fprintf(stderr, "Error: mismatch\n"); + fprintf(stderr, "-- expected: [%s]\n", required); + fprintf(stderr, "-- got: [%s]\n", buf); + goto cleanup; + } + + test_res = true; + +cleanup: + fsm_free(fsm); + + return test_res; +} diff --git a/tests/detect_required/testutil.h b/tests/detect_required/testutil.h new file mode 100644 index 000000000..f9378c190 --- /dev/null +++ b/tests/detect_required/testutil.h @@ -0,0 +1,21 @@ +#ifndef TESTUTIL_H +#define TESTUTIL_H + +#include +#include +#include +#include + +#define DEF_STEP_LIMIT 100000 + +struct testcase { + const char *regex; + const char *required; + size_t max_gen_buffer; /* 0: default */ + size_t step_limit; +}; + +bool +run_test(const struct testcase *tc); + +#endif diff --git a/tests/eager_output/Makefile b/tests/eager_output/Makefile new file mode 100644 index 000000000..a650bf802 --- /dev/null +++ b/tests/eager_output/Makefile @@ -0,0 +1,22 @@ +.include "../../share/mk/top.mk" + +TEST.tests/eager_output != ls -1 tests/eager_output/eager_output*.c +TEST_SRCDIR.tests/eager_output = tests/eager_output +TEST_OUTDIR.tests/eager_output = ${BUILD}/tests/eager_output + +.for n in ${TEST.tests/eager_output:T:R:C/^eager_output//} +INCDIR.${TEST_SRCDIR.tests/eager_output}/eager_output${n}.c += src/adt +.endfor + +SRC += ${TEST_SRCDIR.tests/eager_output}/utils.c + +.for n in ${TEST.tests/eager_output:T:R:C/^eager_output//} +test:: ${TEST_OUTDIR.tests/eager_output}/res${n} +SRC += ${TEST_SRCDIR.tests/eager_output}/eager_output${n}.c +CFLAGS.${TEST_SRCDIR.tests/eager_output}/eager_output${n}.c += -UNDEBUG + +${TEST_OUTDIR.tests/eager_output}/run${n}: ${TEST_OUTDIR.tests/eager_output}/eager_output${n}.o ${TEST_OUTDIR.tests/eager_output}/utils.o ${BUILD}/lib/libfsm.a ${BUILD}/lib/libre.a + ${CC} ${CFLAGS} ${CFLAGS.${TEST_SRCDIR.tests/eager_output}/eager_output${n}.c} -o ${TEST_OUTDIR.tests/eager_output}/run${n} ${TEST_OUTDIR.tests/eager_output}/eager_output${n}.o ${TEST_OUTDIR.tests/eager_output}/utils.o ${BUILD}/lib/libfsm.a ${BUILD}/lib/libre.a +${TEST_OUTDIR.tests/eager_output}/res${n}: ${TEST_OUTDIR.tests/eager_output}/run${n} + ( ${TEST_OUTDIR.tests/eager_output}/run${n} 1>&2 && echo PASS || echo FAIL ) > ${TEST_OUTDIR.tests/eager_output}/res${n} +.endfor diff --git a/tests/eager_output/eager_output1.c b/tests/eager_output/eager_output1.c new file mode 100644 index 000000000..4900f89e0 --- /dev/null +++ b/tests/eager_output/eager_output1.c @@ -0,0 +1,12 @@ +#include "utils.h" + +int main(void) +{ + struct eager_output_test test = { + .patterns = { "abc" }, + .inputs = { + { .input = "abc", .expected_ids = { 1 } }, + }, + }; + return run_test(&test); +} diff --git a/tests/eager_output/eager_output2.c b/tests/eager_output/eager_output2.c new file mode 100644 index 000000000..6a10eec1c --- /dev/null +++ b/tests/eager_output/eager_output2.c @@ -0,0 +1,17 @@ +#include "utils.h" + +int main(void) +{ + struct eager_output_test test = { + .patterns = { "ab(c|d|e)" }, + .inputs = { + { .input = "abc", .expected_ids = { 1 } }, + { .input = "abd", .expected_ids = { 1 } }, + { .input = "abe", .expected_ids = { 1 } }, + { .input = "Xabe", .expected_ids = { 1 } }, + { .input = "abeX", .expected_ids = { 1 } }, + { .input = "XabeX", .expected_ids = { 1 } }, + }, + }; + return run_test(&test); +} diff --git a/tests/eager_output/eager_output3.c b/tests/eager_output/eager_output3.c new file mode 100644 index 000000000..b6320ef79 --- /dev/null +++ b/tests/eager_output/eager_output3.c @@ -0,0 +1,16 @@ +#include "utils.h" + +/* test that eager endids are correctly propagated through fsm_determinise() and fsm_minimise() */ +int main(void) +{ + struct eager_output_test test = { + .patterns = { "ab(c|d|e)?" }, + .inputs = { + { .input = "ab", .expected_ids = { 1 } }, + { .input = "abc", .expected_ids = { 1 } }, + { .input = "abd", .expected_ids = { 1 } }, + { .input = "abe", .expected_ids = { 1 } }, + }, + }; + return run_test(&test); +} diff --git a/tests/eager_output/eager_output4.c b/tests/eager_output/eager_output4.c new file mode 100644 index 000000000..2e0f17f13 --- /dev/null +++ b/tests/eager_output/eager_output4.c @@ -0,0 +1,13 @@ +#include "utils.h" + +int main(void) +{ + struct eager_output_test test = { + .patterns = { "abcde$" }, + .inputs = { + { .input = "abcde", .expected_ids = { 1 } }, + { .input = "Xabcde", .expected_ids = { 1 } }, + }, + }; + return run_test(&test); +} diff --git a/tests/eager_output/eager_output5.c b/tests/eager_output/eager_output5.c new file mode 100644 index 000000000..6d2ce4eb8 --- /dev/null +++ b/tests/eager_output/eager_output5.c @@ -0,0 +1,14 @@ +#include "utils.h" + +int main(void) +{ + struct eager_output_test test = { + .patterns = { "^abc$", "^ab*c$" }, + .inputs = { + { .input = "ac", .expected_ids = { 2 } }, + { .input = "abc", .expected_ids = { 1, 2 } }, + { .input = "abbc", .expected_ids = { 2 } }, + }, + }; + return run_test(&test); +} diff --git a/tests/eager_output/eager_output6.c b/tests/eager_output/eager_output6.c new file mode 100644 index 000000000..188541f39 --- /dev/null +++ b/tests/eager_output/eager_output6.c @@ -0,0 +1,34 @@ +#include "utils.h" + +int main(void) +{ + struct eager_output_test test = { + .patterns = { + "apple", + "banana", + "carrot", + "durian", + "eggplant", + "fig", + "grapefruit", + "hazelnut", + "iceberg lettuce", + "jicama", + }, + .inputs = { + { .input = "apple", .expected_ids = { 1 } }, + { .input = "banana", .expected_ids = { 2 } }, + { .input = "carrot", .expected_ids = { 3 } }, + { .input = "durian", .expected_ids = { 4 } }, + { .input = "eggplant", .expected_ids = { 5 } }, + { .input = "fig", .expected_ids = { 6 } }, + { .input = "grapefruit", .expected_ids = { 7 } }, + { .input = "hazelnut", .expected_ids = { 8 } }, + { .input = "iceberg lettuce", .expected_ids = { 9 } }, + { .input = "jicama", .expected_ids = { 10 } }, + { .input = "apple banana carrot", .expected_ids = { 1, 2, 3 } }, + }, + }; + + return run_test(&test); +} diff --git a/tests/eager_output/eager_output7.c b/tests/eager_output/eager_output7.c new file mode 100644 index 000000000..e764ac946 --- /dev/null +++ b/tests/eager_output/eager_output7.c @@ -0,0 +1,104 @@ +#include "utils.h" + +int main(void) +{ + /* Run this test with env FORCE_ENDIDS=N ... to see how much more + * expensive it is to combine the first N patterns using endids, + * rather than eager_outputs. It becomes VERY slow for >= 9 or so. + * (Note that the checks probably will not pass for N < 4, because + * it will start skipping appear in the early test inputs.) */ + bool force_endids = false; + size_t force_endid_count = 0; + { + const char *str = getenv("FORCE_ENDIDS"); + if (str != NULL) { + force_endid_count = atoi(str); + if (force_endid_count == 0) { + force_endid_count = 26; + } + force_endids = true; + } + } + + struct eager_output_test test = { + .force_endids = force_endids, + .patterns = { + [0] = "apple", + [1] = "banana", + [2] = "carrot", + [3] = "durian", + [4] = "eggplant", + [5] = "fig", + [6] = "grapefruit", + [7] = "hazelnut", + [8] = "iceberg lettuce", + [9] = "jicama", + [10] = "kiwano", + [11] = "lemon", + [12] = "mango", + [13] = "nectarine", + [14] = "orange", + [15] = "plum", + [16] = "quince", + [17] = "radish", + [18] = "strawberry", + [19] = "turnip", + [20] = "ube", + [21] = "vanilla", + [22] = "watermelon", + [23] = "xigua watermelon", + [24] = "yam", + [25] = "zucchini", + }, + .inputs = { + /* Note: expected IDs are shifted by 1, it's 0-terminated. */ + { .input = "apple", .expected_ids = { 1 } }, + { .input = "banana", .expected_ids = { 2 } }, + { .input = "carrot", .expected_ids = { 3 } }, + { .input = "apple banana", .expected_ids = { 1, 2 } }, + { .input = "carrot durian apple", .expected_ids = { 1, 3, 4 } }, + { .input = "carrot fig apple", .expected_ids = { 1, 3, 6 } }, + + /* leading characters and an incomplete trailing match */ + { .input = "mumble mumble fig hazelnut banana xigua watermelo", .expected_ids = { 2, 6, 8 } }, + + /* redundant matches */ + { .input = "ube ube ube ube ube", .expected_ids = { 21 } }, + + /* everything */ + { .input = + "apple banana carrot durian eggplant fig grapefruit " + "hazelnut iceberg lettuce jicamaa kiwano lemon mango " + "nectarine orange plum quince radish strawberry " + "turnip ube vanilla watermelon xigua watermelon yam zucchini", + .expected_ids = { + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, + }, + }, + /* everything, only spaces appearing in patterns */ + { .input = + "applebananacarrotdurianeggplantfiggrapefruit" + "hazelnuticeberg lettucejicamaakiwanolemonmango" + "nectarineorangeplumquinceradishstrawberry" + "turnipubevanillawatermelonxigua watermelonyamzucchini", + .expected_ids = { + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, + }, + }, + }, + }; + + /* truncate patterns to the first N */ + if (force_endids) { + assert(force_endid_count > 0 && force_endid_count <= 26); + test.patterns[force_endid_count] = NULL; + + /* truncate test inputs to just the first couple, since + * later inputs use later patterns */ + test.inputs[5].input = NULL; + } + + return run_test(&test); +} diff --git a/tests/eager_output/eager_output_alt_mixing_anchored_and_unanchored.c b/tests/eager_output/eager_output_alt_mixing_anchored_and_unanchored.c new file mode 100644 index 000000000..ad125b9b1 --- /dev/null +++ b/tests/eager_output/eager_output_alt_mixing_anchored_and_unanchored.c @@ -0,0 +1,29 @@ +#include "utils.h" + +/* Test for false positive matches when combining patterns + * with both anchored and unanchored subtrees. */ + +int main(void) +{ + struct eager_output_test test = { + .patterns = { + "a(?:x|$)", + "a(?:y|$)", + }, + .inputs = { + { .input = "a", .expected_ids = { 1, 2 } }, + { .input = "aZ", .expect_fail = true }, + { .input = "Za", .expected_ids = { 1, 2 } }, + { .input = "ax", .expected_ids = { 1 } }, + { .input = "axZ", .expected_ids = { 1 } }, + { .input = "ay", .expected_ids = { 2 } }, + { .input = "ayZ", .expected_ids = { 2 } }, + { .input = "axa", .expected_ids = { 1, 2 } }, + { .input = "aya", .expected_ids = { 1, 2 } }, + { .input = "axay", .expected_ids = { 1, 2 } }, + { .input = "ayax", .expected_ids = { 1, 2 } }, + }, + }; + + return run_test(&test); +} diff --git a/tests/eager_output/eager_output_at_start.c b/tests/eager_output/eager_output_at_start.c new file mode 100644 index 000000000..8ba5f2ad1 --- /dev/null +++ b/tests/eager_output/eager_output_at_start.c @@ -0,0 +1,12 @@ +#include "utils.h" + +int main(void) +{ + struct eager_output_test test = { + .patterns = { "" }, + .inputs = { + { .input = "", .expected_ids = { 1 } }, + }, + }; + return run_test(&test); +} diff --git a/tests/eager_output/eager_output_at_start_multiple.c b/tests/eager_output/eager_output_at_start_multiple.c new file mode 100644 index 000000000..ddc9530f5 --- /dev/null +++ b/tests/eager_output/eager_output_at_start_multiple.c @@ -0,0 +1,15 @@ +#include "utils.h" + +int main(void) +{ + struct eager_output_test test = { + .patterns = { + [0] = "", + [1] = "", + }, + .inputs = { + { .input = "", .expected_ids = { 1, 2 } }, + }, + }; + return run_test(&test); +} diff --git a/tests/eager_output/eager_output_at_start_multiple_anchored.c b/tests/eager_output/eager_output_at_start_multiple_anchored.c new file mode 100644 index 000000000..0cf7e9b70 --- /dev/null +++ b/tests/eager_output/eager_output_at_start_multiple_anchored.c @@ -0,0 +1,15 @@ +#include "utils.h" + +int main(void) +{ + struct eager_output_test test = { + .patterns = { + [0] = "$", + [1] = "^$", + }, + .inputs = { + { .input = "", .expected_ids = { 1, 2 } }, + }, + }; + return run_test(&test); +} diff --git a/tests/eager_output/eager_output_at_start_single_anchored.c b/tests/eager_output/eager_output_at_start_single_anchored.c new file mode 100644 index 000000000..a9c13ef67 --- /dev/null +++ b/tests/eager_output/eager_output_at_start_single_anchored.c @@ -0,0 +1,22 @@ +#include "utils.h" + +int main(void) +{ + const struct eager_output_test test_unanchored_start = { + .patterns = { "$" }, + .inputs = { + { .input = "", .expected_ids = { 1 } }, + }, + }; + + const struct eager_output_test test_anchored_start = { + .patterns = { "^$" }, + .inputs = { + { .input = "", .expected_ids = { 1 } }, + }, + }; + + bool pass = run_test(&test_unanchored_start); + pass = run_test(&test_anchored_start) && pass; + return pass; +} diff --git a/tests/eager_output/eager_output_fr1.c b/tests/eager_output/eager_output_fr1.c new file mode 100644 index 000000000..97eb34312 --- /dev/null +++ b/tests/eager_output/eager_output_fr1.c @@ -0,0 +1,13 @@ +#include "utils.h" + +/* Fuzzer regresison */ +int main(void) +{ + struct eager_output_test test = { + .patterns = { "ab", "" }, + .inputs = { + { .input = "ab", .expected_ids = { 1, 2 } }, + }, + }; + return run_test(&test); +} diff --git a/tests/eager_output/eager_output_fr2.c b/tests/eager_output/eager_output_fr2.c new file mode 100644 index 000000000..23bd3103c --- /dev/null +++ b/tests/eager_output/eager_output_fr2.c @@ -0,0 +1,13 @@ +#include "utils.h" + +/* Fuzzer regresison */ +int main(void) +{ + struct eager_output_test test = { + .patterns = { "", "" }, + .inputs = { + { .input = "", .expected_ids = { 1, 2 } }, + }, + }; + return run_test(&test); +} diff --git a/tests/eager_output/eager_output_fr3.c b/tests/eager_output/eager_output_fr3.c new file mode 100644 index 000000000..0d15a4a68 --- /dev/null +++ b/tests/eager_output/eager_output_fr3.c @@ -0,0 +1,13 @@ +#include "utils.h" + +/* Fuzzer regresison */ +int main(void) +{ + struct eager_output_test test = { + .patterns = { "^", "" }, + .inputs = { + { .input = "", .expected_ids = { 1, 2 } }, + }, + }; + return run_test(&test); +} diff --git a/tests/eager_output/eager_output_mixed_anchored_and_unanchored_start.c b/tests/eager_output/eager_output_mixed_anchored_and_unanchored_start.c new file mode 100644 index 000000000..d18a67de7 --- /dev/null +++ b/tests/eager_output/eager_output_mixed_anchored_and_unanchored_start.c @@ -0,0 +1,18 @@ +#include "utils.h" + +int main(void) +{ + struct eager_output_test test = { + .patterns = { + "(^|[^A-Z])abc", + }, + .inputs = { + { .input = "abc", .expected_ids = { 1 } }, + { .input = "xabc", .expected_ids = { 1 } }, + { .input = "xyz abc", .expected_ids = { 1 } }, + { .input = "Xabc", .expect_fail = true }, + }, + }; + + return run_test(&test); +} diff --git a/tests/eager_output/eager_output_mixed_anchored_unanchored.c b/tests/eager_output/eager_output_mixed_anchored_unanchored.c new file mode 100644 index 000000000..376b49d8a --- /dev/null +++ b/tests/eager_output/eager_output_mixed_anchored_unanchored.c @@ -0,0 +1,43 @@ +#include "utils.h" + +int main(void) +{ + struct eager_output_test test = { + .patterns = { + "^abc$", + "def", + "^ghi", + "jkl$", + "mno", + }, + .inputs = { + { .input = "abc", .expected_ids = { 1 } }, + { .input = "def", .expected_ids = { 2 } }, + { .input = "ghi", .expected_ids = { 3 } }, + { .input = "jkl", .expected_ids = { 4 } }, + { .input = "mno", .expected_ids = { 5 } }, + + { .input = "defmno", .expected_ids = { 2, 5 } }, + { .input = " def mno ", .expected_ids = { 2, 5 } }, + + /* Matching a start-anchored pattern followed by + * unanchored ones should just work. */ + { .input = "ghi def", .expected_ids = { 2, 3 } }, + + /* An unanchored pattern before a start-anchored pattern + * should only match the unanchored pattern. */ + { .input = "def ghi", .expected_ids = { 2 } }, + + /* Matching an unanchored pattern before an + * end-anchored one is fine. */ + { .input = "mno jkl", .expected_ids = { 4, 5 } }, + + /* This should match "mno" with the "jkl" prefix + * ignored by the unanchored start, which does + * not count as a match for "jkl$". */ + { .input = "jkl mno", .expected_ids = { 5 } }, + }, + }; + + return run_test(&test); +} diff --git a/tests/eager_output/eager_output_mixed_start_anchor.c b/tests/eager_output/eager_output_mixed_start_anchor.c new file mode 100644 index 000000000..2965dc7aa --- /dev/null +++ b/tests/eager_output/eager_output_mixed_start_anchor.c @@ -0,0 +1,20 @@ +#include "utils.h" + +/* Regression: This is a case that requires an anchored_start state set + * rather than a single optional anchored_start state ID to link + * correctly. */ + +int main(void) +{ + struct eager_output_test test = { + .patterns = { + "(^|wax-)((?:banana|^apple))", + "(^|wax-)(orange)", + }, + .inputs = { + { .input = "banana", .expected_ids = { 1 } }, + }, + }; + + return run_test(&test); +} diff --git a/tests/eager_output/eager_output_unanchored_end_plus.c b/tests/eager_output/eager_output_unanchored_end_plus.c new file mode 100644 index 000000000..81354ffcb --- /dev/null +++ b/tests/eager_output/eager_output_unanchored_end_plus.c @@ -0,0 +1,19 @@ +#include "utils.h" + +int main(void) +{ + struct eager_output_test test = { + .patterns = { + "abcx+", + "z", + }, + .inputs = { + { .input = "abc", .expect_fail = true }, + { .input = "abcx", .expected_ids = { 1 } }, + { .input = "abcxx", .expected_ids = { 1 } }, + { .input = "z", .expected_ids = { 2 } }, + }, + }; + + return run_test(&test); +} diff --git a/tests/eager_output/eager_output_unanchored_end_plus_min.c b/tests/eager_output/eager_output_unanchored_end_plus_min.c new file mode 100644 index 000000000..3d270514e --- /dev/null +++ b/tests/eager_output/eager_output_unanchored_end_plus_min.c @@ -0,0 +1,18 @@ +#include "utils.h" + +int main(void) +{ + struct eager_output_test test = { + .patterns = { + "abcx+", + }, + .inputs = { + { .input = "abc", .expect_fail = true }, + { .input = "abcx", .expected_ids = { 1 } }, + { .input = "abcxx", .expected_ids = { 1 } }, + { .input = "z", .expect_fail = true }, + }, + }; + + return run_test(&test); +} diff --git a/tests/eager_output/eager_output_unanchored_end_plus_min2.c b/tests/eager_output/eager_output_unanchored_end_plus_min2.c new file mode 100644 index 000000000..20d735fcc --- /dev/null +++ b/tests/eager_output/eager_output_unanchored_end_plus_min2.c @@ -0,0 +1,19 @@ +#include "utils.h" + +int main(void) +{ + struct eager_output_test test = { + .patterns = { + "z", + }, + .inputs = { + { .input = "abc", .expect_fail = true }, + { .input = "z", .expected_ids = { 1 } }, + { .input = "Xz", .expected_ids = { 1 } }, + { .input = "XzX", .expected_ids = { 1 } }, + { .input = "zX", .expected_ids = { 1 } }, + }, + }; + + return run_test(&test); +} diff --git a/tests/eager_output/eager_output_unanchored_end_plus_multichar.c b/tests/eager_output/eager_output_unanchored_end_plus_multichar.c new file mode 100644 index 000000000..40d3e5d24 --- /dev/null +++ b/tests/eager_output/eager_output_unanchored_end_plus_multichar.c @@ -0,0 +1,19 @@ +#include "utils.h" + +int main(void) +{ + struct eager_output_test test = { + .patterns = { + "abc(xy)+", + "z", + }, + .inputs = { + { .input = "abc", .expect_fail = true }, + { .input = "abcxy", .expected_ids = { 1 } }, + { .input = "abcxyxy", .expected_ids = { 1 } }, + { .input = "z", .expected_ids = { 2 } }, + }, + }; + + return run_test(&test); +} diff --git a/tests/eager_output/utils.c b/tests/eager_output/utils.c new file mode 100644 index 000000000..3a1329e4c --- /dev/null +++ b/tests/eager_output/utils.c @@ -0,0 +1,281 @@ +#include "utils.h" + +void +fsm_eager_output_dump(FILE *f, const struct fsm *fsm); + +void +fsm_endid_dump(FILE *f, const struct fsm *fsm); + +void +append_eager_output_cb(fsm_output_id_t id, void *opaque) +{ + struct cb_info *info = (struct cb_info *)opaque; + assert(info->used < MAX_IDS); + + for (size_t i = 0; i < info->used; i++) { + if (info->ids[i] == id) { + return; /* already present */ + } + } + + info->ids[info->used++] = id; +} + +int +cmp_output(const void *pa, const void *pb) +{ + const fsm_output_id_t a = *(fsm_output_id_t *)pa; + const fsm_output_id_t b = *(fsm_output_id_t *)pb; + return a < b ? -1 : a > b ? 1 : 0; +} + +struct fsm_options print_options = { + .consolidate_edges = 1, + .comments = 0, + .group_edges = 1, +}; + +void +dump(const struct fsm *fsm) +{ + fsm_print(stderr, fsm, + &print_options, NULL, FSM_PRINT_DOT); +} + +int +run_test(const struct eager_output_test *test) +{ + struct fsm *nfas[MAX_PATTERNS] = {0}; + + size_t nfas_used = 0; + int ret = 0; + + int log = 0; + { + const char *logstr = getenv("LOG"); + if (logstr != NULL) { + if (logstr[0] == 'y') { /* make "y" or "yes" non-zero */ + logstr = "1"; + } + log = atoi(logstr); + } + } + + for (size_t i = 0; i < MAX_PATTERNS; i++) { + const char *p = test->patterns[i]; + if (test->patterns[i] == NULL) { break; } + + struct fsm *fsm = re_comp(RE_PCRE, fsm_sgetc, &p, NULL, RE_SAVE_LINKAGE_INFO, NULL); + assert(fsm != NULL); + + if (log) { + fprintf(stderr, "==== source NFA %zd\n", i); + if (log > 1) { + dump(fsm); + fsm_eager_output_dump(stderr, fsm); + fsm_endid_dump(stderr, fsm); + fprintf(stderr, "====\n"); + } + } + + nfas[i] = fsm; + nfas_used++; + } + + const size_t id_base = 1; /* offset by 1 because 0 is used as end-of-list */ + struct fsm *fsm; + if (test->force_endids) { + for (size_t i = 0; i < nfas_used; i++) { + fsm_setendid(nfas[i], i + id_base); + } + fsm = fsm_union_array(nfas_used, nfas, NULL); + } else { + /* This function sets the end & eager output IDs. */ + fsm = fsm_union_repeated_pattern_group(nfas_used, nfas, NULL, id_base); + } + assert(fsm != NULL); + + if (log) { + fprintf(stderr, "==== combined (pre det+min)\n"); + if (log > 1) { + dump(fsm); + fsm_eager_output_dump(stderr, fsm); + fprintf(stderr, "--- endids:\n"); + fsm_endid_dump(stderr, fsm); + fprintf(stderr, "====\n"); + } + } + + if (log) { + fprintf(stderr, "=== determinising combined... NFA has %u states\n", fsm_countstates(fsm)); + } + ret = fsm_determinise(fsm); + assert(ret == 1); + if (log) { + fprintf(stderr, "=== determinising combined...done, DFA has %u states\n", fsm_countstates(fsm)); + } + + if (log) { + fprintf(stderr, "==== combined (post det)\n"); + if (log > 1) { + dump(fsm); + fsm_eager_output_dump(stderr, fsm); + fprintf(stderr, "====\n"); + } + } + + ret = fsm_minimise(fsm); + if (log) { + fprintf(stderr, "=== minimised combined...done, DFA has %u states\n", fsm_countstates(fsm)); + } + assert(ret == 1); + + if (log) { + fprintf(stderr, "==== combined (post det+min)\n"); + if (log > 1) { + dump(fsm); + fsm_eager_output_dump(stderr, fsm); + fprintf(stderr, "--- endids:\n"); + fsm_endid_dump(stderr, fsm); + fprintf(stderr, "====\n"); + } + } + + struct cb_info outputs = { 0 }; + fsm_eager_output_set_cb(fsm, append_eager_output_cb, &outputs); + + for (size_t i_i = 0; i_i < MAX_INPUTS; i_i++) { + outputs.used = 0; + const char *input = test->inputs[i_i].input; + if (input == NULL) { break; } + + size_t expected_id_count = 0; + for (size_t id_i = 0; id_i < MAX_ENDIDS; id_i++) { + const fsm_output_id_t id = test->inputs[i_i].expected_ids[id_i]; + if (id == 0) { break; } + expected_id_count++; + + /* must be ascending */ + if (id_i > 0) { + assert(id > test->inputs[i_i].expected_ids[id_i - 1]); + } + } + + if (log) { + fprintf(stderr, "%s: input %zd: \"%s\", expecting %zd ids:", + __func__, i_i, input, expected_id_count); + for (size_t i = 0; i < expected_id_count; i++) { + fprintf(stderr, " %d", test->inputs[i_i].expected_ids[i]); + } + fprintf(stderr, "\n"); + } + + if (test->inputs[i_i].expect_fail) { + expected_id_count = 0; + } + + fsm_state_t end; /* only set on match */ + ret = fsm_exec(fsm, fsm_sgetc, &input, &end); + + size_t match_id_count = 0; + if (ret == 1) { +#define ENDID_BUF_SIZE 32 + fsm_end_id_t endid_buf[ENDID_BUF_SIZE] = {0}; + const size_t endid_count = fsm_endid_count(fsm, end); + /* fprintf(stderr, "%s: endid_count %zd for state %d\n", __func__, endid_count, end); */ + assert(endid_count < ENDID_BUF_SIZE); + + const enum fsm_getendids_res egres = fsm_endid_get(fsm, end, /*ENDID_BUF_SIZE*/ endid_count, endid_buf); + switch (egres) { + case FSM_GETENDIDS_NOT_FOUND: + /* "not found" can be fine if it only has eager outputs */ + break; + case FSM_GETENDIDS_FOUND: + break; + case FSM_GETENDIDS_ERROR_INSUFFICIENT_SPACE: + default: + fprintf(stderr, "fsm_endid_get res %d\n", egres); + assert(!"fsm_endid_get failed"); + } + + match_id_count += outputs.used; + for (size_t e_i = 0; e_i < endid_count; e_i++) { + fsm_end_id_t endid = endid_buf[e_i]; + bool found = false; + for (size_t o_i = 0; o_i < outputs.used; o_i++) { + if (outputs.ids[o_i] == endid) { + found = true; + break; + } + } + if (!found) { + /* Don't count IDs set by both endids AND eager outputs twice. */ + match_id_count++; + } + } + + /* Copy endid outputs into outputs.ids[], since for testing + * purposes we don't care about the difference between eager + * output and endids here. */ + assert(match_id_count <= MAX_IDS); + for (size_t endid_i = 0; endid_i < endid_count; endid_i++) { + if (log) { + fprintf(stderr, "-- adding endid %zd: %d\n", endid_i, endid_buf[endid_i]); + } + bool found = false; + for (size_t o_i = 0; o_i < outputs.used; o_i++) { + if (outputs.ids[o_i] == endid_buf[endid_i]) { + found = true; + break; + } + } + if (!found) { + outputs.ids[outputs.used++] = (fsm_output_id_t)endid_buf[endid_i]; + } + } + } + + if (ret == 0) { + /* if it didn't match, ignore the eager output IDs. this should + * eventually happen internal to fsm_exec or codegen. */ + outputs.used = 0; + } + + /* NEXT match IDs, sort outputs[] buffer first */ + qsort(outputs.ids, outputs.used, sizeof(outputs.ids[0]), cmp_output); + + if (log) { + fprintf(stderr, "-- got %zd:", outputs.used); + for (size_t i = 0; i < outputs.used; i++) { + fprintf(stderr, " %d", outputs.ids[i]); + } + fprintf(stderr, "\n"); + } + + if (expected_id_count == 0) { + assert(ret == 0 || outputs.used == 0); /* no match */ + continue; + } else { + assert(ret == 1); + } + + assert(match_id_count == expected_id_count); + + size_t floor = 0; + for (size_t exp_i = 0; exp_i < outputs.used; exp_i++) { + bool found = false; + for (size_t got_i = floor; got_i < outputs.used; got_i++) { + if (outputs.ids[got_i] == test->inputs[i_i].expected_ids[exp_i]) { + floor = got_i + 1; + found = true; + break; + } + } + assert(found); + } + } + + fsm_free(fsm); + + return EXIT_SUCCESS;; +} diff --git a/tests/eager_output/utils.h b/tests/eager_output/utils.h new file mode 100644 index 000000000..ee5f941c5 --- /dev/null +++ b/tests/eager_output/utils.h @@ -0,0 +1,65 @@ +#ifndef UTILS_H +#define UTILS_H + +#include +#include +#include +#include +#include + +#include + +#include + +#include + +#include +#include +#include +#include +#include +#include + +#define MAX_IDS 32 + +#include + +#include + +#define MAX_PATTERNS 150 +#define MAX_INPUTS 64 +#define MAX_ENDIDS 32 + +struct eager_output_test { + const char *patterns[MAX_PATTERNS]; + bool force_endids; + + struct { + const char *input; + bool expect_fail; + /* Terminated by 0. pattern[i] => id of i+1. Must be sorted. */ + fsm_output_id_t expected_ids[MAX_ENDIDS]; + } inputs[MAX_INPUTS]; +}; + +void +append_eager_output_cb(fsm_output_id_t id, void *opaque); + +int +cmp_output(const void *pa, const void *pb); + +int +run_test(const struct eager_output_test *test); + +struct cb_info { + size_t used; + fsm_end_id_t ids[MAX_IDS]; +}; + +void +dump(const struct fsm *fsm); + +void +append_eager_output_cb(fsm_end_id_t id, void *opaque); + +#endif diff --git a/tests/endids/endids0.c b/tests/endids/endids0.c index f101f0cc6..1f5e6f9b6 100644 --- a/tests/endids/endids0.c +++ b/tests/endids/endids0.c @@ -43,23 +43,14 @@ int main(void) // memset(all_endids, 0, sizeof all_endids); for (state_ind = 0; state_ind < nstates; state_ind++) { if (fsm_isend(fsm, state_ind)) { - fsm_end_id_t endid = 0; - size_t nwritten; - enum fsm_getendids_res ret; + fsm_end_id_t id = 0; + int ret; - assert( fsm_getendidcount(fsm, state_ind) == 1); + assert( fsm_endid_count(fsm, state_ind) == 1); - nwritten = 0; - ret = fsm_getendids( - fsm, - state_ind, - 1, - &endid, - &nwritten); - - assert(ret == FSM_GETENDIDS_FOUND); - assert(nwritten == 1); - assert( endid == 1 ); + ret = fsm_endid_get(fsm, state_ind, 1, &id); + assert(ret == 1); + assert(id == 1); nend++; } @@ -88,26 +79,26 @@ int main(void) }; for (i=0; i < sizeof matches / sizeof matches[0]; i++) { - fsm_end_id_t *end_ids; - size_t num_end_ids; + fsm_end_id_t *ids; + size_t count; - end_ids = NULL; - num_end_ids = 0; - ret = match_string(fsm, matches[i].s, NULL, &end_ids, &num_end_ids); + ids = NULL; + count = 0; + ret = match_string(fsm, matches[i].s, NULL, &ids, &count); if (matches[i].should_match) { assert( ret == 1 ); - assert( end_ids != NULL ); - assert( end_ids[0] = 1 ); - assert( num_end_ids == 1 ); - assert( end_ids[0] = matches[i].endid ); + assert( ids != NULL ); + assert( ids[0] = 1 ); + assert( count == 1 ); + assert( ids[0] = matches[i].endid ); } else { assert( ret == 0 ); - assert( end_ids == NULL ); - assert( num_end_ids == 0 ); + assert( ids == NULL ); + assert( count == 0 ); } - free(end_ids); + free(ids); } } diff --git a/tests/endids/endids0_many_endids.c b/tests/endids/endids0_many_endids.c index cd5230855..86a66063c 100644 --- a/tests/endids/endids0_many_endids.c +++ b/tests/endids/endids0_many_endids.c @@ -39,18 +39,18 @@ int main(void) * * 17 endids. This should force the array of end ids to resize. */ - static const fsm_end_id_t all_end_ids[17] = { + static const fsm_end_id_t all_ids[17] = { 4, 17, 6, 18, 2, 63, 14, 62, 3, 37, 46, 7, 9, 72, 67, 36, 1, }; - static const fsm_end_id_t sorted_all_end_ids[17] = { + static const fsm_end_id_t sorted_all_ids[17] = { 1, 2, 3, 4, 6, 7, 9, 14, 17, 18, 36, 37, 46, 62, 63, 67, 72, }; - for (size_t i=0; i < sizeof all_end_ids / sizeof all_end_ids[0]; i++) { - ret = fsm_setendid(fsm, all_end_ids[i]); + for (size_t i=0; i < sizeof all_ids / sizeof all_ids[0]; i++) { + ret = fsm_setendid(fsm, all_ids[i]); assert(ret == 1); } @@ -69,52 +69,45 @@ int main(void) assert(end_state < nstates); - assert(fsm_getendidcount(fsm, end_state) == sizeof all_end_ids / sizeof all_end_ids[0]); - for (i=0; i < sizeof all_end_ids / sizeof all_end_ids[0]; i++) { + assert(fsm_endid_count(fsm, end_state) == sizeof all_ids / sizeof all_ids[0]); + for (i=0; i < sizeof all_ids / sizeof all_ids[0]; i++) { /* add duplicate end ids */ - ret = fsm_setendid(fsm, all_end_ids[i]); + ret = fsm_setendid(fsm, all_ids[i]); /* fsm_setendid should succeed */ assert(ret == 1); /* but it shouldn't add a duplicate id */ - assert(fsm_getendidcount(fsm, end_state) == sizeof all_end_ids / sizeof all_end_ids[0]); + assert(fsm_endid_count(fsm, end_state) == sizeof all_ids / sizeof all_ids[0]); } nend = 0; for (state_ind = 0; state_ind < nstates; state_ind++) { if (fsm_isend(fsm, state_ind)) { - fsm_end_id_t endids[sizeof all_end_ids / sizeof all_end_ids[0]]; - size_t i, nwritten; - enum fsm_getendids_res ret; + fsm_end_id_t endids[sizeof all_ids / sizeof all_ids[0]]; + size_t i; + int ret; memset(&endids[0], 0, sizeof endids); - assert(fsm_getendidcount(fsm, state_ind) == sizeof all_end_ids / sizeof all_end_ids[0]); + assert(fsm_endid_count(fsm, state_ind) == sizeof all_ids / sizeof all_ids[0]); - nwritten = 0; - ret = fsm_getendids( + ret = fsm_endid_get( fsm, state_ind, - sizeof endids/sizeof endids[0], - &endids[0], - &nwritten); + sizeof endids / sizeof endids[0], + &endids[0]); - assert(ret == FSM_GETENDIDS_FOUND); - assert(nwritten == sizeof all_end_ids / sizeof all_end_ids[0]); + assert(ret == 1); - /* sort endids and compare */ - qsort(&endids[0], nwritten, sizeof endids[0], cmp_endids); - for (i=0; i < nwritten; i++) { - assert(endids[i] == sorted_all_end_ids[i]); + for (i=0; i < sizeof all_ids / sizeof all_ids[0]; i++) { + assert(endids[i] == sorted_all_ids[i]); } -#if 0 /* endids should be sorted */ - for (i=0; i < nwritten; i++) { - assert(endids[i] == sorted_all_end_ids[i]); + for (i=0; i < sizeof all_ids / sizeof all_ids[0]; i++) { + assert(endids[i] == sorted_all_ids[i]); } -#endif /* 0 */ nend++; } @@ -142,38 +135,35 @@ int main(void) }; for (i=0; i < sizeof matches / sizeof matches[0]; i++) { - fsm_end_id_t *end_ids; - size_t num_end_ids; + fsm_end_id_t *ids; + size_t count; - end_ids = NULL; - num_end_ids = 0; - ret = match_string(fsm, matches[i].s, NULL, &end_ids, &num_end_ids); + ids = NULL; + count = 0; + ret = match_string(fsm, matches[i].s, NULL, &ids, &count); if (matches[i].should_match) { size_t j; assert( ret == 1 ); - assert( end_ids != NULL ); - assert( num_end_ids == sizeof all_end_ids / sizeof all_end_ids[0] ); + assert( ids != NULL ); + assert( count == sizeof all_ids / sizeof all_ids[0] ); #if 0 - assert( end_ids[0] == 1 ); - assert( end_ids[0] == matches[i].endid ); + assert( ids[0] == 1 ); + assert( ids[0] == matches[i].endid ); #endif /* 0 */ - /* sort endids and compare */ - qsort(&end_ids[0], num_end_ids, sizeof end_ids[0], cmp_endids); - - for (j=0; j < num_end_ids; j++) { - assert(end_ids[j] == sorted_all_end_ids[j]); + for (j=0; j < count; j++) { + assert(ids[j] == sorted_all_ids[j]); } } else { assert( ret == 0 ); - assert( end_ids == NULL ); - assert( num_end_ids == 0 ); + assert( ids == NULL ); + assert( count == 0 ); } - free(end_ids); + free(ids); } } diff --git a/tests/endids/endids10_minimise_partial_overlap.c b/tests/endids/endids10_minimise_partial_overlap.c new file mode 100644 index 000000000..5a72d05d9 --- /dev/null +++ b/tests/endids/endids10_minimise_partial_overlap.c @@ -0,0 +1,85 @@ +#include +#include +#include +#include +#include + +#include + +#include + +#include + +#include +#include +#include +#include +#include + +#include "endids_utils.h" + +#define ENDID_AB_STAR_C 1 +#define ENDID_ABC 2 + +int main(void) +{ + /* Union /^ab*c$/ and /^abc$/ with distinct endids set on each + * and verify that inputs of "ac", "abc", "abbc" get the endids + * associated with only the regexes that match. + * + * In other words, check that minimisation does not merge them + * and cause false positives. */ + + const char *regex_ab_star_c = "^ab*c$"; + const char *regex_abc = "^abc$"; + + struct fsm *fsm_ab_star_c = re_comp(RE_NATIVE, fsm_sgetc, (void *)®ex_ab_star_c, NULL, 0, NULL); + assert(fsm_ab_star_c != NULL); + if (!fsm_setendid(fsm_ab_star_c, ENDID_AB_STAR_C)) { assert(!"setendid"); } + + if (!fsm_determinise(fsm_ab_star_c)) { assert(!"determinise"); } + if (!fsm_minimise(fsm_ab_star_c)) { assert(!"minimise"); } + + struct fsm *fsm_abc = re_comp(RE_NATIVE, fsm_sgetc, (void *)®ex_abc, NULL, 0, NULL); + assert(fsm_abc != NULL); + if (!fsm_setendid(fsm_abc, ENDID_ABC)) { assert(!"setendid"); } + + if (!fsm_determinise(fsm_abc)) { assert(!"determinise"); } + if (!fsm_minimise(fsm_abc)) { assert(!"minimise"); } + + struct fsm *combined = fsm_union(fsm_ab_star_c, fsm_abc, NULL); + assert(combined != NULL); + + int ret = fsm_determinise(combined); + assert(ret != 0); + + ret = fsm_minimise(combined); + assert(ret != 0); + + size_t written; + fsm_end_id_t *endids = NULL; + if (match_string(combined, "ac", NULL, &endids, &written) != 1) { + assert(!"'ac' should match"); + } + assert(written == 1); + assert(endids[0] == ENDID_AB_STAR_C); + free(endids); + + if (match_string(combined, "abc", NULL, &endids, &written) != 1) { + assert(!"'abc' should match"); + } + assert(written == 2); + /* result is not sorted */ + assert((endids[0] == ENDID_AB_STAR_C && endids[1] == ENDID_ABC) || + (endids[1] == ENDID_AB_STAR_C && endids[0] == ENDID_ABC)); + free(endids); + + if (match_string(combined, "abbc", NULL, &endids, &written) != 1) { + assert(!"'abbc' should match"); + } + assert(written == 1); + assert(endids[0] == ENDID_AB_STAR_C); + free(endids); + + fsm_free(combined); +} diff --git a/tests/endids/endids1_determinise.c b/tests/endids/endids1_determinise.c index 956e0ee33..472807eed 100644 --- a/tests/endids/endids1_determinise.c +++ b/tests/endids/endids1_determinise.c @@ -40,22 +40,13 @@ int main(void) for (state_ind = 0; state_ind < nstates; state_ind++) { if (fsm_isend(fsm, state_ind)) { fsm_end_id_t endid = 0; - size_t nwritten; - enum fsm_getendids_res ret; - - assert( fsm_getendidcount(fsm, state_ind) == 1); - - nwritten = 0; - ret = fsm_getendids( - fsm, - state_ind, - 1, - &endid, - &nwritten); - - assert(ret == FSM_GETENDIDS_FOUND); - assert(nwritten == 1); - assert( endid == 1 ); + int ret; + + assert( fsm_endid_count(fsm, state_ind) == 1); + + ret = fsm_endid_get(fsm, state_ind, 1, &endid); + assert(ret == 1); + assert( endid == 1 ); } } @@ -79,26 +70,26 @@ int main(void) }; for (i=0; i < sizeof matches / sizeof matches[0]; i++) { - fsm_end_id_t *end_ids; - size_t num_end_ids; + fsm_end_id_t *ids; + size_t count; - end_ids = NULL; - num_end_ids = 0; - ret = match_string(fsm, matches[i].s, NULL, &end_ids, &num_end_ids); + ids = NULL; + count = 0; + ret = match_string(fsm, matches[i].s, NULL, &ids, &count); if (matches[i].should_match) { assert( ret == 1 ); - assert( end_ids != NULL ); - assert( end_ids[0] = 1 ); - assert( num_end_ids == 1 ); - assert( end_ids[0] = matches[i].endid ); + assert( ids != NULL ); + assert( ids[0] = 1 ); + assert( count == 1 ); + assert( ids[0] = matches[i].endid ); } else { assert( ret == 0 ); - assert( end_ids == NULL ); - assert( num_end_ids == 0 ); + assert( ids == NULL ); + assert( count == 0 ); } - free(end_ids); + free(ids); } } diff --git a/tests/endids/endids1_determinise_and_minimise.c b/tests/endids/endids1_determinise_and_minimise.c index 4619af4a8..a2feafb18 100644 --- a/tests/endids/endids1_determinise_and_minimise.c +++ b/tests/endids/endids1_determinise_and_minimise.c @@ -45,22 +45,13 @@ int main(void) for (state_ind = 0; state_ind < nstates; state_ind++) { if (fsm_isend(fsm, state_ind)) { fsm_end_id_t endid = 0; - size_t nwritten; - enum fsm_getendids_res ret; - - assert( fsm_getendidcount(fsm, state_ind) == 1); - - nwritten = 0; - ret = fsm_getendids( - fsm, - state_ind, - 1, - &endid, - &nwritten); - - assert(ret == FSM_GETENDIDS_FOUND); - assert(nwritten == 1); - assert( endid == 1 ); + int ret; + + assert(fsm_endid_count(fsm, state_ind) == 1); + + ret = fsm_endid_get(fsm, state_ind, 1, &endid); + assert(ret == 1); + assert( endid == 1 ); } } @@ -84,26 +75,26 @@ int main(void) }; for (i=0; i < sizeof matches / sizeof matches[0]; i++) { - fsm_end_id_t *end_ids; - size_t num_end_ids; + fsm_end_id_t *ids; + size_t count; - end_ids = NULL; - num_end_ids = 0; - ret = match_string(fsm, matches[i].s, NULL, &end_ids, &num_end_ids); + ids = NULL; + count = 0; + ret = match_string(fsm, matches[i].s, NULL, &ids, &count); if (matches[i].should_match) { assert( ret == 1 ); - assert( end_ids != NULL ); - assert( end_ids[0] = 1 ); - assert( num_end_ids == 1 ); - assert( end_ids[0] = matches[i].endid ); + assert( ids != NULL ); + assert( ids[0] = 1 ); + assert( count == 1 ); + assert( ids[0] = matches[i].endid ); } else { assert( ret == 0 ); - assert( end_ids == NULL ); - assert( num_end_ids == 0 ); + assert( ids == NULL ); + assert( count == 0 ); } - free(end_ids); + free(ids); } } diff --git a/tests/endids/endids2_union.c b/tests/endids/endids2_union.c index d65ef17e8..0d6182437 100644 --- a/tests/endids/endids2_union.c +++ b/tests/endids/endids2_union.c @@ -55,32 +55,22 @@ int main(void) for (state_ind = 0; state_ind < nstates; state_ind++) { if (fsm_isend(comb, state_ind)) { fsm_end_id_t endids[2] = {0,0}; - size_t nwritten; - size_t num_endids; - enum fsm_getendids_res ret; + size_t count; + int ret; - nwritten = 0; - num_endids = fsm_getendidcount(comb, state_ind); - // fprintf(stderr, "state %u, num_endids = %zu\n", state_ind, num_endids); + count = fsm_endid_count(comb, state_ind); + // fprintf(stderr, "state %u, count = %zu\n", state_ind, count); + assert(count > 0 && count <= 2); - assert( num_endids > 0 && num_endids <= 2); + ret = fsm_endid_get( comb, state_ind, count, &endids[0]); + assert(ret == 1); - ret = fsm_getendids( - comb, - state_ind, - 2, - &endids[0], - &nwritten); - - assert(ret == FSM_GETENDIDS_FOUND); - assert(nwritten == num_endids); - - if (nwritten == 1) { + if (count == 1) { assert(endids[0] == 1 || endids[0] == 2); has_endid_1 = has_endid_1 || (endids[0] == 1); has_endid_2 = has_endid_2 || (endids[0] == 2); - } else if (nwritten == 2) { + } else if (count == 2) { assert((endids[0] == 1 && endids[1] == 2) || (endids[0] == 2 && endids[1] == 1)); @@ -111,7 +101,7 @@ int main(void) static const struct { const char *s; int should_match; - size_t num_endids; + size_t count; fsm_end_id_t endid[2]; } matches[] = { { "abc" , 1, 1, { 1, 0 } }, @@ -147,33 +137,31 @@ int main(void) }; for (i=0; i < sizeof matches / sizeof matches[0]; i++) { - fsm_end_id_t *end_ids; - size_t num_end_ids; + fsm_end_id_t *ids; + size_t count; - end_ids = NULL; - num_end_ids = 0; - ret = match_string(comb, matches[i].s, NULL, &end_ids, &num_end_ids); + ids = NULL; + count = 0; + ret = match_string(comb, matches[i].s, NULL, &ids, &count); if (matches[i].should_match) { size_t j; assert( ret == 1 ); - assert( end_ids != NULL ); - - assert( num_end_ids == matches[i].num_endids ); + assert( ids != NULL ); - qsort(&end_ids[0], num_end_ids, sizeof end_ids[0], cmp_endids); + assert( count == matches[i].count ); - for (j=0; j < num_end_ids; j++) { - assert( end_ids[j] == matches[i].endid[j] ); + for (j=0; j < count; j++) { + assert( ids[j] == matches[i].endid[j] ); } } else { assert( ret == 0 ); - assert( end_ids == NULL ); - assert( num_end_ids == 0 ); + assert( ids == NULL ); + assert( count == 0 ); } - free(end_ids); + free(ids); } } diff --git a/tests/endids/endids2_union_many_endids.c b/tests/endids/endids2_union_many_endids.c index 8e39ca93d..09376f0ed 100644 --- a/tests/endids/endids2_union_many_endids.c +++ b/tests/endids/endids2_union_many_endids.c @@ -22,6 +22,7 @@ // to exercise reallocs in the ID list // // Each pattern gets 5 end ids +// Note: These are unanchored and therefore partially overlap static const char *patterns[] = { "abc", // 1-5 "def", // 6-10 @@ -120,18 +121,18 @@ size_t generate_examples(struct example_list *l, const struct fsm *fsm, size_t p return (first_index != SIZE_MAX) ? first_index : 0; } -/* test that endids correctly propagate through union, determinise, and minimise */ +/* test that ids correctly propagate through union, determinise, and minimise */ int main(void) { struct fsm *fsm; - fsm_end_id_t all_endids[NUM_PATTERNS]; + fsm_end_id_t all_ids[NUM_PATTERNS]; size_t nstates, state_ind; size_t i; int ret; struct example_list example_list; - memset(all_endids, 0, sizeof all_endids); + memset(all_ids, 0, sizeof all_ids); init_examples(&example_list); @@ -167,6 +168,7 @@ int main(void) if (fsm == NULL) { fsm = new; } else { + /* TODO: this could use fsm_union_array instead */ fsm = fsm_union(fsm, new, NULL); assert(fsm != NULL); } @@ -175,39 +177,30 @@ int main(void) ret = fsm_determinise(fsm); assert(ret != 0); - // find end states, make sure we have two end states and they each have endids + // find end states, make sure we have multiple end states and they each have ids nstates = fsm_countstates(fsm); for (state_ind = 0; state_ind < nstates; state_ind++) { if (fsm_isend(fsm, state_ind)) { int tested_pattern[NUM_PATTERNS]; - fsm_end_id_t endids[NUM_ENDIDS_TOTAL]; - size_t nwritten, num_endids, j; - enum fsm_getendids_res ret; - - memset(&endids[0], 0, sizeof endids); - - nwritten = 0; - num_endids = fsm_getendidcount(fsm, state_ind); + fsm_end_id_t ids[NUM_ENDIDS_TOTAL]; + size_t count, j; + int ret; - assert(num_endids > 0 && num_endids <= sizeof endids/sizeof endids[0]); + memset(&ids[0], 0, sizeof ids); - ret = fsm_getendids( - fsm, - state_ind, - sizeof endids/sizeof endids[0], - &endids[0], - &nwritten); + count = fsm_endid_count(fsm, state_ind); + assert(count > 0 && count <= sizeof ids/sizeof ids[0]); - assert(ret == FSM_GETENDIDS_FOUND); - assert(nwritten == num_endids); + ret = fsm_endid_get(fsm, state_ind, count, &ids[0]); + assert(ret == 1); memset(&tested_pattern[0], 0, sizeof tested_pattern); - for (j=0; j < num_endids; j++) { + for (j=0; j < count; j++) { size_t k, pattern_index; - pattern_index = (endids[j] - 1)/NUM_ENDIDS_PER_PATTERN; + pattern_index = (ids[j] - 1)/NUM_ENDIDS_PER_PATTERN; assert(pattern_index < NUM_PATTERNS); if (tested_pattern[pattern_index]) { @@ -226,11 +219,11 @@ int main(void) if (ret) { #if 0 printf("end state %zu (end id %u) matches example \"%s\" from pattern /%s/\n", - state_ind, endids[j], ex->example, patterns[pattern_index]); + state_ind, ids[j], ex->example, patterns[pattern_index]); #endif /* 0 */ } else { printf("end state %zu (end id %u) does NOT match example \"%s\" from pattern /%s/\n", - state_ind, endids[j], ex->example, patterns[pattern_index]); + state_ind, ids[j], ex->example, patterns[pattern_index]); abort(); } } @@ -238,41 +231,27 @@ int main(void) } } - /* fsm_minimise currently collapses all end states to the same state. This should - * create a single end state with NUM_ENDIDS_TOTAL end ids - */ ret = fsm_minimise(fsm); assert(ret != 0); - assert( fsm_count(fsm, fsm_isend) == 1 ); + /* fsm_minimise should not collapse all the end states to a + * single end state, because they have distinct ids. */ + assert( fsm_count(fsm, fsm_isend) > 1); nstates = fsm_countstates(fsm); for (state_ind = 0; state_ind < nstates; state_ind++) { if (fsm_isend(fsm, state_ind)) { - fsm_end_id_t endids[NUM_ENDIDS_TOTAL]; - size_t nwritten, num_endids, j; - enum fsm_getendids_res ret; - - memset(&endids[0], 0, sizeof endids); - - nwritten = 0; - num_endids = fsm_getendidcount(fsm, state_ind); - - assert(num_endids == NUM_ENDIDS_TOTAL); + fsm_end_id_t ids[NUM_ENDIDS_TOTAL]; + size_t count; + int ret; - ret = fsm_getendids( - fsm, - state_ind, - sizeof endids/sizeof endids[0], - &endids[0], - &nwritten); + memset(&ids[0], 0, sizeof ids); - assert(ret == FSM_GETENDIDS_FOUND); - assert(nwritten == num_endids); + count = fsm_endid_count(fsm, state_ind); + assert(count <= NUM_ENDIDS_TOTAL); - for (j=0; j < num_endids; j++) { - assert( endids[j] == j+1 ); - } + ret = fsm_endid_get(fsm, state_ind, count, &ids[0]); + assert(ret == 1); } } @@ -283,5 +262,3 @@ int main(void) return EXIT_SUCCESS; } - - diff --git a/tests/endids/endids4.c b/tests/endids/endids4.c index c8f312663..a4530c93b 100644 --- a/tests/endids/endids4.c +++ b/tests/endids/endids4.c @@ -67,29 +67,17 @@ int main(void) for (state_ind = 0; state_ind < nstates; state_ind++) { if (fsm_isend(comb, state_ind)) { fsm_end_id_t endids[2] = {0,0}; - size_t nwritten; - size_t num_endids; - enum fsm_getendids_res ret; + size_t count; + int ret; - nwritten = 0; - num_endids = fsm_getendidcount(comb, state_ind); + count = fsm_endid_count(comb, state_ind); + printf("state %u count = %zu\n", state_ind, count); + assert(count == 2); - printf("state %u num_endids = %zu\n", state_ind, num_endids); + ret = fsm_endid_get(comb, state_ind, count, &endids[0]); + assert(ret == 1); - assert(num_endids == 2); - - ret = fsm_getendids( - comb, - state_ind, - 2, - &endids[0], - &nwritten); - - assert(ret == FSM_GETENDIDS_FOUND); - assert(nwritten == num_endids); - - qsort(&endids[0], num_endids, sizeof endids[0], cmp_endids); - assert(endids[0] == 1 && endids[1] == 2); + assert(endids[0] == 1 && endids[1] == 2); } } @@ -102,7 +90,7 @@ int main(void) static const struct { const char *s; int should_match; - size_t num_endids; + size_t count; fsm_end_id_t endid[2]; } matches[] = { { "abc" , 0, 0, { 0, 0 } }, @@ -138,33 +126,31 @@ int main(void) }; for (i=0; i < sizeof matches / sizeof matches[0]; i++) { - fsm_end_id_t *end_ids; - size_t num_end_ids; + fsm_end_id_t *ids; + size_t count; - end_ids = NULL; - num_end_ids = 0; - ret = match_string(comb, matches[i].s, NULL, &end_ids, &num_end_ids); + ids = NULL; + count = 0; + ret = match_string(comb, matches[i].s, NULL, &ids, &count); if (matches[i].should_match) { size_t j; assert( ret == 1 ); - assert( end_ids != NULL ); - - assert( num_end_ids == matches[i].num_endids ); + assert( ids != NULL ); - qsort(&end_ids[0], num_end_ids, sizeof end_ids[0], cmp_endids); + assert( count == matches[i].count ); - for (j=0; j < num_end_ids; j++) { - assert( end_ids[j] == matches[i].endid[j] ); + for (j=0; j < count; j++) { + assert( ids[j] == matches[i].endid[j] ); } } else { assert( ret == 0 ); - assert( end_ids == NULL ); - assert( num_end_ids == 0 ); + assert( ids == NULL ); + assert( count == 0 ); } - free(end_ids); + free(ids); } } diff --git a/tests/endids/endids5.c b/tests/endids/endids5.c index f6220115a..50030ea0d 100644 --- a/tests/endids/endids5.c +++ b/tests/endids/endids5.c @@ -68,28 +68,17 @@ int main(void) for (state_ind = 0; state_ind < nstates; state_ind++) { if (fsm_isend(comb, state_ind)) { fsm_end_id_t endids[2] = {0,0}; - size_t nwritten; - size_t num_endids; - enum fsm_getendids_res ret; + size_t count; + int ret; - nwritten = 0; - num_endids = fsm_getendidcount(comb, state_ind); + count = fsm_endid_count(comb, state_ind); + printf("state %u count = %zu\n", state_ind, count); + assert(count == 1); - printf("state %u num_endids = %zu\n", state_ind, num_endids); + ret = fsm_endid_get(comb, state_ind, count, &endids[0]); + assert(ret == 1); - assert(num_endids == 1); - - ret = fsm_getendids( - comb, - state_ind, - 2, - &endids[0], - &nwritten); - - assert(ret == FSM_GETENDIDS_FOUND); - assert(nwritten == num_endids); - - assert(endids[0] == 1); + assert(endids[0] == 1); } } @@ -102,7 +91,7 @@ int main(void) static const struct { const char *s; int should_match; - size_t num_endids; + size_t count; fsm_end_id_t endid[2]; } matches[] = { { "abc" , 1, 1, { 1, 0 } }, @@ -138,37 +127,35 @@ int main(void) }; for (i=0; i < sizeof matches / sizeof matches[0]; i++) { - fsm_end_id_t *end_ids; - size_t num_end_ids; + fsm_end_id_t *ids; + size_t count; - end_ids = NULL; - num_end_ids = 0; - ret = match_string(comb, matches[i].s, NULL, &end_ids, &num_end_ids); + ids = NULL; + count = 0; + ret = match_string(comb, matches[i].s, NULL, &ids, &count); - printf("match %zu, \"%s\", should_match=%d, num_endids=%zu, endids={ %u, %u }\n", - i, matches[i].s, matches[i].should_match, matches[i].num_endids, + printf("match %zu, \"%s\", should_match=%d, count=%zu, endids={ %u, %u }\n", + i, matches[i].s, matches[i].should_match, matches[i].count, (unsigned)matches[i].endid[0], (unsigned)matches[i].endid[1]); if (matches[i].should_match) { size_t j; assert( ret == 1 ); - assert( end_ids != NULL ); - - assert( num_end_ids == matches[i].num_endids ); + assert( ids != NULL ); - qsort(&end_ids[0], num_end_ids, sizeof end_ids[0], cmp_endids); + assert( count == matches[i].count ); - for (j=0; j < num_end_ids; j++) { - assert( end_ids[j] == matches[i].endid[j] ); + for (j=0; j < count; j++) { + assert( ids[j] == matches[i].endid[j] ); } } else { assert( ret == 0 ); - assert( end_ids == NULL ); - assert( num_end_ids == 0 ); + assert( ids == NULL ); + assert( count == 0 ); } - free(end_ids); + free(ids); } } diff --git a/tests/endids/endids6.c b/tests/endids/endids6.c index ca089d0f1..e237c2881 100644 --- a/tests/endids/endids6.c +++ b/tests/endids/endids6.c @@ -19,34 +19,34 @@ struct state_info { fsm_state_t state; - unsigned num_endids; - fsm_end_id_t endids[2]; + unsigned count; + fsm_end_id_t ids[2]; }; /* remap 1 -> 512, 2 -> 1024 */ static int -endid_remap_func(fsm_state_t state, size_t num_ids, fsm_end_id_t *endids, size_t *num_written, void *opaque) +endid_remap_func(fsm_state_t state, size_t count, fsm_end_id_t *ids, size_t *num_written, void *opaque) { size_t i; (void)state; (void)opaque; - assert(endids != NULL); - for (i=0; i < num_ids; i++) { - fsm_end_id_t orig = endids[i]; + assert(ids != NULL); + for (i=0; i < count; i++) { + fsm_end_id_t orig = ids[i]; - switch (endids[i]) { - case 1: endids[i] = 512; break; - case 2: endids[i] = 1024; break; + switch (ids[i]) { + case 1: ids[i] = 512; break; + case 2: ids[i] = 1024; break; default: break; } printf("remap: state %u id %u -> %u\n", - (unsigned)state, (unsigned)orig, (unsigned)endids[i]); + (unsigned)state, (unsigned)orig, (unsigned)ids[i]); } - *num_written = num_ids; + *num_written = count; return 1; } @@ -91,37 +91,26 @@ int main(void) for (state_ind = 0; state_ind < nstates; state_ind++) { if (fsm_isend(fsm, state_ind)) { - fsm_end_id_t endids[2] = {0,0}; - size_t nwritten; - size_t num_endids; - enum fsm_getendids_res ret; + fsm_end_id_t ids[2] = {0,0}; + size_t count; + int ret; - nwritten = 0; - num_endids = fsm_getendidcount(fsm, state_ind); + count = fsm_endid_count(fsm, state_ind); + assert(count > 0 && count <= 2); - assert( num_endids > 0 && num_endids <= 2); - - ret = fsm_getendids( - fsm, - state_ind, - sizeof endids / sizeof endids[0], - &endids[0], - &nwritten); - - assert(ret == FSM_GETENDIDS_FOUND); - assert(nwritten == num_endids); + ret = fsm_endid_get(fsm, state_ind, count, &ids[0]); + assert(ret == 1); info[ninfo].state = state_ind; - info[ninfo].num_endids = nwritten; - - if (nwritten == 1) { - assert(endids[0] == 1 || endids[0] == 2); - info[ninfo].endids[0] = endids[0]; - } else if (nwritten == 2) { - qsort(&endids[0], nwritten, sizeof endids[0], cmp_endids); - assert(endids[0] == 1 && endids[1] == 2); - info[ninfo].endids[0] = endids[0]; - info[ninfo].endids[1] = endids[1]; + info[ninfo].count = count; + + if (count == 1) { + assert(ids[0] == 1 || ids[0] == 2); + info[ninfo].ids[0] = ids[0]; + } else if (count == 2) { + assert(ids[0] == 1 && ids[1] == 2); + info[ninfo].ids[0] = ids[0]; + info[ninfo].ids[1] = ids[1]; } ninfo++; @@ -136,35 +125,22 @@ int main(void) for (state_ind = 0, info_ind = 0; state_ind < nstates; state_ind++) { if (fsm_isend(fsm, state_ind)) { - fsm_end_id_t endids[2] = {0,0}; - size_t nwritten, num_endids, ind; - enum fsm_getendids_res ret; + fsm_end_id_t ids[2] = {0,0}; + size_t count, ind; + int ret; assert( state_ind == info[info_ind].state ); - nwritten = 0; - num_endids = fsm_getendidcount(fsm, state_ind); - - assert(num_endids > 0 && num_endids <= 2); + count = fsm_endid_count(fsm, state_ind); + assert(count > 0 && count <= 2); + assert(count == info[info_ind].count); - assert( num_endids == info[info_ind].num_endids ); - ret = fsm_getendids( - fsm, - state_ind, - sizeof endids / sizeof endids[0], - &endids[0], - &nwritten); - - assert(ret == FSM_GETENDIDS_FOUND); - assert(nwritten == num_endids); - - if (nwritten > 1) { - qsort(&endids[0], num_endids, sizeof endids[0], cmp_endids); - } + ret = fsm_endid_get(fsm, state_ind, count, &ids[0]); + assert(ret == 1); - for (ind=0; ind < num_endids; ind++) { - fsm_end_id_t expected = info[info_ind].endids[ind]; - switch (info[info_ind].endids[ind]) { + for (ind=0; ind < count; ind++) { + fsm_end_id_t expected = info[info_ind].ids[ind]; + switch (info[info_ind].ids[ind]) { case 1: expected = 512; break; case 2: expected = 1024; break; default: @@ -172,8 +148,8 @@ int main(void) break; } - printf("state %u, id %u, expected %u\n", (unsigned)state_ind, endids[ind], expected); - assert(endids[ind] == expected); + printf("state %u, id %u, expected %u\n", (unsigned)state_ind, ids[ind], expected); + assert(ids[ind] == expected); } info_ind++; diff --git a/tests/endids/endids7.c b/tests/endids/endids7.c index cc68d02c0..3021be830 100644 --- a/tests/endids/endids7.c +++ b/tests/endids/endids7.c @@ -19,41 +19,41 @@ struct state_info { fsm_state_t state; - unsigned num_endids; - fsm_end_id_t endids[2]; + unsigned count; + fsm_end_id_t ids[2]; }; /* remap 1 -> 7, 2 -> 9, both 1&2 -> 3 */ static int -endid_remap_func(fsm_state_t state, size_t num_ids, fsm_end_id_t *endids, size_t *num_written, void *opaque) +endid_remap_func(fsm_state_t state, size_t num_ids, fsm_end_id_t *ids, size_t *num_written, void *opaque) { (void)state; (void)opaque; - assert(endids != NULL); + assert(ids != NULL); assert(num_ids > 0 && num_ids <= 2); if (num_ids > 1) { - endids[0] = 3; + ids[0] = 3; *num_written = 1; printf("remap: state %u with %zu endids, set to one endid, value %u\n", - (unsigned)state, num_ids, (unsigned)endids[0]); + (unsigned)state, num_ids, (unsigned)ids[0]); return 1; } if (num_ids == 1) { - fsm_end_id_t orig = endids[0]; + fsm_end_id_t orig = ids[0]; - switch (endids[0]) { - case 1: endids[0] = 7; break; - case 2: endids[0] = 9; break; + switch (ids[0]) { + case 1: ids[0] = 7; break; + case 2: ids[0] = 9; break; default: break; } printf("remap: state %u id %u -> %u\n", - (unsigned)state, (unsigned)orig, (unsigned)endids[0]); + (unsigned)state, (unsigned)orig, (unsigned)ids[0]); } *num_written = num_ids; @@ -103,37 +103,26 @@ int main(void) for (state_ind = 0; state_ind < nstates; state_ind++) { if (fsm_isend(fsm, state_ind)) { - fsm_end_id_t endids[2] = {0,0}; - size_t nwritten; - size_t num_endids; - enum fsm_getendids_res ret; + fsm_end_id_t ids[2] = {0,0}; + size_t count; + int ret; - nwritten = 0; - num_endids = fsm_getendidcount(fsm, state_ind); + count = fsm_endid_count(fsm, state_ind); + assert( count > 0 && count <= 2); - assert( num_endids > 0 && num_endids <= 2); - - ret = fsm_getendids( - fsm, - state_ind, - sizeof endids / sizeof endids[0], - &endids[0], - &nwritten); - - assert(ret == FSM_GETENDIDS_FOUND); - assert(nwritten == num_endids); + ret = fsm_endid_get(fsm, state_ind, count, &ids[0]); + assert(ret == 1); info[ninfo].state = state_ind; - info[ninfo].num_endids = nwritten; - - if (nwritten == 1) { - assert(endids[0] == 1 || endids[0] == 2); - info[ninfo].endids[0] = endids[0]; - } else if (nwritten == 2) { - qsort(&endids[0], nwritten, sizeof endids[0], cmp_endids); - assert(endids[0] == 1 && endids[1] == 2); - info[ninfo].endids[0] = endids[0]; - info[ninfo].endids[1] = endids[1]; + info[ninfo].count = count; + + if (count == 1) { + assert(ids[0] == 1 || ids[0] == 2); + info[ninfo].ids[0] = ids[0]; + } else if (count == 2) { + assert(ids[0] == 1 && ids[1] == 2); + info[ninfo].ids[0] = ids[0]; + info[ninfo].ids[1] = ids[1]; } ninfo++; @@ -148,39 +137,31 @@ int main(void) for (state_ind = 0, info_ind = 0; state_ind < nstates; state_ind++) { if (fsm_isend(fsm, state_ind)) { - fsm_end_id_t endid = 0; + fsm_end_id_t id = 0; fsm_end_id_t expected; - size_t nwritten, num_endids; - enum fsm_getendids_res ret; + size_t count; + int ret; assert( state_ind == info[info_ind].state ); - nwritten = 0; - num_endids = fsm_getendidcount(fsm, state_ind); - - assert(num_endids == 1); - ret = fsm_getendids( - fsm, - state_ind, - 1, - &endid, - &nwritten); + count = fsm_endid_count(fsm, state_ind); + assert(count == 1); - assert(ret == FSM_GETENDIDS_FOUND); - assert(nwritten == num_endids); + ret = fsm_endid_get(fsm, state_ind, 1, &id); + assert(ret == 1); - if (info[info_ind].num_endids == 2) { + if (info[info_ind].count == 2) { expected = 3; - } else if (info[info_ind].endids[0] == 1) { + } else if (info[info_ind].ids[0] == 1) { expected = 7; - } else if (info[info_ind].endids[0] == 2) { + } else if (info[info_ind].ids[0] == 2) { expected = 9; } else { assert( ! "unexpected value" ); } - printf("state %u, id %u, expected %u\n", (unsigned)state_ind, endid, expected); - assert(endid == expected);; + printf("state %u, id %u, expected %u\n", (unsigned)state_ind, id, expected); + assert(id == expected);; info_ind++; } diff --git a/tests/endids/endids7_with_duplicates.c b/tests/endids/endids7_with_duplicates.c index b554f7cb6..939c4b306 100644 --- a/tests/endids/endids7_with_duplicates.c +++ b/tests/endids/endids7_with_duplicates.c @@ -19,45 +19,45 @@ struct state_info { fsm_state_t state; - unsigned num_endids; - fsm_end_id_t endids[2]; + unsigned count; + fsm_end_id_t ids[2]; }; /* remap 1 -> 7, 2 -> 9, both 1&2 -> 3,3 */ static int -endid_remap_func(fsm_state_t state, size_t num_ids, fsm_end_id_t *endids, size_t *num_written, void *opaque) +endid_remap_func(fsm_state_t state, size_t num_ids, fsm_end_id_t *ids, size_t *num_written, void *opaque) { (void)state; (void)opaque; - assert(endids != NULL); + assert(ids != NULL); assert(num_ids > 0 && num_ids <= 2); if (num_ids > 1) { size_t i; for (i = 0; i < num_ids; i++) { - endids[i] = 3; + ids[i] = 3; } *num_written = num_ids; printf("remap: state %u with %zu endids, set to %zu endid, value %u\n", - (unsigned)state, num_ids, *num_written, (unsigned)endids[0]); + (unsigned)state, num_ids, *num_written, (unsigned) ids[0]); return 1; } if (num_ids == 1) { - fsm_end_id_t orig = endids[0]; + fsm_end_id_t orig = ids[0]; - switch (endids[0]) { - case 1: endids[0] = 7; break; - case 2: endids[0] = 9; break; + switch (ids[0]) { + case 1: ids[0] = 7; break; + case 2: ids[0] = 9; break; default: break; } printf("remap: state %u id %u -> %u\n", - (unsigned)state, (unsigned)orig, (unsigned)endids[0]); + (unsigned)state, (unsigned)orig, (unsigned) ids[0]); } *num_written = num_ids; @@ -107,37 +107,26 @@ int main(void) for (state_ind = 0; state_ind < nstates; state_ind++) { if (fsm_isend(fsm, state_ind)) { - fsm_end_id_t endids[2] = {0,0}; - size_t nwritten; - size_t num_endids; - enum fsm_getendids_res ret; + fsm_end_id_t ids[2] = {0,0}; + size_t count; + int ret; - nwritten = 0; - num_endids = fsm_getendidcount(fsm, state_ind); + count = fsm_endid_count(fsm, state_ind); + assert( count > 0 && count <= 2); - assert( num_endids > 0 && num_endids <= 2); - - ret = fsm_getendids( - fsm, - state_ind, - sizeof endids / sizeof endids[0], - &endids[0], - &nwritten); - - assert(ret == FSM_GETENDIDS_FOUND); - assert(nwritten == num_endids); + ret = fsm_endid_get(fsm, state_ind, count, &ids[0]); + assert(ret == 1); info[ninfo].state = state_ind; - info[ninfo].num_endids = nwritten; - - if (nwritten == 1) { - assert(endids[0] == 1 || endids[0] == 2); - info[ninfo].endids[0] = endids[0]; - } else if (nwritten == 2) { - qsort(&endids[0], nwritten, sizeof endids[0], cmp_endids); - assert(endids[0] == 1 && endids[1] == 2); - info[ninfo].endids[0] = endids[0]; - info[ninfo].endids[1] = endids[1]; + info[ninfo].count = count; + + if (count == 1) { + assert(ids[0] == 1 || ids[0] == 2); + info[ninfo].ids[0] = ids[0]; + } else if (count == 2) { + assert(ids[0] == 1 && ids[1] == 2); + info[ninfo].ids[0] = ids[0]; + info[ninfo].ids[1] = ids[1]; } ninfo++; @@ -154,30 +143,22 @@ int main(void) if (fsm_isend(fsm, state_ind)) { fsm_end_id_t endid = 0; fsm_end_id_t expected; - size_t nwritten, num_endids; - enum fsm_getendids_res ret; + size_t count; + int ret; assert( state_ind == info[info_ind].state ); - nwritten = 0; - num_endids = fsm_getendidcount(fsm, state_ind); - - assert(num_endids == 1); - ret = fsm_getendids( - fsm, - state_ind, - 1, - &endid, - &nwritten); + count = fsm_endid_count(fsm, state_ind); + assert(count == 1); - assert(ret == FSM_GETENDIDS_FOUND); - assert(nwritten == num_endids); + ret = fsm_endid_get( fsm, state_ind, 1, &endid); + assert(ret == 1); - if (info[info_ind].num_endids == 2) { + if (info[info_ind].count == 2) { expected = 3; - } else if (info[info_ind].endids[0] == 1) { + } else if (info[info_ind].ids[0] == 1) { expected = 7; - } else if (info[info_ind].endids[0] == 2) { + } else if (info[info_ind].ids[0] == 2) { expected = 9; } else { assert( ! "unexpected value" ); diff --git a/tests/endids/endids8.c b/tests/endids/endids8.c index e413e71e7..12b36fae0 100644 --- a/tests/endids/endids8.c +++ b/tests/endids/endids8.c @@ -24,7 +24,7 @@ int main(void) const char *s1, *s2; int ret; - fsm_end_id_t all_endids[2]; + fsm_end_id_t all_ids[2]; unsigned nstates, state_ind; s1 = "abc"; fsm1 = re_comp(RE_NATIVE, fsm_sgetc, &s1, NULL, 0, NULL); @@ -69,33 +69,21 @@ int main(void) nstates = fsm_countstates(comb); // all end states should have BOTH endids - memset(all_endids, 0, sizeof all_endids); + memset(all_ids, 0, sizeof all_ids); for (state_ind = 0; state_ind < nstates; state_ind++) { if (fsm_isend(comb, state_ind)) { - fsm_end_id_t endids[3] = {0,0,0}; - size_t nwritten; - size_t num_endids; - enum fsm_getendids_res ret; + fsm_end_id_t ids[3] = {0,0,0}; + size_t count; + int ret; - nwritten = 0; - num_endids = fsm_getendidcount(comb, state_ind); + count = fsm_endid_count(comb, state_ind); + printf("state %u count = %zu\n", state_ind, count); + assert(count == 3); - printf("state %u num_endids = %zu\n", state_ind, num_endids); + ret = fsm_endid_get(comb, state_ind, count, &ids[0]); + assert(ret == 1); - assert(num_endids == 3); - - ret = fsm_getendids( - comb, - state_ind, - sizeof endids / sizeof endids[0], - &endids[0], - &nwritten); - - assert(ret == FSM_GETENDIDS_FOUND); - assert(nwritten == num_endids); - - qsort(&endids[0], num_endids, sizeof endids[0], cmp_endids); - assert(endids[0] == 1 && endids[1] == 2 && endids[2] == 4); + assert(ids[0] == 1 && ids[1] == 2 && ids[2] == 4); } } diff --git a/tests/endids/endids9.c b/tests/endids/endids9.c index de3f28644..519399973 100644 --- a/tests/endids/endids9.c +++ b/tests/endids/endids9.c @@ -19,8 +19,8 @@ struct state_info { fsm_state_t state; - unsigned num_endids; - fsm_end_id_t endids[2]; + unsigned count; + fsm_end_id_t ids[2]; }; /* use fsm_increndids to change the endids before a union/intersection */ @@ -71,37 +71,26 @@ int main(void) for (state_ind = 0; state_ind < nstates; state_ind++) { if (fsm_isend(fsm, state_ind)) { - fsm_end_id_t endids[2] = {0,0}; - size_t nwritten; - size_t num_endids; - enum fsm_getendids_res ret; + fsm_end_id_t ids[2] = {0,0}; + size_t count; + int ret; - nwritten = 0; - num_endids = fsm_getendidcount(fsm, state_ind); + count = fsm_endid_count(fsm, state_ind); + assert( count > 0 && count <= 2); - assert( num_endids > 0 && num_endids <= 2); - - ret = fsm_getendids( - fsm, - state_ind, - sizeof endids / sizeof endids[0], - &endids[0], - &nwritten); - - assert(ret == FSM_GETENDIDS_FOUND); - assert(nwritten == num_endids); + ret = fsm_endid_get(fsm, state_ind, count, &ids[0]); + assert(ret == 1); info[ninfo].state = state_ind; - info[ninfo].num_endids = nwritten; - - if (nwritten == 1) { - assert(endids[0] == 11 || endids[0] == 12); - info[ninfo].endids[0] = endids[0]; - } else if (nwritten == 2) { - qsort(&endids[0], nwritten, sizeof endids[0], cmp_endids); - assert(endids[0] == 11 && endids[1] == 12); - info[ninfo].endids[0] = endids[0]; - info[ninfo].endids[1] = endids[1]; + info[ninfo].count = count; + + if (count == 1) { + assert(ids[0] == 11 || ids[0] == 12); + info[ninfo].ids[0] = ids[0]; + } else if (count == 2) { + assert(ids[0] == 11 && ids[1] == 12); + info[ninfo].ids[0] = ids[0]; + info[ninfo].ids[1] = ids[1]; } ninfo++; diff --git a/tests/endids/endids_reused_buffer.c b/tests/endids/endids_reused_buffer.c new file mode 100644 index 000000000..14750d276 --- /dev/null +++ b/tests/endids/endids_reused_buffer.c @@ -0,0 +1,57 @@ +#include +#include +#include +#include + +#include + +#include + +#include + +#include +#include +#include +#include +#include + +#include "endids_utils.h" + +/* Test that providing an overly large endid buffer doesn't clobber the result with stale data. */ +int main(void) +{ + const char *input = "banana turtle"; + + const char *s = input; + struct fsm *fsm = re_comp(RE_NATIVE, fsm_sgetc, &s, NULL, 0, NULL); + int ret; + + ret = fsm_setendid(fsm, (fsm_end_id_t) 12345); + assert(ret == 1); + + ret = fsm_determinise(fsm); + assert(ret == 1); + + ret = fsm_minimise(fsm); + assert(ret == 1); + + fsm_state_t end; + ret = fsm_exec(fsm, fsm_sgetc, &input, &end); + assert(ret == 1); + + const size_t endid_count = fsm_endid_count(fsm, end); + assert(endid_count == 1); + + /* Regression: fsm_endid_get previously qsort'd endids with + * a count based on the buffer size (here ENDID_BUF_CEIL), + * rather than the number of results populated, so if the + * buffer was larger than fsm_endid_count's result stale + * contents could displace the correct results. */ +#define ENDID_BUF_CEIL 10 + fsm_end_id_t endids[ENDID_BUF_CEIL] = { 333 }; + ret = fsm_endid_get(fsm, end, ENDID_BUF_CEIL, endids); + assert(endids[0] == 12345); + + fsm_free(fsm); + return EXIT_SUCCESS; +} diff --git a/tests/endids/endids_trim_must_compact_endids.c b/tests/endids/endids_trim_must_compact_endids.c new file mode 100644 index 000000000..fe78969de --- /dev/null +++ b/tests/endids/endids_trim_must_compact_endids.c @@ -0,0 +1,64 @@ +#include +#include + +#include + +#include +#include + +/* Regression: When fsm_trim called fsm_compact_states it previously + * didn't remap endids, so the endid collection could refer to stale + * state IDs or past the end of fsm->states[]. */ +int main(void) +{ + size_t limit = 100; + const fsm_end_id_t end_id = 12345; + + /* Create FSMs with a bunch of garbage states, to increase + * the likelihood that the endid's state not being updated + * after calling fsm_trim triggers the bug. */ + for (size_t end_state = 1; end_state < limit; end_state++) { + struct fsm *fsm = fsm_new(NULL); + assert(fsm != NULL); + + for (size_t i = 0; i < limit; i++) { + if (!fsm_addstate(fsm, NULL)) { + return EXIT_FAILURE; + } + } + + const fsm_state_t start_state = end_state - 1; + fsm_setstart(fsm, start_state); + fsm_setend(fsm, end_state, 1); + + /* add an any edge, so the end state is reachable */ + if (!fsm_addedge_any(fsm, start_state, end_state)) { + return EXIT_FAILURE; + } + + if (!fsm_setendid(fsm, end_id)) { + return EXIT_FAILURE; + } + + /* Call fsm_trim, as fsm_minimise would. */ + (void)fsm_trim(fsm, FSM_TRIM_START_AND_END_REACHABLE, NULL); + + /* Execute the fsm and check that the endid's state was updated. */ + fsm_state_t end; + const char *input = "x"; + int ret = fsm_exec(fsm, fsm_sgetc, &input, &end); + assert(ret == 1); + + size_t count = fsm_endid_count(fsm, end); + assert(count == 1); + + fsm_end_id_t id_buf[1]; + if (fsm_endid_get(fsm, end, 1, id_buf) != FSM_GETENDIDS_FOUND) { + return EXIT_FAILURE; + } + assert(id_buf[0] == end_id); + + fsm_free(fsm); + } + return EXIT_SUCCESS; +} diff --git a/tests/endids/endids_utils.h b/tests/endids/endids_utils.h index f1aaefb5d..e2aad628a 100644 --- a/tests/endids/endids_utils.h +++ b/tests/endids/endids_utils.h @@ -7,8 +7,5 @@ int match_string(const struct fsm *fsm, const char *s, fsm_state_t *end_ptr, fsm_end_id_t **endids_ptr, size_t *num_endids_ptr); -int -cmp_endids(const void *pa, const void *pb); - #endif /* ENDIDS_UTILS_H */ diff --git a/tests/endids/utils.c b/tests/endids/utils.c index 79777e825..e1fca5f0c 100644 --- a/tests/endids/utils.c +++ b/tests/endids/utils.c @@ -4,43 +4,42 @@ #include int -match_string(const struct fsm *fsm, const char *s, fsm_state_t *end_ptr, fsm_end_id_t **endids_ptr, size_t *num_endids_ptr) +match_string(const struct fsm *fsm, const char *s, fsm_state_t *end_ptr, fsm_end_id_t **ids_ptr, size_t *count_ptr) { fsm_state_t end = 0; int ret; - ret = fsm_exec(fsm, fsm_sgetc, &s, &end, NULL); + ret = fsm_exec(fsm, fsm_sgetc, &s, &end); if (ret == 1) { - size_t num_endids; + size_t count; if (end_ptr != NULL) { *end_ptr = end; } - num_endids = fsm_getendidcount(fsm, end); - if (num_endids > 0) { - enum fsm_getendids_res ret; - size_t nwritten = 0; + count = fsm_endid_count(fsm, end); + if (count > 0) { + int ret; - fsm_end_id_t *endids = calloc(num_endids, sizeof *endids); - if (endids == NULL) { + fsm_end_id_t *ids = calloc(count, sizeof *ids); + if (ids == NULL) { return -1; } - ret = fsm_getendids(fsm, end, num_endids, endids, &nwritten); - if (ret != FSM_GETENDIDS_FOUND) { - free(endids); - errno = (ret == FSM_GETENDIDS_NOT_FOUND) ? EINVAL : ENOMEM; + ret = fsm_endid_get(fsm, end, count, ids); + if (ret == 0) { + free(ids); + errno = EINVAL; return -1; } - if (endids_ptr != NULL) { - *endids_ptr = endids; - if (num_endids_ptr != NULL) { - *num_endids_ptr = num_endids; + if (ids_ptr != NULL) { + *ids_ptr = ids; + if (count_ptr != NULL) { + *count_ptr = count; } } else { - free(endids); + free(ids); } } } @@ -48,14 +47,3 @@ match_string(const struct fsm *fsm, const char *s, fsm_state_t *end_ptr, fsm_end return ret; } -int -cmp_endids(const void *pa, const void *pb) { - const fsm_end_id_t *a = pa; - const fsm_end_id_t *b = pb; - - if (*a < *b) { return -1; } - if (*a > *b) { return 1; } - return 0; -} - - diff --git a/tests/fsm/Makefile b/tests/fsm/Makefile new file mode 100755 index 000000000..6ac0a2646 --- /dev/null +++ b/tests/fsm/Makefile @@ -0,0 +1,22 @@ +.include "../../share/mk/top.mk" + +TEST.tests/fsm != ls -1 tests/fsm/out*.fsm +TEST_SRCDIR.tests/fsm = tests/fsm +TEST_OUTDIR.tests/fsm = ${BUILD}/tests/fsm + +FSM=${BUILD}/bin/fsm + +.for n in ${TEST.tests/fsm:T:R:C/^out//} + +${TEST_OUTDIR.tests/fsm}/got${n}.fsm: ${TEST_SRCDIR.tests/fsm}/in${n}.fsm + ${FSM} -p ${.ALLSRC:M*.fsm} \ + > $@ + +${TEST_OUTDIR.tests/fsm}/res${n}: \ + ${TEST_SRCDIR.tests/fsm}/out${n}.fsm \ + ${TEST_OUTDIR.tests/fsm}/got${n}.fsm + +FSMTEST_RESULT += ${TEST_OUTDIR.tests/fsm}/res${n} + +.endfor + diff --git a/tests/fsm/in0.fsm b/tests/fsm/in0.fsm new file mode 100644 index 000000000..e69de29bb diff --git a/tests/fsm/in1.fsm b/tests/fsm/in1.fsm new file mode 100644 index 000000000..88d443fec --- /dev/null +++ b/tests/fsm/in1.fsm @@ -0,0 +1 @@ +# just a comment diff --git a/tests/fsm/in10.fsm b/tests/fsm/in10.fsm new file mode 100644 index 000000000..8b044d3bb --- /dev/null +++ b/tests/fsm/in10.fsm @@ -0,0 +1,10 @@ +# +# Copyright 2008-2017 Katherine Flavel +# +# See LICENCE for the full copyright terms. +# + +# missing seperator +start: a +end: b + diff --git a/tests/fsm/in11.fsm b/tests/fsm/in11.fsm new file mode 100644 index 000000000..bc058b5e2 --- /dev/null +++ b/tests/fsm/in11.fsm @@ -0,0 +1,9 @@ +# +# Copyright 2008-2024 Katherine Flavel +# +# See LICENCE for the full copyright terms. +# + +# missing comma +end: 1 2 3; + diff --git a/tests/fsm/in12.fsm b/tests/fsm/in12.fsm new file mode 100644 index 000000000..973f98c86 --- /dev/null +++ b/tests/fsm/in12.fsm @@ -0,0 +1,9 @@ +# +# Copyright 2008-2024 Katherine Flavel +# +# See LICENCE for the full copyright terms. +# + +start: a; +start: b; + diff --git a/tests/fsm/in13.fsm b/tests/fsm/in13.fsm new file mode 100644 index 000000000..0ce5b760a --- /dev/null +++ b/tests/fsm/in13.fsm @@ -0,0 +1,9 @@ +# +# Copyright 2008-2017 Katherine Flavel +# +# See LICENCE for the full copyright terms. +# + +end: a; +end: a; + diff --git a/tests/fsm/in14.fsm b/tests/fsm/in14.fsm new file mode 100644 index 000000000..82bd7b25c --- /dev/null +++ b/tests/fsm/in14.fsm @@ -0,0 +1,12 @@ +# +# Copyright 2008-2024 Katherine Flavel +# +# See LICENCE for the full copyright terms. +# + +# missing semicolon +0 1; + +start: 0; +end: 1; + diff --git a/tests/fsm/in15.fsm b/tests/fsm/in15.fsm new file mode 100644 index 000000000..f0622bf0f --- /dev/null +++ b/tests/fsm/in15.fsm @@ -0,0 +1,11 @@ +# +# Copyright 2008-2024 Katherine Flavel +# +# See LICENCE for the full copyright terms. +# + +start: a; + +# missing ] +end: a = [ ; + diff --git a/tests/fsm/in16.fsm b/tests/fsm/in16.fsm new file mode 100644 index 000000000..0e7fcec60 --- /dev/null +++ b/tests/fsm/in16.fsm @@ -0,0 +1,11 @@ +# +# Copyright 2008-2024 Katherine Flavel +# +# See LICENCE for the full copyright terms. +# + +start: a; + +# an empty list is okay +end: a = [ ]; + diff --git a/tests/fsm/in17.fsm b/tests/fsm/in17.fsm new file mode 100644 index 000000000..42df382d5 --- /dev/null +++ b/tests/fsm/in17.fsm @@ -0,0 +1,11 @@ +# +# Copyright 2008-2024 Katherine Flavel +# +# See LICENCE for the full copyright terms. +# + +start: a; + +# end-id must be numeric +end: a = [ abc ]; + diff --git a/tests/fsm/in18.fsm b/tests/fsm/in18.fsm new file mode 100644 index 000000000..2cccfc2e5 --- /dev/null +++ b/tests/fsm/in18.fsm @@ -0,0 +1,11 @@ +# +# Copyright 2008-2024 Katherine Flavel +# +# See LICENCE for the full copyright terms. +# + +start: a; + +# missing end-id +end: a = [ , ]; + diff --git a/tests/fsm/in19.fsm b/tests/fsm/in19.fsm new file mode 100644 index 000000000..f0875ad1f --- /dev/null +++ b/tests/fsm/in19.fsm @@ -0,0 +1,11 @@ +# +# Copyright 2008-2024 Katherine Flavel +# +# See LICENCE for the full copyright terms. +# + +start: a; + +# out of order is okay +end: a = [ 999 ]; + diff --git a/tests/fsm/in2.fsm b/tests/fsm/in2.fsm new file mode 100644 index 000000000..345ab6cc7 --- /dev/null +++ b/tests/fsm/in2.fsm @@ -0,0 +1,13 @@ +# +# Copyright 2008-2024 Katherine Flavel +# +# See LICENCE for the full copyright terms. +# + +# states are named by identifiers +abc; +0; +_x; +_; +abc123; + diff --git a/tests/fsm/in20.fsm b/tests/fsm/in20.fsm new file mode 100644 index 000000000..d025dc47e --- /dev/null +++ b/tests/fsm/in20.fsm @@ -0,0 +1,11 @@ +# +# Copyright 2008-2024 Katherine Flavel +# +# See LICENCE for the full copyright terms. +# + +start: a; + +# out of order is okay +end: a = [ 7, 53, 999 ]; + diff --git a/tests/fsm/in21.fsm b/tests/fsm/in21.fsm new file mode 100644 index 000000000..f53082d14 --- /dev/null +++ b/tests/fsm/in21.fsm @@ -0,0 +1,11 @@ +# +# Copyright 2008-2024 Katherine Flavel +# +# See LICENCE for the full copyright terms. +# + +start: a; + +# positive integers only +end: a = [ -1 ]; + diff --git a/tests/fsm/in22.fsm b/tests/fsm/in22.fsm new file mode 100644 index 000000000..c376a9d28 --- /dev/null +++ b/tests/fsm/in22.fsm @@ -0,0 +1,11 @@ +# +# Copyright 2008-2024 Katherine Flavel +# +# See LICENCE for the full copyright terms. +# + +a -> b 'a'; + +start: a; +end: b = [ 7 ]; + diff --git a/tests/fsm/in3.fsm b/tests/fsm/in3.fsm new file mode 100644 index 000000000..b3f86c37b --- /dev/null +++ b/tests/fsm/in3.fsm @@ -0,0 +1,9 @@ +# +# Copyright 2008-2024 Katherine Flavel +# +# See LICENCE for the full copyright terms. +# + +# not a valid identifier ++; + diff --git a/tests/fsm/in4.fsm b/tests/fsm/in4.fsm new file mode 100644 index 000000000..e1edb77b8 --- /dev/null +++ b/tests/fsm/in4.fsm @@ -0,0 +1,9 @@ +# +# Copyright 2008-2024 Katherine Flavel +# +# See LICENCE for the full copyright terms. +# + +# syntax error +abc -> + diff --git a/tests/fsm/in5.fsm b/tests/fsm/in5.fsm new file mode 100644 index 000000000..8550f0566 --- /dev/null +++ b/tests/fsm/in5.fsm @@ -0,0 +1,9 @@ +# +# Copyright 2008-2024 Katherine Flavel +# +# See LICENCE for the full copyright terms. +# + +# syntax error +-> def; + diff --git a/tests/fsm/in6.fsm b/tests/fsm/in6.fsm new file mode 100644 index 000000000..9f6c3c3e8 --- /dev/null +++ b/tests/fsm/in6.fsm @@ -0,0 +1,8 @@ +# +# Copyright 2008-2024 Katherine Flavel +# +# See LICENCE for the full copyright terms. +# + +x -> y; + diff --git a/tests/fsm/in7.fsm b/tests/fsm/in7.fsm new file mode 100644 index 000000000..09d508a27 --- /dev/null +++ b/tests/fsm/in7.fsm @@ -0,0 +1,9 @@ +# +# Copyright 2008-2024 Katherine Flavel +# +# See LICENCE for the full copyright terms. +# + +# implicit state +start: x; + diff --git a/tests/fsm/in8.fsm b/tests/fsm/in8.fsm new file mode 100644 index 000000000..7760b6e3c --- /dev/null +++ b/tests/fsm/in8.fsm @@ -0,0 +1,12 @@ +# +# Copyright 2008-2024 Katherine Flavel +# +# See LICENCE for the full copyright terms. +# + +# explicitly created states +x; +y; + +start: x; + diff --git a/tests/fsm/in9.fsm b/tests/fsm/in9.fsm new file mode 100644 index 000000000..e7cbbcc0d --- /dev/null +++ b/tests/fsm/in9.fsm @@ -0,0 +1,9 @@ +# +# Copyright 2008-2024 Katherine Flavel +# +# See LICENCE for the full copyright terms. +# + +# missing separator +a b c + diff --git a/tests/fsm/out0.fsm b/tests/fsm/out0.fsm new file mode 100644 index 000000000..e69de29bb diff --git a/tests/fsm/out1.fsm b/tests/fsm/out1.fsm new file mode 100644 index 000000000..e69de29bb diff --git a/tests/fsm/out10.err b/tests/fsm/out10.err new file mode 100644 index 000000000..412679b24 --- /dev/null +++ b/tests/fsm/out10.err @@ -0,0 +1 @@ +9:1: Syntax error: expected ';' diff --git a/tests/fsm/out11.err b/tests/fsm/out11.err new file mode 100644 index 000000000..d5b538786 --- /dev/null +++ b/tests/fsm/out11.err @@ -0,0 +1 @@ +8:8: Syntax error: expected ',' diff --git a/tests/fsm/out12.err b/tests/fsm/out12.err new file mode 100644 index 000000000..ebbbb9d1b --- /dev/null +++ b/tests/fsm/out12.err @@ -0,0 +1 @@ +8:1: Syntax error diff --git a/tests/fsm/out13.err b/tests/fsm/out13.err new file mode 100644 index 000000000..ebbbb9d1b --- /dev/null +++ b/tests/fsm/out13.err @@ -0,0 +1 @@ +8:1: Syntax error diff --git a/tests/fsm/out14.err b/tests/fsm/out14.err new file mode 100644 index 000000000..f7e4f861b --- /dev/null +++ b/tests/fsm/out14.err @@ -0,0 +1 @@ +8:3: Syntax error diff --git a/tests/fsm/out15.err b/tests/fsm/out15.err new file mode 100644 index 000000000..fbca0e9a6 --- /dev/null +++ b/tests/fsm/out15.err @@ -0,0 +1 @@ +10:12: Syntax error diff --git a/tests/fsm/out16.fsm b/tests/fsm/out16.fsm new file mode 100644 index 000000000..49f89e1ca --- /dev/null +++ b/tests/fsm/out16.fsm @@ -0,0 +1,8 @@ +# +# Copyright 2008-2024 Katherine Flavel +# +# See LICENCE for the full copyright terms. +# + +start: a; +end: a; diff --git a/tests/fsm/out17.err b/tests/fsm/out17.err new file mode 100644 index 000000000..fbca0e9a6 --- /dev/null +++ b/tests/fsm/out17.err @@ -0,0 +1 @@ +10:12: Syntax error diff --git a/tests/fsm/out18.err b/tests/fsm/out18.err new file mode 100644 index 000000000..fbca0e9a6 --- /dev/null +++ b/tests/fsm/out18.err @@ -0,0 +1 @@ +10:12: Syntax error diff --git a/tests/fsm/out19.fsm b/tests/fsm/out19.fsm new file mode 100644 index 000000000..11c504879 --- /dev/null +++ b/tests/fsm/out19.fsm @@ -0,0 +1,8 @@ +# +# Copyright 2008-2024 Katherine Flavel +# +# See LICENCE for the full copyright terms. +# + +start: a; +end: a = [ 0 ]; diff --git a/tests/fsm/out2.fsm b/tests/fsm/out2.fsm new file mode 100644 index 000000000..5e711912e --- /dev/null +++ b/tests/fsm/out2.fsm @@ -0,0 +1,7 @@ +# +# Copyright 2008-2024 Katherine Flavel +# +# See LICENCE for the full copyright terms. +# + +0; 1; 2; 3; 4; diff --git a/tests/fsm/out20.fsm b/tests/fsm/out20.fsm new file mode 100644 index 000000000..71556c552 --- /dev/null +++ b/tests/fsm/out20.fsm @@ -0,0 +1,8 @@ +# +# Copyright 2008-2024 Katherine Flavel +# +# See LICENCE for the full copyright terms. +# + +start: a; +end: a = [ 0, 1, 2 ]; diff --git a/tests/fsm/out21.err b/tests/fsm/out21.err new file mode 100644 index 000000000..fbca0e9a6 --- /dev/null +++ b/tests/fsm/out21.err @@ -0,0 +1 @@ +10:12: Syntax error diff --git a/tests/fsm/out22.fsm b/tests/fsm/out22.fsm new file mode 100644 index 000000000..50f1bb868 --- /dev/null +++ b/tests/fsm/out22.fsm @@ -0,0 +1,9 @@ +# +# Copyright 2008-2024 Katherine Flavel +# +# See LICENCE for the full copyright terms. +# + +0 -> 1 'a'; +start: 0; +end: 1 = [ 0 ]; diff --git a/tests/fsm/out3.err b/tests/fsm/out3.err new file mode 100644 index 000000000..ebbbb9d1b --- /dev/null +++ b/tests/fsm/out3.err @@ -0,0 +1 @@ +8:1: Syntax error diff --git a/tests/fsm/out4.err b/tests/fsm/out4.err new file mode 100644 index 000000000..31fa45171 --- /dev/null +++ b/tests/fsm/out4.err @@ -0,0 +1 @@ +8:7: Syntax error diff --git a/tests/fsm/out5.err b/tests/fsm/out5.err new file mode 100644 index 000000000..ebbbb9d1b --- /dev/null +++ b/tests/fsm/out5.err @@ -0,0 +1 @@ +8:1: Syntax error diff --git a/tests/fsm/out6.fsm b/tests/fsm/out6.fsm new file mode 100644 index 000000000..b673566f0 --- /dev/null +++ b/tests/fsm/out6.fsm @@ -0,0 +1,7 @@ +# +# Copyright 2008-2024 Katherine Flavel +# +# See LICENCE for the full copyright terms. +# + +0 -> 1; diff --git a/tests/fsm/out7.fsm b/tests/fsm/out7.fsm new file mode 100644 index 000000000..3d4653a70 --- /dev/null +++ b/tests/fsm/out7.fsm @@ -0,0 +1,7 @@ +# +# Copyright 2008-2024 Katherine Flavel +# +# See LICENCE for the full copyright terms. +# + +start: 0; diff --git a/tests/fsm/out8.fsm b/tests/fsm/out8.fsm new file mode 100644 index 000000000..86110f4ba --- /dev/null +++ b/tests/fsm/out8.fsm @@ -0,0 +1,10 @@ +# +# Copyright 2008-2024 Katherine Flavel +# +# See LICENCE for the full copyright terms. +# + +0; +1; + +start: 0; diff --git a/tests/fsm/out9.err b/tests/fsm/out9.err new file mode 100644 index 000000000..f7e4f861b --- /dev/null +++ b/tests/fsm/out9.err @@ -0,0 +1 @@ +8:3: Syntax error diff --git a/tests/gen/gen1.c b/tests/gen/gen1.c index 25b30b82b..b25ab2bbc 100644 --- a/tests/gen/gen1.c +++ b/tests/gen/gen1.c @@ -34,6 +34,7 @@ int main(void) { assert(fsm != NULL); if (!fsm_generate_matches(fsm, MAX_EXP_MATCH + 1 /* for \0 */, + 0, gtest_matches_cb, &matches)) { fprintf(stderr, "fsm_generate_matches: error\n"); exit(EXIT_FAILURE); diff --git a/tests/gen/gen2.c b/tests/gen/gen2.c index 02faa5e50..a475e5395 100644 --- a/tests/gen/gen2.c +++ b/tests/gen/gen2.c @@ -28,7 +28,7 @@ int main(void) { struct fsm *fsm = gtest_fsm_of_matches(&matches); assert(fsm != NULL); - if (!fsm_generate_matches(fsm, MAX_EXP_MATCH + 1, gtest_matches_cb, &matches)) { + if (!fsm_generate_matches(fsm, MAX_EXP_MATCH + 1, 0, gtest_matches_cb, &matches)) { fprintf(stderr, "fsm_generate_matches: error\n"); exit(EXIT_FAILURE); } diff --git a/tests/gen/gen3.c b/tests/gen/gen3.c index 33cbcba66..7aa0aebce 100644 --- a/tests/gen/gen3.c +++ b/tests/gen/gen3.c @@ -11,6 +11,7 @@ #include #include +#include #include #include "gtest.h" @@ -61,7 +62,7 @@ build(void) } if (getenv("PRINT")) { - fsm_print_fsm(stderr, fsm); + fsm_dump(stderr, fsm); } return fsm; @@ -145,7 +146,7 @@ int main(void) { struct fsm *fsm = build(); assert(fsm != NULL); - if (!fsm_generate_matches(fsm, 11, matches_cb, NULL)) { + if (!fsm_generate_matches(fsm, 11, 0, matches_cb, NULL)) { fprintf(stderr, "fsm_generate_matches: error\n"); exit(EXIT_FAILURE); } diff --git a/tests/gen/gtest.c b/tests/gen/gtest.c index 140407943..55babe873 100644 --- a/tests/gen/gtest.c +++ b/tests/gen/gtest.c @@ -9,7 +9,6 @@ #include #include -#include #include /* Note: not "gentest" because then this matches the gen* pattern and @@ -58,21 +57,20 @@ gtest_matches_cb(const struct fsm *fsm, m->found = true; #define ID_BUF_COUNT 1 - fsm_end_id_t id_buf[ID_BUF_COUNT]; - size_t written; - enum fsm_getendids_res gres = fsm_getendids(fsm, - end_state, ID_BUF_COUNT, id_buf, &written); + fsm_end_id_t ids[ID_BUF_COUNT]; + enum fsm_getendids_res gres = fsm_endid_get(fsm, + end_state, fsm_endid_count(fsm, end_state), ids); if (gres != FSM_GETENDIDS_FOUND) { fprintf(stderr, - "ERROR: fsm_getendids: returned %d\n", + "ERROR: fsm_endid_get: returned %d\n", gres); return FSM_GENERATE_MATCHES_CB_RES_HALT; } - if (written != 1 || id_buf[0] != m_i) { + if (fsm_endid_count(fsm, end_state) != 1 || ids[0] != m_i) { fprintf(stderr, "ERROR: endid mismatch, expected %zu, got %u\n", - m_i, id_buf[0]); + m_i, ids[0]); return FSM_GENERATE_MATCHES_CB_RES_HALT; } @@ -88,8 +86,6 @@ gtest_matches_cb(const struct fsm *fsm, return FSM_GENERATE_MATCHES_CB_RES_HALT; } -static const struct fsm_options options; /* use defaults */ - struct fsm * gtest_fsm_of_matches(const struct exp_matches *ms) { @@ -97,7 +93,7 @@ gtest_fsm_of_matches(const struct exp_matches *ms) for (size_t m_i = 0; m_i < ms->count; m_i++) { const struct exp_match *m = &ms->matches[m_i]; - struct fsm *fsm = fsm_new(&options); + struct fsm *fsm = fsm_new(NULL); assert(fsm != NULL); if (!fsm_addstate_bulk(fsm, m->length + 1)) { diff --git a/tests/idmap/Makefile b/tests/idmap/Makefile new file mode 100644 index 000000000..aee01f565 --- /dev/null +++ b/tests/idmap/Makefile @@ -0,0 +1,19 @@ +.include "../../share/mk/top.mk" + +TEST.tests/idmap != ls -1 tests/idmap/idmap*.c +TEST_SRCDIR.tests/idmap = tests/idmap +TEST_OUTDIR.tests/idmap = ${BUILD}/tests/idmap + +.for n in ${TEST.tests/idmap:T:R:C/^idmap//} +INCDIR.${TEST_SRCDIR.tests/idmap}/idmap${n}.c += src/adt +.endfor + +.for n in ${TEST.tests/idmap:T:R:C/^idmap//} +test:: ${TEST_OUTDIR.tests/idmap}/res${n} +SRC += ${TEST_SRCDIR.tests/idmap}/idmap${n}.c +CFLAGS.${TEST_SRCDIR.tests/idmap}/idmap${n}.c += -UNDEBUG -D_DEFAULT_SOURCE -std=c99 +${TEST_OUTDIR.tests/idmap}/run${n}: ${TEST_OUTDIR.tests/idmap}/idmap${n}.o ${BUILD}/lib/adt.o + ${CC} ${CFLAGS} ${CFLAGS.${TEST_SRCDIR.tests/idmap}/idmap${n}.c} -o ${TEST_OUTDIR.tests/idmap}/run${n} ${TEST_OUTDIR.tests/idmap}/idmap${n}.o ${BUILD}/lib/adt.o +${TEST_OUTDIR.tests/idmap}/res${n}: ${TEST_OUTDIR.tests/idmap}/run${n} + ( ${TEST_OUTDIR.tests/idmap}/run${n} 1>&2 && echo PASS || echo FAIL ) > ${TEST_OUTDIR.tests/idmap}/res${n} +.endfor diff --git a/tests/idmap/idmap_basic.c b/tests/idmap/idmap_basic.c new file mode 100644 index 000000000..c7a18856b --- /dev/null +++ b/tests/idmap/idmap_basic.c @@ -0,0 +1,137 @@ +/* + * Copyright 2021 Scott Vokes + * + * See LICENCE for the full copyright terms. + */ + +#include +#include +#include + +#include + +#define DEF_LIMIT 10 +#define DEF_SEED 0 + +/* Thes numbers were chose to get a reasonable variety, + * but also some duplicated values as the input grows. */ +#define MAX_GEN_VALUES 23 +#define ID_MASK ((1 << 9) - 1) +#define VALUE_MASK ((1 << 10) - 1) + +static int +dump_cb(fsm_state_t state_id, unsigned value, void *opaque) +{ + /* fprintf(stderr, " -- state %d, value %u\n", state_id, value); */ + assert(state_id <= ID_MASK); + assert(value <= VALUE_MASK); + (void)opaque; + return 1; +} + +static int +cmp_u(const void *pa, const void *pb) +{ + const unsigned a = *(unsigned *)pa; + const unsigned b = *(unsigned *)pb; + return a < b ? -1 : a > b ? 1 : 0; +} + +int main(int argc, char **argv) { + const size_t limit = (argc > 1 ? atoi(argv[1]) : DEF_LIMIT); + const unsigned seed = (argc > 2 ? atoi(argv[2]) : DEF_SEED); + + (void)argc; + (void)argv; + struct idmap *m = idmap_new(NULL); + + srandom(seed); + + /* Fill the table with random data */ + for (size_t id_i = 0; id_i < limit; id_i++) { + const fsm_state_t id = (fsm_state_t)(random() & ID_MASK); + const size_t value_count = random() % MAX_GEN_VALUES; + + for (size_t v_i = 0; v_i < value_count; v_i++) { + const unsigned v = random() & VALUE_MASK; + if (!idmap_set(m, id, v)) { + assert(!"failed to set"); + } + } + } + + idmap_iter(m, dump_cb, NULL); + + srandom(seed); + + size_t got_buf_ceil = MAX_GEN_VALUES; + unsigned *got_buf = malloc(got_buf_ceil * sizeof(got_buf[0])); + assert(got_buf != NULL); + + /* Reset the PRNG and read back the same data. */ + for (size_t id_i = 0; id_i < limit; id_i++) { + const fsm_state_t id = (fsm_state_t)(random() & ID_MASK); + const size_t generated_value_count = random() % MAX_GEN_VALUES; + + /* Note: This can occasionally differ from + * generated_value_count, because the same id or values + * may have been generated more than once. As long as + * all the values match, it's fine. */ + const size_t value_count = idmap_get_value_count(m, id); + + if (value_count > got_buf_ceil) { + size_t nceil = got_buf_ceil; + while (nceil <= value_count) { + nceil *= 2; + } + free(got_buf); + got_buf = malloc(nceil * sizeof(got_buf[0])); + assert(got_buf != NULL); + got_buf_ceil = nceil; + } + + size_t written; + if (!idmap_get(m, id, + got_buf_ceil * sizeof(got_buf[0]), got_buf, + &written)) { + assert(!"failed to get"); + } + assert(written == value_count); + + unsigned gen_buf[MAX_GEN_VALUES]; + + for (size_t v_i = 0; v_i < generated_value_count; v_i++) { + const unsigned v = random() & VALUE_MASK; + gen_buf[v_i] = v; + } + qsort(gen_buf, generated_value_count, sizeof(gen_buf[0]), cmp_u); + + /* Every generated value should appear in the buffer. + * There may be more in the buffer; ignore them. */ + size_t v_i = 0; + for (size_t gen_i = 0; gen_i < generated_value_count; gen_i++) { + int found = 0; + const unsigned gv = gen_buf[gen_i]; + assert(value_count <= got_buf_ceil); + /* got_buf should be sorted, so we can pick up where we left off */ + while (v_i < value_count) { + if (gv == got_buf[v_i]) { + /* Intentionally don't increment v_i on match, + * because gen_buf can repeat values. */ + found = 1; + break; + } + v_i++; + } + if (!found) { + fprintf(stderr, "NOT FOUND: state %d -- value: %u\n", + id, gv); + return EXIT_FAILURE; + } + } + } + + free(got_buf); + idmap_free(m); + return EXIT_SUCCESS; +} diff --git a/tests/ir/Makefile b/tests/ir/Makefile index 0009c45ec..566d1add8 100755 --- a/tests/ir/Makefile +++ b/tests/ir/Makefile @@ -9,7 +9,7 @@ RE=${BUILD}/bin/re .for n in ${TEST.tests/ir:T:Mout*.json:R:C/^out//} ${TEST_OUTDIR.tests/ir}/got${n}.json: ${TEST_SRCDIR.tests/ir}/in${n}.re - ${RE} -pl irjson -y ${.ALLSRC:M*.re} \ + ${RE} -FC -pl irjson -y ${.ALLSRC:M*.re} \ > $@ ${TEST_OUTDIR.tests/ir}/res${n}: \ diff --git a/tests/lxpos/Makefile b/tests/lxpos/Makefile index da2263d24..abf11f58a 100644 --- a/tests/lxpos/Makefile +++ b/tests/lxpos/Makefile @@ -4,7 +4,7 @@ TEST.tests/lxpos != ls -1 tests/lxpos/out*.dump TEST_SRCDIR.tests/lxpos = tests/lxpos TEST_OUTDIR.tests/lxpos = ${BUILD}/tests/lxpos -LX?=${BUILD}/bin/lx +LX_BIN?=${BUILD}/bin/lx # for lx -l test LEXER += tests/lxpos/lexer.lx @@ -30,14 +30,14 @@ CFLAGS.${BUILD}/tests/lxpos/${buf}-${getc}-${io}-dump += -I ${BUILD}/tests/lxpos .for ext in c h -${BUILD}/tests/lxpos/${buf}-${getc}-${io}-lexer.${ext}: tests/lxpos/lexer.lx - ${LX} -l ${ext} ${LX_CFLAGS} ${LX_CFLAGS.tests/lxpos/${buf}-${getc}-${io}-lexer.lx} < ${.ALLSRC:M*.lx} > $@ \ +${BUILD}/tests/lxpos/${buf}-${getc}-${io}-lexer.${ext}: tests/lxpos/lexer.lx ${LX_BIN} + ${LX_BIN} -l ${ext} ${LX_CFLAGS} ${LX_CFLAGS.tests/lxpos/${buf}-${getc}-${io}-lexer.lx} < ${.ALLSRC:M*.lx} > $@ \ || { rm -f $@; false; } .endfor -${BUILD}/tests/lxpos/${buf}-${getc}-${io}-dump.c: tests/lxpos/lexer.lx - ${LX} -l dump ${LX_CFLAGS} ${LX_CFLAGS.tests/lxpos/${buf}-${getc}-${io}-lexer.lx} < ${.ALLSRC:M*.lx} > $@ \ +${BUILD}/tests/lxpos/${buf}-${getc}-${io}-dump.c: tests/lxpos/lexer.lx ${LX_BIN} + ${LX_BIN} -l dump ${LX_CFLAGS} ${LX_CFLAGS.tests/lxpos/${buf}-${getc}-${io}-lexer.lx} < ${.ALLSRC:M*.lx} > $@ \ || { rm -f $@; false; } ${BUILD}/tests/lxpos/${buf}-${getc}-${io}-dump: ${BUILD}/tests/lxpos/${buf}-${getc}-${io}-lexer.h @@ -53,11 +53,11 @@ CLEAN += ${BUILD}/tests/lxpos/${buf}-${getc}-${io}-dump ${TEST_OUTDIR.tests/lxpos}/${buf}-${getc}-${io}-got${n}.dump: ${BUILD}/tests/lxpos/${buf}-${getc}-${io}-dump ${TEST_SRCDIR.tests/lxpos}/in${n}.txt .if ${getc} != none - cat ${.ALLSRC:M*.txt} getcio=${io} \ + cat ${.ALLSRC:M*.txt} \ | ${BUILD}/tests/lxpos/${buf}-${getc}-${io}-dump \ > $@ .else - cat ${.ALLSRC:M*.txt} io=${io} \ + cat ${.ALLSRC:M*.txt} \ | xargs -0 ${BUILD}/tests/lxpos/${buf}-${getc}-${io}-dump \ > $@ .endif diff --git a/tests/lxpos/in10.txt b/tests/lxpos/in10.txt new file mode 100644 index 000000000..52cab2a4f --- /dev/null +++ b/tests/lxpos/in10.txt @@ -0,0 +1 @@ +d: r * t /*distance = rate * time* diff --git a/tests/lxpos/in8.txt b/tests/lxpos/in8.txt new file mode 100644 index 000000000..e5c01f9dd --- /dev/null +++ b/tests/lxpos/in8.txt @@ -0,0 +1 @@ +hello ` diff --git a/tests/lxpos/in9.txt b/tests/lxpos/in9.txt new file mode 100644 index 000000000..81d7792df --- /dev/null +++ b/tests/lxpos/in9.txt @@ -0,0 +1 @@ +d: r * t /*distance = rate * time diff --git a/tests/lxpos/out0.dump b/tests/lxpos/out0.dump index 347bb87dc..8f1a4ac1d 100644 --- a/tests/lxpos/out0.dump +++ b/tests/lxpos/out0.dump @@ -2,4 +2,4 @@ 6-12:1,7-13: 12-13:1-2,13-1: -12-13:1-2,13-1: +13:2,1: diff --git a/tests/lxpos/out1.dump b/tests/lxpos/out1.dump index bf4d9bcb9..f3432e1e3 100644 --- a/tests/lxpos/out1.dump +++ b/tests/lxpos/out1.dump @@ -6,4 +6,4 @@ 19-25:2,7-13: 25-26:2-3,13-1: -25-26:2-3,13-1: +26:3,1: diff --git a/tests/lxpos/out10.dump b/tests/lxpos/out10.dump new file mode 100644 index 000000000..4314c8beb --- /dev/null +++ b/tests/lxpos/out10.dump @@ -0,0 +1,7 @@ +0-1:1,1-2: +1-2:1,2-3: +3-4:1,4-5: +5-6:1,6-7: +7-8:1,8-9: +34-35:1-2,35-1: lexically uncategorised: ' +' diff --git a/tests/lxpos/out2.dump b/tests/lxpos/out2.dump index cf5aa7013..bdd9e90a6 100644 --- a/tests/lxpos/out2.dump +++ b/tests/lxpos/out2.dump @@ -9,4 +9,4 @@ 12-13:1,13-14: 13-14:1-2,14-1: -13-14:1-2,14-1: +14:2,1: diff --git a/tests/lxpos/out3.dump b/tests/lxpos/out3.dump index 92d9e915a..e7f0f1e70 100644 --- a/tests/lxpos/out3.dump +++ b/tests/lxpos/out3.dump @@ -5,4 +5,4 @@ 7-8:1,8-9: 35-36:1-2,36-1: -35-36:1-2,36-1: +36:2,1: diff --git a/tests/lxpos/out4.dump b/tests/lxpos/out4.dump index 6161924b0..c3bb9bb2d 100644 --- a/tests/lxpos/out4.dump +++ b/tests/lxpos/out4.dump @@ -8,4 +8,4 @@ 27-28:1-2,28-1: 28-29:2,1-2: -28-29:2,1-2: +29:2,2: diff --git a/tests/lxpos/out5.dump b/tests/lxpos/out5.dump index e2506a1df..040b6734a 100644 --- a/tests/lxpos/out5.dump +++ b/tests/lxpos/out5.dump @@ -7,4 +7,4 @@ 4-5:3,1-2: 5-6:3-4,2-1: -5-6:3-4,2-1: +6:4,1: diff --git a/tests/lxpos/out6.dump b/tests/lxpos/out6.dump index 8036b2ad4..c11d09988 100644 --- a/tests/lxpos/out6.dump +++ b/tests/lxpos/out6.dump @@ -1,2 +1,2 @@ 0-1:1,1-2: -4-5:1,5-6: +4-5:1,5-6: lexically uncategorised: ' ' diff --git a/tests/lxpos/out7.dump b/tests/lxpos/out7.dump index 7eeab8bcb..0706b7981 100644 --- a/tests/lxpos/out7.dump +++ b/tests/lxpos/out7.dump @@ -2,4 +2,4 @@ 17-18:2,12-13: 18-19:2-3,13-1: -18-19:2-3,13-1: +19:3,1: diff --git a/tests/lxpos/out8.dump b/tests/lxpos/out8.dump new file mode 100644 index 000000000..3300df2f6 --- /dev/null +++ b/tests/lxpos/out8.dump @@ -0,0 +1,3 @@ +0-5:1,1-6: +6-7:1,7-8: lexically uncategorised: '` +' diff --git a/tests/lxpos/out9.dump b/tests/lxpos/out9.dump new file mode 100644 index 000000000..1f3cf25bc --- /dev/null +++ b/tests/lxpos/out9.dump @@ -0,0 +1,7 @@ +0-1:1,1-2: +1-2:1,2-3: +3-4:1,4-5: +5-6:1,6-7: +7-8:1,8-9: +33-34:1-2,34-1: lexically uncategorised: ' +' diff --git a/tests/minimise/minimise_test_case_list.c b/tests/minimise/minimise_test_case_list.c index 1386f0dcc..bfe25fee2 100644 --- a/tests/minimise/minimise_test_case_list.c +++ b/tests/minimise/minimise_test_case_list.c @@ -22,7 +22,6 @@ const char *test_cases[] = { "(?:a+|b)a+", "(?:a*ba)+", "(?:a|cd)+e?x", - "-> 1 'a';", "(?:abc|def)+", "(?:abc|def)*", "(?:b|a*)", @@ -62,12 +61,6 @@ scanner_next(void *opaque) return (int) c; } -static const struct fsm_options opt = { - .consolidate_edges = 1, - .comments = 1, - .group_edges = 1, -}; - static int check_minimisation(const char *pattern) { @@ -81,7 +74,7 @@ check_minimisation(const char *pattern) .offset = 0 }; - fsm = re_comp(RE_PCRE, scanner_next, &s, &opt, RE_MULTI, &err); + fsm = re_comp(RE_PCRE, scanner_next, &s, NULL, RE_MULTI | RE_NOCAPTURE, &err); assert(fsm != NULL); if (!fsm_determinise(fsm)) { return 0; @@ -110,10 +103,10 @@ check_minimisation(const char *pattern) __func__, pattern, expected_state_count, state_count_min); fprintf(stderr, "== expected:\n"); - fsm_print_fsm(stderr, oracle_min); + fsm_dump(stderr, oracle_min); fprintf(stderr, "== got:\n"); - fsm_print_fsm(stderr, fsm); + fsm_dump(stderr, fsm); } fsm_free(oracle_min); diff --git a/tests/native/Makefile b/tests/native/Makefile index 8712e1588..fbca0ca69 100755 --- a/tests/native/Makefile +++ b/tests/native/Makefile @@ -9,11 +9,11 @@ RE=${BUILD}/bin/re .for n in ${TEST.tests/native:T:Mout*.fsm:R:C/^out//} ${TEST_OUTDIR.tests/native}/got${n}.fsm: ${TEST_SRCDIR.tests/native}/in${n}.re - ${RE} -r native -py ${.ALLSRC:M*.re} \ + ${RE} -FC -r native -py ${.ALLSRC:M*.re} \ > $@ ${TEST_OUTDIR.tests/native}/nfa${n}.fsm: ${TEST_SRCDIR.tests/native}/in${n}.re - ${RE} -r native -n -py ${.ALLSRC:M*.re} \ + ${RE} -FC -r native -n -py ${.ALLSRC:M*.re} \ > $@ ${TEST_OUTDIR.tests/native}/res${n}: \ @@ -27,7 +27,7 @@ FSMTEST_RESULT += ${TEST_OUTDIR.tests/native}/res${n} .for n in ${TEST.tests/native:T:Mout*.err:R:C/^out//} ${TEST_OUTDIR.tests/native}/got${n}.err: ${TEST_SRCDIR.tests/native}/in${n}.re - ${RE} -r native -py ${.ALLSRC:M*.re} \ + ${RE} -FC -r native -py ${.ALLSRC:M*.re} \ 2> $@; [ $$? -ne 0 ] ${TEST_OUTDIR.tests/native}/res${n}: \ diff --git a/tests/pcre-anchor/Makefile b/tests/pcre-anchor/Makefile index bb9954554..1dc4a77bc 100644 --- a/tests/pcre-anchor/Makefile +++ b/tests/pcre-anchor/Makefile @@ -9,11 +9,11 @@ RE=${BUILD}/bin/re .for n in ${TEST.tests/pcre-anchor:T:Mout*.fsm:R:C/^out//} ${TEST_OUTDIR.tests/pcre-anchor}/got${n}.fsm: ${TEST_SRCDIR.tests/pcre-anchor}/in${n}.re - ${RE} -r pcre -py ${.ALLSRC:M*.re} \ + ${RE} -FC -r pcre -py ${.ALLSRC:M*.re} \ > $@ ${TEST_OUTDIR.tests/pcre-anchor}/nfa${n}.fsm: ${TEST_SRCDIR.tests/pcre-anchor}/in${n}.re - ${RE} -r pcre -n -py ${.ALLSRC:M*.re} \ + ${RE} -FC -r pcre -n -py ${.ALLSRC:M*.re} \ > $@ ${TEST_OUTDIR.tests/pcre-anchor}/res${n}: \ diff --git a/tests/pcre-anchor/in100.re b/tests/pcre-anchor/in100.re new file mode 100644 index 000000000..8fea9b9ec --- /dev/null +++ b/tests/pcre-anchor/in100.re @@ -0,0 +1 @@ +$[^a]*^|$^ \ No newline at end of file diff --git a/tests/pcre-anchor/in81.re b/tests/pcre-anchor/in81.re new file mode 100644 index 000000000..8b5fad7c3 --- /dev/null +++ b/tests/pcre-anchor/in81.re @@ -0,0 +1 @@ +($x)* \ No newline at end of file diff --git a/tests/pcre-anchor/in82.re b/tests/pcre-anchor/in82.re new file mode 100644 index 000000000..60c5f181f --- /dev/null +++ b/tests/pcre-anchor/in82.re @@ -0,0 +1 @@ +(^)+a \ No newline at end of file diff --git a/tests/pcre-anchor/in83.re b/tests/pcre-anchor/in83.re new file mode 100644 index 000000000..ef122c481 --- /dev/null +++ b/tests/pcre-anchor/in83.re @@ -0,0 +1 @@ +(?:a|^)*^x \ No newline at end of file diff --git a/tests/pcre-anchor/in84.re b/tests/pcre-anchor/in84.re new file mode 100644 index 000000000..3aac549f7 --- /dev/null +++ b/tests/pcre-anchor/in84.re @@ -0,0 +1 @@ +(?:(?:$|^)a) \ No newline at end of file diff --git a/tests/pcre-anchor/in85.re b/tests/pcre-anchor/in85.re new file mode 100644 index 000000000..d72dc5d4b --- /dev/null +++ b/tests/pcre-anchor/in85.re @@ -0,0 +1 @@ +$(?:a|^)b? \ No newline at end of file diff --git a/tests/pcre-anchor/in86.re b/tests/pcre-anchor/in86.re new file mode 100644 index 000000000..b288d3dc3 --- /dev/null +++ b/tests/pcre-anchor/in86.re @@ -0,0 +1 @@ +(?:(?:^a)*|b^)^c \ No newline at end of file diff --git a/tests/pcre-anchor/in87.re b/tests/pcre-anchor/in87.re new file mode 100644 index 000000000..a8006a0fe --- /dev/null +++ b/tests/pcre-anchor/in87.re @@ -0,0 +1 @@ +(($|^)a)b \ No newline at end of file diff --git a/tests/pcre-anchor/in88.re b/tests/pcre-anchor/in88.re new file mode 100644 index 000000000..3d2764532 --- /dev/null +++ b/tests/pcre-anchor/in88.re @@ -0,0 +1 @@ +(a$b)*|.^ \ No newline at end of file diff --git a/tests/pcre-anchor/in89.re b/tests/pcre-anchor/in89.re new file mode 100644 index 000000000..bf51de7e9 --- /dev/null +++ b/tests/pcre-anchor/in89.re @@ -0,0 +1 @@ +$.| \ No newline at end of file diff --git a/tests/pcre-anchor/in90.re b/tests/pcre-anchor/in90.re new file mode 100644 index 000000000..519e2e283 --- /dev/null +++ b/tests/pcre-anchor/in90.re @@ -0,0 +1 @@ +.^| \ No newline at end of file diff --git a/tests/pcre-anchor/in91.re b/tests/pcre-anchor/in91.re new file mode 100644 index 000000000..3ce75c157 --- /dev/null +++ b/tests/pcre-anchor/in91.re @@ -0,0 +1 @@ +$.^| \ No newline at end of file diff --git a/tests/pcre-anchor/in92.re b/tests/pcre-anchor/in92.re new file mode 100644 index 000000000..e32c3a5f5 --- /dev/null +++ b/tests/pcre-anchor/in92.re @@ -0,0 +1 @@ +(a|$)(b|^) \ No newline at end of file diff --git a/tests/pcre-anchor/in93.re b/tests/pcre-anchor/in93.re new file mode 100644 index 000000000..228f2e8e0 --- /dev/null +++ b/tests/pcre-anchor/in93.re @@ -0,0 +1 @@ +(a|^)(b|$) \ No newline at end of file diff --git a/tests/pcre-anchor/in94.re b/tests/pcre-anchor/in94.re new file mode 100644 index 000000000..349808e15 --- /dev/null +++ b/tests/pcre-anchor/in94.re @@ -0,0 +1 @@ +.$|b. \ No newline at end of file diff --git a/tests/pcre-anchor/in95.re b/tests/pcre-anchor/in95.re new file mode 100644 index 000000000..0b01ad029 --- /dev/null +++ b/tests/pcre-anchor/in95.re @@ -0,0 +1 @@ +a$|^b$(|c)^$ \ No newline at end of file diff --git a/tests/pcre-anchor/in96.re b/tests/pcre-anchor/in96.re new file mode 100644 index 000000000..4968820fe --- /dev/null +++ b/tests/pcre-anchor/in96.re @@ -0,0 +1 @@ +a$|$b(|c)^$ \ No newline at end of file diff --git a/tests/pcre-anchor/in97.re b/tests/pcre-anchor/in97.re new file mode 100644 index 000000000..bff8c08e1 --- /dev/null +++ b/tests/pcre-anchor/in97.re @@ -0,0 +1 @@ +^a$|^$b$c^d$-$(|e)^$ \ No newline at end of file diff --git a/tests/pcre-anchor/in98.re b/tests/pcre-anchor/in98.re new file mode 100644 index 000000000..c711c61cb --- /dev/null +++ b/tests/pcre-anchor/in98.re @@ -0,0 +1 @@ +a$|^b(^|$c) \ No newline at end of file diff --git a/tests/pcre-anchor/in99.re b/tests/pcre-anchor/in99.re new file mode 100644 index 000000000..5c385ad3e --- /dev/null +++ b/tests/pcre-anchor/in99.re @@ -0,0 +1 @@ +^(?:$^|a)(?:$^|^)* \ No newline at end of file diff --git a/tests/pcre-anchor/out100.fsm b/tests/pcre-anchor/out100.fsm new file mode 100644 index 000000000..9ad9510cb --- /dev/null +++ b/tests/pcre-anchor/out100.fsm @@ -0,0 +1,4 @@ +0 -> 1 "\n"; + +start: 0; +end: 0, 1; diff --git a/tests/pcre-anchor/out81.fsm b/tests/pcre-anchor/out81.fsm new file mode 100644 index 000000000..2cdc2f023 --- /dev/null +++ b/tests/pcre-anchor/out81.fsm @@ -0,0 +1,5 @@ +0 -> 0 ?; +0 -> 1 "\n"; + +start: 0; +end: 0, 1; \ No newline at end of file diff --git a/tests/pcre-anchor/out82.fsm b/tests/pcre-anchor/out82.fsm new file mode 100644 index 000000000..84503d994 --- /dev/null +++ b/tests/pcre-anchor/out82.fsm @@ -0,0 +1,5 @@ +0 -> 1 "a"; +1 -> 1 ?; + +start: 0; +end: 1; \ No newline at end of file diff --git a/tests/pcre-anchor/out83.fsm b/tests/pcre-anchor/out83.fsm new file mode 100644 index 000000000..28643e39e --- /dev/null +++ b/tests/pcre-anchor/out83.fsm @@ -0,0 +1,5 @@ +0 -> 1 "x"; +1 -> 1 ?; + +start: 0; +end: 1; \ No newline at end of file diff --git a/tests/pcre-anchor/out84.fsm b/tests/pcre-anchor/out84.fsm new file mode 100644 index 000000000..84503d994 --- /dev/null +++ b/tests/pcre-anchor/out84.fsm @@ -0,0 +1,5 @@ +0 -> 1 "a"; +1 -> 1 ?; + +start: 0; +end: 1; \ No newline at end of file diff --git a/tests/pcre-anchor/out85.fsm b/tests/pcre-anchor/out85.fsm new file mode 100644 index 000000000..9b7e7aba6 --- /dev/null +++ b/tests/pcre-anchor/out85.fsm @@ -0,0 +1,4 @@ +0 -> 1 "\n"; + +start: 0; +end: 0, 1; \ No newline at end of file diff --git a/tests/pcre-anchor/out86.fsm b/tests/pcre-anchor/out86.fsm new file mode 100644 index 000000000..4bcad67dc --- /dev/null +++ b/tests/pcre-anchor/out86.fsm @@ -0,0 +1,5 @@ +0 -> 1 "c"; +1 -> 1 ?; + +start: 0; +end: 1; \ No newline at end of file diff --git a/tests/pcre-anchor/out87.fsm b/tests/pcre-anchor/out87.fsm new file mode 100644 index 000000000..6f6aa823b --- /dev/null +++ b/tests/pcre-anchor/out87.fsm @@ -0,0 +1,6 @@ +0 -> 1 "a"; +1 -> 2 "b"; +2 -> 2 ?; + +start: 0; +end: 2; \ No newline at end of file diff --git a/tests/pcre-anchor/out88.fsm b/tests/pcre-anchor/out88.fsm new file mode 100644 index 000000000..e4b4f22d4 --- /dev/null +++ b/tests/pcre-anchor/out88.fsm @@ -0,0 +1,4 @@ +0 -> 0 ?; + +start: 0; +end: 0; \ No newline at end of file diff --git a/tests/pcre-anchor/out89.fsm b/tests/pcre-anchor/out89.fsm new file mode 100644 index 000000000..e4b4f22d4 --- /dev/null +++ b/tests/pcre-anchor/out89.fsm @@ -0,0 +1,4 @@ +0 -> 0 ?; + +start: 0; +end: 0; \ No newline at end of file diff --git a/tests/pcre-anchor/out90.fsm b/tests/pcre-anchor/out90.fsm new file mode 100644 index 000000000..e4b4f22d4 --- /dev/null +++ b/tests/pcre-anchor/out90.fsm @@ -0,0 +1,4 @@ +0 -> 0 ?; + +start: 0; +end: 0; \ No newline at end of file diff --git a/tests/pcre-anchor/out91.fsm b/tests/pcre-anchor/out91.fsm new file mode 100644 index 000000000..e4b4f22d4 --- /dev/null +++ b/tests/pcre-anchor/out91.fsm @@ -0,0 +1,4 @@ +0 -> 0 ?; + +start: 0; +end: 0; \ No newline at end of file diff --git a/tests/pcre-anchor/out92.fsm b/tests/pcre-anchor/out92.fsm new file mode 100644 index 000000000..e2aafdeb6 --- /dev/null +++ b/tests/pcre-anchor/out92.fsm @@ -0,0 +1,19 @@ +# This is a tricky one -- with '(a|$)(b|^)' the ^ must not be linked to +# the global start or have an unanchored end loop after, because by the +# time execution gets there it has already either matched some input +# ('a') or anchored to the end -- the only matching inputs should be "", +# "\n", and "ab", but "a^", "$b", or anything else +# that does not contain "ab" must not match. + +0 -> 1; +0 -> 4 "\n"; + +1 -> 1 ?; +1 -> 2 "a"; + +2 -> 3 "b"; + +3 -> 3 ?; + +start: 0; +end: 0, 3, 4; diff --git a/tests/pcre-anchor/out93.fsm b/tests/pcre-anchor/out93.fsm new file mode 100644 index 000000000..aed23cab9 --- /dev/null +++ b/tests/pcre-anchor/out93.fsm @@ -0,0 +1,18 @@ +# /(a|^)(b|$)/ => "", "\n", "a" "b "ab" + +0 -> 1; +0 -> 3; + +1 -> 1 ?; +1 -> 2 "a"; + +2 -> 4 "b"; +2 -> 5 "\n"; + +3 -> 4 "b"; +3 -> 5 "\n"; + +4 -> 4 ?; + +start: 0; +end: 2, 3, 4, 5; diff --git a/tests/pcre-anchor/out94.fsm b/tests/pcre-anchor/out94.fsm new file mode 100644 index 000000000..53d73459c --- /dev/null +++ b/tests/pcre-anchor/out94.fsm @@ -0,0 +1,525 @@ +0 -> 1; +0 -> 4; +2 -> 3; +4 -> 5; +4 -> 4 ?; +5 -> 9 "\x00"; +5 -> 9 "\x01"; +5 -> 9 "\x02"; +5 -> 9 "\x03"; +5 -> 9 "\x04"; +5 -> 9 "\x05"; +5 -> 9 "\x06"; +5 -> 9 "\x07"; +5 -> 9 "\x08"; +5 -> 9 "\t"; +5 -> 9 "\v"; +5 -> 9 "\f"; +5 -> 9 "\r"; +5 -> 9 "\x0e"; +5 -> 9 "\x0f"; +5 -> 9 "\x10"; +5 -> 9 "\x11"; +5 -> 9 "\x12"; +5 -> 9 "\x13"; +5 -> 9 "\x14"; +5 -> 9 "\x15"; +5 -> 9 "\x16"; +5 -> 9 "\x17"; +5 -> 9 "\x18"; +5 -> 9 "\x19"; +5 -> 9 "\x1a"; +5 -> 9 "\x1b"; +5 -> 9 "\x1c"; +5 -> 9 "\x1d"; +5 -> 9 "\x1e"; +5 -> 9 "\x1f"; +5 -> 9 " "; +5 -> 9 "!"; +5 -> 9 "\""; +5 -> 9 "#"; +5 -> 9 "$"; +5 -> 9 "%"; +5 -> 9 "&"; +5 -> 9 "'"; +5 -> 9 "("; +5 -> 9 ")"; +5 -> 9 "*"; +5 -> 9 "+"; +5 -> 9 ","; +5 -> 9 "-"; +5 -> 9 "."; +5 -> 9 "/"; +5 -> 9 "0"; +5 -> 9 "1"; +5 -> 9 "2"; +5 -> 9 "3"; +5 -> 9 "4"; +5 -> 9 "5"; +5 -> 9 "6"; +5 -> 9 "7"; +5 -> 9 "8"; +5 -> 9 "9"; +5 -> 9 ":"; +5 -> 9 ";"; +5 -> 9 "<"; +5 -> 9 "="; +5 -> 9 ">"; +5 -> 9 "?"; +5 -> 9 "@"; +5 -> 9 "A"; +5 -> 9 "B"; +5 -> 9 "C"; +5 -> 9 "D"; +5 -> 9 "E"; +5 -> 9 "F"; +5 -> 9 "G"; +5 -> 9 "H"; +5 -> 9 "I"; +5 -> 9 "J"; +5 -> 9 "K"; +5 -> 9 "L"; +5 -> 9 "M"; +5 -> 9 "N"; +5 -> 9 "O"; +5 -> 9 "P"; +5 -> 9 "Q"; +5 -> 9 "R"; +5 -> 9 "S"; +5 -> 9 "T"; +5 -> 9 "U"; +5 -> 9 "V"; +5 -> 9 "W"; +5 -> 9 "X"; +5 -> 9 "Y"; +5 -> 9 "Z"; +5 -> 9 "["; +5 -> 9 "\\"; +5 -> 9 "]"; +5 -> 9 "^"; +5 -> 9 "_"; +5 -> 9 "`"; +5 -> 9 "a"; +5 -> 9 "b"; +5 -> 10 "b"; +5 -> 9 "c"; +5 -> 9 "d"; +5 -> 9 "e"; +5 -> 9 "f"; +5 -> 9 "g"; +5 -> 9 "h"; +5 -> 9 "i"; +5 -> 9 "j"; +5 -> 9 "k"; +5 -> 9 "l"; +5 -> 9 "m"; +5 -> 9 "n"; +5 -> 9 "o"; +5 -> 9 "p"; +5 -> 9 "q"; +5 -> 9 "r"; +5 -> 9 "s"; +5 -> 9 "t"; +5 -> 9 "u"; +5 -> 9 "v"; +5 -> 9 "w"; +5 -> 9 "x"; +5 -> 9 "y"; +5 -> 9 "z"; +5 -> 9 "{"; +5 -> 9 "|"; +5 -> 9 "}"; +5 -> 9 "~"; +5 -> 9 "\x7f"; +5 -> 9 "\x80"; +5 -> 9 "\x81"; +5 -> 9 "\x82"; +5 -> 9 "\x83"; +5 -> 9 "\x84"; +5 -> 9 "\x85"; +5 -> 9 "\x86"; +5 -> 9 "\x87"; +5 -> 9 "\x88"; +5 -> 9 "\x89"; +5 -> 9 "\x8a"; +5 -> 9 "\x8b"; +5 -> 9 "\x8c"; +5 -> 9 "\x8d"; +5 -> 9 "\x8e"; +5 -> 9 "\x8f"; +5 -> 9 "\x90"; +5 -> 9 "\x91"; +5 -> 9 "\x92"; +5 -> 9 "\x93"; +5 -> 9 "\x94"; +5 -> 9 "\x95"; +5 -> 9 "\x96"; +5 -> 9 "\x97"; +5 -> 9 "\x98"; +5 -> 9 "\x99"; +5 -> 9 "\x9a"; +5 -> 9 "\x9b"; +5 -> 9 "\x9c"; +5 -> 9 "\x9d"; +5 -> 9 "\x9e"; +5 -> 9 "\x9f"; +5 -> 9 "\xa0"; +5 -> 9 "\xa1"; +5 -> 9 "\xa2"; +5 -> 9 "\xa3"; +5 -> 9 "\xa4"; +5 -> 9 "\xa5"; +5 -> 9 "\xa6"; +5 -> 9 "\xa7"; +5 -> 9 "\xa8"; +5 -> 9 "\xa9"; +5 -> 9 "\xaa"; +5 -> 9 "\xab"; +5 -> 9 "\xac"; +5 -> 9 "\xad"; +5 -> 9 "\xae"; +5 -> 9 "\xaf"; +5 -> 9 "\xb0"; +5 -> 9 "\xb1"; +5 -> 9 "\xb2"; +5 -> 9 "\xb3"; +5 -> 9 "\xb4"; +5 -> 9 "\xb5"; +5 -> 9 "\xb6"; +5 -> 9 "\xb7"; +5 -> 9 "\xb8"; +5 -> 9 "\xb9"; +5 -> 9 "\xba"; +5 -> 9 "\xbb"; +5 -> 9 "\xbc"; +5 -> 9 "\xbd"; +5 -> 9 "\xbe"; +5 -> 9 "\xbf"; +5 -> 9 "\xc0"; +5 -> 9 "\xc1"; +5 -> 9 "\xc2"; +5 -> 9 "\xc3"; +5 -> 9 "\xc4"; +5 -> 9 "\xc5"; +5 -> 9 "\xc6"; +5 -> 9 "\xc7"; +5 -> 9 "\xc8"; +5 -> 9 "\xc9"; +5 -> 9 "\xca"; +5 -> 9 "\xcb"; +5 -> 9 "\xcc"; +5 -> 9 "\xcd"; +5 -> 9 "\xce"; +5 -> 9 "\xcf"; +5 -> 9 "\xd0"; +5 -> 9 "\xd1"; +5 -> 9 "\xd2"; +5 -> 9 "\xd3"; +5 -> 9 "\xd4"; +5 -> 9 "\xd5"; +5 -> 9 "\xd6"; +5 -> 9 "\xd7"; +5 -> 9 "\xd8"; +5 -> 9 "\xd9"; +5 -> 9 "\xda"; +5 -> 9 "\xdb"; +5 -> 9 "\xdc"; +5 -> 9 "\xdd"; +5 -> 9 "\xde"; +5 -> 9 "\xdf"; +5 -> 9 "\xe0"; +5 -> 9 "\xe1"; +5 -> 9 "\xe2"; +5 -> 9 "\xe3"; +5 -> 9 "\xe4"; +5 -> 9 "\xe5"; +5 -> 9 "\xe6"; +5 -> 9 "\xe7"; +5 -> 9 "\xe8"; +5 -> 9 "\xe9"; +5 -> 9 "\xea"; +5 -> 9 "\xeb"; +5 -> 9 "\xec"; +5 -> 9 "\xed"; +5 -> 9 "\xee"; +5 -> 9 "\xef"; +5 -> 9 "\xf0"; +5 -> 9 "\xf1"; +5 -> 9 "\xf2"; +5 -> 9 "\xf3"; +5 -> 9 "\xf4"; +5 -> 9 "\xf5"; +5 -> 9 "\xf6"; +5 -> 9 "\xf7"; +5 -> 9 "\xf8"; +5 -> 9 "\xf9"; +5 -> 9 "\xfa"; +5 -> 9 "\xfb"; +5 -> 9 "\xfc"; +5 -> 9 "\xfd"; +5 -> 9 "\xfe"; +5 -> 9 "\xff"; +6 -> 3; +6 -> 6 ?; +7 -> 6; +8 -> 3; +8 -> 3 "\n"; +9 -> 8; +10 -> 7 "\x00"; +10 -> 7 "\x01"; +10 -> 7 "\x02"; +10 -> 7 "\x03"; +10 -> 7 "\x04"; +10 -> 7 "\x05"; +10 -> 7 "\x06"; +10 -> 7 "\x07"; +10 -> 7 "\x08"; +10 -> 7 "\t"; +10 -> 7 "\v"; +10 -> 7 "\f"; +10 -> 7 "\r"; +10 -> 7 "\x0e"; +10 -> 7 "\x0f"; +10 -> 7 "\x10"; +10 -> 7 "\x11"; +10 -> 7 "\x12"; +10 -> 7 "\x13"; +10 -> 7 "\x14"; +10 -> 7 "\x15"; +10 -> 7 "\x16"; +10 -> 7 "\x17"; +10 -> 7 "\x18"; +10 -> 7 "\x19"; +10 -> 7 "\x1a"; +10 -> 7 "\x1b"; +10 -> 7 "\x1c"; +10 -> 7 "\x1d"; +10 -> 7 "\x1e"; +10 -> 7 "\x1f"; +10 -> 7 " "; +10 -> 7 "!"; +10 -> 7 "\""; +10 -> 7 "#"; +10 -> 7 "$"; +10 -> 7 "%"; +10 -> 7 "&"; +10 -> 7 "'"; +10 -> 7 "("; +10 -> 7 ")"; +10 -> 7 "*"; +10 -> 7 "+"; +10 -> 7 ","; +10 -> 7 "-"; +10 -> 7 "."; +10 -> 7 "/"; +10 -> 7 "0"; +10 -> 7 "1"; +10 -> 7 "2"; +10 -> 7 "3"; +10 -> 7 "4"; +10 -> 7 "5"; +10 -> 7 "6"; +10 -> 7 "7"; +10 -> 7 "8"; +10 -> 7 "9"; +10 -> 7 ":"; +10 -> 7 ";"; +10 -> 7 "<"; +10 -> 7 "="; +10 -> 7 ">"; +10 -> 7 "?"; +10 -> 7 "@"; +10 -> 7 "A"; +10 -> 7 "B"; +10 -> 7 "C"; +10 -> 7 "D"; +10 -> 7 "E"; +10 -> 7 "F"; +10 -> 7 "G"; +10 -> 7 "H"; +10 -> 7 "I"; +10 -> 7 "J"; +10 -> 7 "K"; +10 -> 7 "L"; +10 -> 7 "M"; +10 -> 7 "N"; +10 -> 7 "O"; +10 -> 7 "P"; +10 -> 7 "Q"; +10 -> 7 "R"; +10 -> 7 "S"; +10 -> 7 "T"; +10 -> 7 "U"; +10 -> 7 "V"; +10 -> 7 "W"; +10 -> 7 "X"; +10 -> 7 "Y"; +10 -> 7 "Z"; +10 -> 7 "["; +10 -> 7 "\\"; +10 -> 7 "]"; +10 -> 7 "^"; +10 -> 7 "_"; +10 -> 7 "`"; +10 -> 7 "a"; +10 -> 7 "b"; +10 -> 7 "c"; +10 -> 7 "d"; +10 -> 7 "e"; +10 -> 7 "f"; +10 -> 7 "g"; +10 -> 7 "h"; +10 -> 7 "i"; +10 -> 7 "j"; +10 -> 7 "k"; +10 -> 7 "l"; +10 -> 7 "m"; +10 -> 7 "n"; +10 -> 7 "o"; +10 -> 7 "p"; +10 -> 7 "q"; +10 -> 7 "r"; +10 -> 7 "s"; +10 -> 7 "t"; +10 -> 7 "u"; +10 -> 7 "v"; +10 -> 7 "w"; +10 -> 7 "x"; +10 -> 7 "y"; +10 -> 7 "z"; +10 -> 7 "{"; +10 -> 7 "|"; +10 -> 7 "}"; +10 -> 7 "~"; +10 -> 7 "\x7f"; +10 -> 7 "\x80"; +10 -> 7 "\x81"; +10 -> 7 "\x82"; +10 -> 7 "\x83"; +10 -> 7 "\x84"; +10 -> 7 "\x85"; +10 -> 7 "\x86"; +10 -> 7 "\x87"; +10 -> 7 "\x88"; +10 -> 7 "\x89"; +10 -> 7 "\x8a"; +10 -> 7 "\x8b"; +10 -> 7 "\x8c"; +10 -> 7 "\x8d"; +10 -> 7 "\x8e"; +10 -> 7 "\x8f"; +10 -> 7 "\x90"; +10 -> 7 "\x91"; +10 -> 7 "\x92"; +10 -> 7 "\x93"; +10 -> 7 "\x94"; +10 -> 7 "\x95"; +10 -> 7 "\x96"; +10 -> 7 "\x97"; +10 -> 7 "\x98"; +10 -> 7 "\x99"; +10 -> 7 "\x9a"; +10 -> 7 "\x9b"; +10 -> 7 "\x9c"; +10 -> 7 "\x9d"; +10 -> 7 "\x9e"; +10 -> 7 "\x9f"; +10 -> 7 "\xa0"; +10 -> 7 "\xa1"; +10 -> 7 "\xa2"; +10 -> 7 "\xa3"; +10 -> 7 "\xa4"; +10 -> 7 "\xa5"; +10 -> 7 "\xa6"; +10 -> 7 "\xa7"; +10 -> 7 "\xa8"; +10 -> 7 "\xa9"; +10 -> 7 "\xaa"; +10 -> 7 "\xab"; +10 -> 7 "\xac"; +10 -> 7 "\xad"; +10 -> 7 "\xae"; +10 -> 7 "\xaf"; +10 -> 7 "\xb0"; +10 -> 7 "\xb1"; +10 -> 7 "\xb2"; +10 -> 7 "\xb3"; +10 -> 7 "\xb4"; +10 -> 7 "\xb5"; +10 -> 7 "\xb6"; +10 -> 7 "\xb7"; +10 -> 7 "\xb8"; +10 -> 7 "\xb9"; +10 -> 7 "\xba"; +10 -> 7 "\xbb"; +10 -> 7 "\xbc"; +10 -> 7 "\xbd"; +10 -> 7 "\xbe"; +10 -> 7 "\xbf"; +10 -> 7 "\xc0"; +10 -> 7 "\xc1"; +10 -> 7 "\xc2"; +10 -> 7 "\xc3"; +10 -> 7 "\xc4"; +10 -> 7 "\xc5"; +10 -> 7 "\xc6"; +10 -> 7 "\xc7"; +10 -> 7 "\xc8"; +10 -> 7 "\xc9"; +10 -> 7 "\xca"; +10 -> 7 "\xcb"; +10 -> 7 "\xcc"; +10 -> 7 "\xcd"; +10 -> 7 "\xce"; +10 -> 7 "\xcf"; +10 -> 7 "\xd0"; +10 -> 7 "\xd1"; +10 -> 7 "\xd2"; +10 -> 7 "\xd3"; +10 -> 7 "\xd4"; +10 -> 7 "\xd5"; +10 -> 7 "\xd6"; +10 -> 7 "\xd7"; +10 -> 7 "\xd8"; +10 -> 7 "\xd9"; +10 -> 7 "\xda"; +10 -> 7 "\xdb"; +10 -> 7 "\xdc"; +10 -> 7 "\xdd"; +10 -> 7 "\xde"; +10 -> 7 "\xdf"; +10 -> 7 "\xe0"; +10 -> 7 "\xe1"; +10 -> 7 "\xe2"; +10 -> 7 "\xe3"; +10 -> 7 "\xe4"; +10 -> 7 "\xe5"; +10 -> 7 "\xe6"; +10 -> 7 "\xe7"; +10 -> 7 "\xe8"; +10 -> 7 "\xe9"; +10 -> 7 "\xea"; +10 -> 7 "\xeb"; +10 -> 7 "\xec"; +10 -> 7 "\xed"; +10 -> 7 "\xee"; +10 -> 7 "\xef"; +10 -> 7 "\xf0"; +10 -> 7 "\xf1"; +10 -> 7 "\xf2"; +10 -> 7 "\xf3"; +10 -> 7 "\xf4"; +10 -> 7 "\xf5"; +10 -> 7 "\xf6"; +10 -> 7 "\xf7"; +10 -> 7 "\xf8"; +10 -> 7 "\xf9"; +10 -> 7 "\xfa"; +10 -> 7 "\xfb"; +10 -> 7 "\xfc"; +10 -> 7 "\xfd"; +10 -> 7 "\xfe"; +10 -> 7 "\xff"; + +start: 0; +end: 3; diff --git a/tests/pcre-anchor/out95.fsm b/tests/pcre-anchor/out95.fsm new file mode 100644 index 000000000..ee435ee3c --- /dev/null +++ b/tests/pcre-anchor/out95.fsm @@ -0,0 +1,771 @@ +0 -> 0 "\x00"; +0 -> 0 "\x01"; +0 -> 0 "\x02"; +0 -> 0 "\x03"; +0 -> 0 "\x04"; +0 -> 0 "\x05"; +0 -> 0 "\x06"; +0 -> 0 "\x07"; +0 -> 0 "\x08"; +0 -> 0 "\t"; +0 -> 0 "\n"; +0 -> 0 "\v"; +0 -> 0 "\f"; +0 -> 0 "\r"; +0 -> 0 "\x0e"; +0 -> 0 "\x0f"; +0 -> 0 "\x10"; +0 -> 0 "\x11"; +0 -> 0 "\x12"; +0 -> 0 "\x13"; +0 -> 0 "\x14"; +0 -> 0 "\x15"; +0 -> 0 "\x16"; +0 -> 0 "\x17"; +0 -> 0 "\x18"; +0 -> 0 "\x19"; +0 -> 0 "\x1a"; +0 -> 0 "\x1b"; +0 -> 0 "\x1c"; +0 -> 0 "\x1d"; +0 -> 0 "\x1e"; +0 -> 0 "\x1f"; +0 -> 0 " "; +0 -> 0 "!"; +0 -> 0 "\""; +0 -> 0 "#"; +0 -> 0 "$"; +0 -> 0 "%"; +0 -> 0 "&"; +0 -> 0 "'"; +0 -> 0 "("; +0 -> 0 ")"; +0 -> 0 "*"; +0 -> 0 "+"; +0 -> 0 ","; +0 -> 0 "-"; +0 -> 0 "."; +0 -> 0 "/"; +0 -> 0 "0"; +0 -> 0 "1"; +0 -> 0 "2"; +0 -> 0 "3"; +0 -> 0 "4"; +0 -> 0 "5"; +0 -> 0 "6"; +0 -> 0 "7"; +0 -> 0 "8"; +0 -> 0 "9"; +0 -> 0 ":"; +0 -> 0 ";"; +0 -> 0 "<"; +0 -> 0 "="; +0 -> 0 ">"; +0 -> 0 "?"; +0 -> 0 "@"; +0 -> 0 "A"; +0 -> 0 "B"; +0 -> 0 "C"; +0 -> 0 "D"; +0 -> 0 "E"; +0 -> 0 "F"; +0 -> 0 "G"; +0 -> 0 "H"; +0 -> 0 "I"; +0 -> 0 "J"; +0 -> 0 "K"; +0 -> 0 "L"; +0 -> 0 "M"; +0 -> 0 "N"; +0 -> 0 "O"; +0 -> 0 "P"; +0 -> 0 "Q"; +0 -> 0 "R"; +0 -> 0 "S"; +0 -> 0 "T"; +0 -> 0 "U"; +0 -> 0 "V"; +0 -> 0 "W"; +0 -> 0 "X"; +0 -> 0 "Y"; +0 -> 0 "Z"; +0 -> 0 "["; +0 -> 0 "\\"; +0 -> 0 "]"; +0 -> 0 "^"; +0 -> 0 "_"; +0 -> 0 "`"; +0 -> 1 "a"; +0 -> 0 "b"; +0 -> 0 "c"; +0 -> 0 "d"; +0 -> 0 "e"; +0 -> 0 "f"; +0 -> 0 "g"; +0 -> 0 "h"; +0 -> 0 "i"; +0 -> 0 "j"; +0 -> 0 "k"; +0 -> 0 "l"; +0 -> 0 "m"; +0 -> 0 "n"; +0 -> 0 "o"; +0 -> 0 "p"; +0 -> 0 "q"; +0 -> 0 "r"; +0 -> 0 "s"; +0 -> 0 "t"; +0 -> 0 "u"; +0 -> 0 "v"; +0 -> 0 "w"; +0 -> 0 "x"; +0 -> 0 "y"; +0 -> 0 "z"; +0 -> 0 "{"; +0 -> 0 "|"; +0 -> 0 "}"; +0 -> 0 "~"; +0 -> 0 "\x7f"; +0 -> 0 "\x80"; +0 -> 0 "\x81"; +0 -> 0 "\x82"; +0 -> 0 "\x83"; +0 -> 0 "\x84"; +0 -> 0 "\x85"; +0 -> 0 "\x86"; +0 -> 0 "\x87"; +0 -> 0 "\x88"; +0 -> 0 "\x89"; +0 -> 0 "\x8a"; +0 -> 0 "\x8b"; +0 -> 0 "\x8c"; +0 -> 0 "\x8d"; +0 -> 0 "\x8e"; +0 -> 0 "\x8f"; +0 -> 0 "\x90"; +0 -> 0 "\x91"; +0 -> 0 "\x92"; +0 -> 0 "\x93"; +0 -> 0 "\x94"; +0 -> 0 "\x95"; +0 -> 0 "\x96"; +0 -> 0 "\x97"; +0 -> 0 "\x98"; +0 -> 0 "\x99"; +0 -> 0 "\x9a"; +0 -> 0 "\x9b"; +0 -> 0 "\x9c"; +0 -> 0 "\x9d"; +0 -> 0 "\x9e"; +0 -> 0 "\x9f"; +0 -> 0 "\xa0"; +0 -> 0 "\xa1"; +0 -> 0 "\xa2"; +0 -> 0 "\xa3"; +0 -> 0 "\xa4"; +0 -> 0 "\xa5"; +0 -> 0 "\xa6"; +0 -> 0 "\xa7"; +0 -> 0 "\xa8"; +0 -> 0 "\xa9"; +0 -> 0 "\xaa"; +0 -> 0 "\xab"; +0 -> 0 "\xac"; +0 -> 0 "\xad"; +0 -> 0 "\xae"; +0 -> 0 "\xaf"; +0 -> 0 "\xb0"; +0 -> 0 "\xb1"; +0 -> 0 "\xb2"; +0 -> 0 "\xb3"; +0 -> 0 "\xb4"; +0 -> 0 "\xb5"; +0 -> 0 "\xb6"; +0 -> 0 "\xb7"; +0 -> 0 "\xb8"; +0 -> 0 "\xb9"; +0 -> 0 "\xba"; +0 -> 0 "\xbb"; +0 -> 0 "\xbc"; +0 -> 0 "\xbd"; +0 -> 0 "\xbe"; +0 -> 0 "\xbf"; +0 -> 0 "\xc0"; +0 -> 0 "\xc1"; +0 -> 0 "\xc2"; +0 -> 0 "\xc3"; +0 -> 0 "\xc4"; +0 -> 0 "\xc5"; +0 -> 0 "\xc6"; +0 -> 0 "\xc7"; +0 -> 0 "\xc8"; +0 -> 0 "\xc9"; +0 -> 0 "\xca"; +0 -> 0 "\xcb"; +0 -> 0 "\xcc"; +0 -> 0 "\xcd"; +0 -> 0 "\xce"; +0 -> 0 "\xcf"; +0 -> 0 "\xd0"; +0 -> 0 "\xd1"; +0 -> 0 "\xd2"; +0 -> 0 "\xd3"; +0 -> 0 "\xd4"; +0 -> 0 "\xd5"; +0 -> 0 "\xd6"; +0 -> 0 "\xd7"; +0 -> 0 "\xd8"; +0 -> 0 "\xd9"; +0 -> 0 "\xda"; +0 -> 0 "\xdb"; +0 -> 0 "\xdc"; +0 -> 0 "\xdd"; +0 -> 0 "\xde"; +0 -> 0 "\xdf"; +0 -> 0 "\xe0"; +0 -> 0 "\xe1"; +0 -> 0 "\xe2"; +0 -> 0 "\xe3"; +0 -> 0 "\xe4"; +0 -> 0 "\xe5"; +0 -> 0 "\xe6"; +0 -> 0 "\xe7"; +0 -> 0 "\xe8"; +0 -> 0 "\xe9"; +0 -> 0 "\xea"; +0 -> 0 "\xeb"; +0 -> 0 "\xec"; +0 -> 0 "\xed"; +0 -> 0 "\xee"; +0 -> 0 "\xef"; +0 -> 0 "\xf0"; +0 -> 0 "\xf1"; +0 -> 0 "\xf2"; +0 -> 0 "\xf3"; +0 -> 0 "\xf4"; +0 -> 0 "\xf5"; +0 -> 0 "\xf6"; +0 -> 0 "\xf7"; +0 -> 0 "\xf8"; +0 -> 0 "\xf9"; +0 -> 0 "\xfa"; +0 -> 0 "\xfb"; +0 -> 0 "\xfc"; +0 -> 0 "\xfd"; +0 -> 0 "\xfe"; +0 -> 0 "\xff"; +1 -> 0 "\x00"; +1 -> 0 "\x01"; +1 -> 0 "\x02"; +1 -> 0 "\x03"; +1 -> 0 "\x04"; +1 -> 0 "\x05"; +1 -> 0 "\x06"; +1 -> 0 "\x07"; +1 -> 0 "\x08"; +1 -> 0 "\t"; +1 -> 2 "\n"; +1 -> 0 "\v"; +1 -> 0 "\f"; +1 -> 0 "\r"; +1 -> 0 "\x0e"; +1 -> 0 "\x0f"; +1 -> 0 "\x10"; +1 -> 0 "\x11"; +1 -> 0 "\x12"; +1 -> 0 "\x13"; +1 -> 0 "\x14"; +1 -> 0 "\x15"; +1 -> 0 "\x16"; +1 -> 0 "\x17"; +1 -> 0 "\x18"; +1 -> 0 "\x19"; +1 -> 0 "\x1a"; +1 -> 0 "\x1b"; +1 -> 0 "\x1c"; +1 -> 0 "\x1d"; +1 -> 0 "\x1e"; +1 -> 0 "\x1f"; +1 -> 0 " "; +1 -> 0 "!"; +1 -> 0 "\""; +1 -> 0 "#"; +1 -> 0 "$"; +1 -> 0 "%"; +1 -> 0 "&"; +1 -> 0 "'"; +1 -> 0 "("; +1 -> 0 ")"; +1 -> 0 "*"; +1 -> 0 "+"; +1 -> 0 ","; +1 -> 0 "-"; +1 -> 0 "."; +1 -> 0 "/"; +1 -> 0 "0"; +1 -> 0 "1"; +1 -> 0 "2"; +1 -> 0 "3"; +1 -> 0 "4"; +1 -> 0 "5"; +1 -> 0 "6"; +1 -> 0 "7"; +1 -> 0 "8"; +1 -> 0 "9"; +1 -> 0 ":"; +1 -> 0 ";"; +1 -> 0 "<"; +1 -> 0 "="; +1 -> 0 ">"; +1 -> 0 "?"; +1 -> 0 "@"; +1 -> 0 "A"; +1 -> 0 "B"; +1 -> 0 "C"; +1 -> 0 "D"; +1 -> 0 "E"; +1 -> 0 "F"; +1 -> 0 "G"; +1 -> 0 "H"; +1 -> 0 "I"; +1 -> 0 "J"; +1 -> 0 "K"; +1 -> 0 "L"; +1 -> 0 "M"; +1 -> 0 "N"; +1 -> 0 "O"; +1 -> 0 "P"; +1 -> 0 "Q"; +1 -> 0 "R"; +1 -> 0 "S"; +1 -> 0 "T"; +1 -> 0 "U"; +1 -> 0 "V"; +1 -> 0 "W"; +1 -> 0 "X"; +1 -> 0 "Y"; +1 -> 0 "Z"; +1 -> 0 "["; +1 -> 0 "\\"; +1 -> 0 "]"; +1 -> 0 "^"; +1 -> 0 "_"; +1 -> 0 "`"; +1 -> 1 "a"; +1 -> 0 "b"; +1 -> 0 "c"; +1 -> 0 "d"; +1 -> 0 "e"; +1 -> 0 "f"; +1 -> 0 "g"; +1 -> 0 "h"; +1 -> 0 "i"; +1 -> 0 "j"; +1 -> 0 "k"; +1 -> 0 "l"; +1 -> 0 "m"; +1 -> 0 "n"; +1 -> 0 "o"; +1 -> 0 "p"; +1 -> 0 "q"; +1 -> 0 "r"; +1 -> 0 "s"; +1 -> 0 "t"; +1 -> 0 "u"; +1 -> 0 "v"; +1 -> 0 "w"; +1 -> 0 "x"; +1 -> 0 "y"; +1 -> 0 "z"; +1 -> 0 "{"; +1 -> 0 "|"; +1 -> 0 "}"; +1 -> 0 "~"; +1 -> 0 "\x7f"; +1 -> 0 "\x80"; +1 -> 0 "\x81"; +1 -> 0 "\x82"; +1 -> 0 "\x83"; +1 -> 0 "\x84"; +1 -> 0 "\x85"; +1 -> 0 "\x86"; +1 -> 0 "\x87"; +1 -> 0 "\x88"; +1 -> 0 "\x89"; +1 -> 0 "\x8a"; +1 -> 0 "\x8b"; +1 -> 0 "\x8c"; +1 -> 0 "\x8d"; +1 -> 0 "\x8e"; +1 -> 0 "\x8f"; +1 -> 0 "\x90"; +1 -> 0 "\x91"; +1 -> 0 "\x92"; +1 -> 0 "\x93"; +1 -> 0 "\x94"; +1 -> 0 "\x95"; +1 -> 0 "\x96"; +1 -> 0 "\x97"; +1 -> 0 "\x98"; +1 -> 0 "\x99"; +1 -> 0 "\x9a"; +1 -> 0 "\x9b"; +1 -> 0 "\x9c"; +1 -> 0 "\x9d"; +1 -> 0 "\x9e"; +1 -> 0 "\x9f"; +1 -> 0 "\xa0"; +1 -> 0 "\xa1"; +1 -> 0 "\xa2"; +1 -> 0 "\xa3"; +1 -> 0 "\xa4"; +1 -> 0 "\xa5"; +1 -> 0 "\xa6"; +1 -> 0 "\xa7"; +1 -> 0 "\xa8"; +1 -> 0 "\xa9"; +1 -> 0 "\xaa"; +1 -> 0 "\xab"; +1 -> 0 "\xac"; +1 -> 0 "\xad"; +1 -> 0 "\xae"; +1 -> 0 "\xaf"; +1 -> 0 "\xb0"; +1 -> 0 "\xb1"; +1 -> 0 "\xb2"; +1 -> 0 "\xb3"; +1 -> 0 "\xb4"; +1 -> 0 "\xb5"; +1 -> 0 "\xb6"; +1 -> 0 "\xb7"; +1 -> 0 "\xb8"; +1 -> 0 "\xb9"; +1 -> 0 "\xba"; +1 -> 0 "\xbb"; +1 -> 0 "\xbc"; +1 -> 0 "\xbd"; +1 -> 0 "\xbe"; +1 -> 0 "\xbf"; +1 -> 0 "\xc0"; +1 -> 0 "\xc1"; +1 -> 0 "\xc2"; +1 -> 0 "\xc3"; +1 -> 0 "\xc4"; +1 -> 0 "\xc5"; +1 -> 0 "\xc6"; +1 -> 0 "\xc7"; +1 -> 0 "\xc8"; +1 -> 0 "\xc9"; +1 -> 0 "\xca"; +1 -> 0 "\xcb"; +1 -> 0 "\xcc"; +1 -> 0 "\xcd"; +1 -> 0 "\xce"; +1 -> 0 "\xcf"; +1 -> 0 "\xd0"; +1 -> 0 "\xd1"; +1 -> 0 "\xd2"; +1 -> 0 "\xd3"; +1 -> 0 "\xd4"; +1 -> 0 "\xd5"; +1 -> 0 "\xd6"; +1 -> 0 "\xd7"; +1 -> 0 "\xd8"; +1 -> 0 "\xd9"; +1 -> 0 "\xda"; +1 -> 0 "\xdb"; +1 -> 0 "\xdc"; +1 -> 0 "\xdd"; +1 -> 0 "\xde"; +1 -> 0 "\xdf"; +1 -> 0 "\xe0"; +1 -> 0 "\xe1"; +1 -> 0 "\xe2"; +1 -> 0 "\xe3"; +1 -> 0 "\xe4"; +1 -> 0 "\xe5"; +1 -> 0 "\xe6"; +1 -> 0 "\xe7"; +1 -> 0 "\xe8"; +1 -> 0 "\xe9"; +1 -> 0 "\xea"; +1 -> 0 "\xeb"; +1 -> 0 "\xec"; +1 -> 0 "\xed"; +1 -> 0 "\xee"; +1 -> 0 "\xef"; +1 -> 0 "\xf0"; +1 -> 0 "\xf1"; +1 -> 0 "\xf2"; +1 -> 0 "\xf3"; +1 -> 0 "\xf4"; +1 -> 0 "\xf5"; +1 -> 0 "\xf6"; +1 -> 0 "\xf7"; +1 -> 0 "\xf8"; +1 -> 0 "\xf9"; +1 -> 0 "\xfa"; +1 -> 0 "\xfb"; +1 -> 0 "\xfc"; +1 -> 0 "\xfd"; +1 -> 0 "\xfe"; +1 -> 0 "\xff"; +2 -> 0 "\x00"; +2 -> 0 "\x01"; +2 -> 0 "\x02"; +2 -> 0 "\x03"; +2 -> 0 "\x04"; +2 -> 0 "\x05"; +2 -> 0 "\x06"; +2 -> 0 "\x07"; +2 -> 0 "\x08"; +2 -> 0 "\t"; +2 -> 0 "\n"; +2 -> 0 "\v"; +2 -> 0 "\f"; +2 -> 0 "\r"; +2 -> 0 "\x0e"; +2 -> 0 "\x0f"; +2 -> 0 "\x10"; +2 -> 0 "\x11"; +2 -> 0 "\x12"; +2 -> 0 "\x13"; +2 -> 0 "\x14"; +2 -> 0 "\x15"; +2 -> 0 "\x16"; +2 -> 0 "\x17"; +2 -> 0 "\x18"; +2 -> 0 "\x19"; +2 -> 0 "\x1a"; +2 -> 0 "\x1b"; +2 -> 0 "\x1c"; +2 -> 0 "\x1d"; +2 -> 0 "\x1e"; +2 -> 0 "\x1f"; +2 -> 0 " "; +2 -> 0 "!"; +2 -> 0 "\""; +2 -> 0 "#"; +2 -> 0 "$"; +2 -> 0 "%"; +2 -> 0 "&"; +2 -> 0 "'"; +2 -> 0 "("; +2 -> 0 ")"; +2 -> 0 "*"; +2 -> 0 "+"; +2 -> 0 ","; +2 -> 0 "-"; +2 -> 0 "."; +2 -> 0 "/"; +2 -> 0 "0"; +2 -> 0 "1"; +2 -> 0 "2"; +2 -> 0 "3"; +2 -> 0 "4"; +2 -> 0 "5"; +2 -> 0 "6"; +2 -> 0 "7"; +2 -> 0 "8"; +2 -> 0 "9"; +2 -> 0 ":"; +2 -> 0 ";"; +2 -> 0 "<"; +2 -> 0 "="; +2 -> 0 ">"; +2 -> 0 "?"; +2 -> 0 "@"; +2 -> 0 "A"; +2 -> 0 "B"; +2 -> 0 "C"; +2 -> 0 "D"; +2 -> 0 "E"; +2 -> 0 "F"; +2 -> 0 "G"; +2 -> 0 "H"; +2 -> 0 "I"; +2 -> 0 "J"; +2 -> 0 "K"; +2 -> 0 "L"; +2 -> 0 "M"; +2 -> 0 "N"; +2 -> 0 "O"; +2 -> 0 "P"; +2 -> 0 "Q"; +2 -> 0 "R"; +2 -> 0 "S"; +2 -> 0 "T"; +2 -> 0 "U"; +2 -> 0 "V"; +2 -> 0 "W"; +2 -> 0 "X"; +2 -> 0 "Y"; +2 -> 0 "Z"; +2 -> 0 "["; +2 -> 0 "\\"; +2 -> 0 "]"; +2 -> 0 "^"; +2 -> 0 "_"; +2 -> 0 "`"; +2 -> 1 "a"; +2 -> 0 "b"; +2 -> 0 "c"; +2 -> 0 "d"; +2 -> 0 "e"; +2 -> 0 "f"; +2 -> 0 "g"; +2 -> 0 "h"; +2 -> 0 "i"; +2 -> 0 "j"; +2 -> 0 "k"; +2 -> 0 "l"; +2 -> 0 "m"; +2 -> 0 "n"; +2 -> 0 "o"; +2 -> 0 "p"; +2 -> 0 "q"; +2 -> 0 "r"; +2 -> 0 "s"; +2 -> 0 "t"; +2 -> 0 "u"; +2 -> 0 "v"; +2 -> 0 "w"; +2 -> 0 "x"; +2 -> 0 "y"; +2 -> 0 "z"; +2 -> 0 "{"; +2 -> 0 "|"; +2 -> 0 "}"; +2 -> 0 "~"; +2 -> 0 "\x7f"; +2 -> 0 "\x80"; +2 -> 0 "\x81"; +2 -> 0 "\x82"; +2 -> 0 "\x83"; +2 -> 0 "\x84"; +2 -> 0 "\x85"; +2 -> 0 "\x86"; +2 -> 0 "\x87"; +2 -> 0 "\x88"; +2 -> 0 "\x89"; +2 -> 0 "\x8a"; +2 -> 0 "\x8b"; +2 -> 0 "\x8c"; +2 -> 0 "\x8d"; +2 -> 0 "\x8e"; +2 -> 0 "\x8f"; +2 -> 0 "\x90"; +2 -> 0 "\x91"; +2 -> 0 "\x92"; +2 -> 0 "\x93"; +2 -> 0 "\x94"; +2 -> 0 "\x95"; +2 -> 0 "\x96"; +2 -> 0 "\x97"; +2 -> 0 "\x98"; +2 -> 0 "\x99"; +2 -> 0 "\x9a"; +2 -> 0 "\x9b"; +2 -> 0 "\x9c"; +2 -> 0 "\x9d"; +2 -> 0 "\x9e"; +2 -> 0 "\x9f"; +2 -> 0 "\xa0"; +2 -> 0 "\xa1"; +2 -> 0 "\xa2"; +2 -> 0 "\xa3"; +2 -> 0 "\xa4"; +2 -> 0 "\xa5"; +2 -> 0 "\xa6"; +2 -> 0 "\xa7"; +2 -> 0 "\xa8"; +2 -> 0 "\xa9"; +2 -> 0 "\xaa"; +2 -> 0 "\xab"; +2 -> 0 "\xac"; +2 -> 0 "\xad"; +2 -> 0 "\xae"; +2 -> 0 "\xaf"; +2 -> 0 "\xb0"; +2 -> 0 "\xb1"; +2 -> 0 "\xb2"; +2 -> 0 "\xb3"; +2 -> 0 "\xb4"; +2 -> 0 "\xb5"; +2 -> 0 "\xb6"; +2 -> 0 "\xb7"; +2 -> 0 "\xb8"; +2 -> 0 "\xb9"; +2 -> 0 "\xba"; +2 -> 0 "\xbb"; +2 -> 0 "\xbc"; +2 -> 0 "\xbd"; +2 -> 0 "\xbe"; +2 -> 0 "\xbf"; +2 -> 0 "\xc0"; +2 -> 0 "\xc1"; +2 -> 0 "\xc2"; +2 -> 0 "\xc3"; +2 -> 0 "\xc4"; +2 -> 0 "\xc5"; +2 -> 0 "\xc6"; +2 -> 0 "\xc7"; +2 -> 0 "\xc8"; +2 -> 0 "\xc9"; +2 -> 0 "\xca"; +2 -> 0 "\xcb"; +2 -> 0 "\xcc"; +2 -> 0 "\xcd"; +2 -> 0 "\xce"; +2 -> 0 "\xcf"; +2 -> 0 "\xd0"; +2 -> 0 "\xd1"; +2 -> 0 "\xd2"; +2 -> 0 "\xd3"; +2 -> 0 "\xd4"; +2 -> 0 "\xd5"; +2 -> 0 "\xd6"; +2 -> 0 "\xd7"; +2 -> 0 "\xd8"; +2 -> 0 "\xd9"; +2 -> 0 "\xda"; +2 -> 0 "\xdb"; +2 -> 0 "\xdc"; +2 -> 0 "\xdd"; +2 -> 0 "\xde"; +2 -> 0 "\xdf"; +2 -> 0 "\xe0"; +2 -> 0 "\xe1"; +2 -> 0 "\xe2"; +2 -> 0 "\xe3"; +2 -> 0 "\xe4"; +2 -> 0 "\xe5"; +2 -> 0 "\xe6"; +2 -> 0 "\xe7"; +2 -> 0 "\xe8"; +2 -> 0 "\xe9"; +2 -> 0 "\xea"; +2 -> 0 "\xeb"; +2 -> 0 "\xec"; +2 -> 0 "\xed"; +2 -> 0 "\xee"; +2 -> 0 "\xef"; +2 -> 0 "\xf0"; +2 -> 0 "\xf1"; +2 -> 0 "\xf2"; +2 -> 0 "\xf3"; +2 -> 0 "\xf4"; +2 -> 0 "\xf5"; +2 -> 0 "\xf6"; +2 -> 0 "\xf7"; +2 -> 0 "\xf8"; +2 -> 0 "\xf9"; +2 -> 0 "\xfa"; +2 -> 0 "\xfb"; +2 -> 0 "\xfc"; +2 -> 0 "\xfd"; +2 -> 0 "\xfe"; +2 -> 0 "\xff"; + +start: 0; +end: 1, 2; diff --git a/tests/pcre-anchor/out96.fsm b/tests/pcre-anchor/out96.fsm new file mode 100644 index 000000000..ee435ee3c --- /dev/null +++ b/tests/pcre-anchor/out96.fsm @@ -0,0 +1,771 @@ +0 -> 0 "\x00"; +0 -> 0 "\x01"; +0 -> 0 "\x02"; +0 -> 0 "\x03"; +0 -> 0 "\x04"; +0 -> 0 "\x05"; +0 -> 0 "\x06"; +0 -> 0 "\x07"; +0 -> 0 "\x08"; +0 -> 0 "\t"; +0 -> 0 "\n"; +0 -> 0 "\v"; +0 -> 0 "\f"; +0 -> 0 "\r"; +0 -> 0 "\x0e"; +0 -> 0 "\x0f"; +0 -> 0 "\x10"; +0 -> 0 "\x11"; +0 -> 0 "\x12"; +0 -> 0 "\x13"; +0 -> 0 "\x14"; +0 -> 0 "\x15"; +0 -> 0 "\x16"; +0 -> 0 "\x17"; +0 -> 0 "\x18"; +0 -> 0 "\x19"; +0 -> 0 "\x1a"; +0 -> 0 "\x1b"; +0 -> 0 "\x1c"; +0 -> 0 "\x1d"; +0 -> 0 "\x1e"; +0 -> 0 "\x1f"; +0 -> 0 " "; +0 -> 0 "!"; +0 -> 0 "\""; +0 -> 0 "#"; +0 -> 0 "$"; +0 -> 0 "%"; +0 -> 0 "&"; +0 -> 0 "'"; +0 -> 0 "("; +0 -> 0 ")"; +0 -> 0 "*"; +0 -> 0 "+"; +0 -> 0 ","; +0 -> 0 "-"; +0 -> 0 "."; +0 -> 0 "/"; +0 -> 0 "0"; +0 -> 0 "1"; +0 -> 0 "2"; +0 -> 0 "3"; +0 -> 0 "4"; +0 -> 0 "5"; +0 -> 0 "6"; +0 -> 0 "7"; +0 -> 0 "8"; +0 -> 0 "9"; +0 -> 0 ":"; +0 -> 0 ";"; +0 -> 0 "<"; +0 -> 0 "="; +0 -> 0 ">"; +0 -> 0 "?"; +0 -> 0 "@"; +0 -> 0 "A"; +0 -> 0 "B"; +0 -> 0 "C"; +0 -> 0 "D"; +0 -> 0 "E"; +0 -> 0 "F"; +0 -> 0 "G"; +0 -> 0 "H"; +0 -> 0 "I"; +0 -> 0 "J"; +0 -> 0 "K"; +0 -> 0 "L"; +0 -> 0 "M"; +0 -> 0 "N"; +0 -> 0 "O"; +0 -> 0 "P"; +0 -> 0 "Q"; +0 -> 0 "R"; +0 -> 0 "S"; +0 -> 0 "T"; +0 -> 0 "U"; +0 -> 0 "V"; +0 -> 0 "W"; +0 -> 0 "X"; +0 -> 0 "Y"; +0 -> 0 "Z"; +0 -> 0 "["; +0 -> 0 "\\"; +0 -> 0 "]"; +0 -> 0 "^"; +0 -> 0 "_"; +0 -> 0 "`"; +0 -> 1 "a"; +0 -> 0 "b"; +0 -> 0 "c"; +0 -> 0 "d"; +0 -> 0 "e"; +0 -> 0 "f"; +0 -> 0 "g"; +0 -> 0 "h"; +0 -> 0 "i"; +0 -> 0 "j"; +0 -> 0 "k"; +0 -> 0 "l"; +0 -> 0 "m"; +0 -> 0 "n"; +0 -> 0 "o"; +0 -> 0 "p"; +0 -> 0 "q"; +0 -> 0 "r"; +0 -> 0 "s"; +0 -> 0 "t"; +0 -> 0 "u"; +0 -> 0 "v"; +0 -> 0 "w"; +0 -> 0 "x"; +0 -> 0 "y"; +0 -> 0 "z"; +0 -> 0 "{"; +0 -> 0 "|"; +0 -> 0 "}"; +0 -> 0 "~"; +0 -> 0 "\x7f"; +0 -> 0 "\x80"; +0 -> 0 "\x81"; +0 -> 0 "\x82"; +0 -> 0 "\x83"; +0 -> 0 "\x84"; +0 -> 0 "\x85"; +0 -> 0 "\x86"; +0 -> 0 "\x87"; +0 -> 0 "\x88"; +0 -> 0 "\x89"; +0 -> 0 "\x8a"; +0 -> 0 "\x8b"; +0 -> 0 "\x8c"; +0 -> 0 "\x8d"; +0 -> 0 "\x8e"; +0 -> 0 "\x8f"; +0 -> 0 "\x90"; +0 -> 0 "\x91"; +0 -> 0 "\x92"; +0 -> 0 "\x93"; +0 -> 0 "\x94"; +0 -> 0 "\x95"; +0 -> 0 "\x96"; +0 -> 0 "\x97"; +0 -> 0 "\x98"; +0 -> 0 "\x99"; +0 -> 0 "\x9a"; +0 -> 0 "\x9b"; +0 -> 0 "\x9c"; +0 -> 0 "\x9d"; +0 -> 0 "\x9e"; +0 -> 0 "\x9f"; +0 -> 0 "\xa0"; +0 -> 0 "\xa1"; +0 -> 0 "\xa2"; +0 -> 0 "\xa3"; +0 -> 0 "\xa4"; +0 -> 0 "\xa5"; +0 -> 0 "\xa6"; +0 -> 0 "\xa7"; +0 -> 0 "\xa8"; +0 -> 0 "\xa9"; +0 -> 0 "\xaa"; +0 -> 0 "\xab"; +0 -> 0 "\xac"; +0 -> 0 "\xad"; +0 -> 0 "\xae"; +0 -> 0 "\xaf"; +0 -> 0 "\xb0"; +0 -> 0 "\xb1"; +0 -> 0 "\xb2"; +0 -> 0 "\xb3"; +0 -> 0 "\xb4"; +0 -> 0 "\xb5"; +0 -> 0 "\xb6"; +0 -> 0 "\xb7"; +0 -> 0 "\xb8"; +0 -> 0 "\xb9"; +0 -> 0 "\xba"; +0 -> 0 "\xbb"; +0 -> 0 "\xbc"; +0 -> 0 "\xbd"; +0 -> 0 "\xbe"; +0 -> 0 "\xbf"; +0 -> 0 "\xc0"; +0 -> 0 "\xc1"; +0 -> 0 "\xc2"; +0 -> 0 "\xc3"; +0 -> 0 "\xc4"; +0 -> 0 "\xc5"; +0 -> 0 "\xc6"; +0 -> 0 "\xc7"; +0 -> 0 "\xc8"; +0 -> 0 "\xc9"; +0 -> 0 "\xca"; +0 -> 0 "\xcb"; +0 -> 0 "\xcc"; +0 -> 0 "\xcd"; +0 -> 0 "\xce"; +0 -> 0 "\xcf"; +0 -> 0 "\xd0"; +0 -> 0 "\xd1"; +0 -> 0 "\xd2"; +0 -> 0 "\xd3"; +0 -> 0 "\xd4"; +0 -> 0 "\xd5"; +0 -> 0 "\xd6"; +0 -> 0 "\xd7"; +0 -> 0 "\xd8"; +0 -> 0 "\xd9"; +0 -> 0 "\xda"; +0 -> 0 "\xdb"; +0 -> 0 "\xdc"; +0 -> 0 "\xdd"; +0 -> 0 "\xde"; +0 -> 0 "\xdf"; +0 -> 0 "\xe0"; +0 -> 0 "\xe1"; +0 -> 0 "\xe2"; +0 -> 0 "\xe3"; +0 -> 0 "\xe4"; +0 -> 0 "\xe5"; +0 -> 0 "\xe6"; +0 -> 0 "\xe7"; +0 -> 0 "\xe8"; +0 -> 0 "\xe9"; +0 -> 0 "\xea"; +0 -> 0 "\xeb"; +0 -> 0 "\xec"; +0 -> 0 "\xed"; +0 -> 0 "\xee"; +0 -> 0 "\xef"; +0 -> 0 "\xf0"; +0 -> 0 "\xf1"; +0 -> 0 "\xf2"; +0 -> 0 "\xf3"; +0 -> 0 "\xf4"; +0 -> 0 "\xf5"; +0 -> 0 "\xf6"; +0 -> 0 "\xf7"; +0 -> 0 "\xf8"; +0 -> 0 "\xf9"; +0 -> 0 "\xfa"; +0 -> 0 "\xfb"; +0 -> 0 "\xfc"; +0 -> 0 "\xfd"; +0 -> 0 "\xfe"; +0 -> 0 "\xff"; +1 -> 0 "\x00"; +1 -> 0 "\x01"; +1 -> 0 "\x02"; +1 -> 0 "\x03"; +1 -> 0 "\x04"; +1 -> 0 "\x05"; +1 -> 0 "\x06"; +1 -> 0 "\x07"; +1 -> 0 "\x08"; +1 -> 0 "\t"; +1 -> 2 "\n"; +1 -> 0 "\v"; +1 -> 0 "\f"; +1 -> 0 "\r"; +1 -> 0 "\x0e"; +1 -> 0 "\x0f"; +1 -> 0 "\x10"; +1 -> 0 "\x11"; +1 -> 0 "\x12"; +1 -> 0 "\x13"; +1 -> 0 "\x14"; +1 -> 0 "\x15"; +1 -> 0 "\x16"; +1 -> 0 "\x17"; +1 -> 0 "\x18"; +1 -> 0 "\x19"; +1 -> 0 "\x1a"; +1 -> 0 "\x1b"; +1 -> 0 "\x1c"; +1 -> 0 "\x1d"; +1 -> 0 "\x1e"; +1 -> 0 "\x1f"; +1 -> 0 " "; +1 -> 0 "!"; +1 -> 0 "\""; +1 -> 0 "#"; +1 -> 0 "$"; +1 -> 0 "%"; +1 -> 0 "&"; +1 -> 0 "'"; +1 -> 0 "("; +1 -> 0 ")"; +1 -> 0 "*"; +1 -> 0 "+"; +1 -> 0 ","; +1 -> 0 "-"; +1 -> 0 "."; +1 -> 0 "/"; +1 -> 0 "0"; +1 -> 0 "1"; +1 -> 0 "2"; +1 -> 0 "3"; +1 -> 0 "4"; +1 -> 0 "5"; +1 -> 0 "6"; +1 -> 0 "7"; +1 -> 0 "8"; +1 -> 0 "9"; +1 -> 0 ":"; +1 -> 0 ";"; +1 -> 0 "<"; +1 -> 0 "="; +1 -> 0 ">"; +1 -> 0 "?"; +1 -> 0 "@"; +1 -> 0 "A"; +1 -> 0 "B"; +1 -> 0 "C"; +1 -> 0 "D"; +1 -> 0 "E"; +1 -> 0 "F"; +1 -> 0 "G"; +1 -> 0 "H"; +1 -> 0 "I"; +1 -> 0 "J"; +1 -> 0 "K"; +1 -> 0 "L"; +1 -> 0 "M"; +1 -> 0 "N"; +1 -> 0 "O"; +1 -> 0 "P"; +1 -> 0 "Q"; +1 -> 0 "R"; +1 -> 0 "S"; +1 -> 0 "T"; +1 -> 0 "U"; +1 -> 0 "V"; +1 -> 0 "W"; +1 -> 0 "X"; +1 -> 0 "Y"; +1 -> 0 "Z"; +1 -> 0 "["; +1 -> 0 "\\"; +1 -> 0 "]"; +1 -> 0 "^"; +1 -> 0 "_"; +1 -> 0 "`"; +1 -> 1 "a"; +1 -> 0 "b"; +1 -> 0 "c"; +1 -> 0 "d"; +1 -> 0 "e"; +1 -> 0 "f"; +1 -> 0 "g"; +1 -> 0 "h"; +1 -> 0 "i"; +1 -> 0 "j"; +1 -> 0 "k"; +1 -> 0 "l"; +1 -> 0 "m"; +1 -> 0 "n"; +1 -> 0 "o"; +1 -> 0 "p"; +1 -> 0 "q"; +1 -> 0 "r"; +1 -> 0 "s"; +1 -> 0 "t"; +1 -> 0 "u"; +1 -> 0 "v"; +1 -> 0 "w"; +1 -> 0 "x"; +1 -> 0 "y"; +1 -> 0 "z"; +1 -> 0 "{"; +1 -> 0 "|"; +1 -> 0 "}"; +1 -> 0 "~"; +1 -> 0 "\x7f"; +1 -> 0 "\x80"; +1 -> 0 "\x81"; +1 -> 0 "\x82"; +1 -> 0 "\x83"; +1 -> 0 "\x84"; +1 -> 0 "\x85"; +1 -> 0 "\x86"; +1 -> 0 "\x87"; +1 -> 0 "\x88"; +1 -> 0 "\x89"; +1 -> 0 "\x8a"; +1 -> 0 "\x8b"; +1 -> 0 "\x8c"; +1 -> 0 "\x8d"; +1 -> 0 "\x8e"; +1 -> 0 "\x8f"; +1 -> 0 "\x90"; +1 -> 0 "\x91"; +1 -> 0 "\x92"; +1 -> 0 "\x93"; +1 -> 0 "\x94"; +1 -> 0 "\x95"; +1 -> 0 "\x96"; +1 -> 0 "\x97"; +1 -> 0 "\x98"; +1 -> 0 "\x99"; +1 -> 0 "\x9a"; +1 -> 0 "\x9b"; +1 -> 0 "\x9c"; +1 -> 0 "\x9d"; +1 -> 0 "\x9e"; +1 -> 0 "\x9f"; +1 -> 0 "\xa0"; +1 -> 0 "\xa1"; +1 -> 0 "\xa2"; +1 -> 0 "\xa3"; +1 -> 0 "\xa4"; +1 -> 0 "\xa5"; +1 -> 0 "\xa6"; +1 -> 0 "\xa7"; +1 -> 0 "\xa8"; +1 -> 0 "\xa9"; +1 -> 0 "\xaa"; +1 -> 0 "\xab"; +1 -> 0 "\xac"; +1 -> 0 "\xad"; +1 -> 0 "\xae"; +1 -> 0 "\xaf"; +1 -> 0 "\xb0"; +1 -> 0 "\xb1"; +1 -> 0 "\xb2"; +1 -> 0 "\xb3"; +1 -> 0 "\xb4"; +1 -> 0 "\xb5"; +1 -> 0 "\xb6"; +1 -> 0 "\xb7"; +1 -> 0 "\xb8"; +1 -> 0 "\xb9"; +1 -> 0 "\xba"; +1 -> 0 "\xbb"; +1 -> 0 "\xbc"; +1 -> 0 "\xbd"; +1 -> 0 "\xbe"; +1 -> 0 "\xbf"; +1 -> 0 "\xc0"; +1 -> 0 "\xc1"; +1 -> 0 "\xc2"; +1 -> 0 "\xc3"; +1 -> 0 "\xc4"; +1 -> 0 "\xc5"; +1 -> 0 "\xc6"; +1 -> 0 "\xc7"; +1 -> 0 "\xc8"; +1 -> 0 "\xc9"; +1 -> 0 "\xca"; +1 -> 0 "\xcb"; +1 -> 0 "\xcc"; +1 -> 0 "\xcd"; +1 -> 0 "\xce"; +1 -> 0 "\xcf"; +1 -> 0 "\xd0"; +1 -> 0 "\xd1"; +1 -> 0 "\xd2"; +1 -> 0 "\xd3"; +1 -> 0 "\xd4"; +1 -> 0 "\xd5"; +1 -> 0 "\xd6"; +1 -> 0 "\xd7"; +1 -> 0 "\xd8"; +1 -> 0 "\xd9"; +1 -> 0 "\xda"; +1 -> 0 "\xdb"; +1 -> 0 "\xdc"; +1 -> 0 "\xdd"; +1 -> 0 "\xde"; +1 -> 0 "\xdf"; +1 -> 0 "\xe0"; +1 -> 0 "\xe1"; +1 -> 0 "\xe2"; +1 -> 0 "\xe3"; +1 -> 0 "\xe4"; +1 -> 0 "\xe5"; +1 -> 0 "\xe6"; +1 -> 0 "\xe7"; +1 -> 0 "\xe8"; +1 -> 0 "\xe9"; +1 -> 0 "\xea"; +1 -> 0 "\xeb"; +1 -> 0 "\xec"; +1 -> 0 "\xed"; +1 -> 0 "\xee"; +1 -> 0 "\xef"; +1 -> 0 "\xf0"; +1 -> 0 "\xf1"; +1 -> 0 "\xf2"; +1 -> 0 "\xf3"; +1 -> 0 "\xf4"; +1 -> 0 "\xf5"; +1 -> 0 "\xf6"; +1 -> 0 "\xf7"; +1 -> 0 "\xf8"; +1 -> 0 "\xf9"; +1 -> 0 "\xfa"; +1 -> 0 "\xfb"; +1 -> 0 "\xfc"; +1 -> 0 "\xfd"; +1 -> 0 "\xfe"; +1 -> 0 "\xff"; +2 -> 0 "\x00"; +2 -> 0 "\x01"; +2 -> 0 "\x02"; +2 -> 0 "\x03"; +2 -> 0 "\x04"; +2 -> 0 "\x05"; +2 -> 0 "\x06"; +2 -> 0 "\x07"; +2 -> 0 "\x08"; +2 -> 0 "\t"; +2 -> 0 "\n"; +2 -> 0 "\v"; +2 -> 0 "\f"; +2 -> 0 "\r"; +2 -> 0 "\x0e"; +2 -> 0 "\x0f"; +2 -> 0 "\x10"; +2 -> 0 "\x11"; +2 -> 0 "\x12"; +2 -> 0 "\x13"; +2 -> 0 "\x14"; +2 -> 0 "\x15"; +2 -> 0 "\x16"; +2 -> 0 "\x17"; +2 -> 0 "\x18"; +2 -> 0 "\x19"; +2 -> 0 "\x1a"; +2 -> 0 "\x1b"; +2 -> 0 "\x1c"; +2 -> 0 "\x1d"; +2 -> 0 "\x1e"; +2 -> 0 "\x1f"; +2 -> 0 " "; +2 -> 0 "!"; +2 -> 0 "\""; +2 -> 0 "#"; +2 -> 0 "$"; +2 -> 0 "%"; +2 -> 0 "&"; +2 -> 0 "'"; +2 -> 0 "("; +2 -> 0 ")"; +2 -> 0 "*"; +2 -> 0 "+"; +2 -> 0 ","; +2 -> 0 "-"; +2 -> 0 "."; +2 -> 0 "/"; +2 -> 0 "0"; +2 -> 0 "1"; +2 -> 0 "2"; +2 -> 0 "3"; +2 -> 0 "4"; +2 -> 0 "5"; +2 -> 0 "6"; +2 -> 0 "7"; +2 -> 0 "8"; +2 -> 0 "9"; +2 -> 0 ":"; +2 -> 0 ";"; +2 -> 0 "<"; +2 -> 0 "="; +2 -> 0 ">"; +2 -> 0 "?"; +2 -> 0 "@"; +2 -> 0 "A"; +2 -> 0 "B"; +2 -> 0 "C"; +2 -> 0 "D"; +2 -> 0 "E"; +2 -> 0 "F"; +2 -> 0 "G"; +2 -> 0 "H"; +2 -> 0 "I"; +2 -> 0 "J"; +2 -> 0 "K"; +2 -> 0 "L"; +2 -> 0 "M"; +2 -> 0 "N"; +2 -> 0 "O"; +2 -> 0 "P"; +2 -> 0 "Q"; +2 -> 0 "R"; +2 -> 0 "S"; +2 -> 0 "T"; +2 -> 0 "U"; +2 -> 0 "V"; +2 -> 0 "W"; +2 -> 0 "X"; +2 -> 0 "Y"; +2 -> 0 "Z"; +2 -> 0 "["; +2 -> 0 "\\"; +2 -> 0 "]"; +2 -> 0 "^"; +2 -> 0 "_"; +2 -> 0 "`"; +2 -> 1 "a"; +2 -> 0 "b"; +2 -> 0 "c"; +2 -> 0 "d"; +2 -> 0 "e"; +2 -> 0 "f"; +2 -> 0 "g"; +2 -> 0 "h"; +2 -> 0 "i"; +2 -> 0 "j"; +2 -> 0 "k"; +2 -> 0 "l"; +2 -> 0 "m"; +2 -> 0 "n"; +2 -> 0 "o"; +2 -> 0 "p"; +2 -> 0 "q"; +2 -> 0 "r"; +2 -> 0 "s"; +2 -> 0 "t"; +2 -> 0 "u"; +2 -> 0 "v"; +2 -> 0 "w"; +2 -> 0 "x"; +2 -> 0 "y"; +2 -> 0 "z"; +2 -> 0 "{"; +2 -> 0 "|"; +2 -> 0 "}"; +2 -> 0 "~"; +2 -> 0 "\x7f"; +2 -> 0 "\x80"; +2 -> 0 "\x81"; +2 -> 0 "\x82"; +2 -> 0 "\x83"; +2 -> 0 "\x84"; +2 -> 0 "\x85"; +2 -> 0 "\x86"; +2 -> 0 "\x87"; +2 -> 0 "\x88"; +2 -> 0 "\x89"; +2 -> 0 "\x8a"; +2 -> 0 "\x8b"; +2 -> 0 "\x8c"; +2 -> 0 "\x8d"; +2 -> 0 "\x8e"; +2 -> 0 "\x8f"; +2 -> 0 "\x90"; +2 -> 0 "\x91"; +2 -> 0 "\x92"; +2 -> 0 "\x93"; +2 -> 0 "\x94"; +2 -> 0 "\x95"; +2 -> 0 "\x96"; +2 -> 0 "\x97"; +2 -> 0 "\x98"; +2 -> 0 "\x99"; +2 -> 0 "\x9a"; +2 -> 0 "\x9b"; +2 -> 0 "\x9c"; +2 -> 0 "\x9d"; +2 -> 0 "\x9e"; +2 -> 0 "\x9f"; +2 -> 0 "\xa0"; +2 -> 0 "\xa1"; +2 -> 0 "\xa2"; +2 -> 0 "\xa3"; +2 -> 0 "\xa4"; +2 -> 0 "\xa5"; +2 -> 0 "\xa6"; +2 -> 0 "\xa7"; +2 -> 0 "\xa8"; +2 -> 0 "\xa9"; +2 -> 0 "\xaa"; +2 -> 0 "\xab"; +2 -> 0 "\xac"; +2 -> 0 "\xad"; +2 -> 0 "\xae"; +2 -> 0 "\xaf"; +2 -> 0 "\xb0"; +2 -> 0 "\xb1"; +2 -> 0 "\xb2"; +2 -> 0 "\xb3"; +2 -> 0 "\xb4"; +2 -> 0 "\xb5"; +2 -> 0 "\xb6"; +2 -> 0 "\xb7"; +2 -> 0 "\xb8"; +2 -> 0 "\xb9"; +2 -> 0 "\xba"; +2 -> 0 "\xbb"; +2 -> 0 "\xbc"; +2 -> 0 "\xbd"; +2 -> 0 "\xbe"; +2 -> 0 "\xbf"; +2 -> 0 "\xc0"; +2 -> 0 "\xc1"; +2 -> 0 "\xc2"; +2 -> 0 "\xc3"; +2 -> 0 "\xc4"; +2 -> 0 "\xc5"; +2 -> 0 "\xc6"; +2 -> 0 "\xc7"; +2 -> 0 "\xc8"; +2 -> 0 "\xc9"; +2 -> 0 "\xca"; +2 -> 0 "\xcb"; +2 -> 0 "\xcc"; +2 -> 0 "\xcd"; +2 -> 0 "\xce"; +2 -> 0 "\xcf"; +2 -> 0 "\xd0"; +2 -> 0 "\xd1"; +2 -> 0 "\xd2"; +2 -> 0 "\xd3"; +2 -> 0 "\xd4"; +2 -> 0 "\xd5"; +2 -> 0 "\xd6"; +2 -> 0 "\xd7"; +2 -> 0 "\xd8"; +2 -> 0 "\xd9"; +2 -> 0 "\xda"; +2 -> 0 "\xdb"; +2 -> 0 "\xdc"; +2 -> 0 "\xdd"; +2 -> 0 "\xde"; +2 -> 0 "\xdf"; +2 -> 0 "\xe0"; +2 -> 0 "\xe1"; +2 -> 0 "\xe2"; +2 -> 0 "\xe3"; +2 -> 0 "\xe4"; +2 -> 0 "\xe5"; +2 -> 0 "\xe6"; +2 -> 0 "\xe7"; +2 -> 0 "\xe8"; +2 -> 0 "\xe9"; +2 -> 0 "\xea"; +2 -> 0 "\xeb"; +2 -> 0 "\xec"; +2 -> 0 "\xed"; +2 -> 0 "\xee"; +2 -> 0 "\xef"; +2 -> 0 "\xf0"; +2 -> 0 "\xf1"; +2 -> 0 "\xf2"; +2 -> 0 "\xf3"; +2 -> 0 "\xf4"; +2 -> 0 "\xf5"; +2 -> 0 "\xf6"; +2 -> 0 "\xf7"; +2 -> 0 "\xf8"; +2 -> 0 "\xf9"; +2 -> 0 "\xfa"; +2 -> 0 "\xfb"; +2 -> 0 "\xfc"; +2 -> 0 "\xfd"; +2 -> 0 "\xfe"; +2 -> 0 "\xff"; + +start: 0; +end: 1, 2; diff --git a/tests/pcre-anchor/out97.fsm b/tests/pcre-anchor/out97.fsm new file mode 100644 index 000000000..1e6bf8fca --- /dev/null +++ b/tests/pcre-anchor/out97.fsm @@ -0,0 +1,5 @@ +0 -> 1 "a"; +1 -> 2 "\n"; + +start: 0; +end: 1, 2; diff --git a/tests/pcre-anchor/out98.fsm b/tests/pcre-anchor/out98.fsm new file mode 100644 index 000000000..ee435ee3c --- /dev/null +++ b/tests/pcre-anchor/out98.fsm @@ -0,0 +1,771 @@ +0 -> 0 "\x00"; +0 -> 0 "\x01"; +0 -> 0 "\x02"; +0 -> 0 "\x03"; +0 -> 0 "\x04"; +0 -> 0 "\x05"; +0 -> 0 "\x06"; +0 -> 0 "\x07"; +0 -> 0 "\x08"; +0 -> 0 "\t"; +0 -> 0 "\n"; +0 -> 0 "\v"; +0 -> 0 "\f"; +0 -> 0 "\r"; +0 -> 0 "\x0e"; +0 -> 0 "\x0f"; +0 -> 0 "\x10"; +0 -> 0 "\x11"; +0 -> 0 "\x12"; +0 -> 0 "\x13"; +0 -> 0 "\x14"; +0 -> 0 "\x15"; +0 -> 0 "\x16"; +0 -> 0 "\x17"; +0 -> 0 "\x18"; +0 -> 0 "\x19"; +0 -> 0 "\x1a"; +0 -> 0 "\x1b"; +0 -> 0 "\x1c"; +0 -> 0 "\x1d"; +0 -> 0 "\x1e"; +0 -> 0 "\x1f"; +0 -> 0 " "; +0 -> 0 "!"; +0 -> 0 "\""; +0 -> 0 "#"; +0 -> 0 "$"; +0 -> 0 "%"; +0 -> 0 "&"; +0 -> 0 "'"; +0 -> 0 "("; +0 -> 0 ")"; +0 -> 0 "*"; +0 -> 0 "+"; +0 -> 0 ","; +0 -> 0 "-"; +0 -> 0 "."; +0 -> 0 "/"; +0 -> 0 "0"; +0 -> 0 "1"; +0 -> 0 "2"; +0 -> 0 "3"; +0 -> 0 "4"; +0 -> 0 "5"; +0 -> 0 "6"; +0 -> 0 "7"; +0 -> 0 "8"; +0 -> 0 "9"; +0 -> 0 ":"; +0 -> 0 ";"; +0 -> 0 "<"; +0 -> 0 "="; +0 -> 0 ">"; +0 -> 0 "?"; +0 -> 0 "@"; +0 -> 0 "A"; +0 -> 0 "B"; +0 -> 0 "C"; +0 -> 0 "D"; +0 -> 0 "E"; +0 -> 0 "F"; +0 -> 0 "G"; +0 -> 0 "H"; +0 -> 0 "I"; +0 -> 0 "J"; +0 -> 0 "K"; +0 -> 0 "L"; +0 -> 0 "M"; +0 -> 0 "N"; +0 -> 0 "O"; +0 -> 0 "P"; +0 -> 0 "Q"; +0 -> 0 "R"; +0 -> 0 "S"; +0 -> 0 "T"; +0 -> 0 "U"; +0 -> 0 "V"; +0 -> 0 "W"; +0 -> 0 "X"; +0 -> 0 "Y"; +0 -> 0 "Z"; +0 -> 0 "["; +0 -> 0 "\\"; +0 -> 0 "]"; +0 -> 0 "^"; +0 -> 0 "_"; +0 -> 0 "`"; +0 -> 1 "a"; +0 -> 0 "b"; +0 -> 0 "c"; +0 -> 0 "d"; +0 -> 0 "e"; +0 -> 0 "f"; +0 -> 0 "g"; +0 -> 0 "h"; +0 -> 0 "i"; +0 -> 0 "j"; +0 -> 0 "k"; +0 -> 0 "l"; +0 -> 0 "m"; +0 -> 0 "n"; +0 -> 0 "o"; +0 -> 0 "p"; +0 -> 0 "q"; +0 -> 0 "r"; +0 -> 0 "s"; +0 -> 0 "t"; +0 -> 0 "u"; +0 -> 0 "v"; +0 -> 0 "w"; +0 -> 0 "x"; +0 -> 0 "y"; +0 -> 0 "z"; +0 -> 0 "{"; +0 -> 0 "|"; +0 -> 0 "}"; +0 -> 0 "~"; +0 -> 0 "\x7f"; +0 -> 0 "\x80"; +0 -> 0 "\x81"; +0 -> 0 "\x82"; +0 -> 0 "\x83"; +0 -> 0 "\x84"; +0 -> 0 "\x85"; +0 -> 0 "\x86"; +0 -> 0 "\x87"; +0 -> 0 "\x88"; +0 -> 0 "\x89"; +0 -> 0 "\x8a"; +0 -> 0 "\x8b"; +0 -> 0 "\x8c"; +0 -> 0 "\x8d"; +0 -> 0 "\x8e"; +0 -> 0 "\x8f"; +0 -> 0 "\x90"; +0 -> 0 "\x91"; +0 -> 0 "\x92"; +0 -> 0 "\x93"; +0 -> 0 "\x94"; +0 -> 0 "\x95"; +0 -> 0 "\x96"; +0 -> 0 "\x97"; +0 -> 0 "\x98"; +0 -> 0 "\x99"; +0 -> 0 "\x9a"; +0 -> 0 "\x9b"; +0 -> 0 "\x9c"; +0 -> 0 "\x9d"; +0 -> 0 "\x9e"; +0 -> 0 "\x9f"; +0 -> 0 "\xa0"; +0 -> 0 "\xa1"; +0 -> 0 "\xa2"; +0 -> 0 "\xa3"; +0 -> 0 "\xa4"; +0 -> 0 "\xa5"; +0 -> 0 "\xa6"; +0 -> 0 "\xa7"; +0 -> 0 "\xa8"; +0 -> 0 "\xa9"; +0 -> 0 "\xaa"; +0 -> 0 "\xab"; +0 -> 0 "\xac"; +0 -> 0 "\xad"; +0 -> 0 "\xae"; +0 -> 0 "\xaf"; +0 -> 0 "\xb0"; +0 -> 0 "\xb1"; +0 -> 0 "\xb2"; +0 -> 0 "\xb3"; +0 -> 0 "\xb4"; +0 -> 0 "\xb5"; +0 -> 0 "\xb6"; +0 -> 0 "\xb7"; +0 -> 0 "\xb8"; +0 -> 0 "\xb9"; +0 -> 0 "\xba"; +0 -> 0 "\xbb"; +0 -> 0 "\xbc"; +0 -> 0 "\xbd"; +0 -> 0 "\xbe"; +0 -> 0 "\xbf"; +0 -> 0 "\xc0"; +0 -> 0 "\xc1"; +0 -> 0 "\xc2"; +0 -> 0 "\xc3"; +0 -> 0 "\xc4"; +0 -> 0 "\xc5"; +0 -> 0 "\xc6"; +0 -> 0 "\xc7"; +0 -> 0 "\xc8"; +0 -> 0 "\xc9"; +0 -> 0 "\xca"; +0 -> 0 "\xcb"; +0 -> 0 "\xcc"; +0 -> 0 "\xcd"; +0 -> 0 "\xce"; +0 -> 0 "\xcf"; +0 -> 0 "\xd0"; +0 -> 0 "\xd1"; +0 -> 0 "\xd2"; +0 -> 0 "\xd3"; +0 -> 0 "\xd4"; +0 -> 0 "\xd5"; +0 -> 0 "\xd6"; +0 -> 0 "\xd7"; +0 -> 0 "\xd8"; +0 -> 0 "\xd9"; +0 -> 0 "\xda"; +0 -> 0 "\xdb"; +0 -> 0 "\xdc"; +0 -> 0 "\xdd"; +0 -> 0 "\xde"; +0 -> 0 "\xdf"; +0 -> 0 "\xe0"; +0 -> 0 "\xe1"; +0 -> 0 "\xe2"; +0 -> 0 "\xe3"; +0 -> 0 "\xe4"; +0 -> 0 "\xe5"; +0 -> 0 "\xe6"; +0 -> 0 "\xe7"; +0 -> 0 "\xe8"; +0 -> 0 "\xe9"; +0 -> 0 "\xea"; +0 -> 0 "\xeb"; +0 -> 0 "\xec"; +0 -> 0 "\xed"; +0 -> 0 "\xee"; +0 -> 0 "\xef"; +0 -> 0 "\xf0"; +0 -> 0 "\xf1"; +0 -> 0 "\xf2"; +0 -> 0 "\xf3"; +0 -> 0 "\xf4"; +0 -> 0 "\xf5"; +0 -> 0 "\xf6"; +0 -> 0 "\xf7"; +0 -> 0 "\xf8"; +0 -> 0 "\xf9"; +0 -> 0 "\xfa"; +0 -> 0 "\xfb"; +0 -> 0 "\xfc"; +0 -> 0 "\xfd"; +0 -> 0 "\xfe"; +0 -> 0 "\xff"; +1 -> 0 "\x00"; +1 -> 0 "\x01"; +1 -> 0 "\x02"; +1 -> 0 "\x03"; +1 -> 0 "\x04"; +1 -> 0 "\x05"; +1 -> 0 "\x06"; +1 -> 0 "\x07"; +1 -> 0 "\x08"; +1 -> 0 "\t"; +1 -> 2 "\n"; +1 -> 0 "\v"; +1 -> 0 "\f"; +1 -> 0 "\r"; +1 -> 0 "\x0e"; +1 -> 0 "\x0f"; +1 -> 0 "\x10"; +1 -> 0 "\x11"; +1 -> 0 "\x12"; +1 -> 0 "\x13"; +1 -> 0 "\x14"; +1 -> 0 "\x15"; +1 -> 0 "\x16"; +1 -> 0 "\x17"; +1 -> 0 "\x18"; +1 -> 0 "\x19"; +1 -> 0 "\x1a"; +1 -> 0 "\x1b"; +1 -> 0 "\x1c"; +1 -> 0 "\x1d"; +1 -> 0 "\x1e"; +1 -> 0 "\x1f"; +1 -> 0 " "; +1 -> 0 "!"; +1 -> 0 "\""; +1 -> 0 "#"; +1 -> 0 "$"; +1 -> 0 "%"; +1 -> 0 "&"; +1 -> 0 "'"; +1 -> 0 "("; +1 -> 0 ")"; +1 -> 0 "*"; +1 -> 0 "+"; +1 -> 0 ","; +1 -> 0 "-"; +1 -> 0 "."; +1 -> 0 "/"; +1 -> 0 "0"; +1 -> 0 "1"; +1 -> 0 "2"; +1 -> 0 "3"; +1 -> 0 "4"; +1 -> 0 "5"; +1 -> 0 "6"; +1 -> 0 "7"; +1 -> 0 "8"; +1 -> 0 "9"; +1 -> 0 ":"; +1 -> 0 ";"; +1 -> 0 "<"; +1 -> 0 "="; +1 -> 0 ">"; +1 -> 0 "?"; +1 -> 0 "@"; +1 -> 0 "A"; +1 -> 0 "B"; +1 -> 0 "C"; +1 -> 0 "D"; +1 -> 0 "E"; +1 -> 0 "F"; +1 -> 0 "G"; +1 -> 0 "H"; +1 -> 0 "I"; +1 -> 0 "J"; +1 -> 0 "K"; +1 -> 0 "L"; +1 -> 0 "M"; +1 -> 0 "N"; +1 -> 0 "O"; +1 -> 0 "P"; +1 -> 0 "Q"; +1 -> 0 "R"; +1 -> 0 "S"; +1 -> 0 "T"; +1 -> 0 "U"; +1 -> 0 "V"; +1 -> 0 "W"; +1 -> 0 "X"; +1 -> 0 "Y"; +1 -> 0 "Z"; +1 -> 0 "["; +1 -> 0 "\\"; +1 -> 0 "]"; +1 -> 0 "^"; +1 -> 0 "_"; +1 -> 0 "`"; +1 -> 1 "a"; +1 -> 0 "b"; +1 -> 0 "c"; +1 -> 0 "d"; +1 -> 0 "e"; +1 -> 0 "f"; +1 -> 0 "g"; +1 -> 0 "h"; +1 -> 0 "i"; +1 -> 0 "j"; +1 -> 0 "k"; +1 -> 0 "l"; +1 -> 0 "m"; +1 -> 0 "n"; +1 -> 0 "o"; +1 -> 0 "p"; +1 -> 0 "q"; +1 -> 0 "r"; +1 -> 0 "s"; +1 -> 0 "t"; +1 -> 0 "u"; +1 -> 0 "v"; +1 -> 0 "w"; +1 -> 0 "x"; +1 -> 0 "y"; +1 -> 0 "z"; +1 -> 0 "{"; +1 -> 0 "|"; +1 -> 0 "}"; +1 -> 0 "~"; +1 -> 0 "\x7f"; +1 -> 0 "\x80"; +1 -> 0 "\x81"; +1 -> 0 "\x82"; +1 -> 0 "\x83"; +1 -> 0 "\x84"; +1 -> 0 "\x85"; +1 -> 0 "\x86"; +1 -> 0 "\x87"; +1 -> 0 "\x88"; +1 -> 0 "\x89"; +1 -> 0 "\x8a"; +1 -> 0 "\x8b"; +1 -> 0 "\x8c"; +1 -> 0 "\x8d"; +1 -> 0 "\x8e"; +1 -> 0 "\x8f"; +1 -> 0 "\x90"; +1 -> 0 "\x91"; +1 -> 0 "\x92"; +1 -> 0 "\x93"; +1 -> 0 "\x94"; +1 -> 0 "\x95"; +1 -> 0 "\x96"; +1 -> 0 "\x97"; +1 -> 0 "\x98"; +1 -> 0 "\x99"; +1 -> 0 "\x9a"; +1 -> 0 "\x9b"; +1 -> 0 "\x9c"; +1 -> 0 "\x9d"; +1 -> 0 "\x9e"; +1 -> 0 "\x9f"; +1 -> 0 "\xa0"; +1 -> 0 "\xa1"; +1 -> 0 "\xa2"; +1 -> 0 "\xa3"; +1 -> 0 "\xa4"; +1 -> 0 "\xa5"; +1 -> 0 "\xa6"; +1 -> 0 "\xa7"; +1 -> 0 "\xa8"; +1 -> 0 "\xa9"; +1 -> 0 "\xaa"; +1 -> 0 "\xab"; +1 -> 0 "\xac"; +1 -> 0 "\xad"; +1 -> 0 "\xae"; +1 -> 0 "\xaf"; +1 -> 0 "\xb0"; +1 -> 0 "\xb1"; +1 -> 0 "\xb2"; +1 -> 0 "\xb3"; +1 -> 0 "\xb4"; +1 -> 0 "\xb5"; +1 -> 0 "\xb6"; +1 -> 0 "\xb7"; +1 -> 0 "\xb8"; +1 -> 0 "\xb9"; +1 -> 0 "\xba"; +1 -> 0 "\xbb"; +1 -> 0 "\xbc"; +1 -> 0 "\xbd"; +1 -> 0 "\xbe"; +1 -> 0 "\xbf"; +1 -> 0 "\xc0"; +1 -> 0 "\xc1"; +1 -> 0 "\xc2"; +1 -> 0 "\xc3"; +1 -> 0 "\xc4"; +1 -> 0 "\xc5"; +1 -> 0 "\xc6"; +1 -> 0 "\xc7"; +1 -> 0 "\xc8"; +1 -> 0 "\xc9"; +1 -> 0 "\xca"; +1 -> 0 "\xcb"; +1 -> 0 "\xcc"; +1 -> 0 "\xcd"; +1 -> 0 "\xce"; +1 -> 0 "\xcf"; +1 -> 0 "\xd0"; +1 -> 0 "\xd1"; +1 -> 0 "\xd2"; +1 -> 0 "\xd3"; +1 -> 0 "\xd4"; +1 -> 0 "\xd5"; +1 -> 0 "\xd6"; +1 -> 0 "\xd7"; +1 -> 0 "\xd8"; +1 -> 0 "\xd9"; +1 -> 0 "\xda"; +1 -> 0 "\xdb"; +1 -> 0 "\xdc"; +1 -> 0 "\xdd"; +1 -> 0 "\xde"; +1 -> 0 "\xdf"; +1 -> 0 "\xe0"; +1 -> 0 "\xe1"; +1 -> 0 "\xe2"; +1 -> 0 "\xe3"; +1 -> 0 "\xe4"; +1 -> 0 "\xe5"; +1 -> 0 "\xe6"; +1 -> 0 "\xe7"; +1 -> 0 "\xe8"; +1 -> 0 "\xe9"; +1 -> 0 "\xea"; +1 -> 0 "\xeb"; +1 -> 0 "\xec"; +1 -> 0 "\xed"; +1 -> 0 "\xee"; +1 -> 0 "\xef"; +1 -> 0 "\xf0"; +1 -> 0 "\xf1"; +1 -> 0 "\xf2"; +1 -> 0 "\xf3"; +1 -> 0 "\xf4"; +1 -> 0 "\xf5"; +1 -> 0 "\xf6"; +1 -> 0 "\xf7"; +1 -> 0 "\xf8"; +1 -> 0 "\xf9"; +1 -> 0 "\xfa"; +1 -> 0 "\xfb"; +1 -> 0 "\xfc"; +1 -> 0 "\xfd"; +1 -> 0 "\xfe"; +1 -> 0 "\xff"; +2 -> 0 "\x00"; +2 -> 0 "\x01"; +2 -> 0 "\x02"; +2 -> 0 "\x03"; +2 -> 0 "\x04"; +2 -> 0 "\x05"; +2 -> 0 "\x06"; +2 -> 0 "\x07"; +2 -> 0 "\x08"; +2 -> 0 "\t"; +2 -> 0 "\n"; +2 -> 0 "\v"; +2 -> 0 "\f"; +2 -> 0 "\r"; +2 -> 0 "\x0e"; +2 -> 0 "\x0f"; +2 -> 0 "\x10"; +2 -> 0 "\x11"; +2 -> 0 "\x12"; +2 -> 0 "\x13"; +2 -> 0 "\x14"; +2 -> 0 "\x15"; +2 -> 0 "\x16"; +2 -> 0 "\x17"; +2 -> 0 "\x18"; +2 -> 0 "\x19"; +2 -> 0 "\x1a"; +2 -> 0 "\x1b"; +2 -> 0 "\x1c"; +2 -> 0 "\x1d"; +2 -> 0 "\x1e"; +2 -> 0 "\x1f"; +2 -> 0 " "; +2 -> 0 "!"; +2 -> 0 "\""; +2 -> 0 "#"; +2 -> 0 "$"; +2 -> 0 "%"; +2 -> 0 "&"; +2 -> 0 "'"; +2 -> 0 "("; +2 -> 0 ")"; +2 -> 0 "*"; +2 -> 0 "+"; +2 -> 0 ","; +2 -> 0 "-"; +2 -> 0 "."; +2 -> 0 "/"; +2 -> 0 "0"; +2 -> 0 "1"; +2 -> 0 "2"; +2 -> 0 "3"; +2 -> 0 "4"; +2 -> 0 "5"; +2 -> 0 "6"; +2 -> 0 "7"; +2 -> 0 "8"; +2 -> 0 "9"; +2 -> 0 ":"; +2 -> 0 ";"; +2 -> 0 "<"; +2 -> 0 "="; +2 -> 0 ">"; +2 -> 0 "?"; +2 -> 0 "@"; +2 -> 0 "A"; +2 -> 0 "B"; +2 -> 0 "C"; +2 -> 0 "D"; +2 -> 0 "E"; +2 -> 0 "F"; +2 -> 0 "G"; +2 -> 0 "H"; +2 -> 0 "I"; +2 -> 0 "J"; +2 -> 0 "K"; +2 -> 0 "L"; +2 -> 0 "M"; +2 -> 0 "N"; +2 -> 0 "O"; +2 -> 0 "P"; +2 -> 0 "Q"; +2 -> 0 "R"; +2 -> 0 "S"; +2 -> 0 "T"; +2 -> 0 "U"; +2 -> 0 "V"; +2 -> 0 "W"; +2 -> 0 "X"; +2 -> 0 "Y"; +2 -> 0 "Z"; +2 -> 0 "["; +2 -> 0 "\\"; +2 -> 0 "]"; +2 -> 0 "^"; +2 -> 0 "_"; +2 -> 0 "`"; +2 -> 1 "a"; +2 -> 0 "b"; +2 -> 0 "c"; +2 -> 0 "d"; +2 -> 0 "e"; +2 -> 0 "f"; +2 -> 0 "g"; +2 -> 0 "h"; +2 -> 0 "i"; +2 -> 0 "j"; +2 -> 0 "k"; +2 -> 0 "l"; +2 -> 0 "m"; +2 -> 0 "n"; +2 -> 0 "o"; +2 -> 0 "p"; +2 -> 0 "q"; +2 -> 0 "r"; +2 -> 0 "s"; +2 -> 0 "t"; +2 -> 0 "u"; +2 -> 0 "v"; +2 -> 0 "w"; +2 -> 0 "x"; +2 -> 0 "y"; +2 -> 0 "z"; +2 -> 0 "{"; +2 -> 0 "|"; +2 -> 0 "}"; +2 -> 0 "~"; +2 -> 0 "\x7f"; +2 -> 0 "\x80"; +2 -> 0 "\x81"; +2 -> 0 "\x82"; +2 -> 0 "\x83"; +2 -> 0 "\x84"; +2 -> 0 "\x85"; +2 -> 0 "\x86"; +2 -> 0 "\x87"; +2 -> 0 "\x88"; +2 -> 0 "\x89"; +2 -> 0 "\x8a"; +2 -> 0 "\x8b"; +2 -> 0 "\x8c"; +2 -> 0 "\x8d"; +2 -> 0 "\x8e"; +2 -> 0 "\x8f"; +2 -> 0 "\x90"; +2 -> 0 "\x91"; +2 -> 0 "\x92"; +2 -> 0 "\x93"; +2 -> 0 "\x94"; +2 -> 0 "\x95"; +2 -> 0 "\x96"; +2 -> 0 "\x97"; +2 -> 0 "\x98"; +2 -> 0 "\x99"; +2 -> 0 "\x9a"; +2 -> 0 "\x9b"; +2 -> 0 "\x9c"; +2 -> 0 "\x9d"; +2 -> 0 "\x9e"; +2 -> 0 "\x9f"; +2 -> 0 "\xa0"; +2 -> 0 "\xa1"; +2 -> 0 "\xa2"; +2 -> 0 "\xa3"; +2 -> 0 "\xa4"; +2 -> 0 "\xa5"; +2 -> 0 "\xa6"; +2 -> 0 "\xa7"; +2 -> 0 "\xa8"; +2 -> 0 "\xa9"; +2 -> 0 "\xaa"; +2 -> 0 "\xab"; +2 -> 0 "\xac"; +2 -> 0 "\xad"; +2 -> 0 "\xae"; +2 -> 0 "\xaf"; +2 -> 0 "\xb0"; +2 -> 0 "\xb1"; +2 -> 0 "\xb2"; +2 -> 0 "\xb3"; +2 -> 0 "\xb4"; +2 -> 0 "\xb5"; +2 -> 0 "\xb6"; +2 -> 0 "\xb7"; +2 -> 0 "\xb8"; +2 -> 0 "\xb9"; +2 -> 0 "\xba"; +2 -> 0 "\xbb"; +2 -> 0 "\xbc"; +2 -> 0 "\xbd"; +2 -> 0 "\xbe"; +2 -> 0 "\xbf"; +2 -> 0 "\xc0"; +2 -> 0 "\xc1"; +2 -> 0 "\xc2"; +2 -> 0 "\xc3"; +2 -> 0 "\xc4"; +2 -> 0 "\xc5"; +2 -> 0 "\xc6"; +2 -> 0 "\xc7"; +2 -> 0 "\xc8"; +2 -> 0 "\xc9"; +2 -> 0 "\xca"; +2 -> 0 "\xcb"; +2 -> 0 "\xcc"; +2 -> 0 "\xcd"; +2 -> 0 "\xce"; +2 -> 0 "\xcf"; +2 -> 0 "\xd0"; +2 -> 0 "\xd1"; +2 -> 0 "\xd2"; +2 -> 0 "\xd3"; +2 -> 0 "\xd4"; +2 -> 0 "\xd5"; +2 -> 0 "\xd6"; +2 -> 0 "\xd7"; +2 -> 0 "\xd8"; +2 -> 0 "\xd9"; +2 -> 0 "\xda"; +2 -> 0 "\xdb"; +2 -> 0 "\xdc"; +2 -> 0 "\xdd"; +2 -> 0 "\xde"; +2 -> 0 "\xdf"; +2 -> 0 "\xe0"; +2 -> 0 "\xe1"; +2 -> 0 "\xe2"; +2 -> 0 "\xe3"; +2 -> 0 "\xe4"; +2 -> 0 "\xe5"; +2 -> 0 "\xe6"; +2 -> 0 "\xe7"; +2 -> 0 "\xe8"; +2 -> 0 "\xe9"; +2 -> 0 "\xea"; +2 -> 0 "\xeb"; +2 -> 0 "\xec"; +2 -> 0 "\xed"; +2 -> 0 "\xee"; +2 -> 0 "\xef"; +2 -> 0 "\xf0"; +2 -> 0 "\xf1"; +2 -> 0 "\xf2"; +2 -> 0 "\xf3"; +2 -> 0 "\xf4"; +2 -> 0 "\xf5"; +2 -> 0 "\xf6"; +2 -> 0 "\xf7"; +2 -> 0 "\xf8"; +2 -> 0 "\xf9"; +2 -> 0 "\xfa"; +2 -> 0 "\xfb"; +2 -> 0 "\xfc"; +2 -> 0 "\xfd"; +2 -> 0 "\xfe"; +2 -> 0 "\xff"; + +start: 0; +end: 1, 2; diff --git a/tests/pcre-anchor/out99.fsm b/tests/pcre-anchor/out99.fsm new file mode 100644 index 000000000..43ecc6bce --- /dev/null +++ b/tests/pcre-anchor/out99.fsm @@ -0,0 +1,8 @@ +# '^(?:$^|a)(?:$^|^)*' is effectively the same as '^(?:$^|a)'. + +0 -> 2 "\n"; +0 -> 1 "a"; +1 -> 1 ?; + +start: 0; +end: 0, 1, 2; diff --git a/tests/pcre-classes/Makefile b/tests/pcre-classes/Makefile index 0d9809d76..0d459e256 100755 --- a/tests/pcre-classes/Makefile +++ b/tests/pcre-classes/Makefile @@ -16,7 +16,7 @@ RE=${BUILD}/bin/re FSM=${BUILD}/bin/fsm ${TEST_OUTDIR.tests/pcre-classes}/dot-all.fsm: - ${RE} -r pcre -p '^[\x00-\xff]$$' | ${FSM} -pm \ + ${RE} -FC -r pcre -p '^[\x00-\xff]$$' | ${FSM} -pm \ > $@ # compl.re tests @@ -32,7 +32,7 @@ ${TEST_OUTDIR.tests/pcre-classes}/dot-all.fsm: .for n in ${TEST.tests/pcre-classes:M*/compl*.re:T:R:C/^compl//} ${TEST_OUTDIR.tests/pcre-classes}/got${n}.fsm: ${TEST_SRCDIR.tests/pcre-classes}/in${n}.re - ${RE} -r pcre -py ${.ALLSRC:M*/in*.re} | ${FSM} -pm \ + ${RE} -FC -r pcre -py ${.ALLSRC:M*/in*.re} | ${FSM} -pm \ > $@ ${TEST_OUTDIR.tests/pcre-classes}/got-compl${n}.fsm: ${TEST_OUTDIR.tests/pcre-classes}/got${n}.fsm ${TEST_OUTDIR.tests/pcre-classes}/dot-all.fsm @@ -40,7 +40,7 @@ ${TEST_OUTDIR.tests/pcre-classes}/got-compl${n}.fsm: ${TEST_OUTDIR.tests/pcre-cl > $@ ${TEST_OUTDIR.tests/pcre-classes}/expect-compl${n}.fsm: ${TEST_SRCDIR.tests/pcre-classes}/compl${n}.re - ${RE} -r pcre -py ${.ALLSRC:M*/compl*.re} | ${FSM} -pm \ + ${RE} -FC -r pcre -py ${.ALLSRC:M*/compl*.re} | ${FSM} -pm \ > $@ ${TEST_OUTDIR.tests/pcre-classes}/res${n}: \ @@ -66,11 +66,11 @@ FSMTEST_RESULT += ${TEST_OUTDIR.tests/pcre-classes}/res${n} @echo x: ${n} ${TEST_OUTDIR.tests/pcre-classes}/got${n}.fsm: ${TEST_SRCDIR.tests/pcre-classes}/in${n}.re - ${RE} -r pcre -py ${.ALLSRC:M*/in*.re} | ${FSM} -pm \ + ${RE} -FC -r pcre -py ${.ALLSRC:M*/in*.re} | ${FSM} -pm \ > $@ ${TEST_OUTDIR.tests/pcre-classes}/out${n}.fsm: ${TEST_SRCDIR.tests/pcre-classes}/equal${n}.re - ${RE} -r pcre -py ${.ALLSRC:M*/equal*.re} | ${FSM} -pm \ + ${RE} -FC -r pcre -py ${.ALLSRC:M*/equal*.re} | ${FSM} -pm \ > $@ ${TEST_OUTDIR.tests/pcre-classes}/res${n}: \ diff --git a/tests/pcre-flags/Makefile b/tests/pcre-flags/Makefile index 67e70fbbb..308571395 100755 --- a/tests/pcre-flags/Makefile +++ b/tests/pcre-flags/Makefile @@ -13,17 +13,17 @@ RE=${BUILD}/bin/re TEST_OUTDIR.tests/pcre-flags/mode${n} != cat ${TEST_SRCDIR.tests/pcre-flags}/mode${n} ${TEST_OUTDIR.tests/pcre-flags}/got${n}.fsm: ${TEST_SRCDIR.tests/pcre-flags}/in${n}.re - ${RE} -F "${TEST_OUTDIR.tests/pcre-flags/mode${n}}" -b -r pcre -py ${.ALLSRC:M*.re} \ + ${RE} -FC -F "${TEST_OUTDIR.tests/pcre-flags/mode${n}}" -b -r pcre -py ${.ALLSRC:M*.re} \ > $@ ${TEST_OUTDIR.tests/pcre-flags}/nfa${n}.fsm: ${TEST_SRCDIR.tests/pcre-flags}/in${n}.re - ${RE} -F "${TEST_OUTDIR.tests/pcre-flags/mode${n}}" -b -r pcre -n -py ${.ALLSRC:M*.re} \ + ${RE} -FC -F "${TEST_OUTDIR.tests/pcre-flags/mode${n}}" -b -r pcre -n -py ${.ALLSRC:M*.re} \ > $@ .else ${TEST_OUTDIR.tests/pcre-flags}/got${n}.fsm: ${TEST_SRCDIR.tests/pcre-flags}/in${n}.re - ${RE} -b -r pcre -py ${.ALLSRC:M*.re} \ + ${RE} -FC -b -r pcre -py ${.ALLSRC:M*.re} \ > $@ ${TEST_OUTDIR.tests/pcre-flags}/nfa${n}.fsm: ${TEST_SRCDIR.tests/pcre-flags}/in${n}.re diff --git a/tests/pcre-repeat/Makefile b/tests/pcre-repeat/Makefile index c325d2f8e..97535b2c3 100755 --- a/tests/pcre-repeat/Makefile +++ b/tests/pcre-repeat/Makefile @@ -12,11 +12,11 @@ RE=${BUILD}/bin/re .for n in ${TEST.tests/pcre-repeat:T:Mout*.fsm:R:C/^out//} ${TEST_OUTDIR.tests/pcre-repeat}/got${n}.fsm: ${TEST_SRCDIR.tests/pcre-repeat}/in${n}.re - ${RE} -r pcre -py ${.ALLSRC:M*.re} \ + ${RE} -FC -r pcre -py ${.ALLSRC:M*.re} \ > $@ ${TEST_OUTDIR.tests/pcre-repeat}/nfa${n}.fsm: ${TEST_SRCDIR.tests/pcre-repeat}/in${n}.re - ${RE} -r pcre -n -py ${.ALLSRC:M*.re} \ + ${RE} -FC -r pcre -n -py ${.ALLSRC:M*.re} \ > $@ ${TEST_OUTDIR.tests/pcre-repeat}/res${n}: \ @@ -30,7 +30,7 @@ FSMTEST_RESULT += ${TEST_OUTDIR.tests/pcre-repeat}/res${n} .for n in ${TEST.tests/pcre-repeat:T:Mout*.err:R:C/^out//} ${TEST_OUTDIR.tests/pcre-repeat}/got${n}.err: ${TEST_SRCDIR.tests/pcre-repeat}/in${n}.re - ${RE} -r pcre -py ${.ALLSRC:M*.re} \ + ${RE} -FC -r pcre -py ${.ALLSRC:M*.re} \ 2> $@; [ $$? -ne 0 ] ${TEST_OUTDIR.tests/pcre-repeat}/res${n}: \ diff --git a/tests/pcre/Makefile b/tests/pcre/Makefile index 23f879a04..239d2c93b 100755 --- a/tests/pcre/Makefile +++ b/tests/pcre/Makefile @@ -21,7 +21,7 @@ PCREGREP ?= pcregrep # bit of a hack. # 2) removes any trailing \n at the end of the input ${TEST_OUTDIR.tests/pcre-pcregrep}/in${n}.txt: ${TEST_SRCDIR.tests/pcre}/in${n}.re - ${RE} -mr pcre -y ${.ALLSRC:M*.re} \ + ${RE} -FC -mr pcre -y ${.ALLSRC:M*.re} \ | perl -0pe 's/\\x([0-9a-zA-z]{2})/chr(hex($$1))/ge;' -e 's/\n\Z//' \ > $@ @@ -41,16 +41,16 @@ test:: ${TEST_OUTDIR.tests/pcre-pcregrep}/res${n} .if exists(${TEST_SRCDIR.tests/pcre}/mode${n}) TEST_OUTDIR.tests/pcre/mode${n} != cat ${TEST_SRCDIR.tests/pcre}/mode$n ${TEST_OUTDIR.tests/pcre}/got${n}.fsm: ${TEST_SRCDIR.tests/pcre}/in${n}.re - ${RE} -F "${TEST_OUTDIR.tests/pcre/mode${n}}" -r pcre -py ${.ALLSRC:M*.re} \ + ${RE} -FC -F "${TEST_OUTDIR.tests/pcre/mode${n}}" -r pcre -py ${.ALLSRC:M*.re} \ > $@ .else ${TEST_OUTDIR.tests/pcre}/got${n}.fsm: ${TEST_SRCDIR.tests/pcre}/in${n}.re - ${RE} -r pcre -py ${.ALLSRC:M*.re} \ + ${RE} -FC -r pcre -py ${.ALLSRC:M*.re} \ > $@ .endif ${TEST_OUTDIR.tests/pcre}/nfa${n}.fsm: ${TEST_SRCDIR.tests/pcre}/in${n}.re - ${RE} -r pcre -n -py ${.ALLSRC:M*.re} \ + ${RE} -FC -r pcre -n -py ${.ALLSRC:M*.re} \ > $@ ${TEST_OUTDIR.tests/pcre}/res${n}: \ @@ -64,7 +64,7 @@ FSMTEST_RESULT += ${TEST_OUTDIR.tests/pcre}/res${n} .for n in ${TEST.tests/pcre:T:Mout*.err:R:C/^out//} ${TEST_OUTDIR.tests/pcre}/got${n}.err: ${TEST_SRCDIR.tests/pcre}/in${n}.re - ${RE} -r pcre -py ${.ALLSRC:M*.re} \ + ${RE} -FC -r pcre -py ${.ALLSRC:M*.re} \ 2> $@; [ $$? -ne 0 ] ${TEST_OUTDIR.tests/pcre}/res${n}: \ diff --git a/tests/pcre/in48.re b/tests/pcre/in48.re new file mode 100644 index 000000000..15490c598 --- /dev/null +++ b/tests/pcre/in48.re @@ -0,0 +1 @@ +^a|$[^x]b* \ No newline at end of file diff --git a/tests/pcre/in49.re b/tests/pcre/in49.re new file mode 100644 index 000000000..07a2d0949 --- /dev/null +++ b/tests/pcre/in49.re @@ -0,0 +1 @@ +(?:(?:|^a)+|^)^b \ No newline at end of file diff --git a/tests/pcre/out48.fsm b/tests/pcre/out48.fsm new file mode 100644 index 000000000..80ce4807e --- /dev/null +++ b/tests/pcre/out48.fsm @@ -0,0 +1,5 @@ +0 -> 1 "a"; # e.g. "a" +1 -> 1 ?; + +start: 0; +end: 1 = [0]; diff --git a/tests/pcre/out49.err b/tests/pcre/out49.err new file mode 100644 index 000000000..b92cc933b --- /dev/null +++ b/tests/pcre/out49.err @@ -0,0 +1 @@ +tests/pcre/in49.re:1-17: Unsupported operator diff --git a/tests/re_interpolate_groups/Makefile b/tests/re_interpolate_groups/Makefile new file mode 100644 index 000000000..41f9e2599 --- /dev/null +++ b/tests/re_interpolate_groups/Makefile @@ -0,0 +1,23 @@ +.include "../../share/mk/top.mk" + +TEST.tests/re_interpolate_groups != ls -1 tests/re_interpolate_groups/re_interpolate_groups*.c +TEST_SRCDIR.tests/re_interpolate_groups = tests/re_interpolate_groups +TEST_OUTDIR.tests/re_interpolate_groups = ${BUILD}/tests/re_interpolate_groups + +.for n in ${TEST.tests/re_interpolate_groups:T:R:C/^re_interpolate_groups//} +test:: ${TEST_OUTDIR.tests/re_interpolate_groups}/res${n} +SRC += ${TEST_SRCDIR.tests/re_interpolate_groups}/re_interpolate_groups${n}.c +#CFLAGS.${TEST_SRCDIR.tests/re_interpolate_groups}/re_interpolate_groups${n}.c = -UNDEBUG +CFLAGS.${TEST_SRCDIR.tests/re_interpolate_groups}/re_interpolate_groups${n}.c = -std=c99 + +${TEST_OUTDIR.tests/re_interpolate_groups}/run${n}: ${TEST_OUTDIR.tests/re_interpolate_groups}/re_interpolate_groups${n}.o + ${CC} ${CFLAGS} -o ${TEST_OUTDIR.tests/re_interpolate_groups}/run${n} ${TEST_OUTDIR.tests/re_interpolate_groups}/re_interpolate_groups${n}.o ${BUILD}/src/libre/re_interpolate_groups.o + +${TEST_OUTDIR.tests/re_interpolate_groups}/res${n}: ${TEST_OUTDIR.tests/re_interpolate_groups}/run${n} + ( ${TEST_OUTDIR.tests/re_interpolate_groups}/run${n} 1>&2 && echo PASS || echo FAIL ) > ${TEST_OUTDIR.tests/re_interpolate_groups}/res${n} + +#.for lib in ${LIB:Mlibfsm} ${LIB:Mlibre} +#${TEST_OUTDIR.tests/re_interpolate_groups}/run${n}: ${BUILD}/lib/${lib:R}.a +#.endfor +.endfor + diff --git a/tests/re_interpolate_groups/re_interpolate_groups0.c b/tests/re_interpolate_groups/re_interpolate_groups0.c new file mode 100644 index 000000000..4b4e0f709 --- /dev/null +++ b/tests/re_interpolate_groups/re_interpolate_groups0.c @@ -0,0 +1,72 @@ +/* + * Copyright 2026 Katherine Flavel + * + * See LICENCE for the full copyright terms. + */ + +#include +#include +#include +#include + +#include +#include + +static unsigned failed; + +static void +test(const char *fmt, size_t groupc, const char *groupv[], const char *expected) +{ + char outs[40]; + bool r; + + assert(fmt != NULL); + assert(expected != NULL); + + if (!re_interpolate_groups(fmt, '$', "", groupc, groupv, "", outs, sizeof outs, NULL, NULL)) { + printf("%s/%zu XXX\n", fmt, groupc); + failed++; + return; + } + + failed += r = 0 != strcmp(outs, expected); + + printf("%s/%zu => %s%s\n", fmt, groupc, outs, + r ? " XXX" : ""); +} + +int main(void) { + const char *gn[] = { "one", "two", "three", "four" }; + const char **g0 = NULL; + const char *ga[] = { "1" }; + const char *gb[] = { "" }; +// const char *gc[] = { NULL }; // XXX: not permitted + + test("", 0, g0, ""); + test("", 4, gn, ""); + + test("x", 0, g0, "x"); + test("x", 4, gn, "x"); + + test("\001", 0, g0, "\001"); + test("\001", 4, gn, "\001"); + + test("$0", 0, gn, ""); + test("x$000000000000000000000x", 0, gn, "xx"); + test("x$000000000000000000001x", 1, gn, "xonex"); + test("x$100000000000000000000x", 1, gn, "xx"); + + test("$$$1$1$2$1$3$4$3$2$1$$$$", 4, gn, "$oneonetwoonethreefourthreetwoone$$"); + test("$$$$$$$$$$$$$$$$$$$$", 4, gn, "$$$$$$$$$$"); + + test("xyz_$1..$0003;$3,$$.$1-$4=$123", 4, gn, "xyz_one..three;three,$.one-four="); + test("xyz_$1..$0003;$3,$$.$1-$4=$123", 3, gn, "xyz_one..three;three,$.one-="); + test("xyz_$1..$0003;$3,$$.$1-$4=$123", 2, gn, "xyz_one..;,$.one-="); + test("xyz_$1..$0003;$3,$$.$1-$4=$123", 1, gn, "xyz_one..;,$.one-="); + test("xyz_$1..$0003;$3,$$.$1-$4=$123", 0, g0, "xyz_..;,$.-="); + test("xyz_$1..$0003;$3,$$.$1-$4=$123", 1, ga, "xyz_1..;,$.1-="); + test("xyz_$1..$0003;$3,$$.$1-$4=$123", 1, gb, "xyz_..;,$.-="); + + return failed; +} + diff --git a/tests/re_interpolate_groups/re_interpolate_groups1.c b/tests/re_interpolate_groups/re_interpolate_groups1.c new file mode 100644 index 000000000..01dc3b344 --- /dev/null +++ b/tests/re_interpolate_groups/re_interpolate_groups1.c @@ -0,0 +1,75 @@ +/* + * Copyright 2026 Katherine Flavel + * + * See LICENCE for the full copyright terms. + */ + +#include +#include +#include +#include + +#include +#include + +static unsigned failed; + +static void +test_err(const char *fmt, size_t groupc, const char *groupv[], const char *ne, + unsigned expected_start, unsigned expected_end) +{ + struct re_pos start, end; + char outs[10]; + bool rs, re; + + assert(fmt != NULL); + + outs[0] = 'x'; + + /* for these tests we're expecting to error */ + if (re_interpolate_groups(fmt, '$', "", groupc, groupv, ne, outs, sizeof outs, &start, &end)) { + printf("%s/%zu XXX\n", fmt, groupc); + failed++; + return; + } + + if (outs[0] != '\0') { + failed++; + } + + failed += rs = expected_start != start.byte; + failed += re = expected_end != end.byte; + + printf("%s/%zu => :%u-%u :%u-%u '%.*s'%s\n", fmt, groupc, + start.byte, end.byte, + expected_start, expected_end, + (int) (end.byte - start.byte), fmt + start.byte, + (rs || re) ? " XXX" : ""); +} + +int main(void) { + const char *ne = ""; + + const char *gn[] = { "one", "two", "three", "four" }; + const char **g0 = NULL; + + test_err("$", 0, g0, ne, 0, 1); + test_err("$x", 0, g0, ne, 0, 1); + test_err("$ ", 4, gn, ne, 0, 1); + test_err("$\\01", 0, g0, ne, 0, 1); + + test_err("$0$", 0, g0, ne, 2, 3); + test_err("$$$x", 4, gn, ne, 2, 3); + + test_err("xyz$1", 0, gn, NULL, 3, 5); + test_err("xyz$2", 1, gn, NULL, 3, 5); + + test_err("01234567890", 1, gn, ne, 0, 10); + test_err("$$$$$$$$$$$$$$$$$$$$", 1, gn, ne, 0, 20); + test_err("$1$1$1$$", 1, gn, ne, 0, 8); + test_err("$1$1$1x", 1, gn, ne, 0, 7); + test_err("xxxyyyzzz$$", 1, gn, ne, 0, 11); + + return failed; +} + diff --git a/tests/re_interpolate_groups/re_interpolate_groups2.c b/tests/re_interpolate_groups/re_interpolate_groups2.c new file mode 100644 index 000000000..d33fe656a --- /dev/null +++ b/tests/re_interpolate_groups/re_interpolate_groups2.c @@ -0,0 +1,40 @@ +/* + * Copyright 2026 Katherine Flavel + * + * See LICENCE for the full copyright terms. + */ + +#include +#include +#include +#include + +#include +#include + +static unsigned failed; + +static void +test(const char *fmt, bool expected) +{ + bool r; + + assert(fmt != NULL); + + r = re_interpolate_groups(fmt, '$', "", 0, NULL, "", NULL, 0, NULL, NULL); + + failed += r != expected; + + printf("%s/%d => %d%s\n", fmt, 0, r, + r != expected ? " XXX" : ""); +} + +int main(void) { + test("", true); + test("abc", true); + test("$$", true); + test("$x", false); + + return failed; +} + diff --git a/tests/re_literal/Makefile b/tests/re_literal/Makefile index 1d1333491..9227a5699 100755 --- a/tests/re_literal/Makefile +++ b/tests/re_literal/Makefile @@ -9,7 +9,7 @@ RE=${BUILD}/bin/re .for n in ${TEST.tests/re_literal:T:Mout*.txt:R:C/^out//} ${TEST_OUTDIR.tests/re_literal}/got${n}.txt: ${TEST_SRCDIR.tests/re_literal}/in${n}.re - ( ${RE} -r pcre -t -y ${.ALLSRC:M*.re} || echo non-literal ) \ + ( ${RE} -FC -r pcre -t -y ${.ALLSRC:M*.re} || echo non-literal ) \ > $@ ${TEST_OUTDIR.tests/re_literal}/res${n}: \ diff --git a/tests/re_literal/in13.re b/tests/re_literal/in13.re new file mode 100644 index 000000000..fa9f7f5ff --- /dev/null +++ b/tests/re_literal/in13.re @@ -0,0 +1 @@ +^(?i)a$ \ No newline at end of file diff --git a/tests/re_literal/in14.re b/tests/re_literal/in14.re new file mode 100644 index 000000000..2500582f0 --- /dev/null +++ b/tests/re_literal/in14.re @@ -0,0 +1 @@ +(?:x(?i)a) \ No newline at end of file diff --git a/tests/re_literal/in15.re b/tests/re_literal/in15.re new file mode 100644 index 000000000..27e49d98b --- /dev/null +++ b/tests/re_literal/in15.re @@ -0,0 +1 @@ +(?i) \ No newline at end of file diff --git a/tests/re_literal/in16.re b/tests/re_literal/in16.re new file mode 100644 index 000000000..33e13db9d --- /dev/null +++ b/tests/re_literal/in16.re @@ -0,0 +1 @@ +(?i)xy \ No newline at end of file diff --git a/tests/re_literal/in17.re b/tests/re_literal/in17.re new file mode 100644 index 000000000..92e7f56a7 --- /dev/null +++ b/tests/re_literal/in17.re @@ -0,0 +1 @@ +a(?i)xy \ No newline at end of file diff --git a/tests/re_literal/in18.re b/tests/re_literal/in18.re new file mode 100644 index 000000000..9fe02bd25 --- /dev/null +++ b/tests/re_literal/in18.re @@ -0,0 +1 @@ +a(?:x(?i)xy) \ No newline at end of file diff --git a/tests/re_literal/in19.re b/tests/re_literal/in19.re new file mode 100644 index 000000000..297e030e4 --- /dev/null +++ b/tests/re_literal/in19.re @@ -0,0 +1 @@ +a(?:x(?-i)xy) \ No newline at end of file diff --git a/tests/re_literal/in20.re b/tests/re_literal/in20.re new file mode 100644 index 000000000..800e172fd --- /dev/null +++ b/tests/re_literal/in20.re @@ -0,0 +1 @@ +(?i)(?:(?-i)a)(?:x(?-i)xy) \ No newline at end of file diff --git a/tests/re_literal/in21.re b/tests/re_literal/in21.re new file mode 100644 index 000000000..4b3072ef2 --- /dev/null +++ b/tests/re_literal/in21.re @@ -0,0 +1 @@ +(?i)(?:(?-i)a)(?:(?-i)x(?-i)xy) \ No newline at end of file diff --git a/tests/re_literal/in22.re b/tests/re_literal/in22.re new file mode 100644 index 000000000..40352311b --- /dev/null +++ b/tests/re_literal/in22.re @@ -0,0 +1 @@ +(?s)a \ No newline at end of file diff --git a/tests/re_literal/in23.re b/tests/re_literal/in23.re new file mode 100644 index 000000000..f53b1b835 --- /dev/null +++ b/tests/re_literal/in23.re @@ -0,0 +1 @@ +(?x)a \ No newline at end of file diff --git a/tests/re_literal/in24.re b/tests/re_literal/in24.re new file mode 100644 index 000000000..bda0b6e86 --- /dev/null +++ b/tests/re_literal/in24.re @@ -0,0 +1 @@ +(?n)a \ No newline at end of file diff --git a/tests/re_literal/out13.txt b/tests/re_literal/out13.txt new file mode 100644 index 000000000..c5a2ff66c --- /dev/null +++ b/tests/re_literal/out13.txt @@ -0,0 +1 @@ +non-literal diff --git a/tests/re_literal/out14.txt b/tests/re_literal/out14.txt new file mode 100644 index 000000000..c5a2ff66c --- /dev/null +++ b/tests/re_literal/out14.txt @@ -0,0 +1 @@ +non-literal diff --git a/tests/re_literal/out15.txt b/tests/re_literal/out15.txt new file mode 100644 index 000000000..82977dfb2 --- /dev/null +++ b/tests/re_literal/out15.txt @@ -0,0 +1,2 @@ +anchors: +literal: diff --git a/tests/re_literal/out16.txt b/tests/re_literal/out16.txt new file mode 100644 index 000000000..c5a2ff66c --- /dev/null +++ b/tests/re_literal/out16.txt @@ -0,0 +1 @@ +non-literal diff --git a/tests/re_literal/out17.txt b/tests/re_literal/out17.txt new file mode 100644 index 000000000..c5a2ff66c --- /dev/null +++ b/tests/re_literal/out17.txt @@ -0,0 +1 @@ +non-literal diff --git a/tests/re_literal/out18.txt b/tests/re_literal/out18.txt new file mode 100644 index 000000000..c5a2ff66c --- /dev/null +++ b/tests/re_literal/out18.txt @@ -0,0 +1 @@ +non-literal diff --git a/tests/re_literal/out19.txt b/tests/re_literal/out19.txt new file mode 100644 index 000000000..f761b32c2 --- /dev/null +++ b/tests/re_literal/out19.txt @@ -0,0 +1,2 @@ +anchors: +literal: axxy diff --git a/tests/re_literal/out20.txt b/tests/re_literal/out20.txt new file mode 100644 index 000000000..c5a2ff66c --- /dev/null +++ b/tests/re_literal/out20.txt @@ -0,0 +1 @@ +non-literal diff --git a/tests/re_literal/out21.txt b/tests/re_literal/out21.txt new file mode 100644 index 000000000..f761b32c2 --- /dev/null +++ b/tests/re_literal/out21.txt @@ -0,0 +1,2 @@ +anchors: +literal: axxy diff --git a/tests/re_literal/out22.txt b/tests/re_literal/out22.txt new file mode 100644 index 000000000..c5a2ff66c --- /dev/null +++ b/tests/re_literal/out22.txt @@ -0,0 +1 @@ +non-literal diff --git a/tests/re_literal/out23.txt b/tests/re_literal/out23.txt new file mode 100644 index 000000000..c5a2ff66c --- /dev/null +++ b/tests/re_literal/out23.txt @@ -0,0 +1 @@ +non-literal diff --git a/tests/re_literal/out24.txt b/tests/re_literal/out24.txt new file mode 100644 index 000000000..e9278cb7d --- /dev/null +++ b/tests/re_literal/out24.txt @@ -0,0 +1,2 @@ +anchors: +literal: a diff --git a/tests/re_strings/Makefile b/tests/re_strings/Makefile new file mode 100644 index 000000000..7fc7f2548 --- /dev/null +++ b/tests/re_strings/Makefile @@ -0,0 +1,26 @@ +.include "../../share/mk/top.mk" + +TEST.tests/re_strings != ls -1 tests/re_strings/re_strings*.c +TEST_SRCDIR.tests/re_strings = tests/re_strings +TEST_OUTDIR.tests/re_strings = ${BUILD}/tests/re_strings + +.for n in ${TEST.tests/re_strings:T:R:C/^re_strings//} +test:: ${TEST_OUTDIR.tests/re_strings}/res${n} +SRC += ${TEST_SRCDIR.tests/re_strings}/re_strings${n}.c +CFLAGS.${TEST_SRCDIR.tests/re_strings}/re_strings${n}.c = -UNDEBUG + +${TEST_OUTDIR.tests/re_strings}/run${n}: ${TEST_OUTDIR.tests/re_strings}/re_strings${n}.o ${TEST_OUTDIR.tests/re_strings}/testutil.o + ${CC} ${CFLAGS} -o ${TEST_OUTDIR.tests/re_strings}/run${n} ${TEST_OUTDIR.tests/re_strings}/re_strings${n}.o ${TEST_OUTDIR.tests/re_strings}/testutil.o ${BUILD}/lib/libfsm.a ${BUILD}/lib/libre.a + +${TEST_OUTDIR.tests/re_strings}/re_strings${n}.o: tests/re_strings/testutil.h + +${TEST_OUTDIR.tests/re_strings}/res${n}: ${TEST_OUTDIR.tests/re_strings}/run${n} + ( ${TEST_OUTDIR.tests/re_strings}/run${n} 1>&2 && echo PASS || echo FAIL ) > ${TEST_OUTDIR.tests/re_strings}/res${n} + +.for lib in ${LIB:Mlibfsm} ${LIB:Mlibre} +${TEST_OUTDIR.tests/re_strings}/run${n}: ${BUILD}/lib/${lib:R}.a +.endfor +.endfor + +${TEST_OUTDIR.tests/re_strings}/testutil.o: tests/re_strings/testutil.c + ${CC} ${CFLAGS} -c -o ${TEST_OUTDIR.tests/re_strings}/testutil.o tests/re_strings/testutil.c diff --git a/tests/re_strings/re_strings1.c b/tests/re_strings/re_strings1.c new file mode 100644 index 000000000..44f10d41d --- /dev/null +++ b/tests/re_strings/re_strings1.c @@ -0,0 +1,21 @@ +#include "testutil.h" + +const char *strings[] = { + "aa", + "ab", + "ac", + "ba", + "bb", + "bc", + "ca", + "cb", + "cc", + NULL, +}; + +int main(int argc, char **argv) +{ + (void)argc; + (void)argv; + return run_test(strings); +} diff --git a/tests/re_strings/re_strings2.c b/tests/re_strings/re_strings2.c new file mode 100644 index 000000000..6e4c80459 --- /dev/null +++ b/tests/re_strings/re_strings2.c @@ -0,0 +1,17 @@ +#include "testutil.h" + +const char *strings[] = { + "first", + "duplicate", + "duplicate", + "duplicate", + "last", + NULL, +}; + +int main(int argc, char **argv) +{ + (void)argc; + (void)argv; + return run_test(strings); +} diff --git a/tests/re_strings/re_strings3.c b/tests/re_strings/re_strings3.c new file mode 100644 index 000000000..3f4f1d052 --- /dev/null +++ b/tests/re_strings/re_strings3.c @@ -0,0 +1,15 @@ +#include "testutil.h" + +const char *strings[] = { + "duplicate", + "duplicate", + "duplicate", + NULL, +}; + +int main(int argc, char **argv) +{ + (void)argc; + (void)argv; + return run_test(strings); +} diff --git a/tests/re_strings/re_strings4.c b/tests/re_strings/re_strings4.c new file mode 100644 index 000000000..b1e00e70f --- /dev/null +++ b/tests/re_strings/re_strings4.c @@ -0,0 +1,13 @@ +#include "testutil.h" + +const char *strings[] = { + /* empty */ + NULL, +}; + +int main(int argc, char **argv) +{ + (void)argc; + (void)argv; + return run_test(strings); +} diff --git a/tests/re_strings/testutil.c b/tests/re_strings/testutil.c new file mode 100644 index 000000000..c22f32da9 --- /dev/null +++ b/tests/re_strings/testutil.c @@ -0,0 +1,70 @@ +#include "testutil.h" + +#include +#include + +#include "fsm/fsm.h" +#include "fsm/options.h" + +#include "re/re.h" +#include "re/strings.h" + +#define MAX_INPUTS 100 +static fsm_end_id_t ids[MAX_INPUTS]; + +int +run_test(const char **strings) +{ + struct re_strings *s = re_strings_new(); + assert(s != NULL); + + fsm_end_id_t id = 0; + const char **input = strings; + while (*input != NULL) { + if (!re_strings_add_str(s, *input, &id)) { + assert(!"re_strings_add_str"); + } + + input++; + id++; + assert(id < MAX_INPUTS); + } + + const int flags = 0; /* not anchored */ + + struct fsm *fsm = re_strings_build(s, NULL, flags); + assert(fsm != NULL); + + /* Each literal string input should match, and the set of + * matching endids should include the expected one. */ + id = 0; + input = strings; + while (*input != NULL) { + fsm_state_t end; + const char **string = input; + const int res = fsm_exec(fsm, fsm_sgetc, string, &end); + assert(res > 0); /* match */ + + size_t count = fsm_endid_count(fsm, end); + assert(count <= MAX_INPUTS); + + int eres = fsm_endid_get(fsm, end, count, ids); + assert(eres == 1); + bool found = false; + for (size_t i = 0; i < count; i++) { + if (ids[i] == id) { + found = true; + break; + } + } + assert(found); + + input++; + id++; + } + + re_strings_free(s); + fsm_free(fsm); + + return EXIT_SUCCESS; +} diff --git a/tests/re_strings/testutil.h b/tests/re_strings/testutil.h new file mode 100644 index 000000000..6898200b7 --- /dev/null +++ b/tests/re_strings/testutil.h @@ -0,0 +1,11 @@ +#ifndef TESTUTIL_H +#define TESTUTIL_H + +#include +#include +#include + +int +run_test(const char **strings); + +#endif diff --git a/tests/regressions/Makefile b/tests/regressions/Makefile new file mode 100644 index 000000000..624353ef6 --- /dev/null +++ b/tests/regressions/Makefile @@ -0,0 +1,20 @@ +.include "../../share/mk/top.mk" + +TEST.tests/regressions != ls -1 tests/regressions/regressions*.c +TEST_SRCDIR.tests/regressions = tests/regressions +TEST_OUTDIR.tests/regressions = ${BUILD}/tests/regressions + +.for n in ${TEST.tests/regressions:T:R:C/^regressions//} +INCDIR.${TEST_SRCDIR.tests/regressions}/regressions${n}.c += src/adt +.endfor + +.for n in ${TEST.tests/regressions:T:R:C/^regressions//} +test:: ${TEST_OUTDIR.tests/regressions}/res${n} +SRC += ${TEST_SRCDIR.tests/regressions}/regressions${n}.c +CFLAGS.${TEST_SRCDIR.tests/regressions}/regressions${n}.c += -UNDEBUG + +${TEST_OUTDIR.tests/regressions}/run${n}: ${TEST_OUTDIR.tests/regressions}/regressions${n}.o ${BUILD}/lib/libfsm.a ${BUILD}/lib/libre.a + ${CC} ${CFLAGS} ${CFLAGS.${TEST_SRCDIR.tests/regressions}/regressions${n}.c} -o ${TEST_OUTDIR.tests/regressions}/run${n} ${TEST_OUTDIR.tests/regressions}/regressions${n}.o ${BUILD}/lib/libfsm.a ${BUILD}/lib/libre.a +${TEST_OUTDIR.tests/regressions}/res${n}: ${TEST_OUTDIR.tests/regressions}/run${n} + ( ${TEST_OUTDIR.tests/regressions}/run${n} 1>&2 && echo PASS || echo FAIL ) > ${TEST_OUTDIR.tests/regressions}/res${n} +.endfor diff --git a/tests/regressions/regressions_determinise_state_limit_leak.c b/tests/regressions/regressions_determinise_state_limit_leak.c new file mode 100644 index 000000000..afdfbd9c5 --- /dev/null +++ b/tests/regressions/regressions_determinise_state_limit_leak.c @@ -0,0 +1,69 @@ +#include +#include +#include +#include + +#include + +#include +#include +#include + +static const char *strings[] = { + [0] = "apple", + [1] = "banana", + [2] = "carrot", + [3] = "durian", + [4] = "eggplant", +}; +#define STRING_COUNT sizeof(strings)/sizeof(strings[0]) + +int main(void) +{ + struct fsm *fsms[STRING_COUNT] = {0}; + + for (size_t i = 0; i < STRING_COUNT; i++) { + fsms[i] = re_comp(RE_PCRE, fsm_sgetc, &strings[i], NULL, 0, NULL); + assert(fsms[i] != NULL); + } + + struct fsm *combined_fsm = fsm_union_array(STRING_COUNT, fsms, NULL); + assert(combined_fsm != NULL); + + size_t state_limit_base = fsm_countstates(combined_fsm); + size_t max_state_limit = state_limit_base + 100; + + bool hit_state_limit = false; + + for (size_t state_limit = state_limit_base; state_limit < max_state_limit; state_limit += 10) { + struct fsm *cp = fsm_clone(combined_fsm); + + const struct fsm_determinise_config det_config = { + .state_limit = state_limit, + }; + + /* Previously this would leak memory when hitting the STATE_LIMIT_REACHED + * early exit, because the edge sets for the DFA being constructed were + * not freed properly. + * + * The first time this should fail immediately because the state limit IS the starting size, + * but later on it should halt in the middle of construction. */ + switch (fsm_determinise_with_config(cp, &det_config)) { + case FSM_DETERMINISE_WITH_CONFIG_OK: + fsm_free(cp); + break; + case FSM_DETERMINISE_WITH_CONFIG_STATE_LIMIT_REACHED: + hit_state_limit = true; + fsm_free(cp); + break; + case FSM_DETERMINISE_WITH_CONFIG_ERRNO: + assert(!"internal error"); + return EXIT_FAILURE; + } + } + + assert(hit_state_limit); + + fsm_free(combined_fsm); + return EXIT_SUCCESS; +} diff --git a/tests/retest/Makefile b/tests/retest/Makefile index ec6cc66a9..94fa19b69 100644 --- a/tests/retest/Makefile +++ b/tests/retest/Makefile @@ -8,9 +8,12 @@ DIR += ${TEST_OUTDIR.tests/retest} RETEST=${BUILD}/bin/retest -.for lang in vm asm c vmc +.for lang in vm asm c vmc llvm .for io in pair str +# XXX: we don't have FSM_IO_STR for asm yet +.if ${lang} != asm || ${io} != str + .for n in ${TEST.tests/retest:T:R:C/^tests_//} ${TEST_OUTDIR.tests/retest}/res-${lang}-${io}${n}: ${TEST_SRCDIR.tests/retest}/tests_${n}.tst @@ -22,6 +25,8 @@ test:: ${TEST_OUTDIR.tests/retest}/res-${lang}-${io}${n} .endfor +.endif + .endfor .endfor diff --git a/theft/Makefile b/theft/Makefile index 0d38d8cfc..921c482a9 100644 --- a/theft/Makefile +++ b/theft/Makefile @@ -6,7 +6,6 @@ SRC += theft/util.c SRC += theft/wrap.c SRC += theft/fuzz_adt_edge_set.c -SRC += theft/fuzz_adt_ipriq.c SRC += theft/fuzz_adt_priq.c SRC += theft/fuzz_capture_string_set.c SRC += theft/fuzz_literals.c diff --git a/theft/fuzz_adt_ipriq.c b/theft/fuzz_adt_ipriq.c deleted file mode 100644 index 1847ef6ce..000000000 --- a/theft/fuzz_adt_ipriq.c +++ /dev/null @@ -1,197 +0,0 @@ -/* - * Copyright 2021 Scott Vokes - * - * See LICENCE for the full copyright terms. - */ - -#include "type_info_adt_ipriq.h" - -#include -#include - -struct model { - size_t used; - size_t entries[]; -}; - -static enum ipriq_cmp_res -cmp_size_t(size_t a, size_t b, void *opaque) -{ - (void)opaque; - return a < b ? IPRIQ_CMP_LT : - a > b ? IPRIQ_CMP_GT : IPRIQ_CMP_EQ; -} - -static int exec_add(size_t x, struct model *m, struct ipriq *pq) -{ - if (!ipriq_add(pq, x)) { - return 0; - } - - m->entries[m->used] = x; - m->used++; - return 1; -} - -static int find_min_pos(const struct model *m, size_t *pos) -{ - size_t i; - if (m->used == 0) { - return 0; - } - - size_t res, min; - res = 0; - min = m->entries[0]; - - for (i = 1; i < m->used; i++) { - if (m->entries[i] < min) { - res = i; - min = m->entries[i]; - } - } - *pos = res; - return 1; -} - -static int exec_peek(struct model *m, struct ipriq *pq) -{ - size_t res; - - if (!ipriq_peek(pq, &res)) { - return m->used == 0; - } - - size_t pos; - if (!find_min_pos(m, &pos)) { - assert(!"unreachable (peek)"); - } - - return res == m->entries[pos]; -} - -static int exec_pop(struct model *m, struct ipriq *pq) -{ - size_t res; - - if (!ipriq_pop(pq, &res)) { - return m->used == 0; - } - - size_t pos; - if (!find_min_pos(m, &pos)) { - assert(!"unreachable (pop)"); - } - - if (res != m->entries[pos]) { - return 0; - } - - assert(m->used > 0); - if (pos < m->used - 1) { - m->entries[pos] = m->entries[m->used - 1]; - } - m->used--; - return 1; -} - -static enum theft_trial_res -compare_against_model(const struct ipriq_scenario *scen) -{ - enum theft_trial_res res = THEFT_TRIAL_FAIL; - size_t i; - - struct model *m = malloc(sizeof(*m) - + scen->count * sizeof(m->entries[0])); - if (m == NULL) { - return THEFT_TRIAL_ERROR; - } - m->used = 0; - - struct ipriq *pq = ipriq_new(NULL, cmp_size_t, NULL); - if (pq == NULL) { - return THEFT_TRIAL_ERROR; - } - - for (i = 0; i < scen->count; i++) { - const struct ipriq_op *op = &scen->ops[i]; - - switch (op->t) { - case IPRIQ_OP_ADD: - if (!exec_add(op->u.add.x, m, pq)) { - goto cleanup; - } - break; - - case IPRIQ_OP_PEEK: - if (!exec_peek(m, pq)) { - goto cleanup; - } - break; - - case IPRIQ_OP_POP: - if (!exec_pop(m, pq)) { - goto cleanup; - } - break; - - default: - assert(false); break; - } - } - - res = THEFT_TRIAL_PASS; - -cleanup: - free(m); - - return res; -} - -static enum theft_trial_res -prop_ipriq_model(struct theft *t, void *arg1) -{ - const struct ipriq_scenario *scen = arg1; - (void)t; - return compare_against_model(scen); -} - -static bool -test_ipriq(theft_seed seed, uintptr_t limit) -{ - enum theft_run_res res; - - struct ipriq_hook_env env = { - .tag = 'I', - .limit = limit, - }; - - struct theft_run_config config = { - .name = __func__, - .prop1 = prop_ipriq_model, - .type_info = { &type_info_adt_ipriq }, - .trials = 1000, - .hooks = { - .trial_pre = theft_hook_first_fail_halt, - .env = &env, - }, - .fork = { - .enable = true, - }, - - .seed = seed, - }; - - (void)limit; - - res = theft_run(&config); - printf("%s: %s\n", __func__, theft_run_res_str(res)); - - return res == THEFT_RUN_PASS; -} - -void -register_test_adt_ipriq(void) -{ - reg_test1("adt_ipriq", test_ipriq, 10000); -} diff --git a/theft/fuzz_capture_string_set.c b/theft/fuzz_capture_string_set.c index 7326356c2..477f608cc 100644 --- a/theft/fuzz_capture_string_set.c +++ b/theft/fuzz_capture_string_set.c @@ -158,11 +158,11 @@ check_capstring_set(struct capture_env *env, return THEFT_TRIAL_ERROR; } - const size_t capture_count = fsm_countcaptures(dfa); + const size_t capture_count = fsm_capture_ceiling(dfa); if (verbosity > 2) { fprintf(stderr, "==== cs '%s'\n", cs->string); - fsm_print_fsm(stderr, dfa); + fsm_print(stderr, dfa, FSM_PRINT_FSM); } struct fsm *cp = fsm_clone(dfa); @@ -172,10 +172,10 @@ check_capstring_set(struct capture_env *env, assert(cp != NULL); fsm_copies[cs_i] = cp; - const size_t cp_capture_count = fsm_countcaptures(cp); + const size_t cp_capture_count = fsm_capture_ceiling(cp); if (verbosity > 2) { fprintf(stderr, "==== min(det(cp))\n"); - fsm_print_fsm(stderr, cp); + fsm_print(stderr, cp, FSM_PRINT_FSM); fsm_capture_dump(stderr, "min(det(cp))", cp); fprintf(stderr, "capture_count: %lu\n", cp_capture_count); } @@ -196,7 +196,7 @@ check_capstring_set(struct capture_env *env, return THEFT_TRIAL_FAIL; } - combined_capture_count = fsm_countcaptures(combined); + combined_capture_count = fsm_capture_ceiling(combined); for (size_t cs_i = 0; cs_i < css->count; cs_i++) { total_captures += capture_counts[cs_i]; } @@ -209,7 +209,7 @@ check_capstring_set(struct capture_env *env, if (verbosity > 2) { fprintf(stderr, "==== combined, pre-det\n"); - fsm_print_fsm(stderr, combined); + fsm_print(stderr, combined, FSM_PRINT_FSM); fsm_capture_dump(stderr, "combined, pre-det", combined); } @@ -227,7 +227,7 @@ check_capstring_set(struct capture_env *env, if (verbosity > 2) { fprintf(stderr, "==== det(combined)\n"); - fsm_print_fsm(stderr, combined); + fsm_print(stderr, combined, FSM_PRINT_FSM); fsm_capture_dump(stderr, "det(combined)", combined); } @@ -295,7 +295,7 @@ check_fsms_for_single_input(struct check_env *env, struct fsm_capture *captures, assert(exec_res >= 0); if (exec_res == 1) { if (LOG_LEVEL > 0) { - const size_t combined_capture_count = fsm_countcaptures(env->combined); + const size_t combined_capture_count = fsm_capture_ceiling(env->combined); for (size_t i = 0; i < combined_capture_count; i++) { fprintf(stderr, "capture[%zu/%zu]: (%ld, %ld)\n", i, combined_capture_count, @@ -415,7 +415,7 @@ compare_captures(const struct check_env *env, const struct fsm_capture *captures_combined, size_t nth_fsm, const struct fsm_capture *captures) { - const size_t combined_capture_count = fsm_countcaptures(env->combined); + const size_t combined_capture_count = fsm_capture_ceiling(env->combined); if (combined_capture_count == 0) { return true; /* no captures */ } @@ -639,7 +639,7 @@ build_capstring_dfa(const struct capstring *cs, uint8_t end_id) goto cleanup; } - if (fsm_countcaptures(fsm) != cs->capture_count) { + if (fsm_capture_ceiling(fsm) != cs->capture_count) { goto cleanup; } diff --git a/theft/fuzz_nfa.c b/theft/fuzz_nfa.c index 6ff698c98..bdc22745d 100644 --- a/theft/fuzz_nfa.c +++ b/theft/fuzz_nfa.c @@ -109,7 +109,7 @@ nfa_operations_should_not_impact_matching(theft_seed seed) do { \ if (verbosity > 2) { \ fprintf(stderr, "==== %s ====\n", BANNER); \ - fsm_print_fsm(stderr, nfa); \ + fsm_print(stderr, nfa, FSM_PRINT_FSM); \ } \ } while(0)