diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9c2d15d..a5f4c0f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -71,3 +71,28 @@ jobs: - name: Tests run: | npm run test + # only added temporarily to run integration tests on Java + integration-tests-java: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Setup node.js + uses: actions/setup-node@v4 + - name: Install + run: | + npm install + - name: Build + run: | + npm run build + - name: Install Java + uses: actions/setup-java@v5 + with: + distribution: temurin + java-version: 21 + - name: Prepare for testing + run: | + npm run + npm run test:setup-java + - name: Integration tests + run: | + JAVA_ES_REQUEST_CONVERTER_JAR=/home/runner/work/request-converter/request-converter/tests/integration/.java-request-converter/build/libs/java-request-converter-1.0.0.jar npm run test:integration-java diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 1566ff1..166d4a6 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -112,3 +112,27 @@ jobs: - name: Integration tests run: | npm run test:integration-ruby + integration-tests-java: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Setup node.js + uses: actions/setup-node@v4 + - name: Install + run: | + npm install + - name: Build + run: | + npm run build + - name: Install Java + uses: actions/setup-java@v5 + with: + distribution: temurin + java-version: 21 + - name: Prepare for testing + run: | + npm run + npm run test:setup-java + - name: Integration tests + run: | + JAVA_ES_REQUEST_CONVERTER_JAR=/home/runner/work/request-converter/request-converter/tests/integration/.java-request-converter/build/libs/java-request-converter-1.0.0.jar npm run test:integration-java diff --git a/package.json b/package.json index 9a30e16..bb282f3 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ "compile-templates": "node scripts/compile-templates.mjs", "copy-files": "copyfiles -u 1 src/**/*.js src/**/*.json dist/", "docs": "typedoc src/index.ts", - "lint": "eslint src tests --ignore-pattern tests/wasm/", + "lint": "eslint src tests --ignore-pattern tests/wasm/ --ignore-pattern 'tests/integration/.*'", "prettier": "prettier \"src/**/*.ts\" \"tests/**/*.ts\" --list-different", "test": "npm run compile-templates && jest --test-path-ignore-patterns integration --coverage", "test:setup": "./tests/integration/run-python.sh && ./tests/integration/run-javascript.sh && ./tests/integration/run-ruby.sh && ./tests/integration/run-php.sh", @@ -29,15 +29,17 @@ "test:setup-javascript": "./tests/integration/run-javascript.sh", "test:setup-php": "./tests/integration/run-php.sh", "test:setup-ruby": "./tests/integration/run-ruby.sh", + "test:setup-java": "./tests/integration/run-java.sh", "test:integration": "jest tests/integration", "test:integration-curl": "./scripts/test-format.sh curl", "test:integration-python": "./scripts/test-format.sh python", "test:integration-javascript": "./scripts/test-format.sh javascript", "test:integration-php": "./scripts/test-format.sh php", "test:integration-ruby": "./scripts/test-format.sh ruby", + "test:integration-java": "./scripts/test-format.sh java", "test:example": "./scripts/test-example.sh", "fix": "npm run fix:lint && npm run fix:prettier", - "fix:lint": "eslint src tests --fix --ignore-pattern tests/wasm/", + "fix:lint": "eslint src tests --fix --ignore-pattern tests/wasm/ --ignore-pattern 'tests/integration/.*'", "fix:prettier": "prettier \"src/**/*.ts\" \"tests/**/*.ts\" --write", "fix:precommit": "./scripts/precommit.sh", "watch:test": "npm run compile-templates && jest --test-path-ignore-patterns integration --coverage --watch", diff --git a/src/convert.ts b/src/convert.ts index 1480277..e474c86 100644 --- a/src/convert.ts +++ b/src/convert.ts @@ -6,6 +6,7 @@ import { CurlExporter } from "./exporters/curl"; import { JavaScriptExporter } from "./exporters/javascript"; import { PHPExporter } from "./exporters/php"; import { RubyExporter } from "./exporters/ruby"; +import { JavaExporter } from "./exporters/java"; import util from "util"; const isBrowser = typeof window !== "undefined"; @@ -35,6 +36,7 @@ export type ConvertOptions = { * This interface defines the structure of a language exporter. */ export interface FormatExporter { + available?(): boolean; check(requests: ParsedRequest[]): Promise; convert(requests: ParsedRequest[], options: ConvertOptions): Promise; } @@ -44,9 +46,10 @@ const EXPORTERS: Record = { php: new PHPExporter(), python: new PythonExporter(), ruby: new RubyExporter(), + java: new JavaExporter(), curl: new CurlExporter(), }; -const LANGUAGES = ["JavaScript", "PHP", "Python", "Ruby", "curl"]; +const LANGUAGES = ["JavaScript", "PHP", "Python", "Ruby", "Java", "curl"]; /** * Return the list of available export formats. @@ -55,7 +58,16 @@ const LANGUAGES = ["JavaScript", "PHP", "Python", "Ruby", "curl"]; * to use in the `convertRequests()` function. */ export function listFormats(): string[] { - return LANGUAGES; + return LANGUAGES.filter((lang) => { + const exporter = EXPORTERS[lang.toLowerCase()]; + if (!exporter) { + return false; + } + if (!exporter.available) { + return true; + } + return exporter.available(); + }); } /** diff --git a/src/exporters/java.ts b/src/exporters/java.ts new file mode 100644 index 0000000..2886bd9 --- /dev/null +++ b/src/exporters/java.ts @@ -0,0 +1,88 @@ +import childProcess from "child_process"; +import { FormatExporter, ConvertOptions } from "../convert"; +import { ParsedRequest, JSONValue } from "../parse"; +import { Request } from "../metamodel"; +import util from "util"; + +const isBrowser = typeof window !== "undefined"; +const execAsync = !isBrowser ? util.promisify(childProcess.exec) : undefined; + +type JavaRequest = { + api?: string; + params: Record; + query?: Record; + body: JSONValue; +}; + +function getCodeGenParamNames( + params: Record, + request: Request, +) { + for (const [key, value] of Object.entries(params)) { + if (request?.path) { + for (const prop of request.path) { + if (prop.name === key && prop.codegenName !== undefined) { + delete params[key]; + params[prop.codegenName] = value; + } + } + } + } + return params; +} + +export class JavaExporter implements FormatExporter { + available(): boolean { + return !isBrowser && !!process.env.JAVA_ES_REQUEST_CONVERTER_JAR; + } + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + async check(requests: ParsedRequest[]): Promise { + // only return true if all requests are for Elasticsearch + return requests + .map((req) => req.service == "es") + .reduce((prev, curr) => prev && curr, true); + } + + async convert( + requests: ParsedRequest[], + options: ConvertOptions, + ): Promise { + const javaRequests: JavaRequest[] = []; + for (const request of requests) { + if (request.request) { + const correctParams = getCodeGenParamNames( + request.params, + request.request, + ); + const body = request.body ?? {}; + + const javaRequest: JavaRequest = { + api: request.api, + params: correctParams, + query: request.query, + body: body, + }; + javaRequests.push(javaRequest); + } + } + + // escape single quotes that may appear in the payload + // (only for bash-style shells for now) + const req = JSON.stringify(javaRequests).replaceAll("'", "'\"'\"'"); + // other arguments to the Java converter + const complete = options.complete ? "true" : "false"; + const url = options.elasticsearchUrl ?? ""; + + if (execAsync === undefined) { + throw new Error("Cannot use exec()"); + } + const { stdout, stderr } = await execAsync( + `java -jar ${process.env.JAVA_ES_REQUEST_CONVERTER_JAR} '${req}' '${complete}' '${url}'`, + ); + if (!stdout) { + throw new Error(`Could not invoke exporter: ${stderr}`); + } + return stdout; + } +} diff --git a/src/java-caller.d.ts b/src/java-caller.d.ts new file mode 100644 index 0000000..70c1c4c --- /dev/null +++ b/src/java-caller.d.ts @@ -0,0 +1 @@ +declare module "java-caller"; diff --git a/tests/integration/convert.test.ts b/tests/integration/convert.test.ts index 78615b1..515205a 100644 --- a/tests/integration/convert.test.ts +++ b/tests/integration/convert.test.ts @@ -17,8 +17,17 @@ const TEST_FORMATS: Record = { php: "php", curl: "sh", ruby: "rb", + java: "java", }; +// For the languages listed, bodies are compared taking into account that some +// array or object properties may have been given in their shortcut form, but +// the language client expands them to full form. +// For these languages, a property that is expected to have a scalar value but +// instead comes back as a single-key dictionary or single-element array is +// compared against the value wrapped in the object or array. +const checkExpandedShortcuts = ["java"]; + interface SchemaExample { method_request: string; value: string; @@ -29,6 +38,94 @@ interface Example { source: string; } +// eslint-disable-next-line @typescript-eslint/no-explicit-any +function deepCompare(actual: any, expected: any): boolean { + try { + // first try a standard comparison + expect(actual).toEqual(expected); + } catch (error) { + return false; + } + return true; +} + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +function compareWithShortcuts(actual: any, expected: any): boolean { + if (deepCompare(actual, expected)) { + return true; + } + + // check for single-element arrays + if (Array.isArray(actual) && actual.length === 1) { + if (deepCompare(actual[0], expected)) { + return true; + } + } + + // check for single-key objects + if (typeof actual === "object" && Object.keys(actual).length === 1) { + if (deepCompare(actual[Object.keys(actual)[0]], expected)) { + return true; + } + } + + // check for integer vs string + if (typeof actual === "number" && typeof expected === "string") { + return actual === parseFloat(expected); + } + + if (typeof actual === "string" && typeof expected === "number") { + return parseFloat(actual) === expected; + } + + // check for boolean vs string + if (typeof actual === "boolean" && typeof expected === "string") { + return actual ? expected === "true" : expected === "false"; + } + if (typeof actual === "string" && typeof expected === "boolean") { + return expected ? actual === "true" : actual === "false"; + } + + // check floats given as strings + const NUMERIC_REGEX = /^[-+]?(\d+\.\d+|\d+|\.\d+)$/; + if ( + typeof actual === "string" && + typeof expected === "string" && + NUMERIC_REGEX.test(actual) && + NUMERIC_REGEX.test(expected) + ) { + return parseFloat(actual) === parseFloat(expected); + } + + // check for strings that have `\n` in them + if (typeof actual === "string" && typeof expected === "string") { + if (actual === expected) { + return true; + } + if (actual.includes("\n") && actual.replaceAll("\n", "\\n") === expected) { + return true; + } + return false; + } + + if (typeof actual !== "object" || typeof expected !== "object") { + return false; // the caller will do a complete assert and report the diff + } + + // for objects we recursively compare its properties + for (const actualProp in actual) { + if (!compareWithShortcuts(actual[actualProp], expected[actualProp])) { + return false; + } + } + for (const expectedProp in expected) { + if (!Object.keys(actual).includes(expectedProp)) { + return false; + } + } + return true; +} + beforeAll(async () => { // start a simple web server that will capture requests sent when running // converted scripts @@ -136,7 +233,9 @@ describe("convert", () => { for (const param in parsedRequest?.query ?? {}) { if ( - capturedRequest.query[param] === undefined && + (capturedRequest.query === undefined || + capturedRequest.query[param] === undefined) && + capturedRequest.body && capturedRequest.body[param] !== undefined ) { // the client moved a query argument to the body @@ -155,21 +254,54 @@ describe("convert", () => { result: parsedRequest?.path, source, }); - expect( - { result: capturedRequest.query, source }, - failureMessage, - ).toEqual({ - result: parsedRequest?.query ?? {}, - source, - }); - expect( - { result: capturedRequest.body, source }, - failureMessage, - ).toEqual({ - result: parsedRequest?.body ?? {}, - source, - }); + if (checkExpandedShortcuts.includes(format)) { + if ( + !compareWithShortcuts(capturedRequest.query, parsedRequest?.query) + ) { + // A comparison accounting for shortcut properties came as different + // so now we do a full assert to report this error + expect( + { result: capturedRequest.query, source }, + failureMessage, + ).toEqual({ + result: parsedRequest?.query ?? {}, + source, + }); + } + } else { + expect( + { result: capturedRequest.query, source }, + failureMessage, + ).toEqual({ + result: parsedRequest?.query ?? {}, + source, + }); + } + if (checkExpandedShortcuts.includes(format)) { + if ( + !compareWithShortcuts(capturedRequest.body, parsedRequest?.body) + ) { + // A comparison accounting for shortcut properties came as different + // so now we do a full assert to report this error + expect( + { result: capturedRequest.body, source }, + failureMessage, + ).toEqual({ + result: parsedRequest?.body ?? {}, + source, + }); + } + } else { + expect( + { result: capturedRequest.body, source }, + failureMessage, + ).toEqual({ + result: parsedRequest?.body ?? {}, + source, + }); + } }, + 30000, // timeout ); } } diff --git a/tests/integration/java-app/pom.xml b/tests/integration/java-app/pom.xml new file mode 100644 index 0000000..5d236e5 --- /dev/null +++ b/tests/integration/java-app/pom.xml @@ -0,0 +1,109 @@ + + + 4.0.0 + + org.example + app + 1.0-SNAPSHOT + + app + + http://www.example.com + + + UTF-8 + 17 + + + + + + org.junit + junit-bom + 5.11.0 + pom + import + + + + + + + org.junit.jupiter + junit-jupiter-api + test + + + + org.junit.jupiter + junit-jupiter-params + test + + + + co.elastic.clients + elasticsearch-java + 9.2.0 + + + + + + + + + maven-clean-plugin + 3.4.0 + + + + maven-resources-plugin + 3.3.1 + + + maven-compiler-plugin + 3.13.0 + + + maven-surefire-plugin + 3.3.0 + + + maven-jar-plugin + 3.4.2 + + + maven-install-plugin + 3.1.2 + + + maven-deploy-plugin + 3.1.2 + + + + maven-site-plugin + 3.12.1 + + + maven-project-info-reports-plugin + 3.6.1 + + + maven-assembly-plugin + + + + org.example.App + + + + jar-with-dependencies + + + + + + + diff --git a/tests/integration/run-java.sh b/tests/integration/run-java.sh new file mode 100755 index 0000000..217fec8 --- /dev/null +++ b/tests/integration/run-java.sh @@ -0,0 +1,38 @@ +#!/bin/bash +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +CURRENT_DIR=$(pwd) +BRANCH=$(jq -r .version package.json | grep -Eo "^[0-9]+\.[0-9]+") + +if [[ "$1" == "" ]]; then + # the `test:setup` command runs this script without arguments to initialize + # the environment, so we first delete any previous one + rm -rf $SCRIPT_DIR/.java-request-converter + rm -rf $SCRIPT_DIR/.java +fi + +if [[ ! -d $SCRIPT_DIR/.java-es-request-converter ]]; then + git clone https://github.com/elastic/java-request-converter $SCRIPT_DIR/.java-request-converter + cd $SCRIPT_DIR/.java-request-converter + ./gradlew jar + cd $CURRENT_DIR +fi + +if [[ ! -d $SCRIPT_DIR/.java ]]; then + mkdir $SCRIPT_DIR/.java + echo "Installing from branch $BRANCH." + git clone -b "$BRANCH" --depth=1 "https://github.com/elastic/elasticsearch-java.git" $SCRIPT_DIR/.java || + (echo "Branch $BRANCH not found. Cloning main branch." && + git clone -b "main" --depth=1 "https://github.com/elastic/elasticsearch-java.git" $SCRIPT_DIR/.java) +fi + +if [[ "$1" != "" ]]; then + cd $SCRIPT_DIR/java-app + mkdir -p src/main/java/org/example + cp $CURRENT_DIR/$1 src/main/java/org/example/App.java + mvn compile assembly:single + if [[ "$?" == "0" ]]; then + java -jar target/app-1.0-SNAPSHOT-jar-with-dependencies.jar || true + else + exit 1 + fi +fi diff --git a/tests/integration/skip.ts b/tests/integration/skip.ts index 04fc379..83051b7 100644 --- a/tests/integration/skip.ts +++ b/tests/integration/skip.ts @@ -112,6 +112,84 @@ const skip: Record = { reason: "removed endpoint", formats: ["ruby", "javascript"], }, + + // the following examples trigger known bugs in the Java request converter or + // the Elasticsearch specification, so for now they are skipped + PutScriptRequestExample1: { + reason: "spec bug", + formats: ["java"], + }, + indicesPutSettingsRequestExample2: { + reason: "spec bug", + formats: ["java"], + }, + IndicesPutDataLifecycleRequestExample2: { + reason: "spec bug", + formats: ["java"], + }, + IndicesCreateFromExample1: { + reason: "java client bug", + formats: ["java"], + }, + indicesSimulateIndexTemplateRequestExample1: { + reason: "java client bug", + formats: ["java"], + }, + + // the following examples are not currently supported by the Java request converter + // so they are skipped + BulkRequestExample1: { + reason: "unsupported", + formats: ["java"], + }, + BulkRequestExample2: { + reason: "unsupported", + formats: ["java"], + }, + BulkRequestExample3: { + reason: "unsupported", + formats: ["java"], + }, + BulkRequestExample4: { + reason: "unsupported", + formats: ["java"], + }, + MultiGetRequestExample2: { + reason: "unsupported", + formats: ["java"], + }, + MsearchRequestExample1: { + reason: "unsupported", + formats: ["java"], + }, + MultiSearchTemplateRequestExample1: { + reason: "unsupported", + formats: ["java"], + }, + ConnectorUpdateLastSyncRequestExample1: { + reason: "unsupported", + formats: ["java"], + }, + AsyncQueryRequestExample1: { + reason: "unsupported", + formats: ["java"], + }, + EsqlAsyncQueryDeleteExample1: { + reason: "unsupported", + formats: ["java"], + }, + EsqlAsyncQueryGetExample1: { + reason: "unsupported", + formats: ["java"], + }, + EsqlAsyncQueryStopExample1: { + reason: "unsupported", + formats: ["java"], + }, + FindStructureRequestExample1: { + reason: "unsupported", + formats: ["java"], + }, }; export function shouldBeSkipped(