diff --git a/.gitignore b/.gitignore index 75150e4..132c33c 100644 --- a/.gitignore +++ b/.gitignore @@ -58,3 +58,5 @@ tests/integration/package.json tests/integration/es-client/ .tmp.request.* __pycache__ +src/exporters/templates.js +src/schema-no-generics.json diff --git a/go-exporter-issues.md b/go-exporter-issues.md new file mode 100644 index 0000000..00853bd --- /dev/null +++ b/go-exporter-issues.md @@ -0,0 +1,281 @@ +# Go Exporter Issues + +Comparison of generated output in `go-testing/` against the actual `go-elasticsearch` typed API in `typedapi/`, plus root-cause analysis traced through `elastic-client-generator-go`. + +## Summary + +| Test | Status | Key Issues | +|------|--------|------------| +| 01 - Info | **OK** | — | +| 02 - Search | **OK** | — | +| 03 - Create Index | **WRONG** | `NumberOfShards`/`NumberOfReplicas` are `*string` not `int`; `Properties` uses wrong map value type and wrong property structs | +| 04 - Aggs | **WRONG** | Go struct field is `Aggregations` not `Aggs`; uses `map[string]interface{}` instead of typed `types.Aggregations` | +| 05 - Bool Query | **WRONG** | Pointer in value slice; wrong `Range` map value type; `Gte`/`Lte` are `json.RawMessage`; `Sort` type completely wrong | +| 06 - Multi Request | **WRONG** | `V()` takes `bool` not `string` | +| 07 - Index Doc | **WRONG** | `index.Request` is `json.RawMessage` not a struct; method is `Id()` not `ID()` | + +--- + +## Detailed Issues + +### 03 - Create Index + +**Generated:** +```go +Settings: &types.IndexSettings{ + NumberOfShards: 3, + NumberOfReplicas: 1, +}, +Mappings: &types.TypeMapping{ + Properties: map[string]types.BinaryProperty{ + "title": &types.BinaryProperty{ Type: "text" }, + ... + }, +}, +``` + +**Problems:** + +1. `IndexSettings.NumberOfShards` is `*string`, not `int`. Should be `some.String("3")`. +2. `IndexSettings.NumberOfReplicas` is `*string`, not `int`. Should be `some.String("1")`. +3. `TypeMapping.Properties` is `map[string]Property` where `Property` is `any`. The output uses `map[string]types.BinaryProperty` which is the wrong map value type. +4. `BinaryProperty` is a specific mapping type for binary fields. Each mapping type has its own struct: `TextProperty`, `DateProperty`, `IntegerNumberProperty`. There is no generic `Type` string field — the struct type itself determines the mapping type. + +**Correct approach:** +```go +Properties: map[string]types.Property{ + "title": types.TextProperty{}, + "timestamp": types.DateProperty{}, + "count": types.IntegerNumberProperty{}, +}, +``` + +### 04 - Aggregations + +**Generated:** +```go +Request(&search.Request{ + Size: some.Int(0), + Aggs: map[string]interface{}{ ... }, +}) +``` + +**Problems:** + +1. The Go struct field is `Aggregations`, not `Aggs`. (`Aggs` is only a JSON deserialization alias, not the Go field name.) +2. Should use typed `types.Aggregations` struct, not `map[string]interface{}`. + +**Correct approach:** +```go +Request(&search.Request{ + Size: some.Int(0), + Aggregations: map[string]types.Aggregations{ + "status_codes": { + Terms: &types.TermsAggregation{ + Field: some.String("response.status_code"), + Size: some.Int(10), + }, + }, + "avg_duration": { + Avg: &types.AverageAggregation{ + Field: some.String("duration"), + }, + }, + }, +}) +``` + +### 05 - Bool Query + +**Generated (excerpt):** +```go +Must: []types.Query{ + &types.Query{ Match: ... }, +}, +Range: map[string]types.UntypedRangeQuery{ + "price": &types.UntypedRangeQuery{ Gte: 10, Lte: 100 }, +}, +Sort: []string{ + &types.SortOptions{ Price: "asc" }, + "_score", +}, +``` + +**Problems:** + +1. `Must`, `Filter`, `Should` are `[]Query` (value slices). Using `&types.Query{...}` produces a pointer — won't compile. +2. `Query.Range` is `map[string]RangeQuery` where `RangeQuery` is `any`. The output uses `map[string]types.UntypedRangeQuery` as the declared map type, which is wrong. +3. `UntypedRangeQuery.Gte` and `.Lte` are `json.RawMessage`, not bare integers. Should be `json.RawMessage("10")`. +4. `Sort` is `[]types.SortCombinations` (where `SortCombinations` is `any`), not `[]string`. The output puts a `&types.SortOptions{}` into a `[]string` which won't compile. +5. `SortOptions` has no `Price` field — custom field sorts go in the `SortOptions map[string]FieldSort` embedded map. `FieldSort` has an `Order` field of type `*sortorder.SortOrder`. + +### 06 - Multi Request + +**Generated:** +```go +es.Cat.Health().V("true") +``` + +**Problem:** `V()` signature is `func (r *Health) V(v bool) *Health`. It takes a `bool`, not a `string`. Should be `V(true)`. + +### 07 - Index Doc + +**Generated:** +```go +es.Index("my-index"). + ID("1"). + Request(&index.Request{ + Title: "Hello World", + Tags: []interface{}{ ... }, + Metadata: map[string]interface{}{ ... }, + }) +``` + +**Problems:** + +1. `index.Request` is defined as `type Request = json.RawMessage`. It is not a typed struct — you cannot initialize it with fields like `Title`, `Tags`, `Metadata`. +2. The method is `Id()` (lowercase d), not `ID()`. +3. The `Request()` method accepts `any`, so the correct approach is to pass a plain Go value (map or anonymous struct). + +**Correct approach:** +```go +es.Index("my-index"). + Id("1"). + Request(map[string]interface{}{ + "title": "Hello World", + "tags": []string{"intro", "welcome"}, + "metadata": map[string]interface{}{ + "author": "test", + "version": 2, + }, + }) +``` + +--- + +## Root Cause Analysis (traced through `elastic-client-generator-go`) + +### 1. `toPascalCase` special-cases `id` → `ID` but the generator does NOT + +**File:** `src/exporters/go/naming.ts` line 9 + +The exporter has: +```typescript +if (part.toLowerCase() === "id") return "ID"; +if (part.toLowerCase() === "ip") return "IP"; +``` + +But the generator (`elastic-client-generator-go/src/pkg/mapping/strings.go`) uses `cases.Title(language.English, cases.NoLower)` which simply title-cases each word. For `id` this produces `Id`, not `ID`. Same for `ip` → `Ip`. + +The generator has no special acronym handling, so the exporter's special-casing is incorrect and should be removed. This causes `ID("1")` in the output instead of the correct `Id("1")`. + +### 2. Query parameter type resolution: boolean vs string methods + +**File:** `src/exporters/go/exporter.ts` `renderQueryParams()` + +The exporter correctly checks for boolean type at line 177 (`ctx.resolver.isBooleanType(inst.type)`), but there's a fallback at line 202 where unresolved params get `methodName("value")` — always passing a string. + +In the generated client, query parameter methods have typed signatures determined by the endpoint template (`endpoint.go.tmpl` lines 486-497): +- `"boolean"` → `func V(v bool)` +- `"integer"` → `func From(from int)` + +The `v` parameter for `cat.health` has `RawType: "boolean"`, so the generated method is `V(v bool)`. If the spec param lookup fails (line 167: `req.request?.query?.find`), the fallback renders a string. The fix is to ensure query parameter matching works, including when the parser sends `"v"` but the spec has `"v"` with boolean type. + +### 3. Union types resolve to first item instead of matching Go generator behavior + +**File:** `src/exporters/go/renderer.ts` `goTypeString()` and `renderUnionOf()` + +For `union_of` types, the exporter takes the first item: +```typescript +case "union_of": { + const union = typeInfo as UnionOf; + if (union.items.length > 0) { + return this.goTypeString(union.items[0], ctx); + } +} +``` + +But the Go generator's `DecisionTree` has complex logic for union resolution. Some examples of how this matters: + +| Spec union | Exporter resolves to | Generator resolves to | Go type | +|---|---|---|---| +| `integer \| string` (`number_of_shards`) | `int` (first item) | `string` | `*string` | +| `SortCombinations` (union) | first item | `any` | `SortCombinations any` | +| `RangeQuery` (union) | first item | `any` | `RangeQuery any` | +| `Property` (union) | first item (`BinaryProperty`) | `any` | `Property any` | +| `MinimumShouldMatch` (union) | first item | `any` | `MinimumShouldMatch any` | + +In the Go generator, unions that contain non-compatible types (not just string+[]string) are rendered as `any` with a type alias. The exporter needs to check: if the generated Go type is a type alias to `any`, it should use that type alias name instead of resolving the first union item. + +### 4. Property alias resolution (JSON `aggs` → Go field `Aggregations`) + +**File:** `src/exporters/go/renderer.ts` `renderStructFields()` + +The spec property for aggregations has: +```json +{ "name": "aggregations", "aliases": ["aggs"], ... } +``` + +When the user input uses `"aggs"`, the field lookup (`properties.find(p => p.name === key)`) fails because it matches against `name` but not `aliases`. The code falls to the `!prop` branch which renders `toPascalCase("aggs")` = `Aggs` and uses `renderLiteralValue` instead of typed rendering. + +**Fix:** Also check `p.aliases?.includes(key)` in the property lookup. + +### 5. `user_defined_value` body → `json.RawMessage` (the `index` API case) + +**Files:** `src/exporters/go/exporter.ts` `renderBody()`, `elastic-client-generator-go/src/pkg/models/request.go` + +In the raw schema (`schema.json`), the index API body is: +```json +{ "kind": "value", "value": { "kind": "instance_of", "type": { "name": "TDocument", ... } } } +``` + +In the expanded schema (`schema-no-generics.json`), after generic expansion, it becomes: +```json +{ "kind": "value", "value": { "kind": "user_defined_value" } } +``` + +The exporter uses `req.request.body` from the raw schema (via parser) where `bodyDef.value.kind === "instance_of"` is true. It then calls `resolver.getInterfaceProperties("TDocument", "_global.index.Request")` which returns nothing (TDocument doesn't exist in the expanded schema). With empty properties, `renderStructFields` falls back to literal rendering for each field, but still wraps everything in `&index.Request{...}`. + +In the generator, `TDocument` resolves to `user_defined_value` → `json.RawMessage`. The `Request()` method accepts `any`. + +**Fix:** The exporter needs to consult the expanded schema's body definition. When the expanded body value is `user_defined_value`, it should skip the typed Request wrapper and pass the body as a plain `map[string]interface{}` or struct literal. Alternatively, detect when `getInterfaceProperties()` returns empty for a generic type parameter and fall back to untyped. + +### 6. Pointer vs value in slices (e.g. `&types.Query{}` in `[]Query`) + +**File:** `src/exporters/go/renderer.ts` `renderInstanceOf()` + +The renderer always uses `&types.Foo{...}` for struct instances (line 249): +```typescript +return `&types.${goName}{\n${fields}\n${ctx.indent()}}`; +``` + +But in Go, `[]Query` is a value slice. Putting `&types.Query{}` (a pointer) into `[]Query` won't compile. The renderer needs context about whether it's rendering into a pointer context (struct field with `*` prefix) or a value context (slice element, map value that's not a pointer type). + +In the generator, the `ShouldBePointer()` method on `Field` determines this: +- Slices and maps are already underlying pointers, so their elements are not pointer-wrapped +- Only optional struct fields get pointers + +### 7. Container variant types rendered as structs with wrong field names + +The `SortOptions` struct in go-elasticsearch has: +```go +type SortOptions struct { + Doc_ *ScoreSort `json:"_doc,omitempty"` + Score_ *ScoreSort `json:"_score,omitempty"` + SortOptions map[string]FieldSort `json:"-"` +} +``` + +The exporter doesn't understand this container variant pattern — custom field sorts go into the `SortOptions` map, not as direct struct fields. It incorrectly tries `SortOptions{Price: "asc"}` which has no `Price` field. + +--- + +## Priority Fix Order + +1. ~~**Property alias resolution** — easy fix, check `aliases` in property lookup. Fixes `Aggs` → `Aggregations`.~~ **FIXED** +2. ~~**Remove `id`/`ip` special-casing in `toPascalCase`** — simple fix. Fixes `ID()` → `Id()`.~~ **FIXED** +3. ~~**Union type resolution** — check if the Go type is a type alias to `any` and use the alias name. Fixes `NumberOfShards`, `Sort`, `Range`, `Property` types.~~ **FIXED** +4. **`user_defined_value` body detection** — consult expanded schema to detect untyped request bodies. Fixes index API output. +5. **Pointer vs value context** — track whether we're in a slice/map element context and omit `&`. Fixes `&types.Query{}` in `[]Query`. +6. **Query parameter boolean detection** — ensure spec param lookup finds the `v` parameter for cat health. Fixes `V("true")` → `V(true)`. +7. **Container variant / sort rendering** — complex; requires understanding the sort DSL structure to emit correct `SortOptions` with `map[string]FieldSort`. diff --git a/package.json b/package.json index fcf90d8..61615bb 100644 --- a/package.json +++ b/package.json @@ -13,34 +13,37 @@ "url": "https://github.com/elastic/request-converter" }, "scripts": { - "update-schema": "node scripts/update-schema.mjs", + "update-schema": "node scripts/update-schema.mjs && node scripts/expand-generics.mjs", "release": "./scripts/release.sh", - "build": "npm run clean && tsc && npm run compile-templates && npm run copy-files", + "build": "npm run clean && tsc && npm run compile-templates && npm run expand-schema && npm run copy-files", "clean": "rimraf dist/", "compile-templates": "node scripts/compile-templates.mjs", + "expand-schema": "node scripts/expand-generics.mjs", "copy-files": "copyfiles -u 1 src/**/*.js src/**/*.json dist/", "docs": "typedoc src/index.ts", "lint": "eslint src tests --ignore-pattern tests/wasm/", "prettier": "prettier \"src/**/*.ts\" \"tests/**/*.ts\" --list-different", - "test": "npm run compile-templates && jest --test-path-ignore-patterns integration --coverage", - "test:setup": "./tests/integration/run-python.sh && ./tests/integration/run-javascript.sh && ./tests/integration/run-ruby.sh && ./tests/integration/run-php.sh", + "test": "npm run compile-templates && npm run expand-schema && jest --test-path-ignore-patterns integration --coverage", + "test:setup": "./tests/integration/run-python.sh && ./tests/integration/run-javascript.sh && ./tests/integration/run-ruby.sh && ./tests/integration/run-php.sh && ./tests/integration/run-go.sh", "test:setup-curl": "echo \"Nothing to do!\"", "test:setup-python": "./tests/integration/run-python.sh", "test:setup-javascript": "./tests/integration/run-javascript.sh", "test:setup-php": "./tests/integration/run-php.sh", "test:setup-ruby": "./tests/integration/run-ruby.sh", + "test:setup-go": "./tests/integration/run-go.sh", "test:integration": "jest tests/integration", "test:integration-curl": "./scripts/test-format.sh curl", "test:integration-python": "./scripts/test-format.sh python", "test:integration-javascript": "./scripts/test-format.sh javascript", "test:integration-php": "./scripts/test-format.sh php", "test:integration-ruby": "./scripts/test-format.sh ruby", + "test:integration-go": "./scripts/test-format.sh go", "test:example": "./scripts/test-example.sh", "fix": "npm run fix:lint && npm run fix:prettier", "fix:lint": "eslint src tests --fix --ignore-pattern tests/wasm/", "fix:prettier": "prettier \"src/**/*.ts\" \"tests/**/*.ts\" --write", "fix:precommit": "./scripts/precommit.sh", - "watch:test": "npm run compile-templates && jest --test-path-ignore-patterns integration --coverage --watch", + "watch:test": "npm run compile-templates && npm run expand-schema && jest --test-path-ignore-patterns integration --coverage --watch", "parser": "peggy --plugin ./node_modules/ts-pegjs/dist/tspegjs -o src/es_parser.ts ./src/es_parser.pegjs", "prepare": "husky install" }, diff --git a/scripts/expand-generics.mjs b/scripts/expand-generics.mjs new file mode 100644 index 0000000..d7fef93 --- /dev/null +++ b/scripts/expand-generics.mjs @@ -0,0 +1,376 @@ +/** + * Ported from elasticsearch-specification/compiler/src/transform/expand-generics.ts + * + * Expands all generics by creating new concrete types for every instantiation + * of a generic type. The resulting model has no generics. Top-level generic + * parameters (e.g. SearchRequest's TDocument) are replaced by + * user_defined_value. + * + * The Go exporter needs this expanded schema because the go-elasticsearch typed + * API is generated from the post-expansion spec via elastic-client-generator-go. + */ + +import { readFile, writeFile } from "fs/promises"; +import { join, dirname } from "path"; +import { fileURLToPath } from "url"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); + +// --------------------------------------------------------------------------- +// Helper functions +// --------------------------------------------------------------------------- + +function nameKey(t) { + if (t.kind !== undefined) { + return nameKey(t.name); + } + return t.namespace + ":" + t.name; +} + +function genericParamMapping(generics, params) { + const mapping = new Map(); + (generics ?? []).forEach((name, i) => { + mapping.set(nameKey(name), params[i]); + }); + return mapping; +} + +function valueTypeName(value) { + switch (value.kind) { + case "literal_value": + return value.value.toString(); + case "user_defined_value": + return "UserDefined"; + case "array_of": + return "Array" + valueTypeName(value.value); + case "dictionary_of": + return "Dict" + valueTypeName(value.value); + case "union_of": + return "Union" + value.items.map((v) => valueTypeName(v)).join(); + case "instance_of": + return value.type.name; + } +} + +function sortTypeDefinitions(types) { + types.sort((a, b) => { + if (a.name.namespace === b.name.namespace) { + if (a.name.name > b.name.name) return 1; + if (a.name.name < b.name.name) return -1; + return 0; + } + if (a.name.namespace > b.name.namespace) return 1; + if (a.name.namespace < b.name.namespace) return -1; + return 0; + }); +} + +// --------------------------------------------------------------------------- +// Core expansion +// --------------------------------------------------------------------------- + +function expandGenerics(inputModel, config) { + const typesToUnwrap = new Set(); + const typesToInline = new Set(); + + for (const name of config?.unwrappedTypes ?? []) { + typesToUnwrap.add(typeof name === "string" ? name : nameKey(name)); + } + for (const name of config?.inlinedTypes ?? []) { + typesToInline.add(typeof name === "string" ? name : nameKey(name)); + } + + const typesSeen = new Set(); + const types = []; + const inputTypeByName = new Map(); + + for (const type of inputModel.types) { + inputTypeByName.set(nameKey(type), type); + } + + function addIfNotSeen(name, build) { + const key = nameKey(name); + if (!typesSeen.has(key)) { + typesSeen.add(key); + const type = build(); + type.name = name; + types.push(type); + } + return name; + } + + function getType(name) { + const result = inputTypeByName.get(nameKey(name)); + if (result === undefined) { + throw Error(`Type ${nameKey(name)} does not exist.`); + } + return result; + } + + function expandRootType(name) { + if (name == null) return; + const type = getType(name); + if (type.kind !== "request" && type.kind !== "response") { + throw Error(`${nameKey(name)} should be a request or a response`); + } + const userDefined = { kind: "user_defined_value" }; + const typeParams = (type.generics ?? []).map(() => userDefined); + expandType(type.name, typeParams); + } + + function expandType(name, params) { + if (name.namespace === "_builtins") return name; + const type = getType(name); + switch (type.kind) { + case "enum": + return addIfNotSeen(type.name, () => type); + case "type_alias": + return expandTypeAlias(type, params); + case "request": + return expandRequest(type, params); + case "response": + return expandResponse(type, params); + case "interface": + return expandInterface(type, params); + } + } + + function addDanglingTypeIfNotSeen(type) { + switch (type.kind) { + case "type_alias": + if (type.generics !== undefined && type.generics.length > 0) return; + break; + case "interface": + if (type.generics !== undefined && type.generics.length > 0) return; + break; + } + addIfNotSeen(type.name, () => type); + } + + function expandInterface(type, params) { + return addIfNotSeen(expandedName(type, params), () => { + const result = { ...type }; + const mappings = genericParamMapping(type.generics, params); + result.inherits = expandInherits(result.inherits, mappings); + + if (result.behaviors != null) { + result.behaviors.forEach((b) => { + if (b.generics == null) { + const type = getType(b.type); + addIfNotSeen(b.type, () => type); + } + }); + result.behaviors = result.behaviors.map((b) => ({ + type: b.type, + generics: (b.generics ?? []).map((g) => expandValueOf(g, mappings)), + })); + } + + result.properties = expandProperties(result.properties, mappings); + result.generics = undefined; + return result; + }); + } + + function expandInherits(inherits, mappings) { + if (inherits == null) return undefined; + const expanded = expandValueOf( + { + kind: "instance_of", + type: inherits.type, + generics: inherits.generics, + }, + mappings, + ); + return { type: expanded.type, generics: undefined }; + } + + function expandTypeAlias(alias, params) { + return addIfNotSeen(expandedName(alias, params), () => { + const result = { ...alias }; + result.type = expandValueOf( + alias.type, + genericParamMapping(alias.generics, params), + ); + result.generics = undefined; + return result; + }); + } + + function expandRequest(req, params) { + return addIfNotSeen(req.name, () => { + const mappings = genericParamMapping(req.generics, params); + const result = { ...req }; + result.inherits = expandInherits(result.inherits, mappings); + result.path = expandProperties(result.path, mappings); + result.query = expandProperties(result.query, mappings); + result.body = expandBody( + req.body, + genericParamMapping(req.generics, params), + ); + result.generics = undefined; + return result; + }); + } + + function expandResponse(resp, params) { + return addIfNotSeen(resp.name, () => { + const result = { ...resp }; + result.body = expandBody( + resp.body, + genericParamMapping(resp.generics, params), + ); + if (resp.exceptions != null) { + result.exceptions = resp.exceptions.map((exception) => ({ + description: exception.description, + statusCodes: exception.statusCodes, + body: expandBody( + exception.body, + genericParamMapping(resp.generics, params), + ), + })); + } + result.generics = undefined; + return result; + }); + } + + function expandProperties(properties, mappings) { + return properties.map((prop) => ({ + ...prop, + type: expandValueOf(prop.type, mappings), + })); + } + + function expandBody(body, mappings) { + switch (body.kind) { + case "no_body": + return body; + case "properties": + return { + kind: "properties", + properties: expandProperties(body.properties, mappings), + }; + case "value": + return { + kind: "value", + value: expandValueOf(body.value, mappings), + codegenName: body.codegenName, + }; + } + } + + function expandValueOf(value, mappings) { + switch (value.kind) { + case "array_of": + return { + kind: "array_of", + value: expandValueOf(value.value, mappings), + }; + + case "dictionary_of": + return { + kind: "dictionary_of", + key: expandValueOf(value.key, mappings), + value: expandValueOf(value.value, mappings), + singleKey: value.singleKey, + }; + + case "instance_of": { + const valueOfType = nameKey(value.type); + + if (typesToUnwrap.has(valueOfType)) { + return expandValueOf(value.generics[0], mappings); + } + + if (typesToInline.has(valueOfType)) { + const inlinedTypeDef = inputTypeByName.get(valueOfType); + if (inlinedTypeDef?.kind !== "type_alias") { + throw Error( + `Inlined type ${valueOfType} should be an alias definition`, + ); + } + const inlineMappings = new Map(); + for (let i = 0; i < (inlinedTypeDef.generics?.length ?? 0); i++) { + inlineMappings.set( + nameKey(inlinedTypeDef.generics[i]), + value.generics[i], + ); + } + return expandValueOf(inlinedTypeDef.type, inlineMappings); + } + + const mapping = mappings.get(nameKey(value.type)); + if (mapping !== undefined) return mapping; + + const params = (value.generics ?? []).map((g) => + expandValueOf(g, mappings), + ); + return { + kind: "instance_of", + type: expandType(value.type, params), + generics: undefined, + }; + } + + case "literal_value": + return value; + + case "union_of": + return { + kind: "union_of", + items: value.items.map((item) => expandValueOf(item, mappings)), + }; + + case "user_defined_value": + return value; + } + } + + function expandedName(type, params) { + let localName = type.name.name; + type.generics?.forEach((_paramName, i) => { + const param = params[i]; + if (param.kind === "user_defined_value") return; + localName = localName + valueTypeName(params[i]); + }); + return { namespace: type.name.namespace, name: localName }; + } + + // --- Main expansion loop --- + + for (const endpoint of inputModel.endpoints) { + expandRootType(endpoint.request); + expandRootType(endpoint.response); + } + + for (const type of inputModel.types) { + addDanglingTypeIfNotSeen(type); + } + + sortTypeDefinitions(types); + + return { + _info: inputModel._info, + endpoints: inputModel.endpoints, + types: types, + }; +} + +// --------------------------------------------------------------------------- +// CLI: read schema.json -> write schema-no-generics.json +// --------------------------------------------------------------------------- + +const inputPath = join(__dirname, "..", "src", "schema.json"); +const outputPath = join(__dirname, "..", "src", "schema-no-generics.json"); + +const inputText = await readFile(inputPath, { encoding: "utf8" }); +const inputModel = JSON.parse(inputText); + +const outputModel = expandGenerics(inputModel, { + inlinedTypes: ["_spec_utils:WithNullValue"], +}); + +await writeFile(outputPath, JSON.stringify(outputModel, null, 2), "utf8"); + +console.log(`Expanded schema written to ${outputPath}`); diff --git a/src/convert.ts b/src/convert.ts index 1480277..e79a028 100644 --- a/src/convert.ts +++ b/src/convert.ts @@ -6,6 +6,7 @@ import { CurlExporter } from "./exporters/curl"; import { JavaScriptExporter } from "./exporters/javascript"; import { PHPExporter } from "./exporters/php"; import { RubyExporter } from "./exporters/ruby"; +import { GoExporter } from "./exporters/go"; import util from "util"; const isBrowser = typeof window !== "undefined"; @@ -45,8 +46,9 @@ const EXPORTERS: Record = { python: new PythonExporter(), ruby: new RubyExporter(), curl: new CurlExporter(), + go: new GoExporter(), }; -const LANGUAGES = ["JavaScript", "PHP", "Python", "Ruby", "curl"]; +const LANGUAGES = ["JavaScript", "PHP", "Python", "Ruby", "curl", "Go"]; /** * Return the list of available export formats. diff --git a/src/exporters/go.ts b/src/exporters/go.ts new file mode 100644 index 0000000..4e93ed3 --- /dev/null +++ b/src/exporters/go.ts @@ -0,0 +1 @@ +export { GoExporter } from "./go/exporter"; diff --git a/src/exporters/go/constants.ts b/src/exporters/go/constants.ts new file mode 100644 index 0000000..5fea8f0 --- /dev/null +++ b/src/exporters/go/constants.ts @@ -0,0 +1,221 @@ +export const GO_BASE_IMPORT = "github.com/elastic/go-elasticsearch/v9"; + +export const UNSUPPORTED_APIS = new RegExp("^_internal.*$"); + +export const NUMERIC_TYPES = new Set([ + "integer", + "long", + "float", + "double", + "number", + "uint", + "short", + "byte", + "ulong", +]); + +export const STRING_ALIAS_TYPES = new Set([ + "Id", + "IndexName", + "Name", + "Field", + "Routing", + "NodeId", + "ScrollId", + "IndexAlias", + "TaskId", + "Namespace", + "NodeName", + "Percentage", + "Duration", + "DurationLarge", + "TimeUnit", + "EpochTime", + "DateTime", + "DateString", + "DateMath", + "MinimumShouldMatch", + "VersionString", + "PipelineName", + "DataStreamName", + "Ip", + "Host", + "Password", + "Username", + "Metadata", + "Uri", + "Uuid", + "SequenceNumber", + "ByteSize", + "HumanReadableByteCount", + "WaitForActiveShards", + "Fuzziness", + "MultiTermQueryRewrite", + "GeoHash", + "GeoTilePrecision", + "Script", + "ScriptLanguage", +]); + +/** + * Naming collision map kept in sync with duplicatesNameList in + * elastic-client-generator-go/src/pkg/mapping/duplicates.go + */ +export const GO_TYPE_RENAMES: Record> = { + "snapshot.repository_analyze": { + NodeInfo: "SnapshotRepositoryAnalyzeNodeInfo", + }, + "_types.query_dsl": { + QueryContainer: "Query", + FunctionScoreContainer: "FunctionScore", + IntervalsContainer: "Intervals", + }, + "_types.aggregations": { + AggregationContainer: "Aggregations", + BucketsQueryContainer: "BucketsQuery", + }, + "_global.get_script_context": { + Context: "GetScriptContext", + }, + "_global.knn_search._types": { + Query: "CoreKnnQuery", + }, + "_global.mget": { + Operation: "MgetOperation", + ResponseItem: "MgetResponseItem", + }, + "_global.msearch": { + RequestItem: "MsearchRequestItem", + ResponseItem: "MsearchResponseItem", + }, + "_global.mtermvectors": { + Operation: "MTermVectorsOperation", + }, + "_global.reindex": { + Destination: "ReindexDestination", + Source: "ReindexSource", + }, + "_global.termvectors": { + Filter: "TermVectorsFilter", + Token: "TermVectorsToken", + }, + "cat.component_templates": { + ComponentTemplate: "CatComponentTemplate", + }, + "ccr._types": { + ShardStats: "CcrShardStats", + }, + "cluster._types": { + ComponentTemplate: "ClusterComponentTemplate", + }, + "cluster.stats": { + IndexingPressure: "ClusterIndexingPressure", + IndexingPressureMemory: "ClusterPressureMemory", + RuntimeFieldTypes: "ClusterRuntimeFieldTypes", + }, + "enrich._types": { + Policy: "EnrichPolicy", + }, + "features._types": { + Feature: "Feature", + }, + "ilm._types": { + Policy: "IlmPolicy", + Actions: "IlmActions", + }, + "indices._types": { + IndexingPressure: "IndicesIndexingPressure", + IndexingPressureMemory: "IndicesIndexingPressureMemory", + }, + "indices.field_usage_stats": { + ShardsStats: "IndicesShardsStats", + }, + "indices.modify_data_stream": { + Action: "IndicesModifyAction", + }, + "indices.stats": { + ShardStats: "IndicesShardStats", + }, + "indices.update_aliases": { + Action: "IndicesAction", + }, + "ingest._types": { + Pipeline: "IngestPipeline", + }, + "ingest.simulate": { + Ingest: "SimulateIngest", + }, + "ingest.get_geoip_database": { + DatabaseConfigurationMetadata: "GeoipDatabaseConfigurationMetadata", + }, + "ingest.get_ip_location_database": { + DatabaseConfigurationMetadata: "IpLocationDatabaseConfigurationMetadata", + }, + "logstash._types": { + Pipeline: "LogstashPipeline", + }, + "migration.get_feature_upgrade_status": { + MigrationFeature: "GetMigrationFeature", + }, + "migration.post_feature_upgrade": { + MigrationFeature: "PostMigrationFeature", + }, + "ml._types": { + Datafeed: "MLDatafeed", + Filter: "MLFilter", + }, + "ml.evaluate_data_frame": { + ResponseBody: "MLEvaluateDataFrameResponseBody", + }, + "nodes._types": { + Context: "NodesContext", + IndexingPressure: "NodesIndexingPressure", + IndexingPressureMemory: "NodesIndexingPressureMemory", + Ingest: "NodesIngest", + }, + "security._types": { + Realm: "SecurityRealm", + RoleMapping: "SecurityRoleMapping", + }, + "security.authenticate": { + Token: "AuthenticateToken", + }, + "security.create_service_token": { + Token: "ServiceToken", + }, + "security.enroll_kibana": { + Token: "KibanaToken", + }, + "security.put_privileges": { + Actions: "PrivilegesActions", + }, + "slm._types": { + Policy: "SLMPolicy", + }, + "snapshot._types": { + ShardsStats: "SnapshotShardsStats", + }, + "transform._types": { + Destination: "TransformDestination", + Source: "TransformSource", + }, + "watcher._types": { + Action: "WatcherAction", + Actions: "WatcherStatusActions", + ConditionContainer: "WatcherCondition", + InputContainer: "WatcherInput", + }, + "xpack.info": { + Feature: "XpackFeature", + Features: "XpackFeatures", + }, + "xpack.usage": { + Datafeed: "XpackDatafeed", + Query: "XpackQuery", + Realm: "XpackRealm", + RoleMapping: "XpackRoleMapping", + RuntimeFieldTypes: "XpackRuntimeFieldTypes", + Phase: "UsagePhase", + Phases: "UsagePhases", + }, +}; diff --git a/src/exporters/go/context.ts b/src/exporters/go/context.ts new file mode 100644 index 0000000..95cece0 --- /dev/null +++ b/src/exporters/go/context.ts @@ -0,0 +1,18 @@ +import { ImportTracker } from "./imports"; +import { TypeResolver } from "./schema"; + +export class RenderContext { + constructor( + readonly resolver: TypeResolver, + readonly imports: ImportTracker, + readonly depth: number = 0, + ) {} + + nested(): RenderContext { + return new RenderContext(this.resolver, this.imports, this.depth + 1); + } + + indent(): string { + return " ".repeat(this.depth); + } +} diff --git a/src/exporters/go/exporter.ts b/src/exporters/go/exporter.ts new file mode 100644 index 0000000..951c56f --- /dev/null +++ b/src/exporters/go/exporter.ts @@ -0,0 +1,291 @@ +import { FormatExporter, ConvertOptions } from "../../convert"; +import { ParsedRequest } from "../../parse"; +import { InstanceOf, Property } from "../../metamodel"; +import { UNSUPPORTED_APIS } from "./constants"; +import { toPascalCase, apiToGoMethod, indent } from "./naming"; +import { TypeResolver } from "./schema"; +import { ImportTracker } from "./imports"; +import { RenderContext } from "./context"; +import { GoValueRenderer } from "./renderer"; + +export class GoExporter implements FormatExporter { + async check(requests: ParsedRequest[]): Promise { + return requests + .map((req) => req.service === "es") + .reduce((prev, curr) => prev && curr, true); + } + + async convert( + requests: ParsedRequest[], + options: ConvertOptions, + ): Promise { + if (!(await this.check(requests))) { + throw new Error("Cannot perform conversion"); + } + const resolver = await TypeResolver.load(); + const imports = new ImportTracker(); + imports.addContext(); + const renderer = new GoValueRenderer(); + + const snippets: string[] = []; + for (let i = 0; i < requests.length; i++) { + const ctx = new RenderContext(resolver, imports, 2); + snippets.push( + this.renderRequest(requests[i], i, imports, ctx, renderer, options), + ); + } + + let output = snippets.join("\n"); + + if (options.complete) { + imports.addElasticsearch(); + imports.addLog(); + const esUrl = options.elasticsearchUrl + ? `"${options.elasticsearchUrl}"` + : `os.Getenv("ELASTICSEARCH_URL")`; + if (!options.elasticsearchUrl) { + imports.add("os"); + } + const header = `package main + +${imports.render()} + +func main() { + cfg := elasticsearch.Config{ + Addresses: []string{${esUrl}}, + } + es, err := elasticsearch.NewTypedClient(cfg) + if err != nil { + log.Fatalf("Error creating client: %s", err) + } + +`; + const footer = `} +`; + output = + header + + output + .split("\n") + .map((line) => (line ? " " + line : line)) + .join("\n") + + footer; + } + + return output; + } + + private renderRequest( + req: ParsedRequest, + index: number, + imports: ImportTracker, + ctx: RenderContext, + renderer: GoValueRenderer, + options: ConvertOptions, + ): string { + const varName = index === 0 ? "res" : `res${index}`; + + if (!req.api || UNSUPPORTED_APIS.test(req.api) || !req.request) { + return this.renderUnsupportedRequest(req, varName, imports); + } + + const { subclient, method } = apiToGoMethod(req.api); + const caller = subclient ? `es.${subclient}.${method}` : `es.${method}`; + + const parts: string[] = []; + + const requiredPathParams = this.getRequiredPathArgs(req); + parts.push(`${varName}, err := ${caller}(${requiredPathParams}).`); + + this.renderPathParams(req, parts); + this.renderQueryParams(req, parts, ctx); + this.renderBody(req, parts, ctx, renderer, imports); + + parts.push(`${indent(1)}Do(context.Background())`); + + let result = parts.join("\n"); + + if (options.printResponse) { + imports.addFmt(); + result += `\nfmt.Println(${varName})`; + } + + return result + "\n"; + } + + private renderUnsupportedRequest( + req: ParsedRequest, + varName: string, + imports: ImportTracker, + ): string { + let body = "nil"; + if (req.body) { + body = `strings.NewReader(\`${JSON.stringify(req.body)}\`)`; + imports.add("strings"); + } + return `${varName}, err := es.Transport.Perform(&http.Request{ + Method: "${req.method}", + URL: &url.URL{Path: "${req.path}"}, + Body: ${body}, +}) +`; + } + + private getRequiredPathArgs(req: ParsedRequest): string { + if (!req.request?.path || Object.keys(req.params).length === 0) { + return ""; + } + const required = req.request.path.filter((p) => p.required); + const args: string[] = []; + for (const param of required) { + const value = req.params[param.name]; + if (value !== undefined) { + args.push(`"${value}"`); + } + } + return args.join(", "); + } + + private renderPathParams(req: ParsedRequest, parts: string[]): void { + if (!req.request?.path) return; + const required = new Set( + req.request.path.filter((p) => p.required).map((p) => p.name), + ); + for (const [name, value] of Object.entries(req.params)) { + if (value === undefined || required.has(name)) continue; + const methodName = toPascalCase(name); + parts.push(`${indent(1)}${methodName}("${value}").`); + } + } + + private renderQueryParams( + req: ParsedRequest, + parts: string[], + ctx: RenderContext, + ): void { + if (!req.query) return; + for (const [name, value] of Object.entries(req.query)) { + let specParam = req.request?.query?.find((q) => q.name === name); + if (!specParam && req.request?.attachedBehaviors) { + const behaviorProps = ctx.resolver.getBehaviorProperties( + req.request.attachedBehaviors, + ); + specParam = behaviorProps.find((p) => p.name === name); + } + const methodName = toPascalCase(name); + if (specParam) { + const typeInfo = specParam.type; + if (typeInfo.kind === "instance_of") { + const inst = typeInfo as InstanceOf; + if (ctx.resolver.isNumericType(inst.type)) { + parts.push(`${indent(1)}${methodName}(${parseInt(value, 10)}).`); + continue; + } + if (ctx.resolver.isBooleanType(inst.type)) { + parts.push(`${indent(1)}${methodName}(${value === "true"}).`); + continue; + } + const enumType = ctx.resolver.isEnumType(inst.type); + if (enumType) { + const enumPkg = inst.type.name.toLowerCase(); + const member = enumType.members.find( + (m) => + m.name === value || + m.aliases?.includes(value) || + m.name.toLowerCase() === value.toLowerCase(), + ); + if (member) { + ctx.imports.addEnumPackage(inst.type); + parts.push( + `${indent(1)}${methodName}(${enumPkg}.${toPascalCase( + member.name, + )}).`, + ); + continue; + } + } + } + } + parts.push(`${indent(1)}${methodName}("${value}").`); + } + } + + private renderBody( + req: ParsedRequest, + parts: string[], + ctx: RenderContext, + renderer: GoValueRenderer, + imports: ImportTracker, + ): void { + if (!req.body || typeof req.body !== "object" || Array.isArray(req.body)) { + return; + } + if (!req.request?.body || req.request.body.kind === "no_body") { + return; + } + + const body = req.body as Record; + if (Object.keys(body).length === 0) return; + + const bodyDef = req.request.body; + + if ( + bodyDef.kind === "value" && + bodyDef.value.kind === "instance_of" && + ctx.resolver.isUserDefinedValueBody( + req.request.name.name, + req.request.name.namespace, + ) + ) { + const lines: string[] = []; + for (const [key, value] of Object.entries(body)) { + lines.push( + `${ctx.indent()}"${renderer.escapeGoString( + key, + )}": ${renderer.renderLiteralValue(value, ctx)},`, + ); + } + parts.push(`${indent(1)}Request(map[string]interface{}{`); + parts.push(lines.join("\n")); + parts.push(`${indent(1)}}).`); + return; + } + + let properties: Property[]; + if (bodyDef.kind === "properties") { + properties = bodyDef.properties; + } else if ( + bodyDef.kind === "value" && + bodyDef.value.kind === "instance_of" + ) { + const inst = bodyDef.value as InstanceOf; + properties = ctx.resolver.getInterfaceProperties( + inst.type.name, + inst.type.namespace, + ); + } else { + return; + } + + if (req.request.inherits) { + const parentProps = ctx.resolver.getInterfaceProperties( + req.request.inherits.type.name, + req.request.inherits.type.namespace, + ); + properties = [...properties, ...parentProps]; + } + + const apiPkg = this.getApiPackageName(req.api!); + imports.addApiPackage(req.api!); + imports.addTypes(); + + const bodyLines = renderer.renderStructFields(body, properties, ctx); + parts.push(`${indent(1)}Request(&${apiPkg}.Request{`); + parts.push(bodyLines); + parts.push(`${indent(1)}}).`); + } + + private getApiPackageName(api: string): string { + const parts = api.split("."); + return parts[parts.length - 1]; + } +} diff --git a/src/exporters/go/imports.ts b/src/exporters/go/imports.ts new file mode 100644 index 0000000..fc393c1 --- /dev/null +++ b/src/exporters/go/imports.ts @@ -0,0 +1,65 @@ +import { TypeName } from "../../metamodel"; +import { GO_BASE_IMPORT } from "./constants"; + +export class ImportTracker { + private imports = new Set(); + + add(pkg: string): void { + this.imports.add(pkg); + } + + addContext(): void { + this.imports.add("context"); + } + + addFmt(): void { + this.imports.add("fmt"); + } + + addLog(): void { + this.imports.add("log"); + } + + addTypes(): void { + this.imports.add(`${GO_BASE_IMPORT}/typedapi/types`); + } + + addSome(): void { + this.imports.add(`${GO_BASE_IMPORT}/typedapi/some`); + } + + addElasticsearch(): void { + this.imports.add(GO_BASE_IMPORT); + } + + addApiPackage(api: string): void { + const parts = api.split("."); + let pkgPath: string; + if (parts.length === 1) { + pkgPath = `${GO_BASE_IMPORT}/typedapi/core/${parts[0]}`; + } else { + pkgPath = `${GO_BASE_IMPORT}/typedapi/${parts.join("/")}`; + } + this.imports.add(pkgPath); + } + + addEnumPackage(typeName: TypeName): void { + const enumPkgName = typeName.name.toLowerCase(); + this.imports.add(`${GO_BASE_IMPORT}/typedapi/types/enums/${enumPkgName}`); + } + + render(): string { + if (this.imports.size === 0) return ""; + const sorted = [...this.imports].sort(); + const stdLib = sorted.filter((i) => !i.includes("/")); + const external = sorted.filter((i) => i.includes("/")); + const parts: string[] = []; + if (stdLib.length > 0) { + parts.push(stdLib.map((i) => ` "${i}"`).join("\n")); + } + if (external.length > 0) { + parts.push(external.map((i) => ` "${i}"`).join("\n")); + } + return `import (\n${parts.join("\n\n")}\n)`; + } +} diff --git a/src/exporters/go/naming.ts b/src/exporters/go/naming.ts new file mode 100644 index 0000000..e3ca850 --- /dev/null +++ b/src/exporters/go/naming.ts @@ -0,0 +1,66 @@ +import { Property, TypeName } from "../../metamodel"; +import { GO_TYPE_RENAMES } from "./constants"; + +export function toPascalCase(name: string): string { + return name + .split("_") + .map((part) => { + if (part.length === 0) return ""; + return part.charAt(0).toUpperCase() + part.slice(1); + }) + .join(""); +} + +export function apiToGoMethod(api: string): { + subclient: string; + method: string; +} { + const parts = api.split("."); + if (parts.length === 1) { + return { subclient: "", method: toPascalCase(parts[0]) }; + } + const method = toPascalCase(parts[parts.length - 1]); + const subclient = parts + .slice(0, -1) + .map((p) => toPascalCase(p)) + .join("."); + return { subclient, method }; +} + +export function resolveGoFieldName(name: string, props?: Property[]): string { + if (props) { + for (const prop of props) { + if (prop.name === name || prop.aliases?.includes(name)) { + if (prop.codegenName != undefined) { + return toPascalCase(prop.codegenName); + } + const canonical = prop.name; + if (canonical.startsWith("_")) { + return toPascalCase(canonical.slice(1)) + "_"; + } + return toPascalCase(canonical); + } + } + } + if (name.startsWith("_")) { + return toPascalCase(name.slice(1)) + "_"; + } + return toPascalCase(name); +} + +export function goTypeName(typeName: TypeName): string { + const nsMap = GO_TYPE_RENAMES[typeName.namespace]; + if (nsMap) { + const renamed = nsMap[typeName.name]; + if (renamed) return renamed; + } + let name = toPascalCase(typeName.name); + if (name.endsWith("Container")) { + name = name.slice(0, -"Container".length); + } + return name; +} + +export function indent(depth: number): string { + return " ".repeat(depth); +} diff --git a/src/exporters/go/renderer.ts b/src/exporters/go/renderer.ts new file mode 100644 index 0000000..d678a1a --- /dev/null +++ b/src/exporters/go/renderer.ts @@ -0,0 +1,553 @@ +import { + ValueOf, + InstanceOf, + ArrayOf, + DictionaryOf, + UnionOf, + Interface, + Property, + Enum, + TypeAlias, + TypeName, +} from "../../metamodel"; +import { RenderContext } from "./context"; +import { toPascalCase, resolveGoFieldName, goTypeName } from "./naming"; + +export class GoValueRenderer { + renderGoValue( + value: unknown, + typeInfo: ValueOf, + ctx: RenderContext, + prop?: Property, + ): string { + if (value === null) { + return "nil"; + } + + switch (typeInfo.kind) { + case "instance_of": + return this.renderInstanceOf(value, typeInfo as InstanceOf, ctx, prop); + case "dictionary_of": + return this.renderDictionaryOf(value, typeInfo as DictionaryOf, ctx); + case "array_of": + return this.renderArrayOf(value, typeInfo as ArrayOf, ctx); + case "union_of": + return this.renderUnionOf(value, typeInfo as UnionOf, ctx, prop); + case "user_defined_value": + return this.renderLiteralValue(value, ctx); + case "literal_value": + return JSON.stringify(value); + default: + return this.renderLiteralValue(value, ctx); + } + } + + renderStructFields( + obj: Record, + properties: Property[], + ctx: RenderContext, + ): string { + const lines: string[] = []; + for (const [key, value] of Object.entries(obj)) { + const prop = properties.find( + (p) => p.name === key || p.aliases?.includes(key), + ); + const fieldName = resolveGoFieldName(key, properties); + if (!prop) { + lines.push( + `${ctx.indent()}${fieldName}: ${this.renderLiteralValue( + value, + ctx, + )},`, + ); + continue; + } + lines.push( + `${ctx.indent()}${fieldName}: ${this.renderGoValue( + value, + prop.type, + ctx, + prop, + )},`, + ); + } + return lines.join("\n"); + } + + renderLiteralValue(value: unknown, ctx: RenderContext): string { + if (value === null) return "nil"; + if (typeof value === "string") return `"${this.escapeGoString(value)}"`; + if (typeof value === "number") return String(value); + if (typeof value === "boolean") return String(value); + if (Array.isArray(value)) { + if (value.length === 0) return "[]interface{}{}"; + const nested = ctx.nested(); + const elements = value.map( + (item) => `${nested.indent()}${this.renderLiteralValue(item, nested)},`, + ); + return `[]interface{}{\n${elements.join("\n")}\n${ctx.indent()}}`; + } + if (typeof value === "object") { + const obj = value as Record; + const entries = Object.entries(obj); + if (entries.length === 0) return `map[string]interface{}{}`; + const nested = ctx.nested(); + const lines = entries.map( + ([k, v]) => + `${nested.indent()}"${this.escapeGoString( + k, + )}": ${this.renderLiteralValue(v, nested)},`, + ); + return `map[string]interface{}{\n${lines.join("\n")}\n${ctx.indent()}}`; + } + return String(value); + } + + goTypeString(typeInfo: ValueOf, ctx: RenderContext): string { + switch (typeInfo.kind) { + case "instance_of": { + const inst = typeInfo as InstanceOf; + if (inst.type.namespace === "_builtins") { + switch (inst.type.name) { + case "string": + return "string"; + case "boolean": + return "bool"; + case "number": + return "int"; + case "null": + return "interface{}"; + } + } + if (ctx.resolver.isNumericType(inst.type)) return "int"; + if (ctx.resolver.isStringType(inst.type)) return "string"; + const enumType = ctx.resolver.isEnumType(inst.type); + if (enumType) { + const enumPkg = inst.type.name.toLowerCase(); + ctx.imports.addEnumPackage(inst.type); + return `${enumPkg}.${toPascalCase(inst.type.name)}`; + } + const typeDef = ctx.resolver.getType( + inst.type.name, + inst.type.namespace, + ); + if (typeDef?.kind === "interface") { + ctx.imports.addTypes(); + return `types.${goTypeName(inst.type)}`; + } + if (typeDef?.kind === "type_alias") { + const alias = typeDef as TypeAlias; + if (alias.type.kind === "union_of") { + const classification = ctx.resolver.classifyUnion( + alias.type as UnionOf, + ); + if (classification === "any") { + ctx.imports.addTypes(); + return `types.${goTypeName(inst.type)}`; + } + if (classification === "integer_string") { + return "string"; + } + } + } + const resolved = ctx.resolver.resolveTypeAlias(typeInfo); + if (resolved !== typeInfo) { + return this.goTypeString(resolved, ctx); + } + ctx.imports.addTypes(); + return `types.${goTypeName(inst.type)}`; + } + case "dictionary_of": { + const dict = typeInfo as DictionaryOf; + const keyType = this.goTypeString(dict.key, ctx); + const valueType = this.goTypeString(dict.value, ctx); + return `map[${keyType}]${valueType}`; + } + case "array_of": { + const arr = typeInfo as ArrayOf; + const elemType = this.goTypeString(arr.value, ctx); + return `[]${elemType}`; + } + case "union_of": { + const union = typeInfo as UnionOf; + if (union.items.length === 0) return "interface{}"; + const classification = ctx.resolver.classifyUnion(union); + if (classification === "integer_string") return "string"; + if (classification === "any") return "interface{}"; + return this.goTypeString(union.items[0], ctx); + } + case "user_defined_value": + return "interface{}"; + default: + return "interface{}"; + } + } + + toGoNumber(value: unknown): number { + if (typeof value === "number") return value; + if (typeof value === "string") return parseInt(value, 10) || 0; + return 0; + } + + escapeGoString(s: string): string { + return s + .replace(/\\/g, "\\\\") + .replace(/"/g, '\\"') + .replace(/\n/g, "\\n") + .replace(/\t/g, "\\t"); + } + + private renderInstanceOf( + value: unknown, + typeInfo: InstanceOf, + ctx: RenderContext, + prop?: Property, + ): string { + const { name, namespace } = typeInfo.type; + + if (namespace === "_builtins") { + return this.renderBuiltin(value, name, ctx); + } + + if (ctx.resolver.isNumericType(typeInfo.type)) { + if (prop && !prop.required) { + ctx.imports.addSome(); + return `some.Int(${this.toGoNumber(value)})`; + } + return String(this.toGoNumber(value)); + } + + if (ctx.resolver.isStringType(typeInfo.type)) { + if (typeof value === "string") { + return `"${this.escapeGoString(value)}"`; + } + return `"${value}"`; + } + + const enumType = ctx.resolver.isEnumType(typeInfo.type); + if (enumType) { + return this.renderEnumValue(value, typeInfo.type, enumType, ctx); + } + + const resolved = ctx.resolver.resolveTypeAlias(typeInfo); + if (resolved !== typeInfo) { + if (resolved.kind === "union_of") { + const classification = ctx.resolver.classifyUnion(resolved as UnionOf); + if (classification === "integer_string") { + const strValue = String(value); + if (prop && !prop.required) { + ctx.imports.addSome(); + return `some.String("${this.escapeGoString(strValue)}")`; + } + return `"${this.escapeGoString(strValue)}"`; + } + } + return this.renderGoValue(value, resolved, ctx, prop); + } + + const typeDef = ctx.resolver.getType(name, namespace); + if (!typeDef) { + return this.renderLiteralValue(value, ctx); + } + + if (typeDef.kind === "interface") { + const iface = typeDef as Interface; + if ( + typeof value === "object" && + value !== null && + !Array.isArray(value) + ) { + const usePointer = prop && !prop.required; + const prefix = usePointer ? "&" : ""; + if (iface.variants?.kind === "container") { + return this.renderContainerVariant( + value as Record, + iface, + ctx, + prefix, + ); + } + const props = ctx.resolver.getInterfaceProperties(name, namespace); + const goName = goTypeName(typeInfo.type); + ctx.imports.addTypes(); + const nested = ctx.nested(); + const fields = this.renderStructFields( + value as Record, + props, + nested, + ); + if (!fields.trim()) { + return `${prefix}types.${goName}{}`; + } + return `${prefix}types.${goName}{\n${fields}\n${ctx.indent()}}`; + } + if ( + typeof value === "string" || + typeof value === "number" || + typeof value === "boolean" + ) { + if (iface.shortcutProperty) { + const shortcutProp = ctx.resolver + .getInterfaceProperties(name, namespace) + .find((p) => p.name === iface.shortcutProperty); + if (shortcutProp) { + const goName = goTypeName(typeInfo.type); + const fieldName = resolveGoFieldName( + iface.shortcutProperty, + ctx.resolver.getInterfaceProperties(name, namespace), + ); + const nested = ctx.nested(); + const renderedVal = this.renderGoValue( + value, + shortcutProp.type, + nested, + ); + ctx.imports.addTypes(); + return `types.${goName}{${fieldName}: ${renderedVal}}`; + } + } + return this.renderLiteralValue(value, ctx); + } + } + + return this.renderLiteralValue(value, ctx); + } + + private renderBuiltin( + value: unknown, + name: string, + ctx: RenderContext, + ): string { + switch (name) { + case "string": + if (typeof value === "string") { + return `"${this.escapeGoString(value)}"`; + } + return `"${value}"`; + case "boolean": + return String(!!value); + case "number": + return String(value); + case "null": + return "nil"; + default: + return this.renderLiteralValue(value, ctx); + } + } + + private renderContainerVariant( + obj: Record, + iface: Interface, + ctx: RenderContext, + prefix: string, + ): string { + const props = ctx.resolver.getInterfaceProperties( + iface.name.name, + iface.name.namespace, + ); + const goName = goTypeName(iface.name); + ctx.imports.addTypes(); + + const additionalProp = ctx.resolver.getAdditionalPropertyBehavior(iface); + + const knownEntries: Record = {}; + const additionalEntries: Record = {}; + for (const [key, value] of Object.entries(obj)) { + const isKnown = props.some( + (p) => p.name === key || p.aliases?.includes(key), + ); + if (isKnown) { + knownEntries[key] = value; + } else if (additionalProp) { + additionalEntries[key] = value; + } else { + knownEntries[key] = value; + } + } + + const nested = ctx.nested(); + const lines: string[] = []; + + const knownFields = this.renderStructFields(knownEntries, props, nested); + if (knownFields.trim()) { + lines.push(knownFields); + } + + if (additionalProp && Object.keys(additionalEntries).length > 0) { + const keyType = this.goTypeString(additionalProp.key, ctx); + const valueType = this.goTypeString(additionalProp.value, ctx); + const mapNested = nested.nested(); + const mapEntries: string[] = []; + for (const [k, v] of Object.entries(additionalEntries)) { + const renderedValue = this.renderGoValue( + v, + additionalProp.value, + mapNested, + ); + mapEntries.push( + `${mapNested.indent()}"${this.escapeGoString(k)}": ${renderedValue},`, + ); + } + const mapLiteral = `map[${keyType}]${valueType}{\n${mapEntries.join( + "\n", + )}\n${nested.indent()}}`; + lines.push(`${nested.indent()}${goName}: ${mapLiteral},`); + } + + if (lines.length === 0) { + return `${prefix}types.${goName}{}`; + } + return `${prefix}types.${goName}{\n${lines.join("\n")}\n${ctx.indent()}}`; + } + + private renderDictionaryOf( + value: unknown, + typeInfo: DictionaryOf, + ctx: RenderContext, + ): string { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + return this.renderLiteralValue(value, ctx); + } + + const obj = value as Record; + const keyType = this.goTypeString(typeInfo.key, ctx); + const valueType = this.goTypeString(typeInfo.value, ctx); + + const nested = ctx.nested(); + const entries: string[] = []; + for (const [k, v] of Object.entries(obj)) { + const renderedValue = this.renderGoValue(v, typeInfo.value, nested); + entries.push( + `${nested.indent()}"${this.escapeGoString(k)}": ${renderedValue},`, + ); + } + + if (entries.length === 0) { + return `map[${keyType}]${valueType}{}`; + } + return `map[${keyType}]${valueType}{\n${entries.join( + "\n", + )}\n${ctx.indent()}}`; + } + + private renderArrayOf( + value: unknown, + typeInfo: ArrayOf, + ctx: RenderContext, + ): string { + if (!Array.isArray(value)) { + return this.renderGoValue(value, typeInfo.value, ctx); + } + + const elemType = this.goTypeString(typeInfo.value, ctx); + const nested = ctx.nested(); + const elements: string[] = []; + for (const item of value) { + elements.push( + `${nested.indent()}${this.renderGoValue( + item, + typeInfo.value, + nested, + )},`, + ); + } + + if (elements.length === 0) { + return `[]${elemType}{}`; + } + return `[]${elemType}{\n${elements.join("\n")}\n${ctx.indent()}}`; + } + + private renderUnionOf( + value: unknown, + typeInfo: UnionOf, + ctx: RenderContext, + prop?: Property, + ): string { + const classification = ctx.resolver.classifyUnion(typeInfo); + if (classification === "integer_string") { + const strValue = String(value); + if (prop && !prop.required) { + ctx.imports.addSome(); + return `some.String("${this.escapeGoString(strValue)}")`; + } + return `"${this.escapeGoString(strValue)}"`; + } + + if (typeInfo.items.length === 2) { + const [a, b] = typeInfo.items; + if ( + a.kind === "instance_of" && + b.kind === "array_of" && + b.value.kind === "instance_of" && + (a as InstanceOf).type.name === + ((b as ArrayOf).value as InstanceOf).type.name + ) { + if (Array.isArray(value)) { + return this.renderArrayOf(value, b as ArrayOf, ctx); + } + return this.renderGoValue(value, a, ctx, prop); + } + } + + for (const item of typeInfo.items) { + if (item.kind === "instance_of") { + const inst = item as InstanceOf; + if (typeof value === "string" && ctx.resolver.isStringType(inst.type)) { + return this.renderGoValue(value, item, ctx); + } + if ( + typeof value === "number" && + ctx.resolver.isNumericType(inst.type) + ) { + return this.renderGoValue(value, item, ctx); + } + if ( + typeof value === "boolean" && + ctx.resolver.isBooleanType(inst.type) + ) { + return this.renderGoValue(value, item, ctx); + } + if ( + typeof value === "object" && + value !== null && + !Array.isArray(value) + ) { + const typeDef = ctx.resolver.getType( + inst.type.name, + inst.type.namespace, + ); + if (typeDef?.kind === "interface") { + return this.renderGoValue(value, item, ctx); + } + } + } + if (item.kind === "array_of" && Array.isArray(value)) { + return this.renderArrayOf(value, item as ArrayOf, ctx); + } + } + + return this.renderLiteralValue(value, ctx); + } + + private renderEnumValue( + value: unknown, + typeName: TypeName, + enumType: Enum, + ctx: RenderContext, + ): string { + const strValue = String(value); + const member = enumType.members.find( + (m) => + m.name === strValue || + m.aliases?.includes(strValue) || + m.name.toLowerCase() === strValue.toLowerCase(), + ); + if (member) { + ctx.imports.addEnumPackage(typeName); + const enumPkg = typeName.name.toLowerCase(); + return `${enumPkg}.${toPascalCase(member.name)}`; + } + return `"${this.escapeGoString(strValue)}"`; + } +} diff --git a/src/exporters/go/schema.ts b/src/exporters/go/schema.ts new file mode 100644 index 0000000..fb90dc3 --- /dev/null +++ b/src/exporters/go/schema.ts @@ -0,0 +1,206 @@ +import { readFile } from "fs/promises"; +import path from "path"; +import { + Model, + TypeDefinition, + Interface, + Request, + Property, + ValueOf, + InstanceOf, + ArrayOf, + UnionOf, + TypeAlias, + Enum, + TypeName, +} from "../../metamodel"; +import { NUMERIC_TYPES, STRING_ALIAS_TYPES } from "./constants"; + +const isBrowser = typeof window !== "undefined"; + +let cachedResolver: TypeResolver | undefined; + +export class TypeResolver { + constructor(private readonly schema: Model) {} + + static async load(): Promise { + if (cachedResolver) return cachedResolver; + if (isBrowser) { + throw new Error("Go expanded specification is not available in browser"); + } + const schemaPath = path.join(__dirname, "../../schema-no-generics.json"); + const schema = JSON.parse( + await readFile(schemaPath, { encoding: "utf-8" }), + ) as Model; + cachedResolver = new TypeResolver(schema); + return cachedResolver; + } + + getType(name: string, namespace: string): TypeDefinition | undefined { + for (const type of this.schema.types) { + if (type.name.name === name && type.name.namespace === namespace) { + return type; + } + } + return undefined; + } + + getInterfaceProperties(name: string, namespace: string): Property[] { + let props: Property[] = []; + const type = this.getType(name, namespace); + if (type?.kind === "interface") { + let i = type as Interface; + props = [...i.properties]; + while (i.inherits) { + const parent = this.getType( + i.inherits.type.name, + i.inherits.type.namespace, + ) as Interface | undefined; + if (!parent) break; + i = parent; + props = [...props, ...i.properties]; + } + } + return props; + } + + isNumericType(typeName: TypeName): boolean { + if (typeName.namespace === "_builtins" && typeName.name === "number") { + return true; + } + if (typeName.namespace === "_types" && NUMERIC_TYPES.has(typeName.name)) { + return true; + } + return false; + } + + isBooleanType(typeName: TypeName): boolean { + return typeName.namespace === "_builtins" && typeName.name === "boolean"; + } + + isStringType(typeName: TypeName): boolean { + if (typeName.namespace === "_builtins" && typeName.name === "string") { + return true; + } + if ( + typeName.namespace === "_types" && + STRING_ALIAS_TYPES.has(typeName.name) + ) { + return true; + } + const type = this.getType(typeName.name, typeName.namespace); + if (type?.kind === "type_alias") { + const alias = type as TypeAlias; + if (alias.type.kind === "instance_of") { + return this.isStringType((alias.type as InstanceOf).type); + } + } + return false; + } + + isEnumType(typeName: TypeName): Enum | undefined { + const type = this.getType(typeName.name, typeName.namespace); + if (type?.kind === "enum") { + return type as Enum; + } + if (type?.kind === "type_alias") { + const alias = type as TypeAlias; + if (alias.type.kind === "instance_of") { + return this.isEnumType((alias.type as InstanceOf).type); + } + } + return undefined; + } + + classifyUnion(union: UnionOf): "integer_string" | "any" | null { + if (union.items.length === 2) { + const [a, b] = union.items; + if ( + a.kind === "instance_of" && + b.kind === "array_of" && + (b as ArrayOf).value.kind === "instance_of" && + (a as InstanceOf).type.name === + ((b as ArrayOf).value as InstanceOf).type.name + ) { + return null; + } + if ( + a.kind === "array_of" && + b.kind === "instance_of" && + (a as ArrayOf).value.kind === "instance_of" && + (b as InstanceOf).type.name === + ((a as ArrayOf).value as InstanceOf).type.name + ) { + return null; + } + + let hasNumeric = false; + let hasString = false; + for (const item of union.items) { + if (item.kind === "instance_of") { + const inst = item as InstanceOf; + if (this.isNumericType(inst.type)) hasNumeric = true; + if ( + this.isStringType(inst.type) || + (inst.type.namespace === "_builtins" && inst.type.name === "string") + ) { + hasString = true; + } + } + } + if (hasNumeric && hasString) return "integer_string"; + } + + if (union.items.length > 1) return "any"; + return null; + } + + getBehaviorProperties(behaviorNames: string[]): Property[] { + const props: Property[] = []; + for (const bName of behaviorNames) { + for (const type of this.schema.types) { + if (type.name.name === bName && type.kind === "interface") { + props.push(...(type as Interface).properties); + } + } + } + return props; + } + + isUserDefinedValueBody(name: string, namespace: string): boolean { + const type = this.getType(name, namespace); + if (type?.kind === "request") { + const req = type as Request; + if ( + req.body.kind === "value" && + req.body.value.kind === "user_defined_value" + ) { + return true; + } + } + return false; + } + + getAdditionalPropertyBehavior( + iface: Interface, + ): { key: ValueOf; value: ValueOf } | undefined { + const behavior = iface.behaviors?.find( + (b) => b.type.name === "AdditionalProperty", + ); + if (behavior?.generics && behavior.generics.length === 2) { + return { key: behavior.generics[0], value: behavior.generics[1] }; + } + return undefined; + } + + resolveTypeAlias(typeInfo: ValueOf): ValueOf { + if (typeInfo.kind === "instance_of") { + const inst = typeInfo as InstanceOf; + const type = this.getType(inst.type.name, inst.type.namespace); + if (type?.kind === "type_alias") { + return (type as TypeAlias).type; + } + } + return typeInfo; + } +} diff --git a/tests/convert.test.ts b/tests/convert.test.ts index 8207f9a..5680f4d 100644 --- a/tests/convert.test.ts +++ b/tests/convert.test.ts @@ -102,6 +102,856 @@ describe("convert", () => { ).toBeFalsy(); }); + it("checks for go", async () => { + expect( + await convertRequests(devConsoleScript, "go", { + checkOnly: true, + }), + ).toBeTruthy(); + expect( + await convertRequests(kibanaScript, "go", { + checkOnly: true, + }), + ).toBeFalsy(); + }); + + it("converts to go", async () => { + expect(await convertRequests(devConsoleScript, "go", {})).toEqual( + `res, err := es.Info(). + Do(context.Background()) + +res1, err := es.Search(). + Index("my-index"). + From(40). + Size(20). + Request(&search.Request{ + Query: &types.Query{ + Term: map[string]types.TermQuery{ + "user.id": types.TermQuery{Value: "kimchy's"}, + }, + }, + }). + Do(context.Background()) +`, + ); + }); + + it("converts to a complete go script", async () => { + expect( + await convertRequests(devConsoleScript, "go", { + complete: true, + elasticsearchUrl: "https://localhost:9999", + }), + ).toEqual( + `package main + +import ( + "context" + "log" + + "github.com/elastic/go-elasticsearch/v9" + "github.com/elastic/go-elasticsearch/v9/typedapi/core/search" + "github.com/elastic/go-elasticsearch/v9/typedapi/types" +) + +func main() { + cfg := elasticsearch.Config{ + Addresses: []string{"https://localhost:9999"}, + } + es, err := elasticsearch.NewTypedClient(cfg) + if err != nil { + log.Fatalf("Error creating client: %s", err) + } + + res, err := es.Info(). + Do(context.Background()) + + res1, err := es.Search(). + Index("my-index"). + From(40). + Size(20). + Request(&search.Request{ + Query: &types.Query{ + Term: map[string]types.TermQuery{ + "user.id": types.TermQuery{Value: "kimchy's"}, + }, + }, + }). + Do(context.Background()) +} +`, + ); + }); + + it("converts an unsupported API to go", async () => { + expect( + await convertRequests("GET /_internal/desired_balance", "go", { + complete: false, + elasticsearchUrl: "https://localhost:9999", + }), + ).toEqual( + `res, err := es.Transport.Perform(&http.Request{ + Method: "GET", + URL: &url.URL{Path: "/_internal/desired_balance"}, + Body: nil, +}) +`, + ); + }); + + it("errors when converting Kibana to go", async () => { + expect( + async () => + await convertRequests(kibanaScript, "go", { + complete: false, + elasticsearchUrl: "https://localhost:9999", + }), + ).rejects.toThrowError("Cannot perform conversion"); + }); + + it("converts subclient call with enum query param to go", async () => { + expect( + await convertRequests( + `GET /_settings?expand_wildcards=all&filter_path=*.settings.index.*.slowlog`, + "go", + {}, + ), + ).toEqual( + `res, err := es.Indices.GetSettings(). + ExpandWildcards("all"). + FilterPath("*.settings.index.*.slowlog"). + Do(context.Background()) +`, + ); + }); + + it("converts cluster reroute with commands to go", async () => { + expect( + await convertRequests( + `POST /_cluster/reroute?metric=none +{"commands":[{"move":{"index":"test","shard":0,"from_node":"node1","to_node":"node2"}},{"allocate_replica":{"index":"test","shard":1,"node":"node3"}}]}`, + "go", + {}, + ), + ).toEqual( + `res, err := es.Cluster.Reroute(). + Metric("none"). + Request(&reroute.Request{ + Commands: []types.Command{ + types.Command{ + Move: &types.CommandMoveAction{ + Index: "test", + Shard: 0, + FromNode: "node1", + ToNode: "node2", + }, + }, + types.Command{ + AllocateReplica: &types.CommandAllocateReplicaAction{ + Index: "test", + Shard: 1, + Node: "node3", + }, + }, + }, + }). + Do(context.Background()) +`, + ); + }); + + it("converts string params that are actually numbers to go", async () => { + expect( + await convertRequests( + `PUT /_ml/trained_models/elastic__distilbert-base-uncased-finetuned-conll03-english/definition/0 +{"definition":"...","total_definition_length":265632637,"total_parts":64}`, + "go", + {}, + ), + ).toEqual( + `res, err := es.Ml.PutTrainedModelDefinitionPart("elastic__distilbert-base-uncased-finetuned-conll03-english", "0"). + Request(&put_trained_model_definition_part.Request{ + Definition: "...", + TotalDefinitionLength: 265632637, + TotalParts: 64, + }). + Do(context.Background()) +`, + ); + }); + + it("converts infer trained model to go", async () => { + expect( + await convertRequests( + `POST /_ml/trained_models/test/_infer +{"docs":[{"text":"The fool doth think he is wise, but the wise man knows himself to be a fool."}]}`, + "go", + {}, + ), + ).toEqual( + `res, err := es.Ml.InferTrainedModel("test"). + Request(&infer_trained_model.Request{ + Docs: []map[string]interface{}{ + map[string]interface{}{ + "text": "The fool doth think he is wise, but the wise man knows himself to be a fool.", + }, + }, + }). + Do(context.Background()) +`, + ); + }); + + it("converts deeply nested role mapping rules to go", async () => { + expect( + await convertRequests( + `PUT /_security/role_mapping/mapping8 +{"roles":["superuser"],"enabled":true,"rules":{"all":[{"any":[{"field":{"dn":"*,ou=admin,dc=example,dc=com"}},{"field":{"username":["es-admin","es-system"]}}]},{"field":{"groups":"cn=people,dc=example,dc=com"}},{"except":{"field":{"metadata.terminated_date":null}}}]}}`, + "go", + {}, + ), + ).toEqual( + `res, err := es.Security.PutRoleMapping("mapping8"). + Request(&put_role_mapping.Request{ + Roles: []string{ + "superuser", + }, + Enabled: true, + Rules: &types.RoleMappingRule{ + All: []types.RoleMappingRule{ + types.RoleMappingRule{ + Any: []types.RoleMappingRule{ + types.RoleMappingRule{ + Field: map[string]types.FieldValue{ + "dn": "*,ou=admin,dc=example,dc=com", + }, + }, + types.RoleMappingRule{ + Field: map[string]types.FieldValue{ + "username": []types.FieldValue{ + "es-admin", + "es-system", + }, + }, + }, + }, + }, + types.RoleMappingRule{ + Field: map[string]types.FieldValue{ + "groups": "cn=people,dc=example,dc=com", + }, + }, + types.RoleMappingRule{ + Except: &types.RoleMappingRule{ + Field: map[string]types.FieldValue{ + "metadata.terminated_date": nil, + }, + }, + }, + }, + }, + }). + Do(context.Background()) +`, + ); + }); + + it("converts match_all with enum and multi-index to go", async () => { + expect( + await convertRequests( + `POST /my-index-000001,my-index-000002/_search?from=40&size=20&default_operator=AND +{"query":{"match_all":{}}}`, + "go", + {}, + ), + ).toEqual( + `res, err := es.Search(). + Index("my-index-000001,my-index-000002"). + From(40). + Size(20). + DefaultOperator(operator.And). + Request(&search.Request{ + Query: &types.Query{ + MatchAll: &types.MatchAllQuery{}, + }, + }). + Do(context.Background()) +`, + ); + }); + + it("converts range query with aggregation to go", async () => { + expect( + await convertRequests( + `POST /my-index-000001/_search?from=40&size=20 +{"query":{"range":{"@timestamp":{"gte":"now-1d/d","lt":"now/d"}}},"aggs":{"my-agg-name":{"terms":{"field":"my-field"}}}}`, + "go", + {}, + ), + ).toEqual( + `res, err := es.Search(). + Index("my-index-000001"). + From(40). + Size(20). + Request(&search.Request{ + Query: &types.Query{ + Range: map[string]types.RangeQuery{ + "@timestamp": types.UntypedRangeQuery{ + Gte: "now-1d/d", + Lt: "now/d", + }, + }, + }, + Aggregations: map[string]types.Aggregations{ + "my-agg-name": types.Aggregations{ + Terms: &types.TermsAggregation{ + Field: "my-field", + }, + }, + }, + }). + Do(context.Background()) +`, + ); + }); + + it("converts nested aggregation to go", async () => { + expect( + await convertRequests( + `POST /_search +{"aggs":{"my-agg-name":{"terms":{"field":"my-field"},"aggs":{"my-sub-agg-name":{"avg":{"field":"my-other-field"}}}}}}`, + "go", + {}, + ), + ).toEqual( + `res, err := es.Search(). + Request(&search.Request{ + Aggregations: map[string]types.Aggregations{ + "my-agg-name": types.Aggregations{ + Terms: &types.TermsAggregation{ + Field: "my-field", + }, + Aggregations: map[string]types.Aggregations{ + "my-sub-agg-name": types.Aggregations{ + Avg: &types.AverageAggregation{ + Field: "my-other-field", + }, + }, + }, + }, + }, + }). + Do(context.Background()) +`, + ); + }); + + it("converts aggregation with metadata to go", async () => { + expect( + await convertRequests( + `POST /_search +{"aggs":{"my-agg-name":{"terms":{"field":"my-field"},"meta":{"my-metadata-field":"foo"}}}}`, + "go", + {}, + ), + ).toEqual( + `res, err := es.Search(). + Request(&search.Request{ + Aggregations: map[string]types.Aggregations{ + "my-agg-name": types.Aggregations{ + Terms: &types.TermsAggregation{ + Field: "my-field", + }, + Meta: "[object Object]", + }, + }, + }). + Do(context.Background()) +`, + ); + }); + + it("converts runtime mappings with script to go", async () => { + expect( + await convertRequests( + `POST /_search +{"runtime_mappings":{"message.length":{"type":"long","script":"emit(doc['message.keyword'].value.length())"}},"aggs":{"message_length":{"histogram":{"interval":10,"field":"message.length"}}}}`, + "go", + {}, + ), + ).toEqual( + `res, err := es.Search(). + Request(&search.Request{ + RuntimeMappings: map[string]types.RuntimeField{ + "message.length": types.RuntimeField{ + Type: runtimefieldtype.Long, + Script: "emit(doc['message.keyword'].value.length())", + }, + }, + Aggregations: map[string]types.Aggregations{ + "message_length": types.Aggregations{ + Histogram: &types.HistogramAggregation{ + Interval: some.Int(10), + Field: "message.length", + }, + }, + }, + }). + Do(context.Background()) +`, + ); + }); + + it("converts function score query to go", async () => { + expect( + await convertRequests( + `POST /_search +{"size":10,"query":{"function_score":{"query":{"bool":{"filter":[{"terms":{"tags.keyword":["Monkey","Lion"]}}]}},"functions":[{"filter":{"term":{"mustHaveTags.keyword":{"value":"Monkey"}}},"weight":1}],"score_mode":"sum","boost_mode":"sum"}}}`, + "go", + {}, + ), + ).toEqual( + `res, err := es.Search(). + Request(&search.Request{ + Size: some.Int(10), + Query: &types.Query{ + FunctionScore: &types.FunctionScoreQuery{ + Query: &types.Query{ + Bool: &types.BoolQuery{ + Filter: []types.Query{ + types.Query{ + Terms: &types.TermsQuery{ + Tags.keyword: []interface{}{ + "Monkey", + "Lion", + }, + }, + }, + }, + }, + }, + Functions: []types.FunctionScore{ + types.FunctionScore{ + Filter: &types.Query{ + Term: map[string]types.TermQuery{ + "mustHaveTags.keyword": types.TermQuery{ + Value: "Monkey", + }, + }, + }, + Weight: some.Int(1), + }, + }, + ScoreMode: functionscoremode.Sum, + BoostMode: functionboostmode.Sum, + }, + }, + }). + Do(context.Background()) +`, + ); + }); + + it("converts ingest pipeline with processors to go", async () => { + expect( + await convertRequests( + `PUT /_ingest/pipeline/my-pipeline +{"description":"My optional pipeline description","processors":[{"set":{"description":"My optional processor description","field":"my-long-field","value":10}},{"set":{"description":"Set 'my-boolean-field' to true","field":"my-boolean-field","value":true}},{"lowercase":{"field":"my-keyword-field"}}]}`, + "go", + {}, + ), + ).toEqual( + `res, err := es.Ingest.PutPipeline("my-pipeline"). + Request(&put_pipeline.Request{ + Description: "My optional pipeline description", + Processors: []types.Processor{ + types.Processor{ + Set: &types.SetProcessor{ + Description: "My optional processor description", + Field: "my-long-field", + Value: 10, + }, + }, + types.Processor{ + Set: &types.SetProcessor{ + Description: "Set 'my-boolean-field' to true", + Field: "my-boolean-field", + Value: true, + }, + }, + types.Processor{ + Lowercase: &types.LowercaseProcessor{ + Field: "my-keyword-field", + }, + }, + }, + }). + Do(context.Background()) +`, + ); + }); + + it("converts simulate ingest to go", async () => { + expect( + await convertRequests( + `POST /_ingest/pipeline/my-pipeline/_simulate?verbose=true +{"docs":[{"_index":"index","_id":"id","_source":{"my-keyword-field":"bar"}},{"_index":"index","_id":"id","_source":{"my-long-field":10}}]}`, + "go", + {}, + ), + ).toEqual( + `res, err := es.Ingest.Simulate(). + Id("my-pipeline"). + Verbose(true). + Request(&simulate.Request{ + Docs: []types.Document{ + types.Document{ + Index_: "index", + Id_: "id", + Source_: map[string]interface{}{ + "my-keyword-field": "bar", + }, + }, + types.Document{ + Index_: "index", + Id_: "id", + Source_: map[string]interface{}{ + "my-long-field": 10, + }, + }, + }, + }). + Do(context.Background()) +`, + ); + }); + + it("converts index creation with analyzers to go", async () => { + expect( + await convertRequests( + `PUT /arabic_example +{"settings":{"analysis":{"filter":{"arabic_stop":{"type":"stop","stopwords":"_arabic_"},"arabic_keywords":{"type":"keyword_marker","keywords":["مثال"]},"arabic_stemmer":{"type":"stemmer","language":"arabic"}},"analyzer":{"rebuilt_arabic":{"tokenizer":"standard","filter":["lowercase","decimal_digit","arabic_stop","arabic_normalization","arabic_keywords","arabic_stemmer"]}}}}}`, + "go", + {}, + ), + ).toEqual( + `res, err := es.Indices.Create("arabic_example"). + Request(&create.Request{ + Settings: &types.IndexSettings{ + Analysis: &types.IndexSettingsAnalysis{ + Filter: map[string]types.TokenFilter{ + "arabic_stop": map[string]interface{}{ + "type": "stop", + "stopwords": "_arabic_", + }, + "arabic_keywords": map[string]interface{}{ + "type": "keyword_marker", + "keywords": []interface{}{ + "\u0645\u062B\u0627\u0644", + }, + }, + "arabic_stemmer": map[string]interface{}{ + "type": "stemmer", + "language": "arabic", + }, + }, + Analyzer: map[string]types.Analyzer{ + "rebuilt_arabic": types.CustomAnalyzer{ + Tokenizer: "standard", + Filter: []string{ + "lowercase", + "decimal_digit", + "arabic_stop", + "arabic_normalization", + "arabic_keywords", + "arabic_stemmer", + }, + }, + }, + }, + }, + }). + Do(context.Background()) +`, + ); + }); + + it("converts multiple aggregations to go", async () => { + expect( + await convertRequests( + `POST /my-index-000001/_search?from=40&size=20 +{"aggs":{"my-first-agg-name":{"terms":{"field":"my-field"}},"my-second-agg-name":{"avg":{"field":"my-other-field"}}}}`, + "go", + {}, + ), + ).toEqual( + `res, err := es.Search(). + Index("my-index-000001"). + From(40). + Size(20). + Request(&search.Request{ + Aggregations: map[string]types.Aggregations{ + "my-first-agg-name": types.Aggregations{ + Terms: &types.TermsAggregation{ + Field: "my-field", + }, + }, + "my-second-agg-name": types.Aggregations{ + Avg: &types.AverageAggregation{ + Field: "my-other-field", + }, + }, + }, + }). + Do(context.Background()) +`, + ); + }); + + it("converts KNN search with rescore to go", async () => { + expect( + await convertRequests( + `POST /my-index-000001/_search?from=40&size=20 +{"knn":{"field":"image-vector","query_vector":[0.1,-2],"k":15,"num_candidates":100},"fields":["title"],"rescore":{"window_size":10,"query":{"rescore_query":{"script_score":{"query":{"match_all":{}},"script":{"source":"cosineSimilarity(params.query_vector, 'image-vector') + 1.0","params":{"query_vector":[0.1,-2]}}}}}}}`, + "go", + {}, + ), + ).toEqual( + `res, err := es.Search(). + Index("my-index-000001"). + From(40). + Size(20). + Request(&search.Request{ + Knn: &types.KnnSearch{ + Field: "image-vector", + QueryVector: []int{ + 0.1, + -2, + }, + K: some.Int(15), + NumCandidates: some.Int(100), + }, + Fields: []types.FieldAndFormat{ + types.FieldAndFormat{Field: "title"}, + }, + Rescore: &types.Rescore{ + WindowSize: some.Int(10), + Query: &types.RescoreQuery{ + Query: types.Query{ + ScriptScore: &types.ScriptScoreQuery{ + Query: types.Query{ + MatchAll: &types.MatchAllQuery{}, + }, + Script: "[object Object]", + }, + }, + }, + }, + }). + Do(context.Background()) +`, + ); + }); + + it("converts deeply nested queries (6 levels) to go", async () => { + expect( + await convertRequests( + `POST /my-index-000001/_search?from=40&size=20 +{"query":{"nested":{"path":"driver","query":{"nested":{"path":"driver.vehicle","query":{"nested":{"path":"driver.vehicle.wheel","query":{"nested":{"path":"driver.vehicle.wheel.nut","query":{"nested":{"path":"driver.vehicle.wheel.nut.metal","query":{"nested":{"path":"driver.vehicle.wheel.nut.metal.atom","query":{"match_all":{}}}}}}}}}}}}}}}`, + "go", + {}, + ), + ).toEqual( + `res, err := es.Search(). + Index("my-index-000001"). + From(40). + Size(20). + Request(&search.Request{ + Query: &types.Query{ + Nested: &types.NestedQuery{ + Path: "driver", + Query: types.Query{ + Nested: &types.NestedQuery{ + Path: "driver.vehicle", + Query: types.Query{ + Nested: &types.NestedQuery{ + Path: "driver.vehicle.wheel", + Query: types.Query{ + Nested: &types.NestedQuery{ + Path: "driver.vehicle.wheel.nut", + Query: types.Query{ + Nested: &types.NestedQuery{ + Path: "driver.vehicle.wheel.nut.metal", + Query: types.Query{ + Nested: &types.NestedQuery{ + Path: "driver.vehicle.wheel.nut.metal.atom", + Query: types.Query{ + MatchAll: &types.MatchAllQuery{}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }). + Do(context.Background()) +`, + ); + }); + + it("converts distance feature query to go", async () => { + expect( + await convertRequests( + `POST /my-index-000001/_search?from=40&size=20 +{"query":{"bool":{"must":{"match":{"name":"chocolate"}},"should":{"distance_feature":{"field":"location","pivot":"1000m","origin":[-71.3,41.15]}}}}}`, + "go", + {}, + ), + ).toEqual( + `res, err := es.Search(). + Index("my-index-000001"). + From(40). + Size(20). + Request(&search.Request{ + Query: &types.Query{ + Bool: &types.BoolQuery{ + Must: &types.Query{ + Match: map[string]types.MatchQuery{ + "name": types.MatchQuery{Query: "chocolate"}, + }, + }, + Should: &types.Query{ + DistanceFeature: types.UntypedDistanceFeatureQuery{ + Field: "location", + Pivot: "1000m", + Origin: []interface{}{ + -71.3, + 41.15, + }, + }, + }, + }, + }, + }). + Do(context.Background()) +`, + ); + }); + + it("converts complex sub-aggregation with ordering to go", async () => { + expect( + await convertRequests( + `POST /_search +{"size":0,"query":{"bool":{"filter":[{"term":{"is_sold":true}},{"term":{"lender_id":4477943}}]}},"aggs":{"group_by_summaryGroup":{"terms":{"field":"group.keyword","order":{"_key":"desc"}},"aggs":{"note_count":{"value_count":{"field":"id"}},"invested_sum":{"sum":{"field":"amount_participation"}},"outstanding_principal_sum":{"sum":{"field":"principal_balance"}},"principal_repaid_sum":{"sum":{"field":"principal_repaid"}},"interest_paid_sum":{"sum":{"field":"interest_paid"}}}}}}`, + "go", + {}, + ), + ).toEqual( + `res, err := es.Search(). + Request(&search.Request{ + Size: some.Int(0), + Query: &types.Query{ + Bool: &types.BoolQuery{ + Filter: []types.Query{ + types.Query{ + Term: map[string]types.TermQuery{ + "is_sold": types.TermQuery{Value: true}, + }, + }, + types.Query{ + Term: map[string]types.TermQuery{ + "lender_id": types.TermQuery{Value: 4477943}, + }, + }, + }, + }, + }, + Aggregations: map[string]types.Aggregations{ + "group_by_summaryGroup": types.Aggregations{ + Terms: &types.TermsAggregation{ + Field: "group.keyword", + Order: map[string]interface{}{ + "_key": "desc", + }, + }, + Aggregations: map[string]types.Aggregations{ + "note_count": types.Aggregations{ + ValueCount: &types.ValueCountAggregation{ + Field: "id", + }, + }, + "invested_sum": types.Aggregations{ + Sum: &types.SumAggregation{ + Field: "amount_participation", + }, + }, + "outstanding_principal_sum": types.Aggregations{ + Sum: &types.SumAggregation{ + Field: "principal_balance", + }, + }, + "principal_repaid_sum": types.Aggregations{ + Sum: &types.SumAggregation{ + Field: "principal_repaid", + }, + }, + "interest_paid_sum": types.Aggregations{ + Sum: &types.SumAggregation{ + Field: "interest_paid", + }, + }, + }, + }, + }, + }). + Do(context.Background()) +`, + ); + }); + + it("converts cat health with boolean query param from behavior to go", async () => { + expect(await convertRequests(`GET /_cat/health?v`, "go", {})).toEqual( + `res, err := es.Cat.Health(). + V(true). + Do(context.Background()) +`, + ); + }); + + it("converts index doc with user_defined_value body to go", async () => { + expect( + await convertRequests( + `PUT /my-index/_doc/1 +{"title":"Hello World","tags":["intro","welcome"],"metadata":{"author":"test","version":2}}`, + "go", + {}, + ), + ).toEqual( + `res, err := es.Index("my-index"). + Id("1"). + Request(map[string]interface{}{ + "title": "Hello World", + "tags": []interface{}{ + "intro", + "welcome", + }, + "metadata": map[string]interface{}{ + "author": "test", + "version": 2, + }, + }). + Do(context.Background()) +`, + ); + }); + it("errors for unknown language", async () => { expect( async () => diff --git a/tests/integration/convert.test.ts b/tests/integration/convert.test.ts index 78615b1..ba66880 100644 --- a/tests/integration/convert.test.ts +++ b/tests/integration/convert.test.ts @@ -17,6 +17,7 @@ const TEST_FORMATS: Record = { php: "php", curl: "sh", ruby: "rb", + go: "go", }; interface SchemaExample { diff --git a/tests/integration/run-go.sh b/tests/integration/run-go.sh new file mode 100755 index 0000000..9ce0b83 --- /dev/null +++ b/tests/integration/run-go.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +set -eo pipefail +set -x + +GO_DIR=/tmp/go-test +BRANCH=$(jq -r .version package.json | grep -Eo "^[0-9]+\.[0-9]+") + +if [[ "$1" == "" ]]; then + rm -rf $GO_DIR +fi + +if [ ! -d "$GO_DIR" ]; then + mkdir -p $GO_DIR + pushd $GO_DIR + go mod init go-test + echo "Installing go-elasticsearch from branch $BRANCH" + go get github.com/elastic/go-elasticsearch/v9@$BRANCH || + (echo "Branch $BRANCH not found. Using main branch." && + go get github.com/elastic/go-elasticsearch/v9@main) + popd +fi + +if [[ "$1" != "" ]]; then + cp "$1" "$GO_DIR/main.go" + pushd $GO_DIR + go run main.go + popd +fi