From 4b4a1661665432e6cf303fa70834756d26665716 Mon Sep 17 00:00:00 2001 From: Kohei Noda Date: Fri, 5 Feb 2021 13:12:24 +0000 Subject: [PATCH 01/23] Fixed non-flat :orelse query crashing OTP 23.0 - Nested `:orelse` query causes SIGBUS signal (https://bugs.erlang.org/browse/ERL-727) - Fixed #10 --- lib/etso/ets/match_specification.ex | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/etso/ets/match_specification.ex b/lib/etso/ets/match_specification.ex index f1430fa..50d47ff 100644 --- a/lib/etso/ets/match_specification.ex +++ b/lib/etso/ets/match_specification.ex @@ -49,9 +49,10 @@ defmodule Etso.ETS.MatchSpecification do field_name = resolve_field_name(field) field_index = get_field_index(field_names, field_name) - resolve_field_values(params, value) - |> Enum.map(&{:==, :"$#{field_index}", &1}) - |> Enum.reduce(&{:orelse, &1, &2}) + case resolve_field_values(params, value) do + [] -> [] + values -> List.to_tuple([:orelse | Enum.map(values, &{:==, :"$#{field_index}", &1})]) + end end defp build_condition(field_names, _, {{:., [], [{:&, [], [0]}, field_name]}, [], []}) do From 70c73fe546774846a97962366c9063f5747621d4 Mon Sep 17 00:00:00 2001 From: Evadne Wu Date: Fri, 5 Feb 2021 13:18:36 +0000 Subject: [PATCH 02/23] =?UTF-8?q?=E2=86=92=200.1.3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- mix.exs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6ef28b5..9fd4f45 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ Using Etso is a two-step process. First, include it in your application’s depe ```elixir defp deps do [ - {:etso, "~> 0.1.2"} + {:etso, "~> 0.1.3"} ] end ``` diff --git a/mix.exs b/mix.exs index 93153d4..765d2d6 100644 --- a/mix.exs +++ b/mix.exs @@ -4,7 +4,7 @@ defmodule Etso.MixProject do def project do [ app: :etso, - version: "0.1.2", + version: "0.1.3", elixir: "~> 1.8", elixirc_paths: elixirc_paths(Mix.env()), start_permanent: Mix.env() == :prod, From b0cfd39f6252718f4e78e9d36a485affdf398850 Mon Sep 17 00:00:00 2001 From: Evadne Wu Date: Tue, 2 Mar 2021 19:09:59 +0000 Subject: [PATCH 03/23] Added LICENSE --- LICENSE | 176 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 176 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d9a10c0 --- /dev/null +++ b/LICENSE @@ -0,0 +1,176 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS From 368e93550bcc58dc7bb47c2fdfaae9554ae0acca Mon Sep 17 00:00:00 2001 From: Evadne Wu Date: Tue, 2 Mar 2021 19:11:21 +0000 Subject: [PATCH 04/23] =?UTF-8?q?=E2=86=92=200.1.4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- mix.exs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9fd4f45..6f2fede 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ Using Etso is a two-step process. First, include it in your application’s depe ```elixir defp deps do [ - {:etso, "~> 0.1.3"} + {:etso, "~> 0.1.4"} ] end ``` diff --git a/mix.exs b/mix.exs index 765d2d6..efac119 100644 --- a/mix.exs +++ b/mix.exs @@ -4,7 +4,7 @@ defmodule Etso.MixProject do def project do [ app: :etso, - version: "0.1.3", + version: "0.1.4", elixir: "~> 1.8", elixirc_paths: elixirc_paths(Mix.env()), start_permanent: Mix.env() == :prod, From daad287664e407709c963a873480840eb0bbcbcd Mon Sep 17 00:00:00 2001 From: Evadne Wu Date: Thu, 4 Mar 2021 22:56:17 +0000 Subject: [PATCH 05/23] Fixed reporting of errors on conflicting primary key --- lib/etso/adapter/behaviour/schema.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/etso/adapter/behaviour/schema.ex b/lib/etso/adapter/behaviour/schema.ex index 6fe3992..fd8f7e1 100644 --- a/lib/etso/adapter/behaviour/schema.ex +++ b/lib/etso/adapter/behaviour/schema.ex @@ -21,7 +21,7 @@ defmodule Etso.Adapter.Behaviour.Schema do ets_field_names = TableStructure.field_names(schema) ets_changes = TableStructure.fields_to_tuple(ets_field_names, fields) ets_result = :ets.insert_new(ets_table, ets_changes) - if ets_result, do: {:ok, []}, else: {:invalid, []} + if ets_result, do: {:ok, []}, else: {:invalid, [unique: "primary_key"]} end def update(%{repo: repo}, %{schema: schema}, fields, filters, [], _) do From 3ec42356309cf72dba04f5b7372ce160d4e10bb3 Mon Sep 17 00:00:00 2001 From: Evadne Wu Date: Sat, 6 Mar 2021 20:04:29 +0000 Subject: [PATCH 06/23] =?UTF-8?q?=E2=86=92=200.1.5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 6 +++++- mix.exs | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6f2fede..5a4ec86 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ Using Etso is a two-step process. First, include it in your application’s depe ```elixir defp deps do [ - {:etso, "~> 0.1.4"} + {:etso, "~> 0.1.5"} ] end ``` @@ -103,6 +103,8 @@ The Author also wishes to thank the following individuals: - [Kohei Noda][pobo380], for [fixing nested `{:orelse, …}` queries][pr-10]. +- [William Martins][wmartins], for [fixing primary key unicity check issues][gh-7]. + [erlang-ets]: http://erlang.org/doc/man/ets.html [northwind]: https://github.com/evadne/etso/tree/master/test/support/northwind [northwind-importer]: https://github.com/evadne/etso/tree/master/test/support/northwind/importer.ex @@ -114,3 +116,5 @@ The Author also wishes to thank the following individuals: [pr-6]: https://github.com/evadne/etso/pull/6 [pobo380]: https://github.com/pobo380 [pr-10]: https://github.com/evadne/etso/pull/10 +[wmartins]: https://github.com/wmartins +[gh-7]: https://github.com/evadne/etso/issues/7 diff --git a/mix.exs b/mix.exs index efac119..9364370 100644 --- a/mix.exs +++ b/mix.exs @@ -4,7 +4,7 @@ defmodule Etso.MixProject do def project do [ app: :etso, - version: "0.1.4", + version: "0.1.5", elixir: "~> 1.8", elixirc_paths: elixirc_paths(Mix.env()), start_permanent: Mix.env() == :prod, From 918a69815d468d8c661c7cb281cf4b5ed17a2365 Mon Sep 17 00:00:00 2001 From: Evadne Wu Date: Fri, 25 Jun 2021 20:03:15 +0100 Subject: [PATCH 07/23] Added support for is_nil in queries --- lib/etso/ets/match_specification.ex | 4 ++++ test/northwind/repo_test.exs | 11 +++++++++++ 2 files changed, 15 insertions(+) diff --git a/lib/etso/ets/match_specification.ex b/lib/etso/ets/match_specification.ex index 50d47ff..5836e88 100644 --- a/lib/etso/ets/match_specification.ex +++ b/lib/etso/ets/match_specification.ex @@ -55,6 +55,10 @@ defmodule Etso.ETS.MatchSpecification do end end + defp build_condition(field_names, params, {:is_nil, [], [field]}) do + {:==, build_condition(field_names, params, field), nil} + end + defp build_condition(field_names, _, {{:., [], [{:&, [], [0]}, field_name]}, [], []}) do :"$#{get_field_index(field_names, field_name)}" end diff --git a/test/northwind/repo_test.exs b/test/northwind/repo_test.exs index e32f70f..721be39 100644 --- a/test/northwind/repo_test.exs +++ b/test/northwind/repo_test.exs @@ -88,6 +88,17 @@ defmodule Northwind.RepoTest do |> Repo.all() end + test "Select Where Is Nil" do + query = Model.Employee |> where([x], is_nil(x.title)) + assert [] = Repo.all(query) + + changes = %{first_name: "Ghost", employee_id: 4096} + changeset = Model.Employee.changeset(changes) + {:ok, %{employee_id: employee_id} = employee} = Repo.insert(changeset) + assert [%{employee_id: ^employee_id}] = Repo.all(query) + Repo.delete(employee) + end + test "Select / Update" do Model.Employee |> where([x], x.title == "Vice President Sales") From 94de59a1382f7b796d8aa0c2564311b09aefc0b1 Mon Sep 17 00:00:00 2001 From: Evadne Wu Date: Mon, 5 Jul 2021 16:24:24 +0100 Subject: [PATCH 08/23] =?UTF-8?q?=E2=86=92=200.1.6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- mix.exs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5a4ec86..ee5bffd 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ Using Etso is a two-step process. First, include it in your application’s depe ```elixir defp deps do [ - {:etso, "~> 0.1.5"} + {:etso, "~> 0.1.6"} ] end ``` diff --git a/mix.exs b/mix.exs index 9364370..6bb58ca 100644 --- a/mix.exs +++ b/mix.exs @@ -4,7 +4,7 @@ defmodule Etso.MixProject do def project do [ app: :etso, - version: "0.1.5", + version: "0.1.6", elixir: "~> 1.8", elixirc_paths: elixirc_paths(Mix.env()), start_permanent: Mix.env() == :prod, From 81329ff988d1eb15c0f0161a12987b68cdf2271f Mon Sep 17 00:00:00 2001 From: Evadne Wu Date: Fri, 20 May 2022 13:59:37 +0200 Subject: [PATCH 09/23] Added insert_all test --- test/northwind/repo_test.exs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/northwind/repo_test.exs b/test/northwind/repo_test.exs index 721be39..5fa48d2 100644 --- a/test/northwind/repo_test.exs +++ b/test/northwind/repo_test.exs @@ -21,6 +21,11 @@ defmodule Northwind.RepoTest do Repo.delete(employee) end + test "Insert Employees" do + changes = [%{first_name: "Fred", employee_id: 1}, %{first_name: "Steven", employee_id: 2}] + Repo.insert_all(Model.Employee, changes) + end + test "List all Employees Again" do Repo.all(Model.Employee) end From 3b1f7e063488598b240d2028ca0742b1639a4cca Mon Sep 17 00:00:00 2001 From: Evadne Wu Date: Fri, 20 May 2022 14:08:44 +0200 Subject: [PATCH 10/23] Raised Ecto requirement to 3.8.3+ --- mix.exs | 2 +- mix.lock | 21 +++++++++++---------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/mix.exs b/mix.exs index 6bb58ca..be716fe 100644 --- a/mix.exs +++ b/mix.exs @@ -27,7 +27,7 @@ defmodule Etso.MixProject do defp deps do [ - {:ecto, "~> 3.0"}, + {:ecto, "~> 3.8.3"}, {:dialyxir, "~> 1.0.0-rc.6", only: :dev, runtime: false}, {:ex_doc, ">= 0.0.0", only: :dev, runtime: false}, {:jason, "~> 1.1", only: :test, runtime: false} diff --git a/mix.lock b/mix.lock index b49d46e..331350b 100644 --- a/mix.lock +++ b/mix.lock @@ -1,12 +1,13 @@ %{ - "decimal": {:hex, :decimal, "1.8.0", "ca462e0d885f09a1c5a342dbd7c1dcf27ea63548c65a65e67334f4b61803822e", [:mix], [], "hexpm"}, - "dialyxir": {:hex, :dialyxir, "1.0.0-rc.6", "78e97d9c0ff1b5521dd68041193891aebebce52fc3b93463c0a6806874557d7d", [:mix], [{:erlex, "~> 0.2.1", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm"}, - "earmark": {:hex, :earmark, "1.3.6", "ce1d0675e10a5bb46b007549362bd3f5f08908843957687d8484fe7f37466b19", [:mix], [], "hexpm"}, - "ecto": {:hex, :ecto, "3.1.7", "fa21d06ef56cdc2fdaa62574e8c3ba34a2751d44ea34c30bc65f0728421043e5", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, - "erlex": {:hex, :erlex, "0.2.4", "23791959df45fe8f01f388c6f7eb733cc361668cbeedd801bf491c55a029917b", [:mix], [], "hexpm"}, - "ex_doc": {:hex, :ex_doc, "0.21.2", "caca5bc28ed7b3bdc0b662f8afe2bee1eedb5c3cf7b322feeeb7c6ebbde089d6", [:mix], [{:earmark, "~> 1.3.3 or ~> 1.4", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm"}, - "jason": {:hex, :jason, "1.1.2", "b03dedea67a99223a2eaf9f1264ce37154564de899fd3d8b9a21b1a6fd64afe7", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm"}, - "makeup": {:hex, :makeup, "1.0.0", "671df94cf5a594b739ce03b0d0316aa64312cee2574b6a44becb83cd90fb05dc", [:mix], [{:nimble_parsec, "~> 0.5.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm"}, - "makeup_elixir": {:hex, :makeup_elixir, "0.14.0", "cf8b7c66ad1cff4c14679698d532f0b5d45a3968ffbcbfd590339cb57742f1ae", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm"}, - "nimble_parsec": {:hex, :nimble_parsec, "0.5.1", "c90796ecee0289dbb5ad16d3ad06f957b0cd1199769641c961cfe0b97db190e0", [:mix], [], "hexpm"}, + "decimal": {:hex, :decimal, "2.0.0", "a78296e617b0f5dd4c6caf57c714431347912ffb1d0842e998e9792b5642d697", [:mix], [], "hexpm", "34666e9c55dea81013e77d9d87370fe6cb6291d1ef32f46a1600230b1d44f577"}, + "dialyxir": {:hex, :dialyxir, "1.0.0-rc.6", "78e97d9c0ff1b5521dd68041193891aebebce52fc3b93463c0a6806874557d7d", [:mix], [{:erlex, "~> 0.2.1", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "49496d63267bc1a4614ffd5f67c45d9fc3ea62701a6797975bc98bc156d2763f"}, + "earmark": {:hex, :earmark, "1.3.6", "ce1d0675e10a5bb46b007549362bd3f5f08908843957687d8484fe7f37466b19", [:mix], [], "hexpm", "1476378df80982302d5a7857b6a11dd0230865057dec6d16544afecc6bc6b4c2"}, + "ecto": {:hex, :ecto, "3.8.3", "5e681d35bc2cbb46dcca1e2675837c7d666316e5ada14eca6c9c609b6232817c", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "af92dd7815967bcaea0daaaccf31c3b23165432b1c7a475d84144efbc703d105"}, + "erlex": {:hex, :erlex, "0.2.4", "23791959df45fe8f01f388c6f7eb733cc361668cbeedd801bf491c55a029917b", [:mix], [], "hexpm", "4a12ebc7cd8f24f2d0fce93d279fa34eb5068e0e885bb841d558c4d83c52c439"}, + "ex_doc": {:hex, :ex_doc, "0.21.2", "caca5bc28ed7b3bdc0b662f8afe2bee1eedb5c3cf7b322feeeb7c6ebbde089d6", [:mix], [{:earmark, "~> 1.3.3 or ~> 1.4", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm", "f1155337ae17ff7a1255217b4c1ceefcd1860b7ceb1a1874031e7a861b052e39"}, + "jason": {:hex, :jason, "1.3.0", "fa6b82a934feb176263ad2df0dbd91bf633d4a46ebfdffea0c8ae82953714946", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "53fc1f51255390e0ec7e50f9cb41e751c260d065dcba2bf0d08dc51a4002c2ac"}, + "makeup": {:hex, :makeup, "1.0.0", "671df94cf5a594b739ce03b0d0316aa64312cee2574b6a44becb83cd90fb05dc", [:mix], [{:nimble_parsec, "~> 0.5.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "a10c6eb62cca416019663129699769f0c2ccf39428b3bb3c0cb38c718a0c186d"}, + "makeup_elixir": {:hex, :makeup_elixir, "0.14.0", "cf8b7c66ad1cff4c14679698d532f0b5d45a3968ffbcbfd590339cb57742f1ae", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "d4b316c7222a85bbaa2fd7c6e90e37e953257ad196dc229505137c5e505e9eff"}, + "nimble_parsec": {:hex, :nimble_parsec, "0.5.1", "c90796ecee0289dbb5ad16d3ad06f957b0cd1199769641c961cfe0b97db190e0", [:mix], [], "hexpm", "00e3ebdc821fb3a36957320d49e8f4bfa310d73ea31c90e5f925dc75e030da8f"}, + "telemetry": {:hex, :telemetry, "1.1.0", "a589817034a27eab11144ad24d5c0f9fab1f58173274b1e9bae7074af9cbee51", [:rebar3], [], "hexpm", "b727b2a1f75614774cff2d7565b64d0dfa5bd52ba517f16543e6fc7efcc0df48"}, } From 34044653c117ce4e0d9cd45ebaa9ae131afa38b8 Mon Sep 17 00:00:00 2001 From: Evadne Wu Date: Fri, 20 May 2022 14:08:50 +0200 Subject: [PATCH 11/23] Added additional test for insert_all --- lib/etso/adapter/behaviour/schema.ex | 4 +++- test/northwind/repo_test.exs | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/etso/adapter/behaviour/schema.ex b/lib/etso/adapter/behaviour/schema.ex index fd8f7e1..b41193f 100644 --- a/lib/etso/adapter/behaviour/schema.ex +++ b/lib/etso/adapter/behaviour/schema.ex @@ -8,7 +8,9 @@ defmodule Etso.Adapter.Behaviour.Schema do def autogenerate(:binary_id), do: Ecto.UUID.bingenerate() def autogenerate(:embed_id), do: Ecto.UUID.bingenerate() - def insert_all(%{repo: repo}, %{schema: schema}, _, entries, _, _, _) do + def insert_all(%{repo: repo}, %{schema: schema}, _, entries, _, _, _placeholders \\ [], _) do + # Ecto 3 had `insert_all/7`, + # This was then changed to `insert_all/8`, adding placeholders as `[term()]` {:ok, ets_table} = TableRegistry.get_table(repo, schema) ets_field_names = TableStructure.field_names(schema) ets_changes = TableStructure.entries_to_tuples(ets_field_names, entries) diff --git a/test/northwind/repo_test.exs b/test/northwind/repo_test.exs index 5fa48d2..7c10b92 100644 --- a/test/northwind/repo_test.exs +++ b/test/northwind/repo_test.exs @@ -22,8 +22,11 @@ defmodule Northwind.RepoTest do end test "Insert Employees" do - changes = [%{first_name: "Fred", employee_id: 1}, %{first_name: "Steven", employee_id: 2}] + changes = [%{first_name: "Fred", employee_id: 100}, %{first_name: "Steven", employee_id: 200}] + nil = Repo.get(Model.Employee, 100) + Repo.insert_all(Model.Employee, changes) + %{first_name: "Fred"} = Repo.get(Model.Employee, 100) end test "List all Employees Again" do From fd6f6a91eb04aa82e13e70c518230034d90c16e1 Mon Sep 17 00:00:00 2001 From: Evadne Wu Date: Fri, 20 May 2022 14:44:11 +0200 Subject: [PATCH 12/23] Amended Etso with Ecto 3.8.2 - Made Adapter Meta stronger typed - Added Access to Adapter Meta - Changed insert_all/7 to insert_all/8 - Fixed how functions are brought into the Adapter --- lib/etso/adapter.ex | 20 +++++++++++++++++--- lib/etso/adapter/behaviour/queryable.ex | 4 ++++ lib/etso/adapter/behaviour/schema.ex | 10 +++++++--- lib/etso/adapter/meta.ex | 16 ++++++++++++++-- 4 files changed, 42 insertions(+), 8 deletions(-) diff --git a/lib/etso/adapter.ex b/lib/etso/adapter.ex index 231f442..c57e724 100644 --- a/lib/etso/adapter.ex +++ b/lib/etso/adapter.ex @@ -17,12 +17,15 @@ defmodule Etso.Adapter do @behaviour Ecto.Adapter.Schema @behaviour Ecto.Adapter.Queryable + @impl Ecto.Adapter defmacro __before_compile__(_opts), do: :ok @doc false + @impl Ecto.Adapter def ensure_all_started(_config, _type), do: {:ok, []} @doc false + @impl Ecto.Adapter def init(config) do {:ok, repo} = Keyword.fetch(config, :repo) child_spec = __MODULE__.Supervisor.child_spec(repo) @@ -31,25 +34,36 @@ defmodule Etso.Adapter do end @doc false + @impl Ecto.Adapter def checkout(_, _, fun), do: fun.() @doc false + @impl Ecto.Adapter + def checked_out?(_), do: true + + @doc false + @impl Ecto.Adapter def loaders(:binary_id, type), do: [Ecto.UUID, type] def loaders(:embed_id, type), do: [Ecto.UUID, type] def loaders(_, type), do: [type] @doc false + @impl Ecto.Adapter def dumpers(:binary_id, type), do: [type, Ecto.UUID] def dumpers(:embed_id, type), do: [type, Ecto.UUID] def dumpers(_, type), do: [type] - for module <- [__MODULE__.Behaviour.Schema, __MODULE__.Behaviour.Queryable] do - for {name, arity} <- module.__info__(:functions) do + for {implementation_module, behaviour_module} <- [ + {__MODULE__.Behaviour.Schema, Ecto.Adapter.Schema}, + {__MODULE__.Behaviour.Queryable, Ecto.Adapter.Queryable} + ] do + for {name, arity} <- implementation_module.__info__(:functions) do args = Enum.map(1..arity, &{:"arg_#{&1}", [], Elixir}) @doc false + @impl behaviour_module def unquote(name)(unquote_splicing(args)) do - unquote(module).unquote(name)(unquote_splicing(args)) + unquote(implementation_module).unquote(name)(unquote_splicing(args)) end end end diff --git a/lib/etso/adapter/behaviour/queryable.ex b/lib/etso/adapter/behaviour/queryable.ex index 52b2c1d..7b2a547 100644 --- a/lib/etso/adapter/behaviour/queryable.ex +++ b/lib/etso/adapter/behaviour/queryable.ex @@ -1,13 +1,16 @@ defmodule Etso.Adapter.Behaviour.Queryable do @moduledoc false + @behaviour Ecto.Adapter.Queryable alias Etso.Adapter.TableRegistry alias Etso.ETS.MatchSpecification + @impl Ecto.Adapter.Queryable def prepare(:all, query) do {:nocache, query} end + @impl Ecto.Adapter.Queryable def execute(%{repo: repo}, _, {:nocache, query}, params, _) do {_, schema} = query.from.source {:ok, ets_table} = TableRegistry.get_table(repo, schema) @@ -16,6 +19,7 @@ defmodule Etso.Adapter.Behaviour.Queryable do {length(ets_objects), ets_objects} end + @impl Ecto.Adapter.Queryable def stream(%{repo: repo}, _, {:nocache, query}, params, options) do {_, schema} = query.from.source {:ok, ets_table} = TableRegistry.get_table(repo, schema) diff --git a/lib/etso/adapter/behaviour/schema.ex b/lib/etso/adapter/behaviour/schema.ex index b41193f..498e071 100644 --- a/lib/etso/adapter/behaviour/schema.ex +++ b/lib/etso/adapter/behaviour/schema.ex @@ -1,16 +1,17 @@ defmodule Etso.Adapter.Behaviour.Schema do @moduledoc false + @behaviour Ecto.Adapter.Schema alias Etso.Adapter.TableRegistry alias Etso.ETS.TableStructure + @impl Ecto.Adapter.Schema def autogenerate(:id), do: :erlang.unique_integer() def autogenerate(:binary_id), do: Ecto.UUID.bingenerate() def autogenerate(:embed_id), do: Ecto.UUID.bingenerate() - def insert_all(%{repo: repo}, %{schema: schema}, _, entries, _, _, _placeholders \\ [], _) do - # Ecto 3 had `insert_all/7`, - # This was then changed to `insert_all/8`, adding placeholders as `[term()]` + @impl Ecto.Adapter.Schema + def insert_all(%{repo: repo}, %{schema: schema}, _, entries, _, _, _, _) do {:ok, ets_table} = TableRegistry.get_table(repo, schema) ets_field_names = TableStructure.field_names(schema) ets_changes = TableStructure.entries_to_tuples(ets_field_names, entries) @@ -18,6 +19,7 @@ defmodule Etso.Adapter.Behaviour.Schema do if ets_result, do: {length(ets_changes), nil}, else: {0, nil} end + @impl Ecto.Adapter.Schema def insert(%{repo: repo}, %{schema: schema}, fields, _, _, _) do {:ok, ets_table} = TableRegistry.get_table(repo, schema) ets_field_names = TableStructure.field_names(schema) @@ -26,6 +28,7 @@ defmodule Etso.Adapter.Behaviour.Schema do if ets_result, do: {:ok, []}, else: {:invalid, [unique: "primary_key"]} end + @impl Ecto.Adapter.Schema def update(%{repo: repo}, %{schema: schema}, fields, filters, [], _) do {:ok, ets_table} = TableRegistry.get_table(repo, schema) [key_name] = schema.__schema__(:primary_key) @@ -35,6 +38,7 @@ defmodule Etso.Adapter.Behaviour.Schema do if ets_result, do: {:ok, []}, else: {:error, :stale} end + @impl Ecto.Adapter.Schema def delete(%{repo: repo}, %{schema: schema}, filters, _) do {:ok, ets_table} = TableRegistry.get_table(repo, schema) [key_name] = schema.__schema__(:primary_key) diff --git a/lib/etso/adapter/meta.ex b/lib/etso/adapter/meta.ex index d7f7898..98d27e0 100644 --- a/lib/etso/adapter/meta.ex +++ b/lib/etso/adapter/meta.ex @@ -1,7 +1,19 @@ defmodule Etso.Adapter.Meta do @moduledoc false - @type t :: %__MODULE__{repo: Ecto.Repo.t()} + @type t :: %__MODULE__{ + repo: Ecto.Repo.t(), + cache: :ets.tab() | nil, + pid: pid() | nil, + stacktrace: true | false + } + @enforce_keys ~w(repo)a - defstruct repo: nil + defstruct repo: nil, cache: nil, pid: nil, stacktrace: false + + @behaviour Access + defdelegate get(v, key, default), to: Map + defdelegate fetch(v, key), to: Map + defdelegate get_and_update(v, key, func), to: Map + defdelegate pop(v, key), to: Map end From f5516f7e1ed6ad4c97a52ef8f07412a89510727c Mon Sep 17 00:00:00 2001 From: Evadne Wu Date: Fri, 20 May 2022 15:29:51 +0200 Subject: [PATCH 13/23] Added ability to sort results --- README.md | 4 +++ lib/etso/adapter/behaviour/queryable.ex | 3 ++- lib/etso/ets/sorter.ex | 35 +++++++++++++++++++++++++ test/northwind/repo_test.exs | 21 +++++++++++++++ 4 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 lib/etso/ets/sorter.ex diff --git a/README.md b/README.md index ee5bffd..67aca1b 100644 --- a/README.md +++ b/README.md @@ -105,6 +105,8 @@ The Author also wishes to thank the following individuals: - [William Martins][wmartins], for [fixing primary key unicity check issues][gh-7]. +- [Soichiro Nishizawa][soichiro-nishizawa], for [providing the implementation of sorted results][gh-12]. + [erlang-ets]: http://erlang.org/doc/man/ets.html [northwind]: https://github.com/evadne/etso/tree/master/test/support/northwind [northwind-importer]: https://github.com/evadne/etso/tree/master/test/support/northwind/importer.ex @@ -118,3 +120,5 @@ The Author also wishes to thank the following individuals: [pr-10]: https://github.com/evadne/etso/pull/10 [wmartins]: https://github.com/wmartins [gh-7]: https://github.com/evadne/etso/issues/7 +[soichiro-nishizawa]: https://github.com/soichiro-nishizawa +[gh-12]: https://github.com/evadne/etso/issues/12 diff --git a/lib/etso/adapter/behaviour/queryable.ex b/lib/etso/adapter/behaviour/queryable.ex index 7b2a547..03fd488 100644 --- a/lib/etso/adapter/behaviour/queryable.ex +++ b/lib/etso/adapter/behaviour/queryable.ex @@ -4,6 +4,7 @@ defmodule Etso.Adapter.Behaviour.Queryable do alias Etso.Adapter.TableRegistry alias Etso.ETS.MatchSpecification + alias Etso.ETS.ObjectsSorter @impl Ecto.Adapter.Queryable def prepare(:all, query) do @@ -15,7 +16,7 @@ defmodule Etso.Adapter.Behaviour.Queryable do {_, schema} = query.from.source {:ok, ets_table} = TableRegistry.get_table(repo, schema) ets_match = MatchSpecification.build(query, params) - ets_objects = :ets.select(ets_table, [ets_match]) + ets_objects = :ets.select(ets_table, [ets_match]) |> ObjectsSorter.sort(query) {length(ets_objects), ets_objects} end diff --git a/lib/etso/ets/sorter.ex b/lib/etso/ets/sorter.ex new file mode 100644 index 0000000..6974fda --- /dev/null +++ b/lib/etso/ets/sorter.ex @@ -0,0 +1,35 @@ +defmodule Etso.ETS.ObjectsSorter do + @moduledoc """ + The Objects Sorter is responsible for sorting results returned from ETS according to the sort + predicates provided in the query. + """ + + def sort(ets_objects, %Ecto.Query{order_bys: []}) do + ets_objects + end + + def sort(ets_objects, %Ecto.Query{} = query) do + sort_predicates = build_sort_predicates(query) + Enum.sort_by(ets_objects, & &1, &compare(&1, &2, sort_predicates)) + end + + defp build_sort_predicates(%Ecto.Query{} = query) do + Enum.flat_map(query.order_bys, fn %Ecto.Query.QueryExpr{expr: list} -> + Enum.map(list, fn {direction, field} -> + {direction, Enum.find_index(query.select.fields, &(&1 == field))} + end) + end) + end + + defp compare(lhs, rhs, [{direction, index} | predicates]) do + case {direction, Enum.at(lhs, index), Enum.at(rhs, index)} do + {_, lhs, rhs} when lhs == rhs -> compare(lhs, rhs, predicates) + {:asc, lhs, rhs} -> lhs < rhs + {:desc, lhs, rhs} -> lhs > rhs + end + end + + defp compare(_lhs, _rhs, []) do + true + end +end diff --git a/test/northwind/repo_test.exs b/test/northwind/repo_test.exs index 7c10b92..b919c03 100644 --- a/test/northwind/repo_test.exs +++ b/test/northwind/repo_test.exs @@ -147,4 +147,25 @@ defmodule Northwind.RepoTest do |> Repo.all() |> Repo.preload(shipper: :orders) end + + test "Order / Shipper / Orders Preloading before all()" do + Model.Order + |> preload([_], shipper: :orders) + |> Repo.all() + end + + test "Order By Desc company_name, Asc phone" do + sorted_etso = + Model.Shipper + |> order_by([x], desc: x.company_name, asc: x.phone) + |> Repo.all() + + sorted_code = + Model.Shipper + |> Repo.all() + |> Enum.sort_by(& &1.company_name, :desc) + |> Enum.sort_by(& &1.phone) + + assert sorted_etso == sorted_code + end end From f1e94c53c1cda5d98add805ada24ad7e5a0191b6 Mon Sep 17 00:00:00 2001 From: Evadne Wu Date: Fri, 20 May 2022 15:52:52 +0200 Subject: [PATCH 14/23] Added support for parallel preloading --- lib/etso/adapter.ex | 2 +- test/northwind/repo_test.exs | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/etso/adapter.ex b/lib/etso/adapter.ex index c57e724..dd6e12c 100644 --- a/lib/etso/adapter.ex +++ b/lib/etso/adapter.ex @@ -39,7 +39,7 @@ defmodule Etso.Adapter do @doc false @impl Ecto.Adapter - def checked_out?(_), do: true + def checked_out?(_), do: false @doc false @impl Ecto.Adapter diff --git a/test/northwind/repo_test.exs b/test/northwind/repo_test.exs index b919c03..6cb26d5 100644 --- a/test/northwind/repo_test.exs +++ b/test/northwind/repo_test.exs @@ -148,6 +148,12 @@ defmodule Northwind.RepoTest do |> Repo.preload(shipper: :orders) end + test "Order / Shipper + Employee Preloading" do + Model.Order + |> Repo.all() + |> Repo.preload([[shipper: :orders], :employee, :customer], in_parallel: true) + end + test "Order / Shipper / Orders Preloading before all()" do Model.Order |> preload([_], shipper: :orders) From 61c83fe5c2aa0505fe83306625dd704f73eb7f8a Mon Sep 17 00:00:00 2001 From: Evadne Wu Date: Fri, 20 May 2022 15:55:48 +0200 Subject: [PATCH 15/23] Amended README --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 67aca1b..95d3d67 100644 --- a/README.md +++ b/README.md @@ -107,6 +107,8 @@ The Author also wishes to thank the following individuals: - [Soichiro Nishizawa][soichiro-nishizawa], for [providing the implementation of sorted results][gh-12]. +- [Doug W.][harmon25], for [providing insights into parallel preloads][gh-13]. + [erlang-ets]: http://erlang.org/doc/man/ets.html [northwind]: https://github.com/evadne/etso/tree/master/test/support/northwind [northwind-importer]: https://github.com/evadne/etso/tree/master/test/support/northwind/importer.ex @@ -122,3 +124,5 @@ The Author also wishes to thank the following individuals: [gh-7]: https://github.com/evadne/etso/issues/7 [soichiro-nishizawa]: https://github.com/soichiro-nishizawa [gh-12]: https://github.com/evadne/etso/issues/12 +[harmon25]: https://github.com/harmon25 +[gh-13]: https://github.com/evadne/etso/issues/13 From 33a65d17b9d37f7c827c0feffd4e4fcbbaf641ad Mon Sep 17 00:00:00 2001 From: Evadne Wu Date: Fri, 20 May 2022 15:57:10 +0200 Subject: [PATCH 16/23] =?UTF-8?q?=E2=86=92=201.0.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- mix.exs | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 95d3d67..d43a1e3 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ Using Etso is a two-step process. First, include it in your application’s depe ```elixir defp deps do [ - {:etso, "~> 0.1.6"} + {:etso, "~> 1.0.0"} ] end ``` diff --git a/mix.exs b/mix.exs index be716fe..fdeb0bb 100644 --- a/mix.exs +++ b/mix.exs @@ -4,7 +4,7 @@ defmodule Etso.MixProject do def project do [ app: :etso, - version: "0.1.6", + version: "1.0.0", elixir: "~> 1.8", elixirc_paths: elixirc_paths(Mix.env()), start_permanent: Mix.env() == :prod, @@ -62,6 +62,7 @@ defmodule Etso.MixProject do defp package_files do ~w( lib/etso/* + lib/etso.ex mix.exs ) end From 010135789bc60eae997bdeb5ec01b7d0fb7ef659 Mon Sep 17 00:00:00 2001 From: Evadne Wu Date: Fri, 20 May 2022 16:44:18 +0200 Subject: [PATCH 17/23] Corrected file name of the Objects Sorter --- lib/etso/ets/{sorter.ex => objects_sorter.ex} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename lib/etso/ets/{sorter.ex => objects_sorter.ex} (86%) diff --git a/lib/etso/ets/sorter.ex b/lib/etso/ets/objects_sorter.ex similarity index 86% rename from lib/etso/ets/sorter.ex rename to lib/etso/ets/objects_sorter.ex index 6974fda..0d100a6 100644 --- a/lib/etso/ets/sorter.ex +++ b/lib/etso/ets/objects_sorter.ex @@ -1,7 +1,7 @@ defmodule Etso.ETS.ObjectsSorter do @moduledoc """ - The Objects Sorter is responsible for sorting results returned from ETS according to the sort - predicates provided in the query. + The ETS Objects Sorter module is responsible for sorting results returned from ETS according to + the sort predicates provided in the query. """ def sort(ets_objects, %Ecto.Query{order_bys: []}) do From e870e10fe5b5f726e13e3027e24c76f77da7d799 Mon Sep 17 00:00:00 2001 From: Evadne Wu Date: Fri, 20 May 2022 16:47:45 +0200 Subject: [PATCH 18/23] =?UTF-8?q?=E2=86=92=201.0.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- mix.exs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d43a1e3..0e7ae87 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ Using Etso is a two-step process. First, include it in your application’s depe ```elixir defp deps do [ - {:etso, "~> 1.0.0"} + {:etso, "~> 1.0.1"} ] end ``` diff --git a/mix.exs b/mix.exs index fdeb0bb..a7bef89 100644 --- a/mix.exs +++ b/mix.exs @@ -4,7 +4,7 @@ defmodule Etso.MixProject do def project do [ app: :etso, - version: "1.0.0", + version: "1.0.1", elixir: "~> 1.8", elixirc_paths: elixirc_paths(Mix.env()), start_permanent: Mix.env() == :prod, From 2af83169d74cd26abc9b5c1bce1f511599535ca5 Mon Sep 17 00:00:00 2001 From: Atanda Rasheed Date: Tue, 28 Jun 2022 20:10:14 +0100 Subject: [PATCH 19/23] Added support for json_extract_path() in select queries - Also refined support for where/in --- lib/etso/ets/match_specification.ex | 75 ++++++++++++++++-------- priv/northwind/employees.json | 18 +++++- test/northwind/repo_test.exs | 60 +++++++++++++++++++ test/support/northwind/model/employee.ex | 1 + 4 files changed, 129 insertions(+), 25 deletions(-) diff --git a/lib/etso/ets/match_specification.ex b/lib/etso/ets/match_specification.ex index 5836e88..f1debb9 100644 --- a/lib/etso/ets/match_specification.ex +++ b/lib/etso/ets/match_specification.ex @@ -1,13 +1,19 @@ defmodule Etso.ETS.MatchSpecification do @moduledoc """ The ETS Match Specifications module contains various functions which convert Ecto queries to - ETS Match Specifications in order to execute the given queries. + [ETS Match Specifications](https://www.erlang.org/doc/apps/erts/match_spec.html) in order to + execute the given queries with ETS with as much pushed down to ETS as possible. + + The basic shape of the match head is `[$1, $2, $3, …]` where each field is a named variable, the + ordering of the fields is determined by `Etso.ETS.TableStructure`. + + Conditions are compiled according to the wheres in the underlying Ecto query, while the body is + compiled based on the selected fields in the underlying Ecto query. """ def build(query, params) do {_, schema} = query.from.source field_names = Etso.ETS.TableStructure.field_names(schema) - match_head = build_head(field_names) match_conditions = build_conditions(field_names, params, query.wheres) match_body = [build_body(field_names, query.select.fields)] @@ -45,13 +51,12 @@ defmodule Etso.ETS.MatchSpecification do end end - defp build_condition(field_names, params, {:in, [], [field, value]}) do - field_name = resolve_field_name(field) - field_index = get_field_index(field_names, field_name) + defp build_condition(field_names, params, {:in, [], [field, values]}) do + field_target = resolve_field_target(field_names, field) - case resolve_field_values(params, value) do + case resolve_param_values(params, values) do [] -> [] - values -> List.to_tuple([:orelse | Enum.map(values, &{:==, :"$#{field_index}", &1})]) + values -> List.to_tuple([:orelse | Enum.map(values, &{:==, field_target, &1})]) end end @@ -59,42 +64,64 @@ defmodule Etso.ETS.MatchSpecification do {:==, build_condition(field_names, params, field), nil} end - defp build_condition(field_names, _, {{:., [], [{:&, [], [0]}, field_name]}, [], []}) do - :"$#{get_field_index(field_names, field_name)}" - end - defp build_condition(_, params, {:^, [], [index]}) do Enum.at(params, index) end - defp build_condition(_, _, value) when not is_tuple(value) do + defp build_condition(field_names, _params, field) when is_tuple(field) do + resolve_field_target(field_names, field) + end + + defp build_condition(_, _, value) do value end - defp build_body(field_names, query_select_fields) do - for select_field <- query_select_fields do - field_name = resolve_field_name(select_field) - field_index = get_field_index(field_names, field_name) - :"$#{field_index}" + defp build_body(field_names, fields) do + for field <- fields do + resolve_field_target(field_names, field) end end - defp resolve_field_name(field) do - {{:., _, [{:&, [], [0]}, field_name]}, [], []} = field - field_name + defp resolve_field_target(field_names, {:json_extract_path, [], [field, path]}) do + field_target = resolve_field_target(field_names, field) + resolve_field_target_path(field_target, path) + end + + defp resolve_field_target(field_names, {{:., _, [{:&, [], [0]}, field_name]}, [], []}) do + field_index = 1 + Enum.find_index(field_names, fn x -> x == field_name end) + :"$#{field_index}" + end + + defp resolve_field_target_path(field_target, path) do + # - If the path component is a key, return {:map_get, key, target} + # - If the path component is a number, return {:hd, target} outside as many {:tl, _} around + # as required. For example, [:metadata, 0] would be {:hd, {:map_get, :metadata, field}}, + # while [:metadata, 1] would be {:hd, {:tl, {:map_get, :metadata, field}}} (with one tl). + + at = fn self -> + fn + condition, 0 -> {:hd, condition} + condition, index -> self.(self).({:tl, condition}, index - 1) + end + end + + Enum.reduce(path, field_target, fn + key, condition when is_atom(key) or is_binary(key) -> {:map_get, key, condition} + index, condition when is_integer(index) -> at.(at).(condition, index) + end) end - defp resolve_field_values(params, {:^, [], [index, count]}) do + defp resolve_param_values(params, {:^, [], [index, count]}) do for index <- index..(index + count - 1) do Enum.at(params, index) end end - defp resolve_field_values(params, {:^, [], [index]}) do + defp resolve_param_values(params, {:^, [], [index]}) do Enum.at(params, index) end - defp get_field_index(field_names, field_name) do - 1 + Enum.find_index(field_names, fn x -> x == field_name end) + defp resolve_param_values(_params, values) when is_list(values) do + values end end diff --git a/priv/northwind/employees.json b/priv/northwind/employees.json index c7bc00d..9ee0c29 100644 --- a/priv/northwind/employees.json +++ b/priv/northwind/employees.json @@ -25,7 +25,23 @@ 1581 ], "title": "Vice President Sales", - "titleOfCourtesy": "Dr." + "titleOfCourtesy": "Dr.", + "metadata": { + "twitter": "@andrew_fuller", + "photos": [ + { + "storage": "a", + "url": "https://example.com/a" + }, + { + "storage": "b", + "url": "https://example.com/b" + } + ], + "documents": { + "passport": "verified" + } + } }, { "address": { diff --git a/test/northwind/repo_test.exs b/test/northwind/repo_test.exs index 6cb26d5..6236ea7 100644 --- a/test/northwind/repo_test.exs +++ b/test/northwind/repo_test.exs @@ -174,4 +174,64 @@ defmodule Northwind.RepoTest do assert sorted_etso == sorted_code end + + describe "json_extract_path" do + test "Support json_extract_path expression" do + Model.Employee + |> where([e], e.metadata["twitter"] == "@andrew_fuller") + |> Repo.one!() + end + + test "Support nested json_extract_path expression" do + Model.Employee + |> where([e], e.metadata["documents"]["passport"] == "verified") + |> Repo.one!() + end + + test "Support variable pinning in nested json_extract_path expression" do + field = "passport" + + Model.Employee + |> where([e], e.metadata["documents"][^field] == "verified") + |> Repo.one!() + + Model.Employee + |> select([e], json_extract_path(e.metadata, ["documents", "passport"])) + |> Repo.all() + |> Enum.any?(&(&1 == "verified")) + |> assert() + end + + test "Support accessing JSON arrays in json_extract_path expression" do + Model.Employee + |> select([e], json_extract_path(e.metadata, ["photos", 0, "url"])) + |> where([e], e.metadata["documents"]["passport"] == "verified") + |> Repo.one!() + |> (&(&1 == "https://example.com/a")).() + |> assert() + + Model.Employee + |> where([e], e.metadata["documents"]["passport"] == "verified") + |> select([e], e.metadata["photos"][0]["url"]) + |> Repo.one!() + |> (&(&1 == "https://example.com/a")).() + |> assert() + + Model.Employee + |> select([e], e.metadata["photos"][1]["url"]) + |> where([e], e.metadata["documents"]["passport"] == "verified") + |> Repo.one!() + |> (&(&1 == "https://example.com/b")).() + |> assert() + end + + test "Support where/in" do + Model.Employee + |> where([e], e.metadata["documents"]["passport"] in ~w(verified)) + |> select([e], e.metadata["photos"][1]["url"]) + |> Repo.one!() + |> (&(&1 == "https://example.com/b")).() + |> assert() + end + end end diff --git a/test/support/northwind/model/employee.ex b/test/support/northwind/model/employee.ex index fda0f2e..d05b51b 100644 --- a/test/support/northwind/model/employee.ex +++ b/test/support/northwind/model/employee.ex @@ -13,6 +13,7 @@ defmodule Northwind.Model.Employee do field :hire_date, :date field :notes, :string field :territory_ids, {:array, :integer} + field :metadata, :map, default: %{} embeds_one :address, Model.Address From 6663c1ed53360f9a3381e46dae45d5de192359e3 Mon Sep 17 00:00:00 2001 From: Evadne Wu Date: Tue, 28 Jun 2022 22:07:43 +0100 Subject: [PATCH 20/23] Added support for delete_all --- lib/etso/adapter/behaviour/queryable.ex | 46 ++++++++++++++++++++--- lib/etso/ets/match_specification.ex | 14 ++++--- test/northwind/repo_test.exs | 50 +++++++++++++++++++++---- 3 files changed, 92 insertions(+), 18 deletions(-) diff --git a/lib/etso/adapter/behaviour/queryable.ex b/lib/etso/adapter/behaviour/queryable.ex index 03fd488..800cb81 100644 --- a/lib/etso/adapter/behaviour/queryable.ex +++ b/lib/etso/adapter/behaviour/queryable.ex @@ -7,21 +7,55 @@ defmodule Etso.Adapter.Behaviour.Queryable do alias Etso.ETS.ObjectsSorter @impl Ecto.Adapter.Queryable - def prepare(:all, query) do - {:nocache, query} + def prepare(:all, %Ecto.Query{} = query) do + {:nocache, {:select, query}} end @impl Ecto.Adapter.Queryable - def execute(%{repo: repo}, _, {:nocache, query}, params, _) do + def prepare(:delete_all, %Ecto.Query{wheres: []} = query) do + {:nocache, {:delete_all_objects, query}} + end + + @impl Ecto.Adapter.Queryable + def prepare(:delete_all, %Ecto.Query{wheres: _} = query) do + {:nocache, {:match_delete, query}} + end + + @impl Ecto.Adapter.Queryable + def execute(%{repo: repo}, _, {:nocache, {:select, query}}, params, _) do + {_, schema} = query.from.source + {:ok, ets_table} = TableRegistry.get_table(repo, schema) + ets_match = MatchSpecification.build(query, params) + ets_objects = :ets.select(ets_table, [ets_match]) + ets_count = length(ets_objects) + {ets_count, ObjectsSorter.sort(ets_objects, query)} + end + + @impl Ecto.Adapter.Queryable + def execute(%{repo: repo}, _, {:nocache, {:delete_all_objects, query}}, params, _) do + {_, schema} = query.from.source + {:ok, ets_table} = TableRegistry.get_table(repo, schema) + ets_match = MatchSpecification.build(query, params) + ets_objects = query.select && ObjectsSorter.sort(:ets.select(ets_table, [ets_match]), query) + ets_count = :ets.info(ets_table, :size) + true = :ets.delete_all_objects(ets_table) + {ets_count, ets_objects || nil} + end + + @impl Ecto.Adapter.Queryable + def execute(%{repo: repo}, _, {:nocache, {:match_delete, query}}, params, _) do {_, schema} = query.from.source {:ok, ets_table} = TableRegistry.get_table(repo, schema) ets_match = MatchSpecification.build(query, params) - ets_objects = :ets.select(ets_table, [ets_match]) |> ObjectsSorter.sort(query) - {length(ets_objects), ets_objects} + ets_objects = query.select && ObjectsSorter.sort(:ets.select(ets_table, [ets_match]), query) + {ets_match_head, ets_match_body, _} = ets_match + ets_match = {ets_match_head, ets_match_body, [true]} + ets_count = :ets.select_delete(ets_table, [ets_match]) + {ets_count, ets_objects || nil} end @impl Ecto.Adapter.Queryable - def stream(%{repo: repo}, _, {:nocache, query}, params, options) do + def stream(%{repo: repo}, _, {:nocache, {:select, query}}, params, options) do {_, schema} = query.from.source {:ok, ets_table} = TableRegistry.get_table(repo, schema) ets_match = MatchSpecification.build(query, params) diff --git a/lib/etso/ets/match_specification.ex b/lib/etso/ets/match_specification.ex index f1debb9..aebe806 100644 --- a/lib/etso/ets/match_specification.ex +++ b/lib/etso/ets/match_specification.ex @@ -15,8 +15,8 @@ defmodule Etso.ETS.MatchSpecification do {_, schema} = query.from.source field_names = Etso.ETS.TableStructure.field_names(schema) match_head = build_head(field_names) - match_conditions = build_conditions(field_names, params, query.wheres) - match_body = [build_body(field_names, query.select.fields)] + match_conditions = build_conditions(field_names, params, query) + match_body = [build_body(field_names, query)] {match_head, match_conditions, match_body} end @@ -24,8 +24,8 @@ defmodule Etso.ETS.MatchSpecification do List.to_tuple(Enum.map(1..length(field_names), fn x -> :"$#{x}" end)) end - defp build_conditions(field_names, params, query_wheres) do - Enum.reduce(query_wheres, [], fn %Ecto.Query.BooleanExpr{expr: expression}, acc -> + defp build_conditions(field_names, params, %Ecto.Query{wheres: wheres}) do + Enum.reduce(wheres, [], fn %Ecto.Query.BooleanExpr{expr: expression}, acc -> [build_condition(field_names, params, expression) | acc] end) end @@ -76,7 +76,11 @@ defmodule Etso.ETS.MatchSpecification do value end - defp build_body(field_names, fields) do + defp build_body(_, %Ecto.Query{select: nil}) do + [] + end + + defp build_body(field_names, %Ecto.Query{select: %{fields: fields}}) do for field <- fields do resolve_field_target(field_names, field) end diff --git a/test/northwind/repo_test.exs b/test/northwind/repo_test.exs index 6236ea7..1a23291 100644 --- a/test/northwind/repo_test.exs +++ b/test/northwind/repo_test.exs @@ -10,7 +10,7 @@ defmodule Northwind.RepoTest do :ok = Importer.perform() end - test "list" do + test "List All" do Repo.all(Model.Employee) end @@ -175,20 +175,44 @@ defmodule Northwind.RepoTest do assert sorted_etso == sorted_code end - describe "json_extract_path" do - test "Support json_extract_path expression" do + test "Delete All" do + assert Repo.delete_all(Model.Employee) + assert [] == Repo.all(Model.Employee) + end + + test "Delete Where" do + query = Model.Employee |> where([e], e.employee_id in [1, 5]) + assert [a, b] = Repo.all(query) + assert {2, nil} = Repo.delete_all(query) + assert [] == Repo.all(query) + refute [] == Repo.all(Model.Employee) + end + + test "Delete Where Select" do + query = Model.Employee |> where([e], e.employee_id in [1, 5]) + assert [a, b] = Repo.all(query) + assert {2, list} = Repo.delete_all(query |> select([e], {e, e.employee_id})) + assert is_list(list) + assert Enum.any?(list, &(elem(&1, 1) == 1)) + assert Enum.any?(list, &(elem(&1, 1) == 5)) + assert [] = Repo.all(query) + refute [] == Repo.all(Model.Employee) + end + + describe "With JSON Extract Paths" do + test "using literal value" do Model.Employee |> where([e], e.metadata["twitter"] == "@andrew_fuller") |> Repo.one!() end - test "Support nested json_extract_path expression" do + test "using brackets" do Model.Employee |> where([e], e.metadata["documents"]["passport"] == "verified") |> Repo.one!() end - test "Support variable pinning in nested json_extract_path expression" do + test "with variable pinning" do field = "passport" Model.Employee @@ -202,7 +226,7 @@ defmodule Northwind.RepoTest do |> assert() end - test "Support accessing JSON arrays in json_extract_path expression" do + test "with arrays" do Model.Employee |> select([e], json_extract_path(e.metadata, ["photos", 0, "url"])) |> where([e], e.metadata["documents"]["passport"] == "verified") @@ -225,7 +249,7 @@ defmodule Northwind.RepoTest do |> assert() end - test "Support where/in" do + test "with where/in" do Model.Employee |> where([e], e.metadata["documents"]["passport"] in ~w(verified)) |> select([e], e.metadata["photos"][1]["url"]) @@ -233,5 +257,17 @@ defmodule Northwind.RepoTest do |> (&(&1 == "https://example.com/b")).() |> assert() end + + test "in deletion" do + Model.Employee + |> where([e], e.metadata["documents"]["passport"] == "verified") + |> Repo.delete_all() + + assert_raise Ecto.NoResultsError, fn -> + Model.Employee + |> where([e], e.metadata["documents"]["passport"] == "verified") + |> Repo.one!() + end + end end end From 3f31f683dfcd2070e7e642d4766a381e9829f8d8 Mon Sep 17 00:00:00 2001 From: Evadne Wu Date: Wed, 29 Jun 2022 22:59:21 +0100 Subject: [PATCH 21/23] Attributed author of json_extract_path() support in README --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 0e7ae87..5cae11b 100644 --- a/README.md +++ b/README.md @@ -109,6 +109,8 @@ The Author also wishes to thank the following individuals: - [Doug W.][harmon25], for [providing insights into parallel preloads][gh-13]. +- [Atanda Rasheed][heywhy], for [support of JSON extract paths][gh-13]. + [erlang-ets]: http://erlang.org/doc/man/ets.html [northwind]: https://github.com/evadne/etso/tree/master/test/support/northwind [northwind-importer]: https://github.com/evadne/etso/tree/master/test/support/northwind/importer.ex @@ -126,3 +128,5 @@ The Author also wishes to thank the following individuals: [gh-12]: https://github.com/evadne/etso/issues/12 [harmon25]: https://github.com/harmon25 [gh-13]: https://github.com/evadne/etso/issues/13 +[heywhy]: https://github.com/heywhy +[gh-20]: https://github.com/evadne/etso/issues/20 From d5022761cf3319c8fcadab3d7265c01728d02275 Mon Sep 17 00:00:00 2001 From: Evadne Wu Date: Wed, 29 Jun 2022 23:03:06 +0100 Subject: [PATCH 22/23] =?UTF-8?q?=E2=86=92=201.1.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 4 ++-- mix.exs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 5cae11b..d92da5b 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ Using Etso is a two-step process. First, include it in your application’s depe ```elixir defp deps do [ - {:etso, "~> 1.0.1"} + {:etso, "~> 1.1.0"} ] end ``` @@ -109,7 +109,7 @@ The Author also wishes to thank the following individuals: - [Doug W.][harmon25], for [providing insights into parallel preloads][gh-13]. -- [Atanda Rasheed][heywhy], for [support of JSON extract paths][gh-13]. +- [Atanda Rasheed][heywhy], for [support of JSON extract paths][gh-20]. [erlang-ets]: http://erlang.org/doc/man/ets.html [northwind]: https://github.com/evadne/etso/tree/master/test/support/northwind diff --git a/mix.exs b/mix.exs index a7bef89..5aceb3b 100644 --- a/mix.exs +++ b/mix.exs @@ -4,7 +4,7 @@ defmodule Etso.MixProject do def project do [ app: :etso, - version: "1.0.1", + version: "1.1.0", elixir: "~> 1.8", elixirc_paths: elixirc_paths(Mix.env()), start_permanent: Mix.env() == :prod, From 67fb68cae23b1906d94581bc3a8105f37b20d755 Mon Sep 17 00:00:00 2001 From: Prem Prakash Date: Mon, 15 Jan 2024 09:19:38 -0300 Subject: [PATCH 23/23] single commit --- lib/etso/adapter.ex | 6 ++-- lib/etso/adapter/behaviour/storage.ex | 7 +++++ lib/etso/ets/match_specification.ex | 4 --- test/northwind/importer_test.exs | 2 +- test/northwind/repo_test.exs | 24 +++++++++++++- test/northwind/storage_test.exs | 45 +++++++++++++++++++++++++++ 6 files changed, 80 insertions(+), 8 deletions(-) create mode 100644 lib/etso/adapter/behaviour/storage.ex create mode 100644 test/northwind/storage_test.exs diff --git a/lib/etso/adapter.ex b/lib/etso/adapter.ex index dd6e12c..a5cd1c0 100644 --- a/lib/etso/adapter.ex +++ b/lib/etso/adapter.ex @@ -14,8 +14,9 @@ defmodule Etso.Adapter do """ @behaviour Ecto.Adapter - @behaviour Ecto.Adapter.Schema @behaviour Ecto.Adapter.Queryable + @behaviour Ecto.Adapter.Schema + @behaviour Ecto.Adapter.Storage @impl Ecto.Adapter defmacro __before_compile__(_opts), do: :ok @@ -55,7 +56,8 @@ defmodule Etso.Adapter do for {implementation_module, behaviour_module} <- [ {__MODULE__.Behaviour.Schema, Ecto.Adapter.Schema}, - {__MODULE__.Behaviour.Queryable, Ecto.Adapter.Queryable} + {__MODULE__.Behaviour.Queryable, Ecto.Adapter.Queryable}, + {__MODULE__.Behaviour.Storage, Ecto.Adapter.Storage} ] do for {name, arity} <- implementation_module.__info__(:functions) do args = Enum.map(1..arity, &{:"arg_#{&1}", [], Elixir}) diff --git a/lib/etso/adapter/behaviour/storage.ex b/lib/etso/adapter/behaviour/storage.ex new file mode 100644 index 0000000..bc6de61 --- /dev/null +++ b/lib/etso/adapter/behaviour/storage.ex @@ -0,0 +1,7 @@ +defmodule Etso.Adapter.Behaviour.Storage do + @moduledoc false + @behaviour Ecto.Adapter.Storage + def storage_status(_options), do: :up + def storage_up(_options), do: :ok + def storage_down(_options), do: :ok +end diff --git a/lib/etso/ets/match_specification.ex b/lib/etso/ets/match_specification.ex index aebe806..e90d1af 100644 --- a/lib/etso/ets/match_specification.ex +++ b/lib/etso/ets/match_specification.ex @@ -121,10 +121,6 @@ defmodule Etso.ETS.MatchSpecification do end end - defp resolve_param_values(params, {:^, [], [index]}) do - Enum.at(params, index) - end - defp resolve_param_values(_params, values) when is_list(values) do values end diff --git a/test/northwind/importer_test.exs b/test/northwind/importer_test.exs index 80c436f..955e30a 100644 --- a/test/northwind/importer_test.exs +++ b/test/northwind/importer_test.exs @@ -6,7 +6,7 @@ defmodule Northwind.ImporterTest do setup do repo_id = __MODULE__ - repo_start = {Northwind.Repo, :start_link, []} + repo_start = {Repo, :start_link, []} {:ok, _} = start_supervised(%{id: repo_id, start: repo_start}) :ok end diff --git a/test/northwind/repo_test.exs b/test/northwind/repo_test.exs index 1a23291..dc25a47 100644 --- a/test/northwind/repo_test.exs +++ b/test/northwind/repo_test.exs @@ -5,7 +5,7 @@ defmodule Northwind.RepoTest do setup do repo_id = __MODULE__ - repo_start = {Northwind.Repo, :start_link, []} + repo_start = {Repo, :start_link, []} {:ok, _} = start_supervised(%{id: repo_id, start: repo_start}) :ok = Importer.perform() end @@ -89,6 +89,28 @@ defmodule Northwind.RepoTest do |> (&assert(&1 == [3])).() end + test "Where In Nested With and Without Pin" do + employee_ids = [3, 5, 7] + + Model.Employee + |> where([x], x.employee_id in ^employee_ids) + |> where([x], x.first_name in ["Janet"]) + |> select([x], x.employee_id) + |> Repo.all() + |> Enum.sort() + |> (&assert(&1 == [3])).() + end + + test "Where In Nested Without Pin" do + Model.Employee + |> where([x], x.employee_id in [3, 5, 7]) + |> where([x], x.first_name in ["Janet"]) + |> select([x], x.employee_id) + |> Repo.all() + |> Enum.sort() + |> (&assert(&1 == [3])).() + end + test "Select Where" do Model.Employee |> where([x], x.title == "Vice President Sales" and x.first_name == "Andrew") diff --git a/test/northwind/storage_test.exs b/test/northwind/storage_test.exs new file mode 100644 index 0000000..9de7c51 --- /dev/null +++ b/test/northwind/storage_test.exs @@ -0,0 +1,45 @@ +defmodule Northwind.StorageTest do + use ExUnit.Case + alias Northwind.{Importer, Model, Repo} + import Ecto.Query + + setup do + repo_id = __MODULE__ + repo_start = {Repo, :start_link, []} + {:ok, _} = start_supervised(%{id: repo_id, start: repo_start}) + :ok + end + + test "repo started with Up status" do + adapter = Repo.__adapter__() + assert :up = adapter.storage_status(Repo.config()) + assert_empty() + end + + test "repo up/down roundtrip is no-op" do + adapter = Repo.__adapter__() + :ok = Importer.perform() + assert_not_empty() + assert :ok = adapter.storage_down(Repo.config()) + assert_not_empty() + assert :ok = adapter.storage_up(Repo.config()) + assert_not_empty() + end + + defp assert_empty do + Model.Employee + |> where([x], x.employee_id in [3, 5, 7]) + |> select([x], x.employee_id) + |> Repo.all() + |> (&assert(&1 == [])).() + end + + defp assert_not_empty do + Model.Employee + |> where([x], x.employee_id in [3, 5, 7]) + |> select([x], x.employee_id) + |> Repo.all() + |> Enum.sort() + |> (&assert(&1 == [3, 5, 7])).() + end +end