From 2af83169d74cd26abc9b5c1bce1f511599535ca5 Mon Sep 17 00:00:00 2001 From: Atanda Rasheed Date: Tue, 28 Jun 2022 20:10:14 +0100 Subject: [PATCH 1/2] Added support for json_extract_path() in select queries - Also refined support for where/in --- lib/etso/ets/match_specification.ex | 75 ++++++++++++++++-------- priv/northwind/employees.json | 18 +++++- test/northwind/repo_test.exs | 60 +++++++++++++++++++ test/support/northwind/model/employee.ex | 1 + 4 files changed, 129 insertions(+), 25 deletions(-) diff --git a/lib/etso/ets/match_specification.ex b/lib/etso/ets/match_specification.ex index 5836e88..f1debb9 100644 --- a/lib/etso/ets/match_specification.ex +++ b/lib/etso/ets/match_specification.ex @@ -1,13 +1,19 @@ defmodule Etso.ETS.MatchSpecification do @moduledoc """ The ETS Match Specifications module contains various functions which convert Ecto queries to - ETS Match Specifications in order to execute the given queries. + [ETS Match Specifications](https://www.erlang.org/doc/apps/erts/match_spec.html) in order to + execute the given queries with ETS with as much pushed down to ETS as possible. + + The basic shape of the match head is `[$1, $2, $3, …]` where each field is a named variable, the + ordering of the fields is determined by `Etso.ETS.TableStructure`. + + Conditions are compiled according to the wheres in the underlying Ecto query, while the body is + compiled based on the selected fields in the underlying Ecto query. """ def build(query, params) do {_, schema} = query.from.source field_names = Etso.ETS.TableStructure.field_names(schema) - match_head = build_head(field_names) match_conditions = build_conditions(field_names, params, query.wheres) match_body = [build_body(field_names, query.select.fields)] @@ -45,13 +51,12 @@ defmodule Etso.ETS.MatchSpecification do end end - defp build_condition(field_names, params, {:in, [], [field, value]}) do - field_name = resolve_field_name(field) - field_index = get_field_index(field_names, field_name) + defp build_condition(field_names, params, {:in, [], [field, values]}) do + field_target = resolve_field_target(field_names, field) - case resolve_field_values(params, value) do + case resolve_param_values(params, values) do [] -> [] - values -> List.to_tuple([:orelse | Enum.map(values, &{:==, :"$#{field_index}", &1})]) + values -> List.to_tuple([:orelse | Enum.map(values, &{:==, field_target, &1})]) end end @@ -59,42 +64,64 @@ defmodule Etso.ETS.MatchSpecification do {:==, build_condition(field_names, params, field), nil} end - defp build_condition(field_names, _, {{:., [], [{:&, [], [0]}, field_name]}, [], []}) do - :"$#{get_field_index(field_names, field_name)}" - end - defp build_condition(_, params, {:^, [], [index]}) do Enum.at(params, index) end - defp build_condition(_, _, value) when not is_tuple(value) do + defp build_condition(field_names, _params, field) when is_tuple(field) do + resolve_field_target(field_names, field) + end + + defp build_condition(_, _, value) do value end - defp build_body(field_names, query_select_fields) do - for select_field <- query_select_fields do - field_name = resolve_field_name(select_field) - field_index = get_field_index(field_names, field_name) - :"$#{field_index}" + defp build_body(field_names, fields) do + for field <- fields do + resolve_field_target(field_names, field) end end - defp resolve_field_name(field) do - {{:., _, [{:&, [], [0]}, field_name]}, [], []} = field - field_name + defp resolve_field_target(field_names, {:json_extract_path, [], [field, path]}) do + field_target = resolve_field_target(field_names, field) + resolve_field_target_path(field_target, path) + end + + defp resolve_field_target(field_names, {{:., _, [{:&, [], [0]}, field_name]}, [], []}) do + field_index = 1 + Enum.find_index(field_names, fn x -> x == field_name end) + :"$#{field_index}" + end + + defp resolve_field_target_path(field_target, path) do + # - If the path component is a key, return {:map_get, key, target} + # - If the path component is a number, return {:hd, target} outside as many {:tl, _} around + # as required. For example, [:metadata, 0] would be {:hd, {:map_get, :metadata, field}}, + # while [:metadata, 1] would be {:hd, {:tl, {:map_get, :metadata, field}}} (with one tl). + + at = fn self -> + fn + condition, 0 -> {:hd, condition} + condition, index -> self.(self).({:tl, condition}, index - 1) + end + end + + Enum.reduce(path, field_target, fn + key, condition when is_atom(key) or is_binary(key) -> {:map_get, key, condition} + index, condition when is_integer(index) -> at.(at).(condition, index) + end) end - defp resolve_field_values(params, {:^, [], [index, count]}) do + defp resolve_param_values(params, {:^, [], [index, count]}) do for index <- index..(index + count - 1) do Enum.at(params, index) end end - defp resolve_field_values(params, {:^, [], [index]}) do + defp resolve_param_values(params, {:^, [], [index]}) do Enum.at(params, index) end - defp get_field_index(field_names, field_name) do - 1 + Enum.find_index(field_names, fn x -> x == field_name end) + defp resolve_param_values(_params, values) when is_list(values) do + values end end diff --git a/priv/northwind/employees.json b/priv/northwind/employees.json index c7bc00d..9ee0c29 100644 --- a/priv/northwind/employees.json +++ b/priv/northwind/employees.json @@ -25,7 +25,23 @@ 1581 ], "title": "Vice President Sales", - "titleOfCourtesy": "Dr." + "titleOfCourtesy": "Dr.", + "metadata": { + "twitter": "@andrew_fuller", + "photos": [ + { + "storage": "a", + "url": "https://example.com/a" + }, + { + "storage": "b", + "url": "https://example.com/b" + } + ], + "documents": { + "passport": "verified" + } + } }, { "address": { diff --git a/test/northwind/repo_test.exs b/test/northwind/repo_test.exs index 6cb26d5..6236ea7 100644 --- a/test/northwind/repo_test.exs +++ b/test/northwind/repo_test.exs @@ -174,4 +174,64 @@ defmodule Northwind.RepoTest do assert sorted_etso == sorted_code end + + describe "json_extract_path" do + test "Support json_extract_path expression" do + Model.Employee + |> where([e], e.metadata["twitter"] == "@andrew_fuller") + |> Repo.one!() + end + + test "Support nested json_extract_path expression" do + Model.Employee + |> where([e], e.metadata["documents"]["passport"] == "verified") + |> Repo.one!() + end + + test "Support variable pinning in nested json_extract_path expression" do + field = "passport" + + Model.Employee + |> where([e], e.metadata["documents"][^field] == "verified") + |> Repo.one!() + + Model.Employee + |> select([e], json_extract_path(e.metadata, ["documents", "passport"])) + |> Repo.all() + |> Enum.any?(&(&1 == "verified")) + |> assert() + end + + test "Support accessing JSON arrays in json_extract_path expression" do + Model.Employee + |> select([e], json_extract_path(e.metadata, ["photos", 0, "url"])) + |> where([e], e.metadata["documents"]["passport"] == "verified") + |> Repo.one!() + |> (&(&1 == "https://example.com/a")).() + |> assert() + + Model.Employee + |> where([e], e.metadata["documents"]["passport"] == "verified") + |> select([e], e.metadata["photos"][0]["url"]) + |> Repo.one!() + |> (&(&1 == "https://example.com/a")).() + |> assert() + + Model.Employee + |> select([e], e.metadata["photos"][1]["url"]) + |> where([e], e.metadata["documents"]["passport"] == "verified") + |> Repo.one!() + |> (&(&1 == "https://example.com/b")).() + |> assert() + end + + test "Support where/in" do + Model.Employee + |> where([e], e.metadata["documents"]["passport"] in ~w(verified)) + |> select([e], e.metadata["photos"][1]["url"]) + |> Repo.one!() + |> (&(&1 == "https://example.com/b")).() + |> assert() + end + end end diff --git a/test/support/northwind/model/employee.ex b/test/support/northwind/model/employee.ex index fda0f2e..d05b51b 100644 --- a/test/support/northwind/model/employee.ex +++ b/test/support/northwind/model/employee.ex @@ -13,6 +13,7 @@ defmodule Northwind.Model.Employee do field :hire_date, :date field :notes, :string field :territory_ids, {:array, :integer} + field :metadata, :map, default: %{} embeds_one :address, Model.Address From 6663c1ed53360f9a3381e46dae45d5de192359e3 Mon Sep 17 00:00:00 2001 From: Evadne Wu Date: Tue, 28 Jun 2022 22:07:43 +0100 Subject: [PATCH 2/2] Added support for delete_all --- lib/etso/adapter/behaviour/queryable.ex | 46 ++++++++++++++++++++--- lib/etso/ets/match_specification.ex | 14 ++++--- test/northwind/repo_test.exs | 50 +++++++++++++++++++++---- 3 files changed, 92 insertions(+), 18 deletions(-) diff --git a/lib/etso/adapter/behaviour/queryable.ex b/lib/etso/adapter/behaviour/queryable.ex index 03fd488..800cb81 100644 --- a/lib/etso/adapter/behaviour/queryable.ex +++ b/lib/etso/adapter/behaviour/queryable.ex @@ -7,21 +7,55 @@ defmodule Etso.Adapter.Behaviour.Queryable do alias Etso.ETS.ObjectsSorter @impl Ecto.Adapter.Queryable - def prepare(:all, query) do - {:nocache, query} + def prepare(:all, %Ecto.Query{} = query) do + {:nocache, {:select, query}} end @impl Ecto.Adapter.Queryable - def execute(%{repo: repo}, _, {:nocache, query}, params, _) do + def prepare(:delete_all, %Ecto.Query{wheres: []} = query) do + {:nocache, {:delete_all_objects, query}} + end + + @impl Ecto.Adapter.Queryable + def prepare(:delete_all, %Ecto.Query{wheres: _} = query) do + {:nocache, {:match_delete, query}} + end + + @impl Ecto.Adapter.Queryable + def execute(%{repo: repo}, _, {:nocache, {:select, query}}, params, _) do + {_, schema} = query.from.source + {:ok, ets_table} = TableRegistry.get_table(repo, schema) + ets_match = MatchSpecification.build(query, params) + ets_objects = :ets.select(ets_table, [ets_match]) + ets_count = length(ets_objects) + {ets_count, ObjectsSorter.sort(ets_objects, query)} + end + + @impl Ecto.Adapter.Queryable + def execute(%{repo: repo}, _, {:nocache, {:delete_all_objects, query}}, params, _) do + {_, schema} = query.from.source + {:ok, ets_table} = TableRegistry.get_table(repo, schema) + ets_match = MatchSpecification.build(query, params) + ets_objects = query.select && ObjectsSorter.sort(:ets.select(ets_table, [ets_match]), query) + ets_count = :ets.info(ets_table, :size) + true = :ets.delete_all_objects(ets_table) + {ets_count, ets_objects || nil} + end + + @impl Ecto.Adapter.Queryable + def execute(%{repo: repo}, _, {:nocache, {:match_delete, query}}, params, _) do {_, schema} = query.from.source {:ok, ets_table} = TableRegistry.get_table(repo, schema) ets_match = MatchSpecification.build(query, params) - ets_objects = :ets.select(ets_table, [ets_match]) |> ObjectsSorter.sort(query) - {length(ets_objects), ets_objects} + ets_objects = query.select && ObjectsSorter.sort(:ets.select(ets_table, [ets_match]), query) + {ets_match_head, ets_match_body, _} = ets_match + ets_match = {ets_match_head, ets_match_body, [true]} + ets_count = :ets.select_delete(ets_table, [ets_match]) + {ets_count, ets_objects || nil} end @impl Ecto.Adapter.Queryable - def stream(%{repo: repo}, _, {:nocache, query}, params, options) do + def stream(%{repo: repo}, _, {:nocache, {:select, query}}, params, options) do {_, schema} = query.from.source {:ok, ets_table} = TableRegistry.get_table(repo, schema) ets_match = MatchSpecification.build(query, params) diff --git a/lib/etso/ets/match_specification.ex b/lib/etso/ets/match_specification.ex index f1debb9..aebe806 100644 --- a/lib/etso/ets/match_specification.ex +++ b/lib/etso/ets/match_specification.ex @@ -15,8 +15,8 @@ defmodule Etso.ETS.MatchSpecification do {_, schema} = query.from.source field_names = Etso.ETS.TableStructure.field_names(schema) match_head = build_head(field_names) - match_conditions = build_conditions(field_names, params, query.wheres) - match_body = [build_body(field_names, query.select.fields)] + match_conditions = build_conditions(field_names, params, query) + match_body = [build_body(field_names, query)] {match_head, match_conditions, match_body} end @@ -24,8 +24,8 @@ defmodule Etso.ETS.MatchSpecification do List.to_tuple(Enum.map(1..length(field_names), fn x -> :"$#{x}" end)) end - defp build_conditions(field_names, params, query_wheres) do - Enum.reduce(query_wheres, [], fn %Ecto.Query.BooleanExpr{expr: expression}, acc -> + defp build_conditions(field_names, params, %Ecto.Query{wheres: wheres}) do + Enum.reduce(wheres, [], fn %Ecto.Query.BooleanExpr{expr: expression}, acc -> [build_condition(field_names, params, expression) | acc] end) end @@ -76,7 +76,11 @@ defmodule Etso.ETS.MatchSpecification do value end - defp build_body(field_names, fields) do + defp build_body(_, %Ecto.Query{select: nil}) do + [] + end + + defp build_body(field_names, %Ecto.Query{select: %{fields: fields}}) do for field <- fields do resolve_field_target(field_names, field) end diff --git a/test/northwind/repo_test.exs b/test/northwind/repo_test.exs index 6236ea7..1a23291 100644 --- a/test/northwind/repo_test.exs +++ b/test/northwind/repo_test.exs @@ -10,7 +10,7 @@ defmodule Northwind.RepoTest do :ok = Importer.perform() end - test "list" do + test "List All" do Repo.all(Model.Employee) end @@ -175,20 +175,44 @@ defmodule Northwind.RepoTest do assert sorted_etso == sorted_code end - describe "json_extract_path" do - test "Support json_extract_path expression" do + test "Delete All" do + assert Repo.delete_all(Model.Employee) + assert [] == Repo.all(Model.Employee) + end + + test "Delete Where" do + query = Model.Employee |> where([e], e.employee_id in [1, 5]) + assert [a, b] = Repo.all(query) + assert {2, nil} = Repo.delete_all(query) + assert [] == Repo.all(query) + refute [] == Repo.all(Model.Employee) + end + + test "Delete Where Select" do + query = Model.Employee |> where([e], e.employee_id in [1, 5]) + assert [a, b] = Repo.all(query) + assert {2, list} = Repo.delete_all(query |> select([e], {e, e.employee_id})) + assert is_list(list) + assert Enum.any?(list, &(elem(&1, 1) == 1)) + assert Enum.any?(list, &(elem(&1, 1) == 5)) + assert [] = Repo.all(query) + refute [] == Repo.all(Model.Employee) + end + + describe "With JSON Extract Paths" do + test "using literal value" do Model.Employee |> where([e], e.metadata["twitter"] == "@andrew_fuller") |> Repo.one!() end - test "Support nested json_extract_path expression" do + test "using brackets" do Model.Employee |> where([e], e.metadata["documents"]["passport"] == "verified") |> Repo.one!() end - test "Support variable pinning in nested json_extract_path expression" do + test "with variable pinning" do field = "passport" Model.Employee @@ -202,7 +226,7 @@ defmodule Northwind.RepoTest do |> assert() end - test "Support accessing JSON arrays in json_extract_path expression" do + test "with arrays" do Model.Employee |> select([e], json_extract_path(e.metadata, ["photos", 0, "url"])) |> where([e], e.metadata["documents"]["passport"] == "verified") @@ -225,7 +249,7 @@ defmodule Northwind.RepoTest do |> assert() end - test "Support where/in" do + test "with where/in" do Model.Employee |> where([e], e.metadata["documents"]["passport"] in ~w(verified)) |> select([e], e.metadata["photos"][1]["url"]) @@ -233,5 +257,17 @@ defmodule Northwind.RepoTest do |> (&(&1 == "https://example.com/b")).() |> assert() end + + test "in deletion" do + Model.Employee + |> where([e], e.metadata["documents"]["passport"] == "verified") + |> Repo.delete_all() + + assert_raise Ecto.NoResultsError, fn -> + Model.Employee + |> where([e], e.metadata["documents"]["passport"] == "verified") + |> Repo.one!() + end + end end end