From 2add8bbbf594937f6d356e9af188e0c54fa888f8 Mon Sep 17 00:00:00 2001 From: atanda rasheed Date: Thu, 5 May 2022 11:33:59 +0100 Subject: [PATCH 1/4] feat: support json extract path expression --- lib/etso/adapter.ex | 18 +++++++++- lib/etso/adapter/behaviour/schema.ex | 2 +- lib/etso/adapter/table_registry.ex | 9 +++++ lib/etso/adapter/types/map.ex | 43 ++++++++++++++++++++++++ lib/etso/ets/match_specification.ex | 9 +++++ mix.lock | 21 ++++++------ priv/northwind/employees.json | 3 +- test/northwind/repo_test.exs | 6 ++++ test/support/northwind/model/employee.ex | 1 + 9 files changed, 99 insertions(+), 13 deletions(-) create mode 100644 lib/etso/adapter/types/map.ex diff --git a/lib/etso/adapter.ex b/lib/etso/adapter.ex index 231f442..803401a 100644 --- a/lib/etso/adapter.ex +++ b/lib/etso/adapter.ex @@ -13,6 +13,8 @@ defmodule Etso.Adapter do end """ + alias Etso.Adapter.TableRegistry + @behaviour Ecto.Adapter @behaviour Ecto.Adapter.Schema @behaviour Ecto.Adapter.Queryable @@ -27,12 +29,15 @@ defmodule Etso.Adapter do {:ok, repo} = Keyword.fetch(config, :repo) child_spec = __MODULE__.Supervisor.child_spec(repo) adapter_meta = %__MODULE__.Meta{repo: repo} - {:ok, child_spec, adapter_meta} + {:ok, child_spec, Map.from_struct(adapter_meta)} end @doc false def checkout(_, _, fun), do: fun.() + @doc false + def checked_out?(_), do: false + @doc false def loaders(:binary_id, type), do: [Ecto.UUID, type] def loaders(:embed_id, type), do: [Ecto.UUID, type] @@ -41,8 +46,19 @@ defmodule Etso.Adapter do @doc false def dumpers(:binary_id, type), do: [type, Ecto.UUID] def dumpers(:embed_id, type), do: [type, Ecto.UUID] + def dumpers(:map, type), do: [type, Etso.Ecto.MapType] def dumpers(_, type), do: [type] + @doc """ + Delete all data from the tables. + """ + @spec flush_tables(module()) :: :ok + def flush_tables(repo) do + repo + |> TableRegistry.active_tables() + |> Enum.each(& :ets.delete_all_objects(&1) == true) + end + for module <- [__MODULE__.Behaviour.Schema, __MODULE__.Behaviour.Queryable] do for {name, arity} <- module.__info__(:functions) do args = Enum.map(1..arity, &{:"arg_#{&1}", [], Elixir}) diff --git a/lib/etso/adapter/behaviour/schema.ex b/lib/etso/adapter/behaviour/schema.ex index fd8f7e1..c43f181 100644 --- a/lib/etso/adapter/behaviour/schema.ex +++ b/lib/etso/adapter/behaviour/schema.ex @@ -8,7 +8,7 @@ defmodule Etso.Adapter.Behaviour.Schema do def autogenerate(:binary_id), do: Ecto.UUID.bingenerate() def autogenerate(:embed_id), do: Ecto.UUID.bingenerate() - def insert_all(%{repo: repo}, %{schema: schema}, _, entries, _, _, _) do + def insert_all(%{repo: repo}, %{schema: schema}, _, entries, _, _, _, _) do {:ok, ets_table} = TableRegistry.get_table(repo, schema) ets_field_names = TableStructure.field_names(schema) ets_changes = TableStructure.entries_to_tuples(ets_field_names, entries) diff --git a/lib/etso/adapter/table_registry.ex b/lib/etso/adapter/table_registry.ex index 375ee54..447de0b 100644 --- a/lib/etso/adapter/table_registry.ex +++ b/lib/etso/adapter/table_registry.ex @@ -38,6 +38,15 @@ defmodule Etso.Adapter.TableRegistry do end end + @spec active_tables(module()) :: [any()] + def active_tables(repo) do + conditions = [{{{:_, :"$1"}, :_, :_}, [{:==, :"$1", :ets_table}], [:"$_"]}] + + build_name(repo) + |> Registry.select(conditions) + |> Enum.map(fn {_key, {_pid, table_reference}} -> table_reference end) + end + defp lookup_table(repo, schema) do case Registry.lookup(build_name(repo), {schema, :ets_table}) do [{_, table_reference}] -> {:ok, table_reference} diff --git a/lib/etso/adapter/types/map.ex b/lib/etso/adapter/types/map.ex new file mode 100644 index 0000000..b52ef91 --- /dev/null +++ b/lib/etso/adapter/types/map.ex @@ -0,0 +1,43 @@ +defmodule Etso.Ecto.MapType do + use Ecto.Type + def type, do: :map + + def cast(map) when is_map(map) do + {:ok, map} + end + + # Everything else is a failure though + def cast(_), do: :error + + # When loading data from the database, as long as it's a map, + # we just put the data back into a URI struct to be stored in + # the loaded schema struct. + def load(data) when is_map(data) do + {:ok, data} + end + + # When dumping data to the database, we *expect* a URI struct + # but any value could be inserted into the schema struct at runtime, + # so we need to guard against them. + def dump(%{} = map), do: {:ok, map_keys_to_string(map)} + def dump(_), do: :error + + defp map_keys_to_string(value) when is_map(value) and not is_struct(value) do + value + |> Enum.map(fn {key, value} -> {to_string(key), map_keys_to_string(value)} end) + |> Enum.into(%{}) + end + + defp map_keys_to_string(%{__struct__: module} = value) + when module not in [Date, DateTime, Decimal, NaiveDateTime, Time] do + value + |> Map.from_struct() + |> map_keys_to_string() + end + + defp map_keys_to_string(value) when is_list(value) do + Enum.map(value, &map_keys_to_string/1) + end + + defp map_keys_to_string(value), do: value +end diff --git a/lib/etso/ets/match_specification.ex b/lib/etso/ets/match_specification.ex index 5836e88..5d08c0f 100644 --- a/lib/etso/ets/match_specification.ex +++ b/lib/etso/ets/match_specification.ex @@ -59,6 +59,15 @@ defmodule Etso.ETS.MatchSpecification do {:==, build_condition(field_names, params, field), nil} end + defp build_condition(field_names, _, {:json_extract_path, [], paths}) do + [parent, fields] = paths + [field | remaining] = fields + acc = {:map_get, field, build_condition(field_names, nil, parent)} + Enum.reduce(remaining, acc, fn field, acc -> + {:map_get, field, acc} + end) + end + defp build_condition(field_names, _, {{:., [], [{:&, [], [0]}, field_name]}, [], []}) do :"$#{get_field_index(field_names, field_name)}" end diff --git a/mix.lock b/mix.lock index b49d46e..fcc4a9e 100644 --- a/mix.lock +++ b/mix.lock @@ -1,12 +1,13 @@ %{ - "decimal": {:hex, :decimal, "1.8.0", "ca462e0d885f09a1c5a342dbd7c1dcf27ea63548c65a65e67334f4b61803822e", [:mix], [], "hexpm"}, - "dialyxir": {:hex, :dialyxir, "1.0.0-rc.6", "78e97d9c0ff1b5521dd68041193891aebebce52fc3b93463c0a6806874557d7d", [:mix], [{:erlex, "~> 0.2.1", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm"}, - "earmark": {:hex, :earmark, "1.3.6", "ce1d0675e10a5bb46b007549362bd3f5f08908843957687d8484fe7f37466b19", [:mix], [], "hexpm"}, - "ecto": {:hex, :ecto, "3.1.7", "fa21d06ef56cdc2fdaa62574e8c3ba34a2751d44ea34c30bc65f0728421043e5", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, - "erlex": {:hex, :erlex, "0.2.4", "23791959df45fe8f01f388c6f7eb733cc361668cbeedd801bf491c55a029917b", [:mix], [], "hexpm"}, - "ex_doc": {:hex, :ex_doc, "0.21.2", "caca5bc28ed7b3bdc0b662f8afe2bee1eedb5c3cf7b322feeeb7c6ebbde089d6", [:mix], [{:earmark, "~> 1.3.3 or ~> 1.4", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm"}, - "jason": {:hex, :jason, "1.1.2", "b03dedea67a99223a2eaf9f1264ce37154564de899fd3d8b9a21b1a6fd64afe7", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm"}, - "makeup": {:hex, :makeup, "1.0.0", "671df94cf5a594b739ce03b0d0316aa64312cee2574b6a44becb83cd90fb05dc", [:mix], [{:nimble_parsec, "~> 0.5.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm"}, - "makeup_elixir": {:hex, :makeup_elixir, "0.14.0", "cf8b7c66ad1cff4c14679698d532f0b5d45a3968ffbcbfd590339cb57742f1ae", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm"}, - "nimble_parsec": {:hex, :nimble_parsec, "0.5.1", "c90796ecee0289dbb5ad16d3ad06f957b0cd1199769641c961cfe0b97db190e0", [:mix], [], "hexpm"}, + "decimal": {:hex, :decimal, "2.0.0", "a78296e617b0f5dd4c6caf57c714431347912ffb1d0842e998e9792b5642d697", [:mix], [], "hexpm", "34666e9c55dea81013e77d9d87370fe6cb6291d1ef32f46a1600230b1d44f577"}, + "dialyxir": {:hex, :dialyxir, "1.0.0-rc.6", "78e97d9c0ff1b5521dd68041193891aebebce52fc3b93463c0a6806874557d7d", [:mix], [{:erlex, "~> 0.2.1", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "49496d63267bc1a4614ffd5f67c45d9fc3ea62701a6797975bc98bc156d2763f"}, + "earmark": {:hex, :earmark, "1.3.6", "ce1d0675e10a5bb46b007549362bd3f5f08908843957687d8484fe7f37466b19", [:mix], [], "hexpm", "1476378df80982302d5a7857b6a11dd0230865057dec6d16544afecc6bc6b4c2"}, + "ecto": {:hex, :ecto, "3.8.1", "35e0bd8c8eb772e14a5191a538cd079706ecb45164ea08a7523b4fc69ab70f56", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "f1b68f8d5fe3ab89e24f57c03db5b5d0aed3602077972098b3a6006a1be4b69b"}, + "erlex": {:hex, :erlex, "0.2.4", "23791959df45fe8f01f388c6f7eb733cc361668cbeedd801bf491c55a029917b", [:mix], [], "hexpm", "4a12ebc7cd8f24f2d0fce93d279fa34eb5068e0e885bb841d558c4d83c52c439"}, + "ex_doc": {:hex, :ex_doc, "0.21.2", "caca5bc28ed7b3bdc0b662f8afe2bee1eedb5c3cf7b322feeeb7c6ebbde089d6", [:mix], [{:earmark, "~> 1.3.3 or ~> 1.4", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm", "f1155337ae17ff7a1255217b4c1ceefcd1860b7ceb1a1874031e7a861b052e39"}, + "jason": {:hex, :jason, "1.3.0", "fa6b82a934feb176263ad2df0dbd91bf633d4a46ebfdffea0c8ae82953714946", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "53fc1f51255390e0ec7e50f9cb41e751c260d065dcba2bf0d08dc51a4002c2ac"}, + "makeup": {:hex, :makeup, "1.0.0", "671df94cf5a594b739ce03b0d0316aa64312cee2574b6a44becb83cd90fb05dc", [:mix], [{:nimble_parsec, "~> 0.5.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "a10c6eb62cca416019663129699769f0c2ccf39428b3bb3c0cb38c718a0c186d"}, + "makeup_elixir": {:hex, :makeup_elixir, "0.14.0", "cf8b7c66ad1cff4c14679698d532f0b5d45a3968ffbcbfd590339cb57742f1ae", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "d4b316c7222a85bbaa2fd7c6e90e37e953257ad196dc229505137c5e505e9eff"}, + "nimble_parsec": {:hex, :nimble_parsec, "0.5.1", "c90796ecee0289dbb5ad16d3ad06f957b0cd1199769641c961cfe0b97db190e0", [:mix], [], "hexpm", "00e3ebdc821fb3a36957320d49e8f4bfa310d73ea31c90e5f925dc75e030da8f"}, + "telemetry": {:hex, :telemetry, "1.1.0", "a589817034a27eab11144ad24d5c0f9fab1f58173274b1e9bae7074af9cbee51", [:rebar3], [], "hexpm", "b727b2a1f75614774cff2d7565b64d0dfa5bd52ba517f16543e6fc7efcc0df48"}, } diff --git a/priv/northwind/employees.json b/priv/northwind/employees.json index c7bc00d..18f93bd 100644 --- a/priv/northwind/employees.json +++ b/priv/northwind/employees.json @@ -25,7 +25,8 @@ 1581 ], "title": "Vice President Sales", - "titleOfCourtesy": "Dr." + "titleOfCourtesy": "Dr.", + "metadata": {"twitter": "@andrew_fuller"} }, { "address": { diff --git a/test/northwind/repo_test.exs b/test/northwind/repo_test.exs index 721be39..f72bd13 100644 --- a/test/northwind/repo_test.exs +++ b/test/northwind/repo_test.exs @@ -139,4 +139,10 @@ defmodule Northwind.RepoTest do |> Repo.all() |> Repo.preload(shipper: :orders) end + + test "Support json_extract_path expression" do + from(e in Model.Employee) + |> where([e], e.metadata["twitter"] == "@andrew_fuller") + |> Repo.one!() + end end diff --git a/test/support/northwind/model/employee.ex b/test/support/northwind/model/employee.ex index fda0f2e..d05b51b 100644 --- a/test/support/northwind/model/employee.ex +++ b/test/support/northwind/model/employee.ex @@ -13,6 +13,7 @@ defmodule Northwind.Model.Employee do field :hire_date, :date field :notes, :string field :territory_ids, {:array, :integer} + field :metadata, :map, default: %{} embeds_one :address, Model.Address From 194aac8c9ee3cbea85b275ac2d67eb792e179fdf Mon Sep 17 00:00:00 2001 From: atanda rasheed Date: Thu, 5 May 2022 12:45:17 +0100 Subject: [PATCH 2/4] fix: don't stringify booleans --- lib/etso/adapter/types/map.ex | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/etso/adapter/types/map.ex b/lib/etso/adapter/types/map.ex index b52ef91..bb2a95e 100644 --- a/lib/etso/adapter/types/map.ex +++ b/lib/etso/adapter/types/map.ex @@ -39,5 +39,7 @@ defmodule Etso.Ecto.MapType do Enum.map(value, &map_keys_to_string/1) end + defp map_keys_to_string(value) when is_boolean(value), do: value + defp map_keys_to_string(value) when is_atom(value), do: to_string(value) defp map_keys_to_string(value), do: value end From eb26a570b477734bdf7d1c75863afc3567f3caca Mon Sep 17 00:00:00 2001 From: atanda rasheed Date: Thu, 5 May 2022 13:00:07 +0100 Subject: [PATCH 3/4] fix: return nil as nil --- lib/etso/adapter/types/map.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/etso/adapter/types/map.ex b/lib/etso/adapter/types/map.ex index bb2a95e..42679a6 100644 --- a/lib/etso/adapter/types/map.ex +++ b/lib/etso/adapter/types/map.ex @@ -39,6 +39,7 @@ defmodule Etso.Ecto.MapType do Enum.map(value, &map_keys_to_string/1) end + defp map_keys_to_string(nil), do: nil defp map_keys_to_string(value) when is_boolean(value), do: value defp map_keys_to_string(value) when is_atom(value), do: to_string(value) defp map_keys_to_string(value), do: value From 8c58b1a76027cdc1fefab2c698a5a81ea8041a81 Mon Sep 17 00:00:00 2001 From: atanda rasheed Date: Tue, 28 Jun 2022 16:00:26 +0100 Subject: [PATCH 4/4] Support JSON extract path expression --- lib/etso/adapter.ex | 15 +--- lib/etso/adapter/behaviour/queryable.ex | 46 +++++++++-- lib/etso/adapter/types/map.ex | 46 ----------- lib/etso/ets/match_specification.ex | 100 +++++++++++++++-------- mix.lock | 2 +- priv/northwind/employees.json | 19 ++++- test/northwind/repo_test.exs | 104 ++++++++++++++++++++++-- 7 files changed, 220 insertions(+), 112 deletions(-) delete mode 100644 lib/etso/adapter/types/map.ex diff --git a/lib/etso/adapter.ex b/lib/etso/adapter.ex index 39667d3..dd6e12c 100644 --- a/lib/etso/adapter.ex +++ b/lib/etso/adapter.ex @@ -13,8 +13,6 @@ defmodule Etso.Adapter do end """ - alias Etso.Adapter.TableRegistry - @behaviour Ecto.Adapter @behaviour Ecto.Adapter.Schema @behaviour Ecto.Adapter.Queryable @@ -32,7 +30,7 @@ defmodule Etso.Adapter do {:ok, repo} = Keyword.fetch(config, :repo) child_spec = __MODULE__.Supervisor.child_spec(repo) adapter_meta = %__MODULE__.Meta{repo: repo} - {:ok, child_spec, Map.from_struct(adapter_meta)} + {:ok, child_spec, adapter_meta} end @doc false @@ -53,19 +51,8 @@ defmodule Etso.Adapter do @impl Ecto.Adapter def dumpers(:binary_id, type), do: [type, Ecto.UUID] def dumpers(:embed_id, type), do: [type, Ecto.UUID] - def dumpers(:map, type), do: [type, Etso.Ecto.MapType] def dumpers(_, type), do: [type] - @doc """ - Delete all data from the tables. - """ - @spec flush_tables(module()) :: :ok - def flush_tables(repo) do - repo - |> TableRegistry.active_tables() - |> Enum.each(& :ets.delete_all_objects(&1) == true) - end - for {implementation_module, behaviour_module} <- [ {__MODULE__.Behaviour.Schema, Ecto.Adapter.Schema}, {__MODULE__.Behaviour.Queryable, Ecto.Adapter.Queryable} diff --git a/lib/etso/adapter/behaviour/queryable.ex b/lib/etso/adapter/behaviour/queryable.ex index 03fd488..800cb81 100644 --- a/lib/etso/adapter/behaviour/queryable.ex +++ b/lib/etso/adapter/behaviour/queryable.ex @@ -7,21 +7,55 @@ defmodule Etso.Adapter.Behaviour.Queryable do alias Etso.ETS.ObjectsSorter @impl Ecto.Adapter.Queryable - def prepare(:all, query) do - {:nocache, query} + def prepare(:all, %Ecto.Query{} = query) do + {:nocache, {:select, query}} end @impl Ecto.Adapter.Queryable - def execute(%{repo: repo}, _, {:nocache, query}, params, _) do + def prepare(:delete_all, %Ecto.Query{wheres: []} = query) do + {:nocache, {:delete_all_objects, query}} + end + + @impl Ecto.Adapter.Queryable + def prepare(:delete_all, %Ecto.Query{wheres: _} = query) do + {:nocache, {:match_delete, query}} + end + + @impl Ecto.Adapter.Queryable + def execute(%{repo: repo}, _, {:nocache, {:select, query}}, params, _) do + {_, schema} = query.from.source + {:ok, ets_table} = TableRegistry.get_table(repo, schema) + ets_match = MatchSpecification.build(query, params) + ets_objects = :ets.select(ets_table, [ets_match]) + ets_count = length(ets_objects) + {ets_count, ObjectsSorter.sort(ets_objects, query)} + end + + @impl Ecto.Adapter.Queryable + def execute(%{repo: repo}, _, {:nocache, {:delete_all_objects, query}}, params, _) do + {_, schema} = query.from.source + {:ok, ets_table} = TableRegistry.get_table(repo, schema) + ets_match = MatchSpecification.build(query, params) + ets_objects = query.select && ObjectsSorter.sort(:ets.select(ets_table, [ets_match]), query) + ets_count = :ets.info(ets_table, :size) + true = :ets.delete_all_objects(ets_table) + {ets_count, ets_objects || nil} + end + + @impl Ecto.Adapter.Queryable + def execute(%{repo: repo}, _, {:nocache, {:match_delete, query}}, params, _) do {_, schema} = query.from.source {:ok, ets_table} = TableRegistry.get_table(repo, schema) ets_match = MatchSpecification.build(query, params) - ets_objects = :ets.select(ets_table, [ets_match]) |> ObjectsSorter.sort(query) - {length(ets_objects), ets_objects} + ets_objects = query.select && ObjectsSorter.sort(:ets.select(ets_table, [ets_match]), query) + {ets_match_head, ets_match_body, _} = ets_match + ets_match = {ets_match_head, ets_match_body, [true]} + ets_count = :ets.select_delete(ets_table, [ets_match]) + {ets_count, ets_objects || nil} end @impl Ecto.Adapter.Queryable - def stream(%{repo: repo}, _, {:nocache, query}, params, options) do + def stream(%{repo: repo}, _, {:nocache, {:select, query}}, params, options) do {_, schema} = query.from.source {:ok, ets_table} = TableRegistry.get_table(repo, schema) ets_match = MatchSpecification.build(query, params) diff --git a/lib/etso/adapter/types/map.ex b/lib/etso/adapter/types/map.ex deleted file mode 100644 index 42679a6..0000000 --- a/lib/etso/adapter/types/map.ex +++ /dev/null @@ -1,46 +0,0 @@ -defmodule Etso.Ecto.MapType do - use Ecto.Type - def type, do: :map - - def cast(map) when is_map(map) do - {:ok, map} - end - - # Everything else is a failure though - def cast(_), do: :error - - # When loading data from the database, as long as it's a map, - # we just put the data back into a URI struct to be stored in - # the loaded schema struct. - def load(data) when is_map(data) do - {:ok, data} - end - - # When dumping data to the database, we *expect* a URI struct - # but any value could be inserted into the schema struct at runtime, - # so we need to guard against them. - def dump(%{} = map), do: {:ok, map_keys_to_string(map)} - def dump(_), do: :error - - defp map_keys_to_string(value) when is_map(value) and not is_struct(value) do - value - |> Enum.map(fn {key, value} -> {to_string(key), map_keys_to_string(value)} end) - |> Enum.into(%{}) - end - - defp map_keys_to_string(%{__struct__: module} = value) - when module not in [Date, DateTime, Decimal, NaiveDateTime, Time] do - value - |> Map.from_struct() - |> map_keys_to_string() - end - - defp map_keys_to_string(value) when is_list(value) do - Enum.map(value, &map_keys_to_string/1) - end - - defp map_keys_to_string(nil), do: nil - defp map_keys_to_string(value) when is_boolean(value), do: value - defp map_keys_to_string(value) when is_atom(value), do: to_string(value) - defp map_keys_to_string(value), do: value -end diff --git a/lib/etso/ets/match_specification.ex b/lib/etso/ets/match_specification.ex index 5d08c0f..d5a42da 100644 --- a/lib/etso/ets/match_specification.ex +++ b/lib/etso/ets/match_specification.ex @@ -1,16 +1,22 @@ defmodule Etso.ETS.MatchSpecification do @moduledoc """ The ETS Match Specifications module contains various functions which convert Ecto queries to - ETS Match Specifications in order to execute the given queries. + [ETS Match Specifications](https://www.erlang.org/doc/apps/erts/match_spec.html) in order to + execute the given queries with ETS with as much pushed down to ETS as possible. + + The basic shape of the match head is `[$1, $2, $3, …]` where each field is a named variable, the + ordering of the fields is determined by `Etso.ETS.TableStructure`. + + Conditions are compiled according to the wheres in the underlying Ecto query, while the body is + compiled based on the selected fields in the underlying Ecto query. """ def build(query, params) do {_, schema} = query.from.source field_names = Etso.ETS.TableStructure.field_names(schema) - match_head = build_head(field_names) - match_conditions = build_conditions(field_names, params, query.wheres) - match_body = [build_body(field_names, query.select.fields)] + match_conditions = build_conditions(field_names, params, query) + match_body = [build_body(field_names, query)] {match_head, match_conditions, match_body} end @@ -18,8 +24,8 @@ defmodule Etso.ETS.MatchSpecification do List.to_tuple(Enum.map(1..length(field_names), fn x -> :"$#{x}" end)) end - defp build_conditions(field_names, params, query_wheres) do - Enum.reduce(query_wheres, [], fn %Ecto.Query.BooleanExpr{expr: expression}, acc -> + defp build_conditions(field_names, params, %Ecto.Query{wheres: wheres}) do + Enum.reduce(wheres, [], fn %Ecto.Query.BooleanExpr{expr: expression}, acc -> [build_condition(field_names, params, expression) | acc] end) end @@ -45,13 +51,12 @@ defmodule Etso.ETS.MatchSpecification do end end - defp build_condition(field_names, params, {:in, [], [field, value]}) do - field_name = resolve_field_name(field) - field_index = get_field_index(field_names, field_name) + defp build_condition(field_names, params, {:in, [], [field, values]}) do + field_target = resolve_field_target(field_names, field) - case resolve_field_values(params, value) do + case resolve_param_values(params, values) do [] -> [] - values -> List.to_tuple([:orelse | Enum.map(values, &{:==, :"$#{field_index}", &1})]) + values -> List.to_tuple([:orelse | Enum.map(values, &{:==, field_target, &1})]) end end @@ -59,51 +64,74 @@ defmodule Etso.ETS.MatchSpecification do {:==, build_condition(field_names, params, field), nil} end - defp build_condition(field_names, _, {:json_extract_path, [], paths}) do - [parent, fields] = paths - [field | remaining] = fields - acc = {:map_get, field, build_condition(field_names, nil, parent)} - Enum.reduce(remaining, acc, fn field, acc -> - {:map_get, field, acc} - end) + defp build_condition(_, params, {:^, [], [index]}) do + Enum.at(params, index) end - defp build_condition(field_names, _, {{:., [], [{:&, [], [0]}, field_name]}, [], []}) do - :"$#{get_field_index(field_names, field_name)}" + defp build_condition(field_names, _params, field) when is_tuple(field) do + resolve_field_target(field_names, field) end - defp build_condition(_, params, {:^, [], [index]}) do - Enum.at(params, index) + defp build_condition(_, _, value) do + value end - defp build_condition(_, _, value) when not is_tuple(value) do - value + defp build_body(_, %Ecto.Query{select: nil}) do + [] end - defp build_body(field_names, query_select_fields) do - for select_field <- query_select_fields do - field_name = resolve_field_name(select_field) - field_index = get_field_index(field_names, field_name) - :"$#{field_index}" + defp build_body(field_names, %Ecto.Query{select: %{fields: fields}}) do + for field <- fields do + resolve_field_target(field_names, field) end end - defp resolve_field_name(field) do - {{:., _, [{:&, [], [0]}, field_name]}, [], []} = field - field_name + defp resolve_field_target(field_names, {:json_extract_path, [], [field, path]}) do + field_target = resolve_field_target(field_names, field) + resolve_field_target_path(field_target, path) end - defp resolve_field_values(params, {:^, [], [index, count]}) do + defp resolve_field_target(field_names, {{:., _, [{:&, [], [0]}, field_name]}, [], []}) do + field_index = 1 + Enum.find_index(field_names, fn x -> x == field_name end) + :"$#{field_index}" + end + + defp resolve_field_target_path(field_target, path) do + # - If the path component is a key, return {:map_get, key, target} + # - If the path component is a number, return {:hd, target} outside as many {:tl, _} around + # as required. For example, [:metadata, 0] would be {:hd, {:map_get, :metadata, field}}, + # while [:metadata, 1] would be {:hd, {:tl, {:map_get, :metadata, field}}} (with one tl). + + at = fn self -> + fn + condition, 0 -> {:hd, condition} + condition, index -> self.(self).({:tl, condition}, index - 1) + end + end + + Enum.reduce(path, field_target, fn + key, condition when is_atom(key) or is_binary(key) -> {:map_get, key, condition} + index, condition when is_integer(index) -> at.(at).(condition, index) + end) + end + + defp resolve_param_values(params, {:^, [], [index, count]}) do for index <- index..(index + count - 1) do Enum.at(params, index) end end - defp resolve_field_values(params, {:^, [], [index]}) do + defp resolve_param_values(params, {:^, [], [index]}) do Enum.at(params, index) end - defp get_field_index(field_names, field_name) do - 1 + Enum.find_index(field_names, fn x -> x == field_name end) + defp resolve_param_values(_params, values) when is_list(values) do + values + end + + def list_spec(0, acc), do: {:hd, acc} + + def list_spec(index, acc) do + list_spec(index - 1, {:tl, acc}) end end diff --git a/mix.lock b/mix.lock index 331350b..d35839f 100644 --- a/mix.lock +++ b/mix.lock @@ -2,7 +2,7 @@ "decimal": {:hex, :decimal, "2.0.0", "a78296e617b0f5dd4c6caf57c714431347912ffb1d0842e998e9792b5642d697", [:mix], [], "hexpm", "34666e9c55dea81013e77d9d87370fe6cb6291d1ef32f46a1600230b1d44f577"}, "dialyxir": {:hex, :dialyxir, "1.0.0-rc.6", "78e97d9c0ff1b5521dd68041193891aebebce52fc3b93463c0a6806874557d7d", [:mix], [{:erlex, "~> 0.2.1", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "49496d63267bc1a4614ffd5f67c45d9fc3ea62701a6797975bc98bc156d2763f"}, "earmark": {:hex, :earmark, "1.3.6", "ce1d0675e10a5bb46b007549362bd3f5f08908843957687d8484fe7f37466b19", [:mix], [], "hexpm", "1476378df80982302d5a7857b6a11dd0230865057dec6d16544afecc6bc6b4c2"}, - "ecto": {:hex, :ecto, "3.8.3", "5e681d35bc2cbb46dcca1e2675837c7d666316e5ada14eca6c9c609b6232817c", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "af92dd7815967bcaea0daaaccf31c3b23165432b1c7a475d84144efbc703d105"}, + "ecto": {:hex, :ecto, "3.8.4", "e06b8b87e62b27fea17fd2ff6041572ddd10339fd16cdf58446e402c6c90a74b", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "f9244288b8d42db40515463a008cf3f4e0e564bb9c249fe87bf28a6d79fe82d4"}, "erlex": {:hex, :erlex, "0.2.4", "23791959df45fe8f01f388c6f7eb733cc361668cbeedd801bf491c55a029917b", [:mix], [], "hexpm", "4a12ebc7cd8f24f2d0fce93d279fa34eb5068e0e885bb841d558c4d83c52c439"}, "ex_doc": {:hex, :ex_doc, "0.21.2", "caca5bc28ed7b3bdc0b662f8afe2bee1eedb5c3cf7b322feeeb7c6ebbde089d6", [:mix], [{:earmark, "~> 1.3.3 or ~> 1.4", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm", "f1155337ae17ff7a1255217b4c1ceefcd1860b7ceb1a1874031e7a861b052e39"}, "jason": {:hex, :jason, "1.3.0", "fa6b82a934feb176263ad2df0dbd91bf633d4a46ebfdffea0c8ae82953714946", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "53fc1f51255390e0ec7e50f9cb41e751c260d065dcba2bf0d08dc51a4002c2ac"}, diff --git a/priv/northwind/employees.json b/priv/northwind/employees.json index 18f93bd..c6400f9 100644 --- a/priv/northwind/employees.json +++ b/priv/northwind/employees.json @@ -26,7 +26,22 @@ ], "title": "Vice President Sales", "titleOfCourtesy": "Dr.", - "metadata": {"twitter": "@andrew_fuller"} + "metadata": { + "twitter": "@andrew_fuller", + "photos": [ + { + "storage": "a", + "url": "https://example.com/a" + }, + { + "storage": "b", + "url": "https://example.com/b" + } + ], + "documents": { + "passport": "verified" + } + } }, { "address": { @@ -238,4 +253,4 @@ "title": "Sales Representative", "titleOfCourtesy": "Ms." } -] +] \ No newline at end of file diff --git a/test/northwind/repo_test.exs b/test/northwind/repo_test.exs index 1a93db0..1a23291 100644 --- a/test/northwind/repo_test.exs +++ b/test/northwind/repo_test.exs @@ -10,7 +10,7 @@ defmodule Northwind.RepoTest do :ok = Importer.perform() end - test "list" do + test "List All" do Repo.all(Model.Employee) end @@ -148,12 +148,6 @@ defmodule Northwind.RepoTest do |> Repo.preload(shipper: :orders) end - test "Support json_extract_path expression" do - from(e in Model.Employee) - |> where([e], e.metadata["twitter"] == "@andrew_fuller") - |> Repo.one!() - end - test "Order / Shipper + Employee Preloading" do Model.Order |> Repo.all() @@ -180,4 +174,100 @@ defmodule Northwind.RepoTest do assert sorted_etso == sorted_code end + + test "Delete All" do + assert Repo.delete_all(Model.Employee) + assert [] == Repo.all(Model.Employee) + end + + test "Delete Where" do + query = Model.Employee |> where([e], e.employee_id in [1, 5]) + assert [a, b] = Repo.all(query) + assert {2, nil} = Repo.delete_all(query) + assert [] == Repo.all(query) + refute [] == Repo.all(Model.Employee) + end + + test "Delete Where Select" do + query = Model.Employee |> where([e], e.employee_id in [1, 5]) + assert [a, b] = Repo.all(query) + assert {2, list} = Repo.delete_all(query |> select([e], {e, e.employee_id})) + assert is_list(list) + assert Enum.any?(list, &(elem(&1, 1) == 1)) + assert Enum.any?(list, &(elem(&1, 1) == 5)) + assert [] = Repo.all(query) + refute [] == Repo.all(Model.Employee) + end + + describe "With JSON Extract Paths" do + test "using literal value" do + Model.Employee + |> where([e], e.metadata["twitter"] == "@andrew_fuller") + |> Repo.one!() + end + + test "using brackets" do + Model.Employee + |> where([e], e.metadata["documents"]["passport"] == "verified") + |> Repo.one!() + end + + test "with variable pinning" do + field = "passport" + + Model.Employee + |> where([e], e.metadata["documents"][^field] == "verified") + |> Repo.one!() + + Model.Employee + |> select([e], json_extract_path(e.metadata, ["documents", "passport"])) + |> Repo.all() + |> Enum.any?(&(&1 == "verified")) + |> assert() + end + + test "with arrays" do + Model.Employee + |> select([e], json_extract_path(e.metadata, ["photos", 0, "url"])) + |> where([e], e.metadata["documents"]["passport"] == "verified") + |> Repo.one!() + |> (&(&1 == "https://example.com/a")).() + |> assert() + + Model.Employee + |> where([e], e.metadata["documents"]["passport"] == "verified") + |> select([e], e.metadata["photos"][0]["url"]) + |> Repo.one!() + |> (&(&1 == "https://example.com/a")).() + |> assert() + + Model.Employee + |> select([e], e.metadata["photos"][1]["url"]) + |> where([e], e.metadata["documents"]["passport"] == "verified") + |> Repo.one!() + |> (&(&1 == "https://example.com/b")).() + |> assert() + end + + test "with where/in" do + Model.Employee + |> where([e], e.metadata["documents"]["passport"] in ~w(verified)) + |> select([e], e.metadata["photos"][1]["url"]) + |> Repo.one!() + |> (&(&1 == "https://example.com/b")).() + |> assert() + end + + test "in deletion" do + Model.Employee + |> where([e], e.metadata["documents"]["passport"] == "verified") + |> Repo.delete_all() + + assert_raise Ecto.NoResultsError, fn -> + Model.Employee + |> where([e], e.metadata["documents"]["passport"] == "verified") + |> Repo.one!() + end + end + end end