Collect metrics through datadog for the perf project (#1765)

.circleci/config.yml

@@ -663,6 +663,7 @@ jobs:
       image: ubuntu-1604:201903-01
     environment:
       PERF_IMAGE_NAME: "omisego/perf:latest"
+      STATIX_TAG: "env:perf_circleci"
     steps:
       - checkout
       - run:
@@ -674,6 +675,7 @@ jobs:
       - run:
           name: Build perf docker image
           command: make docker-perf IMAGE_NAME=$PERF_IMAGE_NAME
+      - install_elixir
       - run:
           name: Start daemon services
           command: |
@@ -685,7 +687,6 @@ jobs:
           command: |
             cd priv/perf
             make log-services
-      - install_elixir
       - run: sh .circleci/status.sh
       - run:
           name: Run load test
@@ -698,10 +699,13 @@ jobs:
           name: Show help information
           command: docker run -it $PERF_IMAGE_NAME mix run -e "LoadTest.TestRunner.run()" -- help
       - run:
-          name: Run perf smoke test
+          name: Run perf smoke test (deposits)
+          command: |
+            docker run -it --env-file ./localchain_contract_addresses.env --env DD_API_KEY --env DD_APP_KEY --env STATIX_TAG --network host $PERF_IMAGE_NAME mix run -e "LoadTest.TestRunner.run()" -- "deposits" 1 200
+      - run:
+          name: Run perf smoke test (transactions)
           command: |
-            export $(cat ./localchain_contract_addresses.env | xargs) &&
-            docker run -it --env-file ./localchain_contract_addresses.env --network host $PERF_IMAGE_NAME mix run -e "LoadTest.TestRunner.run()" -- "deposits" 10 10
+            docker run -it --env-file ./localchain_contract_addresses.env --env DD_API_KEY --env DD_APP_KEY --env STATIX_TAG --network host $PERF_IMAGE_NAME mix run -e "LoadTest.TestRunner.run()" -- "transactions" 1 200
       - run:
           name: (Perf) Format generated code and check for warnings
           command: |

Makefile

@@ -412,10 +412,10 @@ docker-start-cluster-with-infura: localchain_contract_addresses.env
 	fi
 
 docker-start-cluster-with-datadog: localchain_contract_addresses.env
-	docker-compose -f docker-compose.yml -f docker-compose.dev.yml up watcher watcher_info childchain
+	docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.datadog.yml up watcher watcher_info childchain
 
 docker-stop-cluster-with-datadog: localchain_contract_addresses.env
-	docker-compose -f docker-compose.yml -f docker-compose.dev.yml down
+	docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.datadog.yml down
 
 docker-nuke: localchain_contract_addresses.env
 	docker-compose down --remove-orphans --volumes

README.md

@@ -86,7 +86,7 @@ Docker building of source code and dependencies used to directly use common `mix
 You can setup the docker environment to run testing and development tasks:
 
 ```sh
-docker-compose -f docker-compose.yml -f docker-compose.dev.yml run --rm --entrypoint bash elixir-omg
+docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.datadog.yml run --rm --entrypoint bash elixir-omg
 ```
 
 Once the shell has loaded, you can continue and run additional tasks.

docker-compose.datadog.yml

@@ -0,0 +1,20 @@
+version: "2.3"
+services:
+  datadog:
+    image: datadog/agent:latest
+    restart: always
+    environment:
+      - DD_API_KEY=${DD_API_KEY}
+      - DD_DOGSTATSD_NON_LOCAL_TRAFFIC=true
+      - DD_LOG_LEVEL=debug
+      - DOCKER_CONTENT_TRUST=1
+      - DD_APM_ENABLED=true
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - /proc/:/host/proc/:ro
+      - /sys/fs/cgroup:/host/sys/fs/cgroup:ro
+    ports:
+      - "2003-2004:2003-2004"
+      - "2023-2024:2023-2024"
+      - "8125:8125/udp"
+      - "8126:8126/tcp"

docker-compose.dev.yml

@@ -24,23 +24,3 @@ services:
     depends_on:
       datadog:
         condition: service_healthy
-  datadog:
-    image: datadog/agent:latest
-    restart: always
-    environment:
-      - DD_API_KEY=${DD_API_KEY}
-      - DD_DOGSTATSD_NON_LOCAL_TRAFFIC=true
-      - DD_DOGSTATSD_TAGS=["env:local_development"]
-      #- DD_LOG_LEVEL=debug
-      - DOCKER_CONTENT_TRUST=1
-      - DD_APM_ENABLED=true
-    volumes:
-      - /var/run/docker.sock:/var/run/docker.sock
-      - /proc/:/host/proc/:ro
-      - /sys/fs/cgroup:/host/sys/fs/cgroup:ro
-    ports:
-      - "80:80"
-      - "2003-2004:2003-2004"
-      - "2023-2024:2023-2024"
-      - "8125:8125/udp"
-      - "8126:8126/tcp"

docker-compose.feefeed.yml

@@ -38,10 +38,10 @@ services:
     restart: always
     healthcheck:
       test: curl -v --silent http://localhost:4000/api/v1/fees 2>&1 | grep contract_address
-      interval: 30s
-      timeout: 1s
-      retries: 5
-      start_period: 30s
+      interval: 4s
+      timeout: 2s
+      retries: 30
+      start_period: 60s
     networks:
       chain_net:
         ipv4_address: 172.27.0.110

priv/perf/Makefile

@@ -1,6 +1,6 @@
 .PHONY: list
 
-COMPOSE_FULL_SERVICES=-f ../../docker-compose.yml -f ../../docker-compose.feefeed.yml
+COMPOSE_FULL_SERVICES=-f ../../docker-compose.yml -f ../../docker-compose.feefeed.yml -f ../../docker-compose.datadog.yml
 
 list:
 	@$(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$'

priv/perf/apps/load_test/lib/application.ex

@@ -4,24 +4,32 @@ defmodule LoadTest.Application do
   """
   use Application
 
+  alias LoadTest.Connection.ConnectionDefaults
+  alias LoadTest.Ethereum.NonceTracker
+  alias LoadTest.Service.Datadog
+  alias LoadTest.Service.Faucet
+
   def start(_type, _args) do
-    pool_size = Application.fetch_env!(:load_test, :pool_size)
-    max_connections = Application.fetch_env!(:load_test, :max_connection)
+    :ok = start_hackney_pool()
+
+    NonceTracker.init()
 
-    :ok =
-      :hackney_pool.start_pool(
-        LoadTest.Connection.ConnectionDefaults.pool_name(),
-        timeout: 180_000,
-        connect_timeout: 30_000,
-        pool_size: pool_size,
-        max_connections: max_connections
-      )
+    children = [{Faucet, fetch_faucet_config()}, {Datadog, []}]
 
-    LoadTest.Ethereum.NonceTracker.init()
+    Supervisor.start_link(children, strategy: :one_for_one, restart: :temporary)
+  end
+
+  defp start_hackney_pool() do
+    pool_size = Application.fetch_env!(:load_test, :pool_size)
+    max_connections = Application.fetch_env!(:load_test, :max_connection)
 
-    faucet_config = fetch_faucet_config()
-    # using temporary strategy as it creates and funds a new Ethereum account on each start
-    Supervisor.start_link([{LoadTest.Service.Faucet, faucet_config}], strategy: :one_for_one, restart: :temporary)
+    :hackney_pool.start_pool(
+      ConnectionDefaults.pool_name(),
+      timeout: 180_000,
+      connect_timeout: 30_000,
+      pool_size: pool_size,
+      max_connections: max_connections
+    )
   end
 
   def stop(_app) do

priv/perf/apps/load_test/lib/child_chain/transaction.ex

@@ -23,6 +23,7 @@ defmodule LoadTest.ChildChain.Transaction do
   alias ExPlasma.Transaction
   alias ExPlasma.Utxo
   alias LoadTest.Connection.ChildChain, as: Connection
+  alias LoadTest.Service.Metrics
   alias LoadTest.Service.Sync
 
   # safe, reasonable amount, equal to the testnet block gas limit
@@ -212,6 +213,15 @@ defmodule LoadTest.ChildChain.Transaction do
   end
 
   defp do_submit_tx(tx) do
+    Metrics.run_with_metrics(
+      fn ->
+        submit_request(tx)
+      end,
+      "Childchain.submit"
+    )
+  end
+
+  defp submit_request(tx) do
     {:ok, response} =
       tx
       |> Transaction.encode()

priv/perf/apps/load_test/lib/runner/utxos.ex → priv/perf/apps/load_test/lib/runner/transactions.ex

@@ -12,15 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-defmodule LoadTest.Runner.Utxos do
+defmodule LoadTest.Runner.Transactions do
   @moduledoc """
-  Utxos tests runner.
+  Transactions tests runner.
   """
   use Chaperon.LoadTest
 
   def scenarios do
     [
-      {{1, LoadTest.Scenario.Utxos}, %{}}
+      {{1, LoadTest.Scenario.Transactions}, %{}}
     ]
   end
 end

priv/perf/apps/load_test/lib/scenario/deposits.ex

@@ -32,6 +32,7 @@ defmodule LoadTest.Scenario.Deposits do
   alias LoadTest.Ethereum
   alias LoadTest.Ethereum.Account
   alias LoadTest.Service.Faucet
+  alias LoadTest.Service.Metrics
   alias LoadTest.WatcherInfo.Balance
   alias LoadTest.WatcherInfo.Transaction
 
@@ -53,17 +54,24 @@ defmodule LoadTest.Scenario.Deposits do
   end
 
   def create_deposit_and_make_assertions(session) do
+    {_, session} =
+      Metrics.run_with_metrics(
+        fn ->
+          do_create_deposit_and_make_assertions(session)
+        end,
+        "deposits_test"
+      )
+
+    session
+  end
+
+  defp do_create_deposit_and_make_assertions(session) do
     with {:ok, from, to} <- create_accounts(session),
          :ok <- create_deposit(from, session),
          :ok <- send_value_to_receiver(from, to, session) do
-      Session.add_metric(session, "error_rate", 0)
+      {:ok, session}
     else
-      error ->
-        log_error(session, "#{__MODULE__} failed with #{inspect(error)}")
-
-        session
-        |> Session.add_metric("error_rate", 1)
-        |> Session.add_error(:error, error)
+      _error -> {:error, session}
     end
   end
 

priv/perf/apps/load_test/lib/scenario/utxos.ex → priv/perf/apps/load_test/lib/scenario/transactions.ex

@@ -12,9 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-defmodule LoadTest.Scenario.Utxos do
+defmodule LoadTest.Scenario.Transactions do
   @moduledoc """
-  The scenario for utxos tests:
+  The scenario for transactions tests:
 
   1. It creates two accounts: the sender and the receiver.
   2. It funds sender with the specified amount on the childchain, checks utxos and balance.
@@ -30,6 +30,7 @@ defmodule LoadTest.Scenario.Utxos do
   alias LoadTest.ChildChain.Transaction
   alias LoadTest.Ethereum.Account
   alias LoadTest.Service.Faucet
+  alias LoadTest.Service.Metrics
   alias LoadTest.WatcherInfo.Balance
   alias LoadTest.WatcherInfo.Utxo
 
@@ -51,17 +52,24 @@ defmodule LoadTest.Scenario.Utxos do
   end
 
   def create_utxos_and_make_assertions(session) do
+    {_, session} =
+      Metrics.run_with_metrics(
+        fn ->
+          do_create_utxos_and_make_assertions(session)
+        end,
+        "transactions_test"
+      )
+
+    session
+  end
+
+  defp do_create_utxos_and_make_assertions(session) do
     with {:ok, sender, receiver} <- create_accounts(),
          {:ok, utxo} <- fund_account(session, sender),
          :ok <- spend_utxo(session, utxo, sender, receiver) do
-      Session.add_metric(session, "error_rate", 0)
+      {:ok, session}
     else
-      error ->
-        log_error(session, "#{__MODULE__} failed with #{inspect(error)}")
-
-        session
-        |> Session.add_metric("error_rate", 1)
-        |> Session.add_error(:error, error)
+      _ -> {:error, session}
     end
   end
 

priv/perf/apps/load_test/lib/service/datadog.ex

@@ -0,0 +1,47 @@
+# Copyright 2019-2020 OmiseGO Pte Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+defmodule LoadTest.Service.Datadog do
+  @moduledoc """
+  Datadog connection wrapper
+  """
+
+  # we want to override Statix
+  # because we don't want to send metrics in unittests
+  case Application.get_env(:load_test, :record_metrics) do
+    true -> use Statix, runtime_config: true
+    _ -> use LoadTest.Service.Datadog.DummyStatix
+  end
+
+  use GenServer
+  require Logger
+
+  def start_link(_params), do: GenServer.start_link(__MODULE__, [], [])
+
+  def init(_opts) do
+    _ = Process.flag(:trap_exit, true)
+    _ = Logger.info("Starting #{inspect(__MODULE__)} and connecting to Datadog.")
+
+    :ok = __MODULE__.connect()
+
+    _ = Logger.info("Datadog Connection for Statix was opened")
+
+    {:ok, []}
+  end
+
+  def handle_info({:EXIT, port, reason}, %Statix.Conn{sock: __MODULE__} = state) do
+    _ = Logger.error("Port in #{inspect(__MODULE__)} #{inspect(port)} exited with reason #{reason}")
+    {:stop, :normal, state}
+  end
+end