From 8c632a0c35b18eb14409e289507af726c7d04092 Mon Sep 17 00:00:00 2001 From: John Lotoski Date: Sat, 17 Dec 2022 20:24:48 -0600 Subject: [PATCH 1/4] fix: conn string for updated graphql-engine --- Dockerfile | 2 ++ nix/nixos/cardano-graphql-background-service.nix | 15 ++++----------- nix/nixos/graphql-engine-service.nix | 15 +++++++-------- .../src/HasuraBackgroundClient.ts | 3 ++- packages/api-cardano-db-hasura/src/background.ts | 7 +++++++ 5 files changed, 22 insertions(+), 20 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0159d120..f6c51bfb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -65,8 +65,10 @@ RUN curl --proto '=https' --tlsv1.2 -sSf -L https://www.postgresql.org/media/key apt-get update && apt-get install -y --no-install-recommends \ ca-certificates COPY --from=downloader /usr/local/bin/hasura /usr/local/bin/hasura +COPY --from=downloader /root/.hasura/plugins/bin/hasura-cli_ext /usr/local/bin/hasura-cli_ext ENV \ HASURA_CLI_PATH=/usr/local/bin/hasura \ + HASURA_CLI_EXT_PATH=/usr/local/bin/hasura-cli_ext \ HASURA_URI="http://hasura:8080" \ LD_LIBRARY_PATH="/usr/local/lib:$LD_LIBRARY_PATH" \ METADATA_SERVER_URI=${METADATA_SERVER_URI} \ diff --git a/nix/nixos/cardano-graphql-background-service.nix b/nix/nixos/cardano-graphql-background-service.nix index abda1eba..16757c9d 100644 --- a/nix/nixos/cardano-graphql-background-service.nix +++ b/nix/nixos/cardano-graphql-background-service.nix @@ -84,22 +84,19 @@ in { pluginLibPath = pkgs.lib.makeLibraryPath [ pkgs.stdenv.cc.cc.lib ]; - installHasuraCLI = '' - # always start with no plugins so future upgrades will work - rm -rf ~/.hasura/plugins - mkdir -p ~/.hasura/plugins/store/cli-ext/v${hasura-cli-ext.version} - ln -s ${hasura-cli-ext}/bin/cli-ext-hasura-linux ~/.hasura/plugins/store/cli-ext/v${hasura-cli-ext.version}/cli-ext-hasura-linux - ''; in lib.mkIf cfg.enable { - systemd.services.cardano-graphql = { + systemd.services.cardano-graphql-background = { wantedBy = [ "multi-user.target" ]; wants = [ "graphql-engine.service" ]; after = [ "graphql-engine.service" ]; environment = lib.filterAttrs (k: v: v != null) { HASURA_CLI_PATH = hasura-cli + "/bin/hasura"; + HASURA_CLI_EXT_PATH = hasura-cli-ext + "/bin/cli-ext-hasura-linux"; HASURA_GRAPHQL_ENABLE_TELEMETRY = toString false; HASURA_URI = hasuraBaseUri; LOGGER_MIN_SEVERITY = cfg.loggerMinSeverity; + OGMIOS_HOST = cfg.ogmiosHost; + OGMIOS_PORT = toString cfg.ogmiosPort; POSTGRES_DB = cfg.db; POSTGRES_HOST = cfg.dbHost; POSTGRES_PASSWORD = cfg.dbPassword; @@ -109,10 +106,6 @@ in { (lib.optionalAttrs (cfg.assetMetadataUpdateInterval != null) { ASSET_METADATA_UPDATE_INTERVAL = toString cfg.assetMetadataUpdateInterval; }) // (lib.optionalAttrs (cfg.metadataServerUri != null) { METADATA_SERVER_URI = toString cfg.metadataServerUri; }); path = with pkgs; [ netcat curl postgresql frontend hasura-cli glibc.bin patchelf ]; - preStart = '' - set -exuo pipefail - ${installHasuraCLI} - ''; script = '' exec cardano-graphql-background ''; diff --git a/nix/nixos/graphql-engine-service.nix b/nix/nixos/graphql-engine-service.nix index 5aef7f6b..dddb5184 100644 --- a/nix/nixos/graphql-engine-service.nix +++ b/nix/nixos/graphql-engine-service.nix @@ -9,7 +9,7 @@ in { host = lib.mkOption { type = lib.types.str; - default = "/var/run/postgresql"; + default = ""; }; dbUser = lib.mkOption { @@ -19,7 +19,7 @@ in { password = lib.mkOption { type = lib.types.str; - default = ''""''; + default = ""; }; dbAdminUser = lib.mkOption { @@ -54,7 +54,7 @@ in { GRANT SELECT ON ALL TABLES IN SCHEMA information_schema TO ${cfg.dbUser}; GRANT SELECT ON ALL TABLES IN SCHEMA pg_catalog TO ${cfg.dbUser}; ''; - postgresqlIp = if ((__head (pkgs.lib.stringToCharacters cfg.host)) == "/") + postgresqlIp = if (cfg.host == "" || (__head (pkgs.lib.stringToCharacters cfg.host)) == "/") then "127.0.0.1" else cfg.host; in lib.mkIf cfg.enable { @@ -62,6 +62,10 @@ in { wantedBy = [ "multi-user.target" ]; requires = [ "postgresql.service" ]; path = with pkgs; [ curl netcat postgresql sudo ]; + environment = { + HASURA_GRAPHQL_DATABASE_URL = "postgres://${cfg.dbUser}:${cfg.password}@${cfg.host}${if cfg.host == "" then "" else toString ":${toString cfg.dbPort}"}/${cfg.db}"; + CARDANO_GRAPHQL_DB_URL = "postgres://${cfg.dbUser}:${cfg.password}@${cfg.host}${if cfg.host == "" then "" else ":${toString cfg.dbPort}"}/cgql"; + }; preStart = '' for x in {1..10}; do nc -z ${postgresqlIp} ${toString cfg.dbPort} && break @@ -72,11 +76,6 @@ in { ''; script = '' exec ${graphqlEngine}/bin/graphql-engine \ - --host ${cfg.host} \ - -u ${cfg.dbUser} \ - --password ${cfg.password} \ - -d ${cfg.db} \ - --port ${toString cfg.dbPort} \ serve \ --server-port ${toString cfg.enginePort} \ --enable-telemetry=false \ diff --git a/packages/api-cardano-db-hasura/src/HasuraBackgroundClient.ts b/packages/api-cardano-db-hasura/src/HasuraBackgroundClient.ts index 2b045cb4..d6f61408 100644 --- a/packages/api-cardano-db-hasura/src/HasuraBackgroundClient.ts +++ b/packages/api-cardano-db-hasura/src/HasuraBackgroundClient.ts @@ -21,6 +21,7 @@ export class HasuraBackgroundClient { constructor ( readonly hasuraCliPath: string, + readonly hasuraCliExtPath: string, readonly hasuraUri: string, private logger: Logger = dummyLogger ) { @@ -39,7 +40,7 @@ export class HasuraBackgroundClient { private async hasuraCli (command: string) { return new Promise((resolve, reject) => { exec( - `${this.hasuraCliPath} --skip-update-check --project ${path.resolve(__dirname, '..', 'hasura', 'project')} --endpoint ${this.hasuraUri} ${command}`, + `${this.hasuraCliPath} --cli-ext-path ${this.hasuraCliExtPath} --skip-update-check --project ${path.resolve(__dirname, '..', 'hasura', 'project')} --endpoint ${this.hasuraUri} ${command}`, (error, stdout) => { if (error) { reject(error) diff --git a/packages/api-cardano-db-hasura/src/background.ts b/packages/api-cardano-db-hasura/src/background.ts index 442507d6..85df9222 100644 --- a/packages/api-cardano-db-hasura/src/background.ts +++ b/packages/api-cardano-db-hasura/src/background.ts @@ -17,6 +17,7 @@ export class MissingConfig extends CustomError { export interface BackgroundConfig { db: DbConfig, hasuraCliPath: string, + hasuraCliExtPath: string, hasuraUri: string, loggerMinSeverity: LogLevelString, metadataServerUri: string, @@ -34,6 +35,9 @@ async function getConfig (): Promise { if (!env.hasuraCliPath) { throw new MissingConfig('HASURA_CLI_PATH env not set') } + if (!env.hasuraCliExtPath) { + throw new MissingConfig('HASURA_CLI_EXT_PATH env not set') + } if (!env.hasuraUri) { throw new MissingConfig('HASURA_URI env not set') } @@ -79,6 +83,7 @@ function filterAndTypecastEnvs (env: any) { const { ASSET_METADATA_UPDATE_INTERVAL, HASURA_CLI_PATH, + HASURA_CLI_EXT_PATH, HASURA_URI, LOGGER_MIN_SEVERITY, METADATA_SERVER_URI, @@ -95,6 +100,7 @@ function filterAndTypecastEnvs (env: any) { } = env as NodeJS.ProcessEnv return { hasuraCliPath: HASURA_CLI_PATH, + hasuraCliExtPath: HASURA_CLI_EXT_PATH, hasuraUri: HASURA_URI, loggerMinSeverity: LOGGER_MIN_SEVERITY as LogLevelString, metadataServerUri: METADATA_SERVER_URI, @@ -127,6 +133,7 @@ function filterAndTypecastEnvs (env: any) { try { const hasuraBackgroundClient = new HasuraBackgroundClient( config.hasuraCliPath, + config.hasuraCliExtPath, config.hasuraUri, logger ) From ed105b4b02392a1cf9936f32bd2f04287424c3ec Mon Sep 17 00:00:00 2001 From: John Lotoski Date: Tue, 7 Feb 2023 14:04:24 -0600 Subject: [PATCH 2/4] imp: fixup graphql-engine with package option --- .../cardano-graphql-background-service.nix | 29 +++++++++++++++---- nix/nixos/cardano-graphql-service.nix | 15 ++++++++-- nix/nixos/graphql-engine-service.nix | 8 +++-- 3 files changed, 42 insertions(+), 10 deletions(-) diff --git a/nix/nixos/cardano-graphql-background-service.nix b/nix/nixos/cardano-graphql-background-service.nix index 16757c9d..688b5075 100644 --- a/nix/nixos/cardano-graphql-background-service.nix +++ b/nix/nixos/cardano-graphql-background-service.nix @@ -2,12 +2,31 @@ { lib, pkgs, config, ... }: let cfg = config.services.cardano-graphql-background; - selfPkgs = import ../pkgs.nix {}; in { options = { services.cardano-graphql-background = { enable = lib.mkEnableOption "cardano-explorer graphql background service"; + frontendPkg = lib.mkOption { + type = lib.types.package; + default = (import ../pkgs.nix {}).packages.cardano-graphql; + }; + + persistPkg = lib.mkOption { + type = lib.types.package; + default = (import ../pkgs.nix {}).packages.persistgraphql; + }; + + hasuraCliPkg = lib.mkOption { + type = lib.types.package; + default = (import ../pkgs.nix {}).packages.hasura-cli; + }; + + hasuraCliExtPkg = lib.mkOption { + type = lib.types.package; + default = (import ../pkgs.nix {}).packages.hasura-cli-ext; + }; + assetMetadataUpdateInterval = lib.mkOption { type = lib.types.nullOr lib.types.int; default = null; @@ -76,10 +95,10 @@ in { }; config = let boolToNodeJSEnv = bool: if bool then "true" else "false"; - frontend = selfPkgs.packages.cardano-graphql; - persistgraphql = selfPkgs.packages.persistgraphql; - hasura-cli = selfPkgs.packages.hasura-cli; - hasura-cli-ext = selfPkgs.packages.hasura-cli-ext; + frontend = cfg.frontendPkg; + persistgraphql = cfg.persistPkg; + hasura-cli = cfg.hasuraCliPkg; + hasura-cli-ext = cfg.hasuraCliExtPkg; hasuraBaseUri = "${cfg.hasuraProtocol}://${cfg.hasuraIp}:${toString cfg.enginePort}"; pluginLibPath = pkgs.lib.makeLibraryPath [ pkgs.stdenv.cc.cc.lib diff --git a/nix/nixos/cardano-graphql-service.nix b/nix/nixos/cardano-graphql-service.nix index ec85b310..c1563ac5 100644 --- a/nix/nixos/cardano-graphql-service.nix +++ b/nix/nixos/cardano-graphql-service.nix @@ -2,12 +2,21 @@ { lib, pkgs, config, ... }: let cfg = config.services.cardano-graphql; - selfPkgs = import ../pkgs.nix {}; in { options = { services.cardano-graphql = { enable = lib.mkEnableOption "cardano-explorer graphql service"; + frontendPkg = lib.mkOption { + type = lib.types.package; + default = (import ../pkgs.nix {}).packages.cardano-graphql; + }; + + persistPkg = lib.mkOption { + type = lib.types.package; + default = (import ../pkgs.nix {}).packages.persistgraphql; + }; + enginePort = lib.mkOption { type = lib.types.int; default = 9999; @@ -93,8 +102,8 @@ in { config = let # TODO: there has to be a better way to handle boolean env vars in nodejs??? boolToNodeJSEnv = bool: if bool then "true" else "false"; - frontend = selfPkgs.packages.cardano-graphql; - persistgraphql = selfPkgs.packages.persistgraphql; + frontend = cfg.frontendPkg; + persistgraphql = cfg.persistPkg; hasuraBaseUri = "${cfg.hasuraProtocol}://${cfg.hasuraIp}:${toString cfg.enginePort}"; pluginLibPath = pkgs.lib.makeLibraryPath [ pkgs.stdenv.cc.cc.lib diff --git a/nix/nixos/graphql-engine-service.nix b/nix/nixos/graphql-engine-service.nix index dddb5184..5968324f 100644 --- a/nix/nixos/graphql-engine-service.nix +++ b/nix/nixos/graphql-engine-service.nix @@ -7,6 +7,11 @@ in { services.graphql-engine = { enable = lib.mkEnableOption "graphql engine service"; + package = lib.mkOption { + type = lib.types.package; + default = (import ../pkgs.nix {}).packages.graphql-engine; + }; + host = lib.mkOption { type = lib.types.str; default = ""; @@ -44,7 +49,6 @@ in { }; }; config = let - graphqlEngine = (import ../pkgs.nix {}).packages.graphql-engine; hasuraDbPerms = pkgs.writeScript "hasuraDbPerms.sql" '' CREATE EXTENSION IF NOT EXISTS pgcrypto; CREATE SCHEMA IF NOT EXISTS hdb_catalog; @@ -75,7 +79,7 @@ in { sudo -u ${cfg.dbAdminUser} -- psql ${cfg.db} < ${hasuraDbPerms} ''; script = '' - exec ${graphqlEngine}/bin/graphql-engine \ + exec ${cfg.package}/bin/graphql-engine \ serve \ --server-port ${toString cfg.enginePort} \ --enable-telemetry=false \ From 0f1f007204454d0d923424d2e3efc4b8ee6c9a65 Mon Sep 17 00:00:00 2001 From: John Lotoski Date: Thu, 9 Feb 2023 13:39:59 -0600 Subject: [PATCH 3/4] cleanup: remove env var for obsolete cgql db use --- nix/nixos/graphql-engine-service.nix | 1 - 1 file changed, 1 deletion(-) diff --git a/nix/nixos/graphql-engine-service.nix b/nix/nixos/graphql-engine-service.nix index 5968324f..21346af1 100644 --- a/nix/nixos/graphql-engine-service.nix +++ b/nix/nixos/graphql-engine-service.nix @@ -68,7 +68,6 @@ in { path = with pkgs; [ curl netcat postgresql sudo ]; environment = { HASURA_GRAPHQL_DATABASE_URL = "postgres://${cfg.dbUser}:${cfg.password}@${cfg.host}${if cfg.host == "" then "" else toString ":${toString cfg.dbPort}"}/${cfg.db}"; - CARDANO_GRAPHQL_DB_URL = "postgres://${cfg.dbUser}:${cfg.password}@${cfg.host}${if cfg.host == "" then "" else ":${toString cfg.dbPort}"}/cgql"; }; preStart = '' for x in {1..10}; do From ea305eab98b8a4476202faf5e7326a56bc264ae8 Mon Sep 17 00:00:00 2001 From: John Lotoski Date: Mon, 13 Feb 2023 10:38:02 -0600 Subject: [PATCH 4/4] fix: ci, reduce to x86_64-linux arch only, bump RAM to avoid oom --- flake.nix | 1 + nix/cells/automation/pipelines.nix | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 1493678f..8f9e0b68 100644 --- a/flake.nix +++ b/flake.nix @@ -59,6 +59,7 @@ } @ inputs: std.growOn { inherit inputs; + systems = ["x86_64-linux"]; cellsFrom = nix/cells; cellBlocks = [ (std.functions "library") diff --git a/nix/cells/automation/pipelines.nix b/nix/cells/automation/pipelines.nix index 7e70b242..bdaa6e04 100644 --- a/nix/cells/automation/pipelines.nix +++ b/nix/cells/automation/pipelines.nix @@ -27,7 +27,7 @@ skippedDescription = lib.escapeShellArg "No nix builder for this system"; }; - memory = 1024 * 8; + memory = 1024 * 16; nomad.resources.cpu = 10000; }; }