feat: added configuration to use a local token registry (#873)

* feat: added configuration to use a local token registry

* feat: made local token registry default.

* chore: adjusted token registry port to not disturb any process at port 8080, so switched to 8091

* chore: adjusted token registry port

* feat: added configurability which repository to use for the token-registry

Dockerfile

@@ -59,7 +59,8 @@ WORKDIR /src
 
 FROM ubuntu-nodejs as background
 ARG NETWORK=mainnet
-ARG METADATA_SERVER_URI="https://tokens.cardano.org"
+# using local token registry as default
+ARG METADATA_SERVER_URI="http://token-metadata-registry:8091"
 RUN apt-get update -y && apt-get install lsb-release -y
 RUN curl --proto '=https' --tlsv1.2 -sSf -L https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - &&\
   echo "deb http://apt.postgresql.org/pub/repos/apt/ `lsb_release -cs`-pgdg main" | tee  /etc/apt/sources.list.d/pgdg.list &&\
@@ -115,3 +116,7 @@ COPY config/network/${NETWORK}/cardano-node /config/cardano-node/
 WORKDIR /app/packages/server/dist
 EXPOSE 3100
 CMD ["node", "index.js"]
+
+FROM cardanofoundation/cf-token-metadata-registry-api:latest as token-registry
+ADD scripts/token-registry-init.sh /app/entrypoint.sh
+ENTRYPOINT sh /app/entrypoint.sh

README.md

@@ -207,6 +207,14 @@ docker compose -p preview down
 
 </details>
 
+### Use global Token Metadata Registry
+The public Token metadata registry has a limit of daily requests, this can lead to long sync times, when resyncing from scratch.
+If it's still needed to run with the global environment it's possible by removing the `token-metadata-registry` from `docker-compose.yml`.
+And start it with:
+```
+METADATA_SERVER_URI="https://tokens.cardano.org" docker compose up -d
+```
+
 ### Upgrade Database to Postgres 14
 If you are upgrading from Postgres 11 to 14: A resync will be needed.
 To speed up the process you can use the following snapshots:

docker-compose.yml

@@ -106,7 +106,7 @@ services:
       - "postgres"
     environment:
       - LOGGER_MIN_SEVERITY=${LOGGER_MIN_SEVERITY:-info}
-      - METADATA_SERVER_URI=${METADATA_SERVER_URI:-https://tokens.cardano.org}
+      - METADATA_SERVER_URI=${METADATA_SERVER_URI:-http://token-metadata-registry:8080}
     restart: on-failure
     secrets:
       - postgres_db
@@ -142,6 +142,23 @@ services:
         max-size: "200k"
         max-file: "10"
 
+  token-metadata-registry:
+    build:
+      context: .
+      target: token-registry
+    ports:
+      - ${TOKEN_REGISTRY_PORT:-8080}:8080
+    environment:
+      - TOKEN_METADATA_SYNC_JOB=true
+      - POSTGRES_PORT=${POSTGRES_PORT:-5432}
+      - POSTGRES_HOST=postgres
+      - DB_SCHEMA=tokenregistry
+      - NETWORK=${NETWORK:-mainnet}
+    secrets:
+      - postgres_db
+      - postgres_password
+      - postgres_user
+
 secrets:
   postgres_db:
     file: ./placeholder-secrets/postgres_db

packages/api-cardano-db-hasura/src/MetadataClient.ts

@@ -3,6 +3,7 @@ import { errors, RunnableModuleState } from '@cardano-graphql/util'
 import { dummyLogger, Logger } from 'ts-log'
 import { AssetMetadata } from './AssetMetadata'
 import { Asset } from './graphql_types'
+import pRetry from 'p-retry'
 
 const MODULE_NAME = 'MetadataFetchClient'
 
@@ -20,16 +21,51 @@ export class MetadataClient {
     })
   }
 
-  private async ensureMetadataServerIsAvailable (): Promise<void> {
-    try {
-      await this.axiosClient.get('/metadata/healthcheck')
-    } catch (error) {
-      if (error.code === 'ENOTFOUND') {
-        throw new errors.HostDoesNotExist('metadata server')
-      } else if (error.response?.status !== 404) {
-        throw error
+  private async ensureLocalMetadataServerIsAvailable (): Promise<void> {
+    await pRetry(
+      async () => {
+        try {
+          await this.axiosClient.get('/health')
+        } catch (error) {
+          if (error.code === 'ENOTFOUND') {
+            this.logger.info('Waiting for TokenRegistry to be available')
+            throw new errors.HostDoesNotExist('metadata server')
+          } else if (error.response?.status === 400) { // Needed until TokenRegistry is updated
+            this.logger.info('Token Registry is up')
+          } else if (error.response?.status !== 404) {
+            this.logger.info('Metadata Server unreachable.')
+            throw error
+          }
+        }
+      }, {
+        factor: 1.5,
+        retries: 10
       }
-    }
+    )
+  }
+
+  private async waitForLocalMetadataServerSynced (): Promise<void> {
+    await pRetry(
+      async () => {
+        try {
+          const result = await this.axiosClient.get('/health')
+          if (!result.data.synced) {
+            this.logger.info('Metadata registry is still syncing. This can take up to 15min...')
+            throw new Error('')
+          }
+        } catch (error) {
+          if (error.response?.status === 400) {
+            this.logger.info('external Registry is up and running') // Needed until TokenRegistry is updated
+          } else {
+            throw new Error('')
+          }
+        }
+      }, {
+        factor: 1.5,
+        retries: 1000,
+        minTimeout: 60000 // first try after one minute
+      }
+    )
   }
 
   public async fetch (assetIds: Asset['assetId'][]): Promise<AssetMetadata[]> {
@@ -62,7 +98,9 @@ export class MetadataClient {
     if (this.state !== null) return
     this.state = 'initializing'
     this.logger.info({ module: MODULE_NAME }, 'Initializing')
-    await this.ensureMetadataServerIsAvailable()
+    await this.ensureLocalMetadataServerIsAvailable()
+    this.logger.info({ module: MODULE_NAME }, 'Metadata Server is up and running. Checking Sync Status.')
+    await this.waitForLocalMetadataServerSynced()
     this.state = 'initialized'
     this.logger.info({ module: MODULE_NAME }, 'Initialized')
   }

scripts/export_env.sh

@@ -11,10 +11,11 @@ case "$NETWORK" in
         mainnet)
             API_PORT=3100
             HASURA_PORT=8090
-            METADATA_SERVER_URI="https://tokens.cardano.org"
+            METADATA_SERVER_URI="http://localhost:8080"
             OGMIOS_PORT=1337
             PG_ADMIN_PORT=8442
             POSTGRES_PORT=5432
+            TOKEN_REGISTRY_PORT=8080
             ;;
         testnet)
             API_PORT=3101
@@ -23,6 +24,7 @@ case "$NETWORK" in
             OGMIOS_PORT=1338
             PG_ADMIN_PORT=8443
             POSTGRES_PORT=5443
+            TOKEN_REGISTRY_PORT=8081
             export CARDANO_NODE_VERSION=8.7.3
             ;;
         preprod)
@@ -32,6 +34,7 @@ case "$NETWORK" in
             OGMIOS_PORT=1339
             PG_ADMIN_PORT=8444
             POSTGRES_PORT=5444
+            TOKEN_REGISTRY_PORT=8082
             ;;
         preview)
             API_PORT=3103
@@ -40,6 +43,7 @@ case "$NETWORK" in
             OGMIOS_PORT=1340
             PG_ADMIN_PORT=8445
             POSTGRES_PORT=5445
+            TOKEN_REGISTRY_PORT=8083
             ;;
         vasil-dev)
             API_PORT=3104
@@ -48,6 +52,7 @@ case "$NETWORK" in
             OGMIOS_PORT=1341
             PG_ADMIN_PORT=8446
             POSTGRES_PORT=5446
+            TOKEN_REGISTRY_PORT=8084
             ;;
 esac
 
@@ -70,3 +75,4 @@ export PG_ADMIN_PORT
 export POSTGRES_PORT
 export POSTGRES_USER_FILE=${SECRETS_DIR}/postgres_user
 export POSTGRES_HOST=localhost
+export TOKEN_REGISTRY_TAG=latest

scripts/token-registry-init.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+export DB_USERNAME=$(cat /run/secrets/postgres_user)
+export DB_PASSWORD=$(cat /run/secrets/postgres_password)
+export DB_NAME=$(cat /run/secrets/postgres_db)
+export DB_URL=jdbc:postgresql://${POSTGRES_HOST}:${POSTGRES_PORT}/${DB_NAME}?currentSchema=${DB_SCHEMA}
+
+case "$NETWORK" in
+    mainnet)
+      GITHUB_ORGANIZATION=cardano-foundation
+      GITHUB_PROJECT_NAME=cardano-token-registry
+      GITHUB_MAPPINGS_FOLDER=mappings
+      ;;
+    preprod|preview|testnet)
+      GITHUB_ORGANIZATION=input-output-hk
+      GITHUB_PROJECT_NAME=metadata-registry-testnet
+      GITHUB_MAPPINGS_FOLDER=registry
+      ;;
+esac
+echo Using Github Repository $GITHUB_ORGANIZATION/$GITHUB_PROJECT_NAME as token registry
+java -jar /app/app.jar