V1.1 (#83)

* Add trace classifier (#82)
* Added golang API
* Fixed dialect problem
* Docker standalone setup update
* Updated docker setup
* Updated submodule
* Updated readme
---------
Co-authored-by: verdigos <chdm.ps@gmail.com>

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "ton-index-cpp"]
 	path = ton-index-cpp
 	url = https://github.com/toncenter/ton-index-worker
+[submodule "ton-index-go"]
+	path = ton-index-go
+	url = https://github.com/toncenter/ton-index-go

README.md

@@ -3,6 +3,9 @@
 > [!NOTE]  
 > This repository's master branch hosts the TON Indexer designed for direct reading from the TON Node database. If you are looking for an indexer that operates through `tonlib + liteserver`, please refer to the branch [legacy](https://github.com/toncenter/ton-indexer/tree/legacy).
 
+> [!NOTE]
+> Be careful upgrading the indexer. Versions with different major and minor have different schemas and they are incompatible with each other.
+
 
 TON Indexer stores blocks, transactions, messages, NFTs, Jettons and DNS domains in PostgreSQL database and provides convenient API.
 
@@ -11,7 +14,7 @@ TON node stores data in a key-value database RocksDB.  While RocksDB excels in s
 TON Indexer stack consists of:
 1. `postgres`: PostgreSQL server to store indexed data and perform queries.
 2. `index-api`: FastAPI server with convenient endpoints to access the database.
-3. `alembic`: alembic service to run database migrations.
+3. `event-classifier`: trace classification service.
 4. `index-worker`: TON Index worker to read and parse data from TON node database. This service must be run on the machine with a working TON Node.
 
 ## How to run
@@ -30,8 +33,8 @@ Do the following steps to setup TON Indexer:
   * ./configure.sh will create .env file only with indexer and PostgreSQL configuration data. Use --worker flag to add TON Index worker configuration data too.
 * Adjust parameters in *.env* file (see [list of available parameters](#available-parameters)).
 * Set PostgreSQL password `echo -n "MY_PASSWORD" > private/postgres_password`
-* Build docker images: `docker compose build postgres alembic index-api`.
-* Run stack: `docker compose up -d postgres alembic index-api`.
+* Build docker images: `docker compose build postgres event-classifier index-api`.
+* Run stack: `docker compose up -d postgres event-classifier event-cache index-api`.
   * To start worker use command `docker compose up -d index-worker` after creating all services.
 
 **NOTE:** we recommend to setup indexer stack and index worker on separate servers. To install index worker to **Systemd** check this [instruction](https://github.com/toncenter/ton-index-worker).
@@ -53,9 +56,7 @@ Do the following steps to setup TON Indexer:
 * TON worker parameters:
   * `TON_WORKER_DBROOT`: path to TON full node database. Use default value if you've installed node with `mytonctrl`. Default: `/var/ton-work/db`.
   * `TON_WORKER_FROM`: masterchain seqno to start indexing. Set 1 to full index, set last masterchain seqno to collect only new blocks (use [/api/v2/getMasterchainInfo](https://toncenter.com/api/v2/getMasterchainInfo) to get last masterchain block seqno).
-  * `TON_WORKER_MAX_PARALLEL_TASKS`: max parallel reading actors. Adjust this parameter to decrease RAM usage. Default: `1024`.
-  * `TON_WORKER_INSERT_BATCH_SIZE`: max masterchain seqnos per INSERT query. Small value will decrease indexing performance. Great value will increase RAM usage. Default: `512`.
-  * `TON_WORKER_INSERT_PARALLEL_ACTORS`: number of parallel INSERT transactions. Increasing this number will increase PostgreSQL server RAM usage. Default: `3`.
+  * `TON_WORKER_ADDITIONAL_ARGS`: additional args to pass into index worker.
 
 ## Swagger
 
@@ -66,13 +67,12 @@ To test API, built-in swagger can be used. It is available after running `docker
 ## How to point TON Index worker to existing PostgreSQL instance
 * Remove PostgreSQL container: `docker compose rm postgres` (add flag `-v` to remove volumes).
 * Setup PostgreSQL credentials in *.env* file.
-* Run alembic migration: `docker compose up alembic`.
-* Run index worker: `docker compose up -d index-worker`.
+* Run index worker: `docker compose up -d index-worker index-api event-classifier event-cache`.
 
 ## How to update code
 * Pull new commits: `git pull`.
 * Update submodules: `git submodule update --recursive --init`.
-* Build new image: `docker compose build postgres alembic index-api`.
+* Build new image: `docker compose build postgres event-classifier event-cache index-api`.
   * Build new image of worker: `docker compose build index-worker`
-* Run new version: `docker compose up -d postgres alembic index-api`
+* Run new version: `docker compose up -d postgres event-classifier event-cache index-api`
   * Run new version of worker: `docker compose up -d index-worker`

configure.sh

@@ -47,8 +47,6 @@ TON_INDEXER_WORKERS=4
 
 TON_INDEXER_TON_HTTP_API_ENDPOINT=${TON_INDEXER_TON_HTTP_API_ENDPOINT}
 
-TON_WORKER_DBROOT=
-
 EOF
 
 if [[ "$WORKER" -eq 1 ]]; then
@@ -57,8 +55,10 @@ echo "Configure Worker"
 cat <<EOF >> .env
 TON_WORKER_DBROOT=${TON_WORKER_DBROOT:-/var/ton-work/db/}
 TON_WORKER_FROM=${TON_WORKER_FROM:-1}
-TON_WORKER_MAX_PARALLEL_TASKS=${TON_WORKER_MAX_PARALLEL_TASKS:-1024}
-TON_WORKER_INSERT_BATCH_SIZE=${TON_WORKER_INSERT_BATCH_SIZE:-512}
-TON_WORKER_INSERT_PARALLEL_ACTORS=${TON_WORKER_INSERT_PARALLEL_ACTORS:-3}
+TON_WORKER_ADDITIONAL_ARGS=${TON_WORKER_ADDITIONAL_ARGS}
+EOF
+else 
+cat <<EOF >> .env
+TON_WORKER_DBROOT=
 EOF
 fi

docker-compose.api.yaml

@@ -1,7 +1,7 @@
 version: '3.9'
 
 x-indexer-environment: &indexer-environment
-  POSTGRES_DIALECT: ${POSTGRES_DIALECT:-postgresql+asyncpg}
+  POSTGRES_DIALECT: ${POSTGRES_DIALECT:-postgresql}
   POSTGRES_HOST:
   POSTGRES_PORT:
   POSTGRES_USER:
@@ -13,31 +13,32 @@ x-indexer-environment: &indexer-environment
   TON_INDEXER_IS_TESTNET:
 
 services:
-  index-api:
-    image: ${DOCKER_REGISTRY:-localhost:5000}/ton-indexer-cpp:${IMAGE_TAG:?}
+  index-api-go:
+    image: ${DOCKER_REGISTRY:-localhost:5000}/ton-index-go:${IMAGE_TAG:?}
     build:
-      context: indexer
+      context: ton-index-go
       dockerfile: Dockerfile
     secrets:
       - postgres_password
-    command: gunicorn indexer.api.main:app -k uvicorn.workers.UvicornWorker --bind=0.0.0.0:8081 -w ${TON_INDEXER_WORKERS:-1}
+    command: -prefork -threads ${TON_INDEXER_WORKERS:-1}
+    init: true
     ports:
       - target: 8081
         published: ${TON_INDEXER_API_PORT:-8081}
         protocol: tcp
         mode: host
-    environment: *indexer-environment 
+    environment: *indexer-environment
     deploy:
       mode: replicated
       replicas: ${SWARM_REPLICAS:-1}
       placement:
         constraints:
-          - "node.labels.${TONCENTER_ENV:?}.indexer-cpp.api==true"
+          - "node.labels.${TONCENTER_ENV:?}.indexer-cpp.api-go==true"
     networks:
       internal:
       toncenter-global:
         aliases:
-          - ${TONCENTER_ENV:?}-indexer-cpp-api
+          - ${TONCENTER_ENV:?}-indexer-cpp-api-go
 
 networks:
   internal:
@@ -51,4 +52,3 @@ networks:
 secrets:
   postgres_password:
     file: ${POSTGRES_PASSWORD_FILE:-private/postgres_password}
-

docker-compose.database.yaml

@@ -2,7 +2,7 @@ version: '3.9'
 
 services:
   postgres:
-    image: postgres:15
+    image: postgres:16
     environment: 
       POSTGRES_USER:
       POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password

docker-compose.events.yaml

@@ -0,0 +1,57 @@
+version: '3.9'
+
+x-indexer-environment: &indexer-environment
+  POSTGRES_DIALECT:
+  POSTGRES_HOST:
+  POSTGRES_PORT:
+  POSTGRES_USER:
+  POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
+  POSTGRES_DBNAME:
+  TON_INDEXER_API_ROOT_PATH:
+  TON_INDEXER_API_TITLE:
+  TON_INDEXER_TON_HTTP_API_ENDPOINT:
+  TON_INDEXER_REDIS_DSN: redis://event-cache:6379
+
+services:
+  event-cache:
+    image: redis:latest
+    networks:
+      - internal
+    command: redis-server --maxclients 40000
+    deploy:
+      mode: replicated
+      replicas: 1
+      placement:
+        constraints:
+          - "node.labels.${TONCENTER_ENV:?}.indexer-cpp.events==true"
+  event-classifier:
+    image: ${DOCKER_REGISTRY:-localhost:5000}/ton-index-event-classfier:${IMAGE_TAG:?}
+    build:
+      context: indexer
+      dockerfile: Dockerfile
+    secrets:
+      - postgres_password
+    command: python3 /app/event_classifier.py --pool-size ${TON_INDEXER_WORKERS:-8} --fetch-size 64000 --batch-size 2000
+    environment: *indexer-environment 
+    deploy:
+      mode: replicated
+      replicas: 1
+      placement:
+        constraints:
+          - "node.labels.${TONCENTER_ENV:?}.indexer-cpp.events==true"
+    networks:
+      internal:
+
+networks:
+  internal:
+    attachable: true
+    external: false
+    driver_opts:
+      com.docker.network.driver.mtu: 1350
+  toncenter-global:
+    external: true
+
+secrets:
+  postgres_password:
+    file: private/postgres_password
+

docker-compose.yaml

@@ -15,43 +15,47 @@ x-index-api: &index-api
   TON_INDEXER_API_PORT:
   TON_INDEXER_TON_HTTP_API_ENDPOINT:
   TON_INDEXER_IS_TESTNET:
+  TON_INDEXER_REDIS_DSN: redis://event-cache:6379
   <<: *index-common
 
 x-index-worker: &index-worker
   TON_WORKER_FROM:
+  TON_WORKER_DBROOT: /tondb
+  TON_WORKER_BINARY: ton-index-postgres-v2
+  TON_WORKER_ADDITIONAL_ARGS:
   <<: *index-common
 
 services:
-  index-api:
+  event-cache:
+    image: redis:latest
+    networks:
+      - internal
+    command: redis-server --maxclients 40000
+  event-classifier:
     build:
       context: indexer
       dockerfile: Dockerfile
     secrets:
       - postgres_password
-    command: gunicorn indexer.api.main:app -k uvicorn.workers.UvicornWorker --bind=0.0.0.0:8081 -w ${TON_INDEXER_WORKERS:-1}
-    ports:
-      - target: 8081
-        published: ${TON_INDEXER_API_PORT:-8081}
-    environment: *index-api
-    restart: always
+    command: python3 /app/event_classifier.py --pool-size ${TON_INDEXER_WORKERS:-8} --fetch-size 10000 --batch-size 500
+    environment: *index-api 
     networks:
       internal:
-    depends_on:
-      - alembic
 
-  alembic:
+  index-api:
     build:
-      context: indexer
+      context: ton-index-go
       dockerfile: Dockerfile
-    environment: *index-api 
     secrets:
       - postgres_password
-    command: alembic upgrade head-1
+    command: -prefork -threads ${TON_INDEXER_WORKERS:-1}
+    ports:
+      - target: 8081
+        published: ${TON_INDEXER_API_PORT:-8081}
+    environment: *index-api
+    restart: always
     networks:
       internal:
-    depends_on:
-      postgres:
-        condition: service_healthy
 
   index-worker:
     build:
@@ -66,9 +70,6 @@ services:
       internal:
     command: --from ${TON_WORKER_FROM:-1}
     restart: unless-stopped
-    depends_on:
-      alembic:
-        condition: service_completed_successfully
 
   postgres:
     image: postgres:16

indexer/Dockerfile

@@ -1,8 +1,4 @@
-FROM ubuntu:20.04
-
-RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get update && apt-get -y install tzdata && rm -rf /var/lib/{apt,dpkg,cache,log}/
-RUN apt-get update && apt-get install -y git cmake wget python3 python3-pip && rm -rf /var/lib/{apt,dpkg,cache,log}/
-RUN python3 -m pip install -U pip
+FROM python:3.12-bookworm
 
 # python requirements
 ADD requirements.txt /tmp/requirements.txt

indexer/classifier.Dockerfile

@@ -0,0 +1,13 @@
+FROM python:3.10
+
+# python requirements
+ADD requirements.txt /tmp/requirements.txt
+RUN python3 -m pip install --no-cache-dir -r /tmp/requirements.txt
+
+# app
+COPY . /app
+WORKDIR /app
+
+# entrypoint
+ENV C_FORCE_ROOT 1
+ENTRYPOINT [ "/app/entrypoint.sh" ]

indexer/entrypoint.sh

@@ -13,13 +13,14 @@ else
     echo "Postgres password file not specified!"
     exit 1
 fi
-
-if [ "$POSTGRES_PASSWORD" -eq 0 ]; then
+export TQDM_NCOLS=0
+export TQDM_POSITION=-1
+if [ -z "$POSTGRES_PASSWORD" ]; then
     echo "Using postgres connection without password"
-    export TON_INDEXER_PG_DSN="${POSTGRES_DIALECT}://${POSTGRES_USER}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DBNAME}"
+    export TON_INDEXER_PG_DSN="${POSTGRES_DIALECT:-postgresql+asyncpg}://${POSTGRES_USER}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DBNAME}"
 else
     echo "Using postgres connection with password"
-    export TON_INDEXER_PG_DSN="${POSTGRES_DIALECT}://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DBNAME}"
+    export TON_INDEXER_PG_DSN="${POSTGRES_DIALECT:-postgresql+asyncpg}://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DBNAME}"
 fi
 printenv