DO 2449 / ask to enable engine_state_api (#120)

* ask for engine state api enabled * fix bug with the template * add autoapprove to ask engine_state_api * fix some unit tests * add engine state to systemd setup * update stuff * change botocore version requirement * remove cryptography hook * rm hook * pinning pyinstaller-hooks-contrib==2023.12 * do not fix pyinstaller version * do not fix pyinstaller version * do not delete crypto hook * change order of prompts * fix systemd config parameters * remove obsolete expose command --------- Co-authored-by: Santiago Baldassin <santiago.baldassin@rdx.works>
radixdlt · Apr 29, 2024 · d85f1e4 · d85f1e4
1 parent 4213783
commit d85f1e4
Show file tree

Hide file tree

Showing 27 changed files with 359 additions and 274 deletions.
diff --git a/node-runner-cli/.dockerignore b/node-runner-cli/.dockerignore
@@ -1,4 +1,7 @@
 .python-version
 junit
 htmlcov
-coverage.xml
+coverage.xml
+.pytest_cache
+.venv
+build
diff --git a/node-runner-cli/Dockerfile.ubuntufocal b/node-runner-cli/Dockerfile.ubuntufocal
@@ -17,15 +17,15 @@ RUN set -ex \
     && curl https://pyenv.run | bash
 
 RUN /root/.pyenv/bin/pyenv install -v 3.10.6
-RUN pip install pipenv
+RUN pip install pipenv==2023.7.23
 
 WORKDIR /app
 COPY Pipfile Pipfile.lock /app/
 RUN PIPENV_VENV_IN_PROJECT=1 pipenv --python=/root/.pyenv/versions/3.10.6/bin/python run pipenv install
-RUN pipenv --python=/root/.pyenv/versions/3.10.6/bin/python run pip install pyinstaller==4.10
+RUN pipenv --python=/root/.pyenv/versions/3.10.6/bin/python run pip install pyinstaller
 
 COPY . /app
-RUN pipenv run --python=/root/.pyenv/versions/3.10.6/bin/python pyinstaller --onefile --windowed babylonnode.spec
+RUN pipenv run --python=/root/.pyenv/versions/3.10.6/bin/python pyinstaller babylonnode.spec
 
 RUN DISABLE_VERSION_CHECK=true /app/dist/babylonnode version
 

diff --git a/node-runner-cli/Dockerfile.ubuntujammy b/node-runner-cli/Dockerfile.ubuntujammy
@@ -14,15 +14,15 @@ RUN apt-get update \
 RUN set -ex \
     && curl https://pyenv.run | bash
 
-RUN pip install pipenv
+RUN pip install pipenv==2023.7.23
 
 WORKDIR /app
 COPY Pipfile Pipfile.lock /app/
 RUN PIPENV_VENV_IN_PROJECT=1 pipenv run pipenv install
-RUN pipenv run pip install pyinstaller==4.10
+RUN pipenv run pip install pyinstaller
 
 COPY . /app
-RUN pipenv run pyinstaller --onefile --windowed babylonnode.spec
+RUN pipenv run pyinstaller babylonnode.spec
 
 RUN DISABLE_VERSION_CHECK=true /app/dist/babylonnode version
 

diff --git a/node-runner-cli/Pipfile b/node-runner-cli/Pipfile
@@ -4,19 +4,21 @@ verify_ssl = true
 name = "pypi"
 
 [packages]
-responses= "==0.23.3"
+responses= "==0.25.0"
 requests = "==2.31.0"
-PyYAML = "==6.0.0"
-deepmerge = "==0.3.0"
+PyYAML = "==6.0.1"
+deepmerge = "==1.1.1"
 bech32 = "==1.2.0"
-ecdsa = "==0.17.0"
+ecdsa = "==0.19.0"
 pyopenssl = "==23.2.0"
-deepdiff = "==5.8.0"
-Jinja2 = "==3.1.2"
-boto3 = "==1.20.33"
-botocore = "==1.23.33"
-pytest = "==7.4.0"
+deepdiff = "==7.0.1"
+Jinja2 = "==3.1.3"
+boto3 = ">=1.20.33"
+botocore = ">=1.20.33"
+pytest = "==8.1.1"
 radix-engine-toolkit = "==0.12.1.dev3"
+cryptography = "==41.0.4"
+pyinstaller-hooks-contrib = "==2023.12"
 
 [requires]
 python_version = "3.10.6"
diff --git a/node-runner-cli/Pipfile.lock b/node-runner-cli/Pipfile.lock
diff --git a/node-runner-cli/config/CoreDockerConfig.py b/node-runner-cli/config/CoreDockerConfig.py
@@ -22,6 +22,9 @@ def __init__(self, config_dict: dict):
         self.trusted_node: str = ""
         self.memory_limit: str = "14000m"
         self.validator_address: str = ""
+        self.engine_state_enabled: bool = False
+        self.engine_state_port: str = "3336"
+        self.engine_state_address: str = "0.0.0.0"
         self.java_opts: str = (
             "--enable-preview -server -Xms12g -Xmx12g  "
             "-XX:MaxDirectMemorySize=2048m "

diff --git a/node-runner-cli/config/CoreSystemDConfig.py b/node-runner-cli/config/CoreSystemDConfig.py
@@ -24,6 +24,9 @@ def __init__(self, config_dict: dict):
         self.node_dir: str = "/etc/radixdlt/node"
         self.node_secrets_dir: str = "/etc/radixdlt/node/secrets"
         self.validator_address: str = ""
+        self.engine_state_enabled: bool = False
+        self.engine_state_port: str = "3336"
+        self.engine_state_address: str = "0.0.0.0"
         self.java_opts: str = (
             "--enable-preview -server -Xms12g -Xmx12g  "
             "-XX:MaxDirectMemorySize=2048m "

diff --git a/node-runner-cli/setup/BaseSetup.py b/node-runner-cli/setup/BaseSetup.py
@@ -140,6 +140,10 @@ def ask_keydetails(ks_password=None, new_keystore=False, ks_file=None):
         )
         return keydetails
 
+    @staticmethod
+    def ask_engine_state_api(auto_approve: bool):
+        return Prompts.ask_engine_state_api(auto_approve)
+
     @staticmethod
     def setup_node_optimisation_config(
         version, setup_ulimit: bool, setup_swap_space_argument: bool, swap_space: str

diff --git a/node-runner-cli/setup/DockerSetup.py b/node-runner-cli/setup/DockerSetup.py
@@ -330,6 +330,8 @@ def questionary(argument_object: DockerConfigArguments) -> DockerConfig:
                 docker_config.common_config.ask_enable_nginx_for_core(
                     argument_object.nginx_on_core
                 )
+                docker_config.core_node.engine_state_enabled = BaseSetup.ask_engine_state_api(
+                    argument_object.autoapprove)
             else:
                 del docker_config.core_node
 
@@ -343,6 +345,7 @@ def questionary(argument_object: DockerConfigArguments) -> DockerConfig:
             else:
                 del docker_config.gateway
 
+
         if (
             "MIGRATION" in argument_object.setupmode.mode
             and docker_config.core_node is not None

diff --git a/node-runner-cli/setup/SystemDCommandArguments.py b/node-runner-cli/setup/SystemDCommandArguments.py
@@ -21,6 +21,7 @@ class SystemDConfigArguments:
     networkid: str
     hostip: str
     validator: str
+    auto_approve: bool
 
     def __init__(self, args):
         validate_ip(args.hostip)
@@ -44,6 +45,7 @@ def __init__(self, args):
         self.config_file = f"{args.configdir}/config.yaml"
         self.networkid = args.networkid
         self.validator = args.validator
+        self.auto_approve = args.autoapprove
 
 
 def validate_ip(hostip: str):

diff --git a/node-runner-cli/setup/SystemDSetup.py b/node-runner-cli/setup/SystemDSetup.py
@@ -513,6 +513,7 @@ def ask_core_node(argument_object: SystemDConfigArguments) -> CoreSystemdConfig:
         systemd_config.core_node.keydetails = BaseSetup.ask_keydetails(
             argument_object.keystore_password, argument_object.new_keystore
         )
+        systemd_config.core_node.engine_state_enabled = BaseSetup.ask_engine_state_api(argument_object.auto_approve)
         return systemd_config.core_node
 
     @staticmethod

diff --git a/node-runner-cli/templates/radix-fullnode-compose.yml.j2 b/node-runner-cli/templates/radix-fullnode-compose.yml.j2
@@ -38,6 +38,8 @@ services:
       {% if core_node.validator_address is defined %}
       RADIXDLT_CONSENSUS_VALIDATOR_ADDRESS: '{{core_node.validator_address}}'
       {% endif %}
+      RADIXDLT_ENGINE_STATE_API_PORT: {{core_node.engine_state_port}}
+      RADIXDLT_ENGINE_STATE_API_BIND_ADDRESS: {{core_node.engine_state_address}}
     image: {{core_node.repo}}:{{core_node.core_release}}
     init: true
     mem_limit: {{core_node.memory_limit or '14000m'}}
@@ -72,6 +74,10 @@ services:
     {% endif %}
       RADIXDLT_GATEWAY_API_ENABLE: '{{common_config.nginx_settings.protect_gateway or 'false'}}'
       RADIXDLT_GATEWAY_BEHIND_AUTH: '{{common_config.nginx_settings.gateway_behind_auth or 'false'}}'
+    {% if core_node is not none and core_node is defined %}
+      RADIXDLT_ENGINE_STATE_ENABLE: '{{core_node.engine_state_enabled or 'false'}}'
+      RADIXDLT_ENGINE_STATE_PORT: '{{core_node.engine_state_port}}'
+    {% endif %}
     image: {{common_config.nginx_settings.repo}}:{{common_config.nginx_settings.release}}
     ports:
     - 443:443

diff --git a/node-runner-cli/templates/systemd-default.config.j2 b/node-runner-cli/templates/systemd-default.config.j2
@@ -41,4 +41,13 @@ genesis.olympia.node_end_state_api_auth_user={{migration.olympia_node_auth_user}
 genesis.olympia.node_end_state_api_auth_password={{migration.olympia_node_auth_password}}
 genesis.olympia.node_bech32_address={{migration.olympia_node_bech32_address}}
 {% endif %}
-{% endif %}
+{% endif %}
+
+
+{% if core_node.engine_state_enabled %}
+api.engine_state.port={{core_node.engine_state_port}}
+api.engine_state.bind_address={{core_node.engine_state_address}}
+db.re_node_listing_indices.enable={{core_node.engine_state_enabled}}
+db.historical_substate_values.enable={{core_node.engine_state_enabled}}
+state_hash_tree.state_version_history_length=60000
+{% endif %}
diff --git a/node-runner-cli/test-prompts/core-gateway-all-local.yml b/node-runner-cli/test-prompts/core-gateway-all-local.yml
@@ -9,6 +9,8 @@
 - input_ledger_path: "/tmp/babylon-ledger"
 - core_nginx_setup: "true"
 - setup_gateway: N
+- input_nginx_release: "development-latest"
+- enable_engine_state_api: N
 #- input_core_api_address: "http://core_node"
 #- core_api_node_name: "core"
 #- postgres_location: "local"
@@ -18,4 +20,4 @@
 #- aggregator_release: "1.1.7"
 #- gateway_release: "1.1.7"
 #- gateway_nginx_setup: "true"
-- input_nginx_release: "development-latest"
+
diff --git a/node-runner-cli/test-prompts/core.yml b/node-runner-cli/test-prompts/core.yml
@@ -9,3 +9,4 @@
 - core_nginx_setup: "true"
 - setup_gateway: N
 - input_nginx_release: "development-latest"
+- enable_engine_state_api: N
diff --git a/node-runner-cli/test-prompts/corenode-01.yml b/node-runner-cli/test-prompts/corenode-01.yml
@@ -9,4 +9,5 @@
 - input_ledger_path: "/tmp/babylon-ledger"
 - core_nginx_setup: "true"
 - setup_gateway: "false"
-- input_nginx_release: "1.3.1"
+- input_nginx_release: "1.3.1"
+- enable_engine_state_api: N
diff --git a/node-runner-cli/test-prompts/corenode-02.yml b/node-runner-cli/test-prompts/corenode-02.yml
@@ -9,3 +9,4 @@
 - input_ledger_path: "/tmp/babylon-ledger"
 - core_nginx_setup: "false"
 - setup_gateway: "false"
+- enable_engine_state_api: N
diff --git a/node-runner-cli/test-prompts/gateway-local-core-remote-postgress.yml b/node-runner-cli/test-prompts/gateway-local-core-remote-postgress.yml
@@ -21,5 +21,6 @@
 - migration_release: ""
 - gateway_nginx_setup: "true"
 - input_nginx_release: ""
+- enable_engine_state_api: N
 
 
diff --git a/node-runner-cli/test-prompts/gateway-remote-core-local-postgress.yml b/node-runner-cli/test-prompts/gateway-remote-core-local-postgress.yml
@@ -16,5 +16,6 @@
 - migration_release: "1.1.7"
 - gateway_nginx_setup: "true"
 - input_nginx_release: "1.3.1"
+- enable_engine_state_api: N
 
 
diff --git a/node-runner-cli/test-prompts/gateway-remote-core-remote-postgress.yml b/node-runner-cli/test-prompts/gateway-remote-core-remote-postgress.yml
@@ -18,3 +18,4 @@
 - migration_release: "1.1.7"
 - gateway_nginx_setup: "true"
 - input_nginx_release: "1.3.1"
+- enable_engine_state_api: N
diff --git a/node-runner-cli/test-prompts/gateway.yml b/node-runner-cli/test-prompts/gateway.yml
@@ -9,3 +9,4 @@
 - core_nginx_setup: "true"
 - setup_gateway: Y
 - input_nginx_release: "development-latest"
+- enable_engine_state_api: N
diff --git a/node-runner-cli/tests/userflows/install-docker-all-same-host.sh b/node-runner-cli/tests/userflows/install-docker-all-same-host.sh
@@ -129,6 +129,9 @@ curl -X POST -k -u "gateway:${NGINX_GATEWAY_PASSWORD}" https://localhost/status/
 echo "Metrics endpoint"
 curl -k -f -u "metrics:${NGINX_METRICS_PASSWORD}" https://localhost/gateway/metrics
 
+echo "Test engine state api"
+curl -v -k -u "admin:${NGINX_ADMIN_PASSWORD}" -X POST --header "Content-Type: application/json" https://localhost:443/engine-state/entity/info --data '{"entity_address": "account_rdx12y386tpv29ndwslkf4drm9d5tsn6eulxg2uuxjq5ykhanuudcs7827"}'
+
 # Endpoint does not exist
 #curl -k --user "gateway:${NGINX_GATEWAY_PASSWORD}" https://localhost/token/native --header 'Content-Type: application/json' -d '{ "network_identifier":{"network":"${NETWORK_NAME}"}}'
 

diff --git a/node-runner-cli/tests/userflows/install-systemd-simple.sh b/node-runner-cli/tests/userflows/install-systemd-simple.sh
@@ -99,6 +99,9 @@ NGINX_ADMIN_PASSWORD=${NGINX_ADMIN_PASSWORD} ./babylonnode api system peers
 echo "Testing Core node identity"
 NGINX_ADMIN_PASSWORD=${NGINX_ADMIN_PASSWORD} ./babylonnode api system identity
 
+echo "Test engine state api"
+curl -v -k -u "admin:${NGINX_ADMIN_PASSWORD}" -X POST --header "Content-Type: application/json" https://localhost:443/engine-state/entity/info --data '{"entity_address": "account_rdx12y386tpv29ndwslkf4drm9d5tsn6eulxg2uuxjq5ykhanuudcs7827"}'
+
 echo "Restarting systemd service"
 ./babylonnode systemd restart
 
@@ -111,7 +114,7 @@ echo "Checking radixdlt service status again"
 sudo systemctl status radixdlt-node.service --no-pager | true
 
 echo "Checking nginx service status again"
-sudo systemctl status nginx.service --no-pager| true
+sudo systemctl status nginx.service --no-pager | true
 
 echo "Starting systemd service"
 ./babylonnode systemd start

diff --git a/node-runner-cli/utils/PromptFeeder.py b/node-runner-cli/utils/PromptFeeder.py
@@ -41,6 +41,7 @@ class QuestionKeys:
     genesis_bin_data_file = "genesis_bin_data_file"
     have_validator_address = "have_validator_address"
     validator_address = "validator_address"
+    enable_engine_state_api = "enable_engine_state_api"
 
 
 class PromptFeeder:

diff --git a/node-runner-cli/utils/Prompts.py b/node-runner-cli/utils/Prompts.py
@@ -248,6 +248,19 @@ def ask_keyfile_name() -> str:
 
         return keyfile_name
 
+    @staticmethod
+    def ask_engine_state_api(auto_approve: bool) -> bool:
+        if auto_approve:
+            return False
+        answer = Helpers.input_guestion(
+            "Do you want to enable the engine state api? (Y/N) (default: false):",
+            QuestionKeys.enable_engine_state_api
+        )
+        if answer == "":
+            return False
+        else:
+            return True
+
     @staticmethod
     def ask_trusted_node() -> str:
         Helpers.section_headline("Trusted node settings")

diff --git a/node-runner-cli/utils/utils.py b/node-runner-cli/utils/utils.py
@@ -339,7 +339,7 @@ def input_guestion(question, question_key=None):
         if not prompt_feed:
             return input(f"\n{bcolors.WARNING}{question}{bcolors.ENDC}")
         else:
-            print("Got from promptfeeder")
+            print(f"Got from question_key from promptfeeder {question_key}")
             print(f"Question is {question}")
             print(f"Answer is {prompt_feed}")
             return prompt_feed

diff --git a/node-runner-cli/version/__init__.py b/node-runner-cli/version/__init__.py
@@ -1,2 +1,2 @@
-__version__ = "2.0.1-rc1-2-ga3d5b6f"
-__base_version__ = "2.0.1-rc1"
+__version__= "2.1.2-5-g67e404b"
+__base_version__= "2.1.2"