diff --git a/docs/command_reference.adoc b/docs/command_reference.adoc index c9601f57..635b30cf 100644 --- a/docs/command_reference.adoc +++ b/docs/command_reference.adoc @@ -5,172 +5,26 @@ Below are the list of commands that can be used with cli to setup a core node or ==== babylonnode docker dependencies [source, bash,subs="+quotes, +attributes" ] ---- -usage: babylonnode docker dependencies [-h] - -This commands installs all necessary software on the Virtual Machine(VM). Run -this command on fresh VM or on a existing VM as the command is tested to be -idempotent - -options: - -h, --help show this help message and exit ---- ==== babylonnode docker config [source, bash,subs="+quotes, +attributes" ] ---- -usage: babylonnode docker config [-h] [-a] [-d CONFIGDIR] - [-k KEYSTOREPASSWORD] -m - {CORE,GATEWAY,DETAILED,MIGRATION} - [{CORE,GATEWAY,DETAILED,MIGRATION} ...] - [-miu MIGRATION_URL] - [-miau MIGRATION_AUTH_USER] - [-miap MIGRATION_AUTH_PASSWORD] - [-miba MIGRATION_BECH_ADDRESS] [-n NETWORKID] - [-nk] [-p POSTGRESPASSWORD] [-t TRUSTEDNODE] - [-v VALIDATOR] [-xc {true,false}] - [-xg {true,false}] - -This commands allows node-runners and gateway admins to create a config file, -which can persist their custom settings. Thus it allows is to decouple the -updates from docker_config. Config is created only once as such and if there -is a version change in the config file, then it updated by doing a migration -to newer version - -options: - -h, --help show this help message and exit - -required arguments: - -m {CORE,GATEWAY,DETAILED,MIGRATION} [{CORE,GATEWAY,DETAILED,MIGRATION} ...], --setupmode {CORE,GATEWAY,DETAILED,MIGRATION} [{CORE,GATEWAY,DETAILED,MIGRATION} ...] - Quick config mode with assumed defaults. It supports - two quick modes and a detailed config mode. CORE: Use - this value to setup CORE using defaults. GATEWAY: Use - this value to setup GATEWAY using defaults. DETAILED: - Default value if not provided. This mode takes your - through series of questions. - -optional arguments: - -a, --autoapprove Set this to true to run without any prompts and in - mode CORE or GATEWAY.Prompts still appear if you run - in DETAILED mode Use this for automation purpose only - -d CONFIGDIR, --configdir CONFIGDIR - Path to node-config directory where config file will - stored. Default value is /Users/kim.fehrs/babylon- - node-config - -k KEYSTOREPASSWORD, --keystorepassword KEYSTOREPASSWORD - Core Node requires a keystore. This is the password - for the keystore file the CLI will create new key with - name `node-keystore.ks` in config directory. If the - keystore exists in config directory, CLI finds it , - and uses the password from -k parameter or prompt for - the password to be entered. CLI never modifies the - password of keystore - -miu MIGRATION_URL, --migration_url MIGRATION_URL - The root url of the olympia node to migrate the ledger - from. Do not add /olympia-end-state. - -miau MIGRATION_AUTH_USER, --migration_auth_user MIGRATION_AUTH_USER - The user to authenticate to the olympia node for - migration - -miap MIGRATION_AUTH_PASSWORD, --migration_auth_password MIGRATION_AUTH_PASSWORD - The password to authenticate to the olympia node for - migration - -miba MIGRATION_BECH_ADDRESS, --migration_bech_address MIGRATION_BECH_ADDRESS - The bech address of the olympia node to migrate the - ledger from - -n NETWORKID, --networkid NETWORKID - Network id of network you want to connect.For stokenet - it is 2 and for mainnet it is 1.If not provided you - will be prompted to enter a value - -nk, --newkeystore Set this to true to create a new store without any - prompts using location defined in argument configdir - -p POSTGRESPASSWORD, --postgrespassword POSTGRESPASSWORD - Network Gateway uses Postgres as datastore. This is - password for the user `postgres`. - -t TRUSTEDNODE, --trustednode TRUSTEDNODE - Trusted node on radix network.Example format: 'radix:/ - /node_tdx_e_1q0gm3fwqh8ggl09g7l8ru96krzlxdyrc694mqw8cf - 227v62vjyrmccv8md5@13.126.65.118'.This is required - only if you are creating config to run a CORE node and - if not provided you will be prompted to enter a value - -v VALIDATOR, --validator VALIDATOR - Address of the validator - -xc {true,false}, --disablenginxforcore {true,false} - Core Node API's are protected by Basic auth - setting.Set this to disable to nginx for core - -xg {true,false}, --disablenginxforgateway {true,false} - GateWay API's end points are protected by Basic auth - settings. Set this to disable to nginx for gateway ---- ==== babylonnode docker install [source, bash,subs="+quotes, +attributes" ] ---- -usage: babylonnode docker install [-h] [-a] [-aue ADVANCEDUSERENVS] - [-f CONFIGFILE] [-u] - -This commands setups up the software and deploys it based on what is stored in -the config.yaml file. To update software versions, most of the time it is -required to update the versions in config file and run this command - -options: - -h, --help show this help message and exit - -optional arguments: - -a, --autoapprove Pass this option to run without any prompts. Use this - for automation purpose only - -aue ADVANCEDUSERENVS, --advanceduserenvs ADVANCEDUSERENVS - Path to advanced config file. This file can directly - configure the core node. It is templated into - default.config and acts as custom configuration that - is not overwritten on install. The default value is - `/Users/kim.fehrs/babylon-node-config/advanced- - user.env-file` if not provided - -f CONFIGFILE, --configfile CONFIGFILE - Path to config file. This file is generated by running - 'babylonnode docker config'The default value is - `/Users/kim.fehrs/babylon-node-config/config.yaml` if - not provided - -u, --update Pass this option to update the deployed softwares to - latest version. CLI prompts to confirm the versions if - '-a' is not passed ---- ==== babylonnode docker start [source, bash,subs="+quotes, +attributes" ] ---- -usage: babylonnode docker start [-h] [-f CONFIGFILE] - -This commands starts the docker containers based on what is stored in the -config.yaml file. If you have modified the config file, it is advised to use -setup command. - -options: - -h, --help show this help message and exit - -optional arguments: - -f CONFIGFILE, --configfile CONFIGFILE - Path to config file. This file is generated by running - 'babylonnode docker config'The default value is - `/Users/kim.fehrs/babylon-node-config/config.yaml` if - not provided ---- ==== babylonnode docker stop [source, bash,subs="+quotes, +attributes" ] ---- -usage: babylonnode docker stop [-h] [-f COMPOSEFILE] [-v] - -This commands stops the docker containers - -options: - -h, --help show this help message and exit - -optional arguments: - -f COMPOSEFILE, --composefile COMPOSEFILE - Path to docker-compose file. This file is generated by - running 'babylonnode docker install'The default value - is `/Users/kim.fehrs/docker-compose.yml` if not - provided - -v, --removevolumes Remove the volumes ---- === Radix node CLI command reference Below are the list of commands supported in cli to setup a core node process as a systemd process @@ -178,166 +32,26 @@ Below are the list of commands supported in cli to setup a core node process as ==== babylonnode systemd dependencies [source, bash,subs="+quotes, +attributes" ] ---- -usage: babylonnode systemd dependencies [-h] [-s] - -This commands installs all necessary software on the Virtual Machine(VM). Run -this command on fresh VM or on an existing VM as the command is tested to be -idempotent - -options: - -h, --help show this help message and exit - -optional arguments: - -s, --skip Skip installation of base dependencies ---- ==== babylonnode systemd config [source, bash,subs="+quotes, +attributes" ] ---- -usage: babylonnode systemd config [-h] [-a] [-d CONFIGDIR] - [-dd DATA_DIRECTORY] [-i HOSTIP] - [-k KEYSTOREPASSWORD] -m - {CORE,MIGRATION,GATEWAY} - [{CORE,MIGRATION,GATEWAY} ...] - [-miu MIGRATION_URL] - [-miau MIGRATION_AUTH_USER] - [-miap MIGRATION_AUTH_PASSWORD] - [-miba MIGRATION_BECH_ADDRESS] - [-n NETWORKID] [-nk] [-r RELEASE] - [-t TRUSTEDNODE] [-v VALIDATOR] - [-x NGINXRELEASE] [-xc {true,false}] - -This commands allows node-runners and gateway admins to create a config file, -which can persist their custom settings. Thus, it allows is to decouple the -updates from systemd_config. Config is created only once as such and if there -is a version change in the config file, then it updated by doing a migration -to newer version - -options: - -h, --help show this help message and exit - -required arguments: - -m {CORE,MIGRATION,GATEWAY} [{CORE,MIGRATION,GATEWAY} ...], --setupmode {CORE,MIGRATION,GATEWAY} [{CORE,MIGRATION,GATEWAY} ...] - Quick config mode with assumed defaults. It supports - two quick modes and a detailed config mode. CORE: Use - this value to setup CORE using defaults. GATEWAY: This - mode adds questions regarding the Network Gateway API - and enables it for installation MIGRATION: This mode - adds questions regarding the migration from an Olympia - End-State node to a Babylon node - -optional arguments: - -a, --autoapprove Set this to true to run without any prompts and in - mode CORE.Prompts still appear if you run in DETAILED - mode Use this for automation purpose only - -d CONFIGDIR, --configdir CONFIGDIR - Path to node-config directory where config file will - stored. Default value is /Users/kim.fehrs/babylon- - node-config - -dd DATA_DIRECTORY, --data_directory DATA_DIRECTORY - Folder for data generated by the node - -i HOSTIP, --hostip HOSTIP - Static Public IP of the node - -k KEYSTOREPASSWORD, --keystorepassword KEYSTOREPASSWORD - Core Node requires a keystore. This is the password - for the keystore file the CLI will create new key with - name `node-keystore.ks` in config directory. If the - keystore exists in config directory, CLI finds it , - and uses the password from -k parameter or prompt for - the password to be entered. CLI never modifies the - password of keystore - -miu MIGRATION_URL, --migration_url MIGRATION_URL - The root url of the olympia node to migrate the ledger - from. Do not add /olympia-end-state. - -miau MIGRATION_AUTH_USER, --migration_auth_user MIGRATION_AUTH_USER - The user to authenticate to the olympia node for - migration - -miap MIGRATION_AUTH_PASSWORD, --migration_auth_password MIGRATION_AUTH_PASSWORD - The password to authenticate to the olympia node for - migration - -miba MIGRATION_BECH_ADDRESS, --migration_bech_address MIGRATION_BECH_ADDRESS - The bech address of the olympia node to migrate the - ledger from - -n NETWORKID, --networkid NETWORKID - Network id of network you want to connect.For stokenet - it is 2 and for mainnet it is 1.If not provided you - will be prompted to enter a value - -nk, --newkeystore Set this to true to create a new store without any - prompts using location defined in argument configdir - -r RELEASE, --release RELEASE - Version of node software to install - -t TRUSTEDNODE, --trustednode TRUSTEDNODE - Trusted node on radix network - -v VALIDATOR, --validator VALIDATOR - Address of the validator - -x NGINXRELEASE, --nginxrelease NGINXRELEASE - Version of radixdlt nginx release - -xc {true,false}, --disablenginxforcore {true,false} - Core Node API's are protected by Basic auth - setting.Set this to disable to nginx for core ---- ==== babylonnode systemd install [source, bash,subs="+quotes, +attributes" ] ---- -usage: babylonnode systemd install [-h] [-a] [-u] [-f CONFIGFILE] - [-auc ADVANCEDUSERCONFIG] [-m] - -This sets up the systemd service for the core node. - -options: - -h, --help show this help message and exit - -optional arguments: - -a, --auto Automatically approve all Yes/No prompts - -u, --update Update the node to new version of node - -f CONFIGFILE, --configfile CONFIGFILE - Path to config file. This file is generated by running - 'babylonnode systemd config'The default value is - `/Users/kim.fehrs/babylon-node-config/config.yaml` if - not provided - -auc ADVANCEDUSERCONFIG, --advanceduserconfig ADVANCEDUSERCONFIG - Path to advanced config file. This file can directly - configure the core node. It is templated into - default.config and acts as custom configuration that - is not overwritten on install. The default value is - `/Users/kim.fehrs/babylon-node-config/advanced- - user.default.config` if not provided - -m, --manual Only generate systemd file but not put it into systemd - folder.This is mainly used for automation in - unprivileged environments. ---- ==== babylonnode systemd restart [source, bash,subs="+quotes, +attributes" ] ---- -usage: babylonnode systemd restart [-h] [-s {all,nginx,radixdlt-node}] - -This restarts the CORE node systemd service. - -options: - -h, --help show this help message and exit - -optional arguments: - -s {all,nginx,radixdlt-node}, --services {all,nginx,radixdlt-node} - Name of the service either to be started. Valid values - nginx or radixdlt-node ---- ==== babylonnode systemd stop [source, bash,subs="+quotes, +attributes" ] ---- -usage: babylonnode systemd stop [-h] [-s {all,nginx,radixdlt-node}] - -This stops the CORE node systemd service. - -options: - -h, --help show this help message and exit - -optional arguments: - -s {all,nginx,radixdlt-node}, --services {all,nginx,radixdlt-node} - Name of the service either to be stopped. Valid values - nginx or radixdlt-node ---- === Ledger sync using an external S3 bucket Below are the list of commands that can be used with cli to sync the ledger with a S3 bucket. @@ -345,26 +59,6 @@ Below are the list of commands that can be used with cli to sync the ledger with ==== babylonnode ledger sync [source, bash,subs="+quotes, +attributes" ] ---- -usage: babylonnode ledger sync [-h] -d DEST -bn BUCKETNAME [-bf BUCKETFOLDER] - -This commands allows to download the content of an external S3 bucket to the -ledger folder. The ledger folder and the name and folder of an external S3 -bucket should be indicated - -options: - -h, --help show this help message and exit - -required arguments: - -d DEST, --dest DEST Destination path where the backup of the ledger will - be downloaded - -bn BUCKETNAME, --bucketname BUCKETNAME - S3 bucket name to download the backup of the ledger - from - -optional arguments: - -bf BUCKETFOLDER, --bucketfolder BUCKETFOLDER - S3 bucket folder to download the backup of the ledger - from ---- === Core node setup using systemd Below are the list of commands supported in cli to setup a core node process as a systemd process @@ -372,80 +66,21 @@ Below are the list of commands supported in cli to setup a core node process as ==== babylonnode systemd dependencies [source, bash,subs="+quotes, +attributes" ] ---- -usage: babylonnode systemd dependencies [-h] [-s] - -This commands installs all necessary software on the Virtual Machine(VM). Run -this command on fresh VM or on an existing VM as the command is tested to be -idempotent - -options: - -h, --help show this help message and exit - -optional arguments: - -s, --skip Skip installation of base dependencies ---- ==== babylonnode systemd install [source, bash,subs="+quotes, +attributes" ] ---- -usage: babylonnode systemd install [-h] [-a] [-u] [-f CONFIGFILE] - [-auc ADVANCEDUSERCONFIG] [-m] - -This sets up the systemd service for the core node. - -options: - -h, --help show this help message and exit - -optional arguments: - -a, --auto Automatically approve all Yes/No prompts - -u, --update Update the node to new version of node - -f CONFIGFILE, --configfile CONFIGFILE - Path to config file. This file is generated by running - 'babylonnode systemd config'The default value is - `/Users/kim.fehrs/babylon-node-config/config.yaml` if - not provided - -auc ADVANCEDUSERCONFIG, --advanceduserconfig ADVANCEDUSERCONFIG - Path to advanced config file. This file can directly - configure the core node. It is templated into - default.config and acts as custom configuration that - is not overwritten on install. The default value is - `/Users/kim.fehrs/babylon-node-config/advanced- - user.default.config` if not provided - -m, --manual Only generate systemd file but not put it into systemd - folder.This is mainly used for automation in - unprivileged environments. ---- ==== babylonnode systemd restart [source, bash,subs="+quotes, +attributes" ] ---- -usage: babylonnode systemd restart [-h] [-s {all,nginx,radixdlt-node}] - -This restarts the CORE node systemd service. - -options: - -h, --help show this help message and exit - -optional arguments: - -s {all,nginx,radixdlt-node}, --services {all,nginx,radixdlt-node} - Name of the service either to be started. Valid values - nginx or radixdlt-node ---- ==== babylonnode systemd stop [source, bash,subs="+quotes, +attributes" ] ---- -usage: babylonnode systemd stop [-h] [-s {all,nginx,radixdlt-node}] - -This stops the CORE node systemd service. - -options: - -h, --help show this help message and exit - -optional arguments: - -s {all,nginx,radixdlt-node}, --services {all,nginx,radixdlt-node} - Name of the service either to be stopped. Valid values - nginx or radixdlt-node ---- === Set passwords for the Nginx server This will set up the admin user and password for access to the general system endpoints. @@ -453,97 +88,21 @@ This will set up the admin user and password for access to the general system en ==== babylonnode auth set-admin-password [source, bash,subs="+quotes, +attributes" ] ---- -usage: babylonnode auth set-admin-password [-h] -m {DOCKER,SYSTEMD} - [-u USERNAME] [-p PASSWORD] - -This sets up admin password on nginx basic auth. Refer this link for all the -paths. https://docs.radixdlt.com/main/node-and-gateway/port- -reference.html#_endpoint_usage - -options: - -h, --help show this help message and exit - -required arguments: - -m {DOCKER,SYSTEMD}, --setupmode {DOCKER,SYSTEMD} - Setup type whether it is DOCKER or SYSTEMD - -optional arguments: - -u USERNAME, --username USERNAME - Name of admin user. Default value is `admin` - -p PASSWORD, --password PASSWORD - Password of admin user ---- ==== babylonnode auth set-superadmin-password [source, bash,subs="+quotes, +attributes" ] ---- -usage: babylonnode auth set-superadmin-password [-h] -m {DOCKER,SYSTEMD} - [-u USERNAME] [-p PASSWORD] - -This sets up superadmin password on nginx basic auth. Refer this link for all -the paths. https://docs.radixdlt.com/main/node-and-gateway/port- -reference.html#_endpoint_usage - -options: - -h, --help show this help message and exit - -required arguments: - -m {DOCKER,SYSTEMD}, --setupmode {DOCKER,SYSTEMD} - Setup type whether it is DOCKER or SYSTEMD - -optional arguments: - -u USERNAME, --username USERNAME - Name of superadmin user. Default value is `superadmin` - -p PASSWORD, --password PASSWORD - Password of superadmin user ---- ==== babylonnode auth set-metrics-password [source, bash,subs="+quotes, +attributes" ] ---- -usage: babylonnode auth set-metrics-password [-h] -m {DOCKER,SYSTEMD} - [-u USERNAME] [-p PASSWORD] - -This sets up metrics password on nginx basic auth. Refer this link for all the -paths. https://docs.radixdlt.com/main/node-and-gateway/port- -reference.html#_endpoint_usage - -options: - -h, --help show this help message and exit - -required arguments: - -m {DOCKER,SYSTEMD}, --setupmode {DOCKER,SYSTEMD} - Setup type whether it is DOCKER or SYSTEMD - -optional arguments: - -u USERNAME, --username USERNAME - Name of metrics user. Default value is `metrics` - -p PASSWORD, --password PASSWORD - Password of metrics user ---- ==== babylonnode auth set-gateway-password [source, bash,subs="+quotes, +attributes" ] ---- -usage: babylonnode auth set-gateway-password [-h] -m {DOCKER,SYSTEMD} - [-u USERNAME] [-p PASSWORD] - -This sets up gateway password on nginx basic auth. Refer this link for all the -paths. https://docs.radixdlt.com/main/node-and-gateway/port- -reference.html#_endpoint_usage - -options: - -h, --help show this help message and exit - -required arguments: - -m {DOCKER,SYSTEMD}, --setupmode {DOCKER,SYSTEMD} - Setup type whether it is DOCKER or SYSTEMD - -optional arguments: - -u USERNAME, --username USERNAME - Name of gateway user. Default value is `gateway` - -p PASSWORD, --password PASSWORD - Password of gateway user ---- === Accessing core endpoints using CLI Once the nginx basic auth passwords for admin, superadmin, metrics users are setup , babylonnode cli can be used to access the node endpoints @@ -551,81 +110,36 @@ Once the nginx basic auth passwords for admin, superadmin, metrics users are set ==== babylonnode system health [source, bash,subs="+quotes, +attributes" ] ---- -usage: babylonnode api system health [-h] - -This command displays the health of the node on whether it is syncing, or -booting or up - -options: - -h, --help show this help message and exit ---- ==== babylonnode system version [source, bash,subs="+quotes, +attributes" ] ---- -usage: babylonnode api system version [-h] - -This command displays the version of node software that is currently running - -options: - -h, --help show this help message and exit ---- ==== babylonnode system configuration [source, bash,subs="+quotes, +attributes" ] ---- -usage: babylonnode api system configuration [-h] - -This command displays the configuration of the node - -options: - -h, --help show this help message and exit ---- ==== babylonnode system peers [source, bash,subs="+quotes, +attributes" ] ---- -usage: babylonnode api system peers [-h] - -This command displays peers that node sees on the network - -options: - -h, --help show this help message and exit ---- ==== babylonnode system addressbook [source, bash,subs="+quotes, +attributes" ] ---- -usage: babylonnode api system addressbook [-h] - -This command displays address book on the data the node has stored - -options: - -h, --help show this help message and exit ---- ==== babylonnode system network-sync-status [source, bash,subs="+quotes, +attributes" ] ---- -usage: babylonnode api system network-sync-status [-h] - -This command displays information on the status with respect to syncing to -network. - -options: - -h, --help show this help message and exit ---- ==== babylonnode system identity [source, bash,subs="+quotes, +attributes" ] ---- -usage: babylonnode api system identity [-h] - -This command displays information on the status with respect to syncing to -network. - -options: - -h, --help show this help message and exit ---- === Setup monitoring using CLI Using CLI , one can setup monitoring of the node or gateway. @@ -633,115 +147,26 @@ Using CLI , one can setup monitoring of the node or gateway. ==== babylonnode monitoring config [source, bash,subs="+quotes, +attributes" ] ---- -usage: babylonnode monitoring config [-h] - [-m {MONITOR_CORE,MONITOR_GATEWAY,DETAILED} [{MONITOR_CORE,MONITOR_GATEWAY,DETAILED} ...]] - [-cm COREMETRICSPASSWORD] - [-gm GATEWAYAPIMETRICSPASSWORD] - [-am AGGREGATORMETRICSPASSWORD] - [-d MONITORINGCONFIGDIR] - -This commands allows to create a config file, which can persist custom -settings for monitoring. Thus it allows is to decouple the updates from -configuration. Config is created only once as such and if there is a version -change in the config file, then it updated by doing a migration to newer -version - -options: - -h, --help show this help message and exit - -optional arguments: - -m {MONITOR_CORE,MONITOR_GATEWAY,DETAILED} [{MONITOR_CORE,MONITOR_GATEWAY,DETAILED} ...], --setupmode {MONITOR_CORE,MONITOR_GATEWAY,DETAILED} [{MONITOR_CORE,MONITOR_GATEWAY,DETAILED} ...] - Quick setup with assumed defaults. It supports three - quick setup mode and a detailed setup mode. - MONITOR_CORE: Use this value to monitor Core using - defaults which assume core is run on same machine as - monitoring. MONITOR_GATEWAY: Use this value to monitor - GATEWAY using defaults which assume network gateway is - run on same machine. DETAILED: Default value if not - provided. This mode takes your through series of - questions. - -cm COREMETRICSPASSWORD, --coremetricspassword COREMETRICSPASSWORD - Password for core metrics basic auth user - -gm GATEWAYAPIMETRICSPASSWORD, --gatewayapimetricspassword GATEWAYAPIMETRICSPASSWORD - Password for gateway api metrics basic auth user - -am AGGREGATORMETRICSPASSWORD, --aggregatormetricspassword AGGREGATORMETRICSPASSWORD - Password for aggregator metrics basic auth user - -d MONITORINGCONFIGDIR, --monitoringconfigdir MONITORINGCONFIGDIR - Path to monitoring directory where config file will - stored ---- ==== babylonnode monitoring install [source, bash,subs="+quotes, +attributes" ] ---- -usage: babylonnode monitoring install [-h] [-f MONITORINGCONFIGFILE] [-a] - -This commands setups up the software and deploys it based on what is stored in -the config.yaml file. To update software versions, most of the time it is -required to update the versions in config file and run this command - -options: - -h, --help show this help message and exit - -optional arguments: - -f MONITORINGCONFIGFILE, --monitoringconfigfile MONITORINGCONFIGFILE - Path to config file. Default is - '/Users/kim.fehrs/monitoring/monitoring_config.yaml' - -a, --autoapprove Set this to true to run without any prompts ---- ==== babylonnode monitoring start [source, bash,subs="+quotes, +attributes" ] ---- -usage: babylonnode monitoring start [-h] [-f MONITORINGCONFIGFILE] [-a] - -This commands starts the docker containers based on what is stored in the -config.yaml file. If you have modified the config file, it is advised to use -setup command. - -options: - -h, --help show this help message and exit - -optional arguments: - -f MONITORINGCONFIGFILE, --monitoringconfigfile MONITORINGCONFIGFILE - Path to config file. Default is - '/Users/kim.fehrs/monitoring/monitoring_config.yaml' - -a, --autoapprove Set this to true to run without any prompts ---- ==== babylonnode monitoring stop [source, bash,subs="+quotes, +attributes" ] ---- -usage: babylonnode monitoring stop [-h] [-f MONITORINGCONFIGFILE] [-v] - -This commands stops the docker containers - -options: - -h, --help show this help message and exit - -optional arguments: - -f MONITORINGCONFIGFILE, --monitoringconfigfile MONITORINGCONFIGFILE - Path to config file. Default is - '/Users/kim.fehrs/monitoring/monitoring_config.yaml' - -v, --removevolumes Remove the volumes ---- ==== babylonnode key info [source, bash,subs="+quotes, +attributes" ] ---- -usage: babylonnode key info [-h] -p PASSWORD -f FILELOCATION - -Using CLI, for a key file, you can print out the validator address. This -feature is in beta. - -options: - -h, --help show this help message and exit - -required arguments: - -p PASSWORD, --password PASSWORD - Password of the keystore - -f FILELOCATION, --filelocation FILELOCATION - Location of keystore on the disk ---- === Ret commands supported by CLI You can derive babylon addresses from both private key and olympia address using this cli. @@ -749,37 +174,11 @@ You can derive babylon addresses from both private key and olympia address using ==== babylonnode ret derive [source, bash,subs="+quotes, +attributes" ] ---- -usage: babylonnode ret derive [-h] -k KEYSTORE -p PASSWORD -n NETWORK - -Derive a babylon address from a private key. - -options: - -h, --help show this help message and exit - -required arguments: - -k KEYSTORE, --keystore KEYSTORE - Keystore path - -p PASSWORD, --password PASSWORD - Keystore password - -n NETWORK, --network NETWORK - Network id ---- ==== babylonnode ret derive-from-olympia [source, bash,subs="+quotes, +attributes" ] ---- -usage: babylonnode ret derive-from-olympia [-h] -oa OLYMPIA_ADDRESS -n NETWORK - -Derive a babylon address from a private key. - -options: - -h, --help show this help message and exit - -required arguments: - -oa OLYMPIA_ADDRESS, --olympia-address OLYMPIA_ADDRESS - Olympia address - -n NETWORK, --network NETWORK - Network id. Int format ---- === Other commands supported by CLI List of other commands supported by cli are to check the version of CLI being used and optimise-node @@ -788,28 +187,9 @@ to setup some of the OS tweaks on ubuntu ==== babylonnode version [source, bash,subs="+quotes, +attributes" ] ---- -usage: babylonnode.py version [-h] - -Run this command td display the version of CLI been used. - -options: - -h, --help show this help message and exit ---- ==== babylonnode optimise-node [source, bash,subs="+quotes, +attributes" ] ---- -usage: babylonnode.py optimise-node [-h] [-u] [-s] [-ss {1G,3G,8G}] - -Run this command to setup ulimits and swap size on the fresh ubuntu machine . -Prompts asking to setup limits . Prompts asking to setup swap and size of swap -in GB - -options: - -h, --help show this help message and exit - -optional arguments: - -u, --setup_ulimit - -s, --setup_swap - -ss {1G,3G,8G}, --swap_space {1G,3G,8G} ---- diff --git a/node-runner-cli/Pipfile b/node-runner-cli/Pipfile index cb4a9a59..828eaf93 100644 --- a/node-runner-cli/Pipfile +++ b/node-runner-cli/Pipfile @@ -19,6 +19,7 @@ pytest = "==8.1.1" radix-engine-toolkit = "==0.12.1.dev3" cryptography = "==41.0.4" pyinstaller-hooks-contrib = "==2023.12" +zstandard= "==0.15.2" [requires] python_version = "3.10.6" diff --git a/node-runner-cli/babylonnode.py b/node-runner-cli/babylonnode.py index 590457d3..eebd695c 100755 --- a/node-runner-cli/babylonnode.py +++ b/node-runner-cli/babylonnode.py @@ -120,11 +120,7 @@ def main(): if ledgercli_args.ledgercommand is None: ledgercli.print_help() else: - if ledgercli_args.ledgercommand == "sync": - logger.info(f"Syncing fullnode ledger {sys.argv[3:]}") - ledgercli_args.func(ledgercli_args) - else: - logger.info(f"Invalid ledger command {ledgercli_args.ledgercommand}") + ledgercli_args.func(ledgercli_args) elif args.subcommand == "monitoring": monitoringcli_args = monitoringcli.parse_args(sys.argv[2:]) diff --git a/node-runner-cli/commands/dockercommand.py b/node-runner-cli/commands/dockercommand.py index 80e333c4..414dca2f 100644 --- a/node-runner-cli/commands/dockercommand.py +++ b/node-runner-cli/commands/dockercommand.py @@ -237,6 +237,7 @@ def install(args): This commands setups up the software and deploys it based on what is stored in the config.yaml file. To update software versions, most of the time it is required to update the versions in config file and run this command """ + ########## Parse Arguments argument_object = DockerInstallArguments(args) diff --git a/node-runner-cli/commands/ledgercommand.py b/node-runner-cli/commands/ledgercommand.py index dd8d32e1..fc75ef2f 100644 --- a/node-runner-cli/commands/ledgercommand.py +++ b/node-runner-cli/commands/ledgercommand.py @@ -1,13 +1,14 @@ from argparse import ArgumentParser from argparse import RawTextHelpFormatter - +from ledger_snapshot import download_and_extract_snapshot +from utils.utils import Helpers import boto3 import botocore from commands.subcommand import get_decorator, argument ledgercli = ArgumentParser( - description="Subcommand to help to sync up the ledger from a S3 bucket", + description="Subcommand to assist with ledger download", usage="babylonnode ledger ", formatter_class=RawTextHelpFormatter, ) @@ -43,11 +44,21 @@ def ledgercommand(ledgercommand_args=[], parent=ledger_parser): ), ] ) -def sync(args): +def s3_download(args): """ - This commands allows to download the content of an external S3 bucket to the ledger folder. - The ledger folder and the name and folder of an external S3 bucket should be indicated + Downloads a backuped ledger from an S3 bucket. + Args: + args: An object containing the following attributes: + - bucketname (str): The name of the S3 bucket. + - bucketfolder (str): The folder within the S3 bucket. + - dest (str): The destination path where the backup ledger will be downloaded. + Raises: + ValueError: If the bucket name is not provided. + Example: + args = Namespace(bucketname='my-bucket', bucketfolder='backups', dest='/local/path') + s3_download(args) """ + bucketName = args.bucketname bucketFolder = args.bucketfolder dest = args.dest @@ -55,12 +66,10 @@ def sync(args): print("No S3 bucket was indicated") else: print(f"Downloading backup ledger {bucketName} {bucketFolder} ...") - download_mainnet_backup_ledger(bucketName, bucketFolder, dest) + s3_fetch_ledger_files(bucketName, bucketFolder, dest) -def download_mainnet_backup_ledger( - bucketName: str, bucketFolder: str, destinationPath: str -): +def s3_fetch_ledger_files(bucketName: str, bucketFolder: str, destinationPath: str): # """ # Downloads validator or fullnode mainnet ledger into destinationPath # :param bucketName: Indicates the name of the s3 bucket that contains the backup ledger to download @@ -99,3 +108,37 @@ def download_mainnet_backup_ledger( error.response["Error"]["Message"] ) # explanation of what went wrong return False + + +@ledgercommand( + [ + argument( + "-d", + "--dest", + help="Destination path where the backup of the ledger will be downloaded ", + action="store", + default=f"{Helpers.get_default_ledger_dir()}", + ), + argument( + "-s", + "--source", + default="radix.live", + choices=["radix.live"], + action="store", + help="Source to download the ledger from. Radix.live is the only supported source at the moment.", + ), + ] +) +def fetch_community_snapshot(args): + """ + Downloads the latest community snapshot of the ledger. + Args: + args: An object containing the following attributes: + - dest (str): The destination path where the backup ledger will be downloaded. + Raises: + ValueError: If the source is not provided. + Example: + args = Namespace(dest='/local/path') + download_and_extract_snapshot(args) + """ + download_and_extract_snapshot(args.dest) diff --git a/node-runner-cli/config/CoreDockerConfig.py b/node-runner-cli/config/CoreDockerConfig.py index 6d77aa99..b4e678fc 100644 --- a/node-runner-cli/config/CoreDockerConfig.py +++ b/node-runner-cli/config/CoreDockerConfig.py @@ -18,7 +18,7 @@ def __init__(self, config_dict: dict): self.keydetails: KeyDetails = KeyDetails(config_dict.get("keydetails")) self.core_release: str = "" self.repo: str = os.getenv(CORE_DOCKER_REPO_OVERRIDE, "radixdlt/babylon-node") - self.data_directory: str = f"{Helpers.get_home_dir()}/babylon-ledger" + self.data_directory: str = f"{Helpers.get_default_ledger_dir()}" self.trusted_node: str = "" self.memory_limit: str = "14000m" self.validator_address: str = "" diff --git a/node-runner-cli/config/CoreSystemDConfig.py b/node-runner-cli/config/CoreSystemDConfig.py index ccf988ad..58f94cbc 100644 --- a/node-runner-cli/config/CoreSystemDConfig.py +++ b/node-runner-cli/config/CoreSystemDConfig.py @@ -19,7 +19,7 @@ def __init__(self, config_dict: dict): self.core_release: str = "" self.core_binary_url: str = "" self.core_library_url: str = "" - self.data_directory: str = f"{Helpers.get_home_dir()}/babylon-ledger" + self.data_directory: str = f"{Helpers.get_default_ledger_dir()}" self.trusted_node: str = "" self.node_dir: str = "/etc/radixdlt/node" self.node_secrets_dir: str = "/etc/radixdlt/node/secrets" diff --git a/node-runner-cli/ledger_snapshot/__init__.py b/node-runner-cli/ledger_snapshot/__init__.py new file mode 100644 index 00000000..f6658807 --- /dev/null +++ b/node-runner-cli/ledger_snapshot/__init__.py @@ -0,0 +1,81 @@ +import os +import subprocess +import tarfile +import zstandard as zstd +import requests +import time + + +def download_and_extract_snapshot(data_dir): + if not os.path.isdir(data_dir): + print(f"Creating {data_dir} directory...") + os.makedirs(data_dir) + + if not os.listdir(data_dir): + os.chdir(data_dir) + print(f"Directory {data_dir} is empty. Downloading LedgerSnapshot...") + radix_live_download() + else: + print( + f"Directory {data_dir} is not empty. Downloading Ledger Snapshot aborted:" + ) + for item in os.listdir(data_dir): + print(item) + + +def radix_live_download(): + print("Fetching and executing the latest snapshot script from Radix...") + if not os.path.isfile("RADIXDB-INDEX.tar.zst"): + # refactor to use requests + url = "https://snapshots.radix.live/latest-snapshot-INDEX.sh" + response = requests.get(url, verify=False) + if response.status_code == 200: + with open("latest-snapshot-INDEX.sh", "wb") as file: + file.write(response.content) + else: + print( + f"Failed to download the snapshot script. Status code: {response.status_code}" + ) + exit(1) + print("Starting snapshot download") + + result = subprocess.run(["bash", "latest-snapshot-INDEX.sh"]) + if result.returncode == 0: + print("Snapshot download and execution completed successfully.") + else: + print("Snapshot download or execution failed.") + exit(1) + else: + print("RADIXDB-INDEX.tar.zst already exists, skipping download.") + + print("Extracting snapshot...") + dctx = zstd.ZstdDecompressor() + download_start_time = time.time() + with open("RADIXDB-INDEX.tar.zst", "rb") as compressed_file: + start_time = time.time() + total_extracted_size = 0 + + with dctx.stream_reader(compressed_file) as reader: + with tarfile.open(fileobj=reader, mode="r|") as tar: + extracted_members = 0 + for member in tar: + if member.name.startswith("./address_book"): + continue + member_path = os.path.join(".", member.name) + if not os.path.exists(member_path): + tar.extract(member, path=".") + extracted_members += 1 + extracted_size = os.path.getsize(member_path) + total_extracted_size += extracted_size + # print(f"Extracted {member.name} with size {extracted_size} bytes.") + if time.time() - start_time >= 180: # 3 minutes + print( + f"{total_extracted_size / (1024 * 1024):.2f} MB extracted so far." + ) + start_time = time.time() + + os.remove("RADIXDB-INDEX.tar.zst") + os.remove("latest-snapshot-INDEX.sh") + print( + f"Snapshot extracted successfully in {time.time() - download_start_time:.2f} seconds." + ) diff --git a/node-runner-cli/ledger_snapshot/radix_live_snapshot.sh b/node-runner-cli/ledger_snapshot/radix_live_snapshot.sh new file mode 100644 index 00000000..5b217369 --- /dev/null +++ b/node-runner-cli/ledger_snapshot/radix_live_snapshot.sh @@ -0,0 +1,42 @@ + +apt-get update && apt-get install -y \ + wget \ + zstd \ + bash \ + build-essential \ + curl \ + bash \ + aria2 \ + zstd \ + && rm -rf /var/lib/apt/lists/* + # Create the /usr/local/scripts directory +mkdir -p /usr/local/scripts + RUN echo '#!/bin/bash + DATA_DIR=$${1:-"/data"} + if [ ! -d "$$DATA_DIR" ]; then + echo "Creating $$DATA_DIR directory..." + mkdir -p "\$$DATA_DIR" + fi + + if [ -z "$(ls -A "$$DATA_DIR")" ]; then + cd $$DATA_DIR + echo "Directory $$DATA_DIR is empty. Downloading LedgerSnapshot..." + echo "Fetching and executing the latest snapshot script from Radix..." + wget https://snapshots.radix.live/latest-snapshot-INDEX.sh --no-check-certificate + echo "Starting snapshot download" + bash latest-snapshot-INDEX.sh + + if [ $? -eq 0 ]; then + echo "Snapshot download and execution completed successfully." + else + echo "Snapshot download or execution failed." + exit 1 + fi + tar --use-compress-program=zstdmt -xvf RADIXDB-INDEX.tar.zst --exclude=./address_book -C . + rm -rf RADIXDB-INDEX.* + rm -rf latest-snapshot-INDEX.sh* + echo "Snapshot restored" + else + echo "Directory $$DATA_DIR is not empty. Downloading Ledger Snapshot aborted:" + ls -l "$$DATA_DIR" + fi \ No newline at end of file diff --git a/node-runner-cli/setup/BaseSetup.py b/node-runner-cli/setup/BaseSetup.py index 1bde8064..63336eaf 100644 --- a/node-runner-cli/setup/BaseSetup.py +++ b/node-runner-cli/setup/BaseSetup.py @@ -29,7 +29,7 @@ def dependencies(install_docker=True): BaseSetup.add_user_docker_group() logger.info("Docker successfully installed") run_shell_command( - "sudo apt install -y wget unzip rng-tools ansible", shell=True + "sudo apt install -y wget unzip rng-tools ansible zstd", shell=True ) run_shell_command("sudo rngd -r /dev/random | true", shell=True) try: @@ -179,12 +179,12 @@ def get_data_dir(create_dir=True): f"\nRadix node stores all the ledger data on a folder. " f"Mounting this location as a docker volume, " f"will allow to restart the node without a need to download the ledger." - f'\n{bcolors.WARNING}Press Enter to store ledger in the "{Helpers.get_home_dir()}/babylon-ledger" directory OR ' + f'\n{bcolors.WARNING}Press Enter to store ledger in the "{Helpers.get_default_ledger_dir()}" directory OR ' f"type the absolute path of an existing ledger data folder:{bcolors.ENDC}", QuestionKeys.input_ledger_path, ) if data_dir_path == "": - data_dir_path = f"{Helpers.get_home_dir()}/babylon-ledger" + data_dir_path = f"{Helpers.get_default_ledger_dir()}" if create_dir: run_shell_command(f"sudo mkdir -p {data_dir_path}", shell=True) return data_dir_path diff --git a/node-runner-cli/setup/DockerSetup.py b/node-runner-cli/setup/DockerSetup.py index 6201dc04..5a5a2280 100644 --- a/node-runner-cli/setup/DockerSetup.py +++ b/node-runner-cli/setup/DockerSetup.py @@ -35,7 +35,6 @@ def print_questionary_header(config_file): class DockerSetup(BaseSetup): - @staticmethod def save_config(config: DockerConfig, config_file: str, autoapprove=False): to_update = "" diff --git a/node-runner-cli/templates/radix-fullnode-compose.yml.j2 b/node-runner-cli/templates/radix-fullnode-compose.yml.j2 index 1ddaad4a..37cfb599 100644 --- a/node-runner-cli/templates/radix-fullnode-compose.yml.j2 +++ b/node-runner-cli/templates/radix-fullnode-compose.yml.j2 @@ -12,7 +12,7 @@ services: {% if core_node.advanced_user_envs is defined and core_node.advanced_user_envs is not none %} env_file: - {{core_node.advanced_user_envs}} - {% endif %} + {% endif %} environment: JAVA_OPTS: {{core_node.java_opts or '--enable-preview -server -Xms12g -Xmx12g -XX:MaxDirectMemorySize=2048m -XX:+HeapDumpOnOutOfMemoryError -XX:+UseCompressedOops -Djavax.net.ssl.trustStore=/etc/ssl/certs/java/cacerts -Djavax.net.ssl.trustStoreType=jks -Djava.security.egd=file:/dev/urandom -DLog4jContextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector'}} RADIXDLT_CORE_API_PORT: {{core_node.core_api_port}} diff --git a/node-runner-cli/templates/systemd-default.config.j2 b/node-runner-cli/templates/systemd-default.config.j2 index ddb405f1..d06d18e7 100644 --- a/node-runner-cli/templates/systemd-default.config.j2 +++ b/node-runner-cli/templates/systemd-default.config.j2 @@ -5,7 +5,6 @@ ntp=false ntp.pool=pool.ntp.org - network.id={{common_config.network_id}} {% if common_config.genesis_bin_data_file is not none %} network.genesis_data_file={{common_config.genesis_bin_data_file}} diff --git a/node-runner-cli/tests/unit/test_prompt_feeder_local.py b/node-runner-cli/tests/unit/test_prompt_feeder_local.py index 9d405bbe..ad7c7949 100644 --- a/node-runner-cli/tests/unit/test_prompt_feeder_local.py +++ b/node-runner-cli/tests/unit/test_prompt_feeder_local.py @@ -98,6 +98,7 @@ def test_docker_config_all_local(self, mockout): PROMPT_KEYSTORE_FILE_NAME = "node-keystore.ks" PROMPT_LEDGER_LOCATION = "/tmp/data" PROMPT_NGINX_PROTECT_CORE_API = "true" + PROMPT_NGINX_PROTECT_CORE_API = "true" PROMPT_ENABLE_GATEWAY = "Y" PROMPT_GATEWAY_CORE_API_URL = "http://core:3333/core" PROMPT_GATEWAY_CORE_API_NAME = "core" diff --git a/node-runner-cli/tests/unit/test_systemd.py b/node-runner-cli/tests/unit/test_systemd.py index 6ae16e40..0de90af3 100644 --- a/node-runner-cli/tests/unit/test_systemd.py +++ b/node-runner-cli/tests/unit/test_systemd.py @@ -33,9 +33,9 @@ def file_contains_regular_expression(re_str: str, file: str) -> bool: class SystemdUnitTests(unittest.TestCase): @unittest.skip("Tests with PROMPT_FEEDS can only be run individually") def test_systemd_install_continue_prompt_feed(self): - os.environ["PROMPT_FEEDS"] = ( - "test-prompts/individual-prompts/systemd_install_continue.yml" - ) + os.environ[ + "PROMPT_FEEDS" + ] = "test-prompts/individual-prompts/systemd_install_continue.yml" PromptFeeder.instance().load_prompt_feeds() SystemDSetup.confirm_config("dummy1", "dummy2", "dummy3", "dummy4") @@ -182,7 +182,6 @@ def test_systemd_setup_default_config(self, mockout): ntp=false ntp.pool=pool.ntp.org - network.id=1 node.key.path=/home/radixdlt/babylon-node-config/node-keystore.ks @@ -236,7 +235,6 @@ def test_systemd_setup_default_config_without_validator(self, mockout): ntp=false ntp.pool=pool.ntp.org - network.id=1 node.key.path=/home/radixdlt/babylon-node-config/node-keystore.ks @@ -290,7 +288,6 @@ def test_systemd_setup_default_config_jinja(self, mockout): ntp=false ntp.pool=pool.ntp.org - network.id=1 node.key.path=/home/radixdlt/babylon-node-config/node-keystore.ks diff --git a/node-runner-cli/tests/unit/test_validator.py b/node-runner-cli/tests/unit/test_validator.py index 93f91939..714afa83 100644 --- a/node-runner-cli/tests/unit/test_validator.py +++ b/node-runner-cli/tests/unit/test_validator.py @@ -84,9 +84,9 @@ def test_validator_address_included_in_dict_from_object(self): @mock.patch("sys.stdout", new_callable=StringIO) def test_validator_promptfeed(self, mock_out): - os.environ["PROMPT_FEEDS"] = ( - "test-prompts/individual-prompts/validator_address.yml" - ) + os.environ[ + "PROMPT_FEEDS" + ] = "test-prompts/individual-prompts/validator_address.yml" PromptFeeder.prompts_feed = PromptFeeder.instance().load_prompt_feeds() address = Prompts.ask_validator_address() self.assertEqual("validator_mock", address) diff --git a/node-runner-cli/utils/utils.py b/node-runner-cli/utils/utils.py index b9e24480..2644f65c 100644 --- a/node-runner-cli/utils/utils.py +++ b/node-runner-cli/utils/utils.py @@ -319,6 +319,10 @@ def get_home_dir(): def get_default_node_config_dir(): return f"{Path.home()}/babylon-node-config" + @staticmethod + def get_default_ledger_dir(): + return f"{Path.home()}/babylon-ledger" + @staticmethod def get_default_monitoring_config_dir(): return f"{Path.home()}/monitoring"