From 10473e4c0711c455ac0beeb143c88b67a06f6023 Mon Sep 17 00:00:00 2001 From: Mikolaj Kucharski Date: Tue, 17 Jun 2025 18:04:36 +0000 Subject: [PATCH 001/198] Fix check_ssl_connection() function Function doesn't wait for `openssl s_client ...` to finish. It assumes that when the command is still running that is the successful condition. However the function should wait for exit code from the binary. We saw in production intermittent and very often `skale ssl upload` failures. This change should fix this problem and underlying race condition. --- node_cli/core/ssl/check.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/node_cli/core/ssl/check.py b/node_cli/core/ssl/check.py index d512c777..3c8593bc 100644 --- a/node_cli/core/ssl/check.py +++ b/node_cli/core/ssl/check.py @@ -201,8 +201,15 @@ def check_ssl_connection(host, port, silent=False): ] expose_output = not silent with detached_subprocess(ssl_check_cmd, expose_output=expose_output) as dp: - time.sleep(1) - code = dp.poll() - if code is not None: + for _ in range(10): + code = dp.poll() + if code is None: + logger.info('Healthcheck process still running...') + time.sleep(2) + continue + elif code == 0: + return logger.error('Healthcheck connection failed') raise SSLHealthcheckError('OpenSSL connection verification failed') + logger.error('Healthcheck timed-out') + raise SSLHealthcheckError('OpenSSL connection verification timed-out') From 533044c08f1116a2822de0233ecc522aeb933d52 Mon Sep 17 00:00:00 2001 From: Mikolaj Kucharski Date: Mon, 23 Jun 2025 18:55:53 +0000 Subject: [PATCH 002/198] Move to dp.wait() in check_ssl_connection() Replace for loop and dp.poll() with more straightforward dp.wait() with a timeout, as requested during diff review. --- node_cli/core/ssl/check.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/node_cli/core/ssl/check.py b/node_cli/core/ssl/check.py index 3c8593bc..a5c7a0a5 100644 --- a/node_cli/core/ssl/check.py +++ b/node_cli/core/ssl/check.py @@ -20,6 +20,7 @@ import time import socket import logging +import subprocess from contextlib import contextmanager from node_cli.core.ssl.utils import detached_subprocess @@ -201,15 +202,15 @@ def check_ssl_connection(host, port, silent=False): ] expose_output = not silent with detached_subprocess(ssl_check_cmd, expose_output=expose_output) as dp: - for _ in range(10): - code = dp.poll() - if code is None: - logger.info('Healthcheck process still running...') - time.sleep(2) - continue - elif code == 0: - return - logger.error('Healthcheck connection failed') - raise SSLHealthcheckError('OpenSSL connection verification failed') - logger.error('Healthcheck timed-out') - raise SSLHealthcheckError('OpenSSL connection verification timed-out') + timeout = 20 + try: + dp.wait(timeout=timeout) + except subprocess.TimeoutExpired: + logger.error('Healthcheck timed-out after %s s', timeout) + raise SSLHealthcheckError('OpenSSL connection verification timed-out') + + if dp.returncode == 0: # success + return + + logger.error('Healthcheck connection failed (code %s)', dp.returncode) + raise SSLHealthcheckError('OpenSSL connection verification failed') From 230835c794733e2a27a787f05f57928e8ec56149 Mon Sep 17 00:00:00 2001 From: Mikolaj Kucharski Date: Mon, 23 Jun 2025 20:25:04 +0000 Subject: [PATCH 003/198] Read from /dev/null in detached_subprocess() Redirect the child's standard input to subprocess.DEVNULL, so it starts with no stdin attached. This prevents the OpenSSL health-check process from reading from, or blocking on, the parent's terminal or execution environment stdin stream. --- node_cli/core/ssl/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node_cli/core/ssl/utils.py b/node_cli/core/ssl/utils.py index a2e71e1f..c11ebda8 100644 --- a/node_cli/core/ssl/utils.py +++ b/node_cli/core/ssl/utils.py @@ -50,7 +50,7 @@ def detached_subprocess(cmd, expose_output=False): logger.debug(f'Starting detached subprocess: {cmd}') p = subprocess.Popen( cmd, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, stdin=subprocess.DEVNULL, encoding='utf-8' ) try: From fcadfd378d5bf804168929c5ae92c43db50cfac7 Mon Sep 17 00:00:00 2001 From: badrogger Date: Tue, 24 Jun 2025 17:56:01 +0100 Subject: [PATCH 004/198] Remove docker-compose requirement --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a0e3c2b3..12741e27 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ SKALE Node CLI, part of the SKALE suite of validator tools, is the command line * Prerequisites -Ensure that the following package is installed: **docker**, **docker-compose** (1.27.4+) +Ensure that the following package is installed: **docker**, **docker-compose** * Download the executable From c03df761dca59305e859e74ba6c0ffb126621d9b Mon Sep 17 00:00:00 2001 From: badrogger Date: Tue, 15 Jul 2025 17:09:54 +0100 Subject: [PATCH 005/198] Improve cleanup --- node_cli/core/schains.py | 45 +++++++++++++++++++------------------ node_cli/operations/fair.py | 6 +++-- node_cli/utils/helper.py | 11 +++++++++ 3 files changed, 38 insertions(+), 24 deletions(-) diff --git a/node_cli/core/schains.py b/node_cli/core/schains.py index eb420fdb..c103d5bb 100644 --- a/node_cli/core/schains.py +++ b/node_cli/core/schains.py @@ -263,25 +263,26 @@ def cleanup_datadir_for_single_chain_node( f'No data directory found in {base_path}. ' 'Please check the path or specify a chain name.' ) - chain_name = folders[0] - base_path = os.path.join(base_path, chain_name) - regular_folders_pattern = f'{base_path}/[!snapshots]*' - logger.info('Removing regular folders') - for filepath in glob.glob(regular_folders_pattern): - if os.path.isdir(filepath): - logger.debug('Removing recursively %s', filepath) - shutil.rmtree(filepath) - if os.path.isfile(filepath): - os.remove(filepath) - - logger.info('Removing subvolumes') - subvolumes_pattern = f'{base_path}/snapshots/*/*' - for filepath in glob.glob(subvolumes_pattern): - logger.debug('Deleting subvolume %s', filepath) - if os.path.isdir(filepath): - rm_btrfs_subvolume(filepath) - else: - os.remove(filepath) - logger.info('Cleaning up snapshots folder') - if os.path.isdir(base_path): - shutil.rmtree(base_path) + for folder_name in folders[0]: + base_path = os.path.join(base_path, folder_name) + if folder_name != 'shared-space': + regular_folders_pattern = f'{base_path}/[!snapshots]*' + logger.info('Removing regular folders') + for filepath in glob.glob(regular_folders_pattern): + if os.path.isdir(filepath): + logger.debug('Removing recursively %s', filepath) + shutil.rmtree(filepath) + if os.path.isfile(filepath): + os.remove(filepath) + + logger.info('Removing subvolumes') + subvolumes_pattern = f'{base_path}/snapshots/*/*' + for filepath in glob.glob(subvolumes_pattern): + logger.debug('Deleting subvolume %s', filepath) + if os.path.isdir(filepath): + rm_btrfs_subvolume(filepath) + else: + os.remove(filepath) + logger.info('Cleaning up snapshots folder') + if os.path.isdir(base_path): + shutil.rmtree(base_path) diff --git a/node_cli/operations/fair.py b/node_cli/operations/fair.py index 0471c399..795ee741 100644 --- a/node_cli/operations/fair.py +++ b/node_cli/operations/fair.py @@ -27,6 +27,7 @@ from node_cli.configs import ( CONTAINER_CONFIG_PATH, GLOBAL_SKALE_DIR, + NFTABLES_CHAIN_FOLDER_PATH, SKALE_DIR, ) from node_cli.core.checks import CheckType @@ -36,8 +37,8 @@ from node_cli.core.nftables import configure_nftables from node_cli.core.nginx import generate_nginx_config from node_cli.core.schains import cleanup_datadir_for_single_chain_node -from node_cli.migrations.fair.from_boot import migrate_nftables_from_boot from node_cli.fair.record.chain_record import migrate_chain_record +from node_cli.migrations.fair.from_boot import migrate_nftables_from_boot from node_cli.operations.base import checked_host, turn_off from node_cli.operations.common import configure_filebeat, configure_flask, unpack_backup_archive from node_cli.operations.config_repo import ( @@ -55,7 +56,7 @@ remove_dynamic_containers, wait_for_container, ) -from node_cli.utils.helper import rm_dir, str_to_bool +from node_cli.utils.helper import cleanup_dir_content, rm_dir, str_to_bool from node_cli.utils.meta import FairCliMetaManager from node_cli.utils.print_formatters import print_failed_requirements_checks @@ -244,4 +245,5 @@ def cleanup(env) -> None: cleanup_datadir_for_single_chain_node() rm_dir(GLOBAL_SKALE_DIR) rm_dir(SKALE_DIR) + cleanup_dir_content(NFTABLES_CHAIN_FOLDER_PATH) cleanup_docker_configuration() diff --git a/node_cli/utils/helper.py b/node_cli/utils/helper.py index 7adfba3b..bff97fa6 100644 --- a/node_cli/utils/helper.py +++ b/node_cli/utils/helper.py @@ -324,6 +324,17 @@ def rm_dir(folder: str) -> None: logger.info(f"{folder} doesn't exist, skipping...") +def cleanup_dir_content(folder: str) -> None: + if os.path.exists(folder): + logger.info('Removing contents of %s') + for filename in os.listdir(folder): + file_path = os.path.join(folder, filename) + if os.path.isfile(file_path) or os.path.islink(file_path): + os.unlink(file_path) + elif os.path.isdir(file_path): + shutil.rmtree(file_path) + + def safe_mkdir(path: str, print_res: bool = False) -> None: if os.path.exists(path): logger.debug(f'Directory {path} already exists') From 761edf844370657ba099381bd12a1e6e41dec641 Mon Sep 17 00:00:00 2001 From: badrogger Date: Thu, 17 Jul 2025 13:13:15 +0100 Subject: [PATCH 006/198] Fix and improve datadir cleanup code --- node_cli/core/schains.py | 56 +++++++++++++++++++-------------- node_cli/operations/base.py | 4 +-- node_cli/operations/fair.py | 4 +-- tests/core/core_schains_test.py | 4 +-- 4 files changed, 38 insertions(+), 30 deletions(-) diff --git a/node_cli/core/schains.py b/node_cli/core/schains.py index c103d5bb..17bc0c42 100644 --- a/node_cli/core/schains.py +++ b/node_cli/core/schains.py @@ -253,36 +253,44 @@ def ensure_schain_volume(schain: str, schain_type: str, env_type: str) -> None: logger.warning('Volume %s already exists', schain) -def cleanup_datadir_for_single_chain_node( +def cleanup_datadir_content(datadir_path: str) -> None: + regular_folders_pattern = f'{datadir_path}/[!snapshots]*' + logger.info('Removing regular folders') + for path in glob.glob(regular_folders_pattern): + if os.path.isdir(path): + logger.debug('Removing recursively %s', path) + shutil.rmtree(path) + if os.path.isfile(path): + os.remove(path) + + logger.info('Removing subvolumes') + subvolumes_pattern = f'{datadir_path}/snapshots/*/*' + for path in glob.glob(subvolumes_pattern): + logger.debug('Deleting subvolume %s', path) + if os.path.isdir(path): + rm_btrfs_subvolume(path) + else: + os.remove(path) + + logger.info('Removing snapshots folder') + shutil.rmtree(os.path.join(datadir_path, 'snapshots'), ignore_errors=True) + + +def cleanup_no_lvm_datadir( chain_name: str = '', base_path: str = SCHAINS_MNT_DIR_SINGLE_CHAIN ) -> None: - if not chain_name: + if chain_name: + folders = [chain_name] + else: folders = [f for f in os.listdir(base_path) if os.path.isdir(os.path.join(base_path, f))] if not folders: raise NoDataDirForChainError( f'No data directory found in {base_path}. ' 'Please check the path or specify a chain name.' ) - for folder_name in folders[0]: - base_path = os.path.join(base_path, folder_name) + for folder_name in folders: + folder_path = os.path.join(base_path, folder_name) if folder_name != 'shared-space': - regular_folders_pattern = f'{base_path}/[!snapshots]*' - logger.info('Removing regular folders') - for filepath in glob.glob(regular_folders_pattern): - if os.path.isdir(filepath): - logger.debug('Removing recursively %s', filepath) - shutil.rmtree(filepath) - if os.path.isfile(filepath): - os.remove(filepath) - - logger.info('Removing subvolumes') - subvolumes_pattern = f'{base_path}/snapshots/*/*' - for filepath in glob.glob(subvolumes_pattern): - logger.debug('Deleting subvolume %s', filepath) - if os.path.isdir(filepath): - rm_btrfs_subvolume(filepath) - else: - os.remove(filepath) - logger.info('Cleaning up snapshots folder') - if os.path.isdir(base_path): - shutil.rmtree(base_path) + cleanup_datadir_content(folder_path) + if os.path.isdir(folder_path): + shutil.rmtree(folder_path) diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index a5e29a4c..cd9a51b2 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -44,7 +44,7 @@ from node_cli.core.node_options import NodeOptions from node_cli.core.resources import init_shared_space_volume, update_resource_allocation from node_cli.core.schains import ( - cleanup_datadir_for_single_chain_node, + cleanup_no_lvm_datadir, update_node_cli_schain_status, ) from node_cli.operations.common import configure_filebeat, configure_flask, unpack_backup_archive @@ -425,6 +425,6 @@ def restore(env, backup_path, node_type: NodeType, config_only=False): def cleanup_sync(env, schain_name: str) -> None: turn_off(env, node_type=NodeType.SYNC) - cleanup_datadir_for_single_chain_node(schain_name=schain_name) + cleanup_no_lvm_datadir(schain_name=schain_name) rm_dir(GLOBAL_SKALE_DIR) rm_dir(SKALE_DIR) diff --git a/node_cli/operations/fair.py b/node_cli/operations/fair.py index 795ee741..4bc841d9 100644 --- a/node_cli/operations/fair.py +++ b/node_cli/operations/fair.py @@ -36,7 +36,7 @@ from node_cli.core.host import ensure_btrfs_kernel_module_autoloaded, link_env_file, prepare_host from node_cli.core.nftables import configure_nftables from node_cli.core.nginx import generate_nginx_config -from node_cli.core.schains import cleanup_datadir_for_single_chain_node +from node_cli.core.schains import cleanup_no_lvm_datadir from node_cli.fair.record.chain_record import migrate_chain_record from node_cli.migrations.fair.from_boot import migrate_nftables_from_boot from node_cli.operations.base import checked_host, turn_off @@ -242,7 +242,7 @@ def restore_fair(env, backup_path, config_only=False): def cleanup(env) -> None: turn_off(env, node_type=NodeType.FAIR) - cleanup_datadir_for_single_chain_node() + cleanup_no_lvm_datadir() rm_dir(GLOBAL_SKALE_DIR) rm_dir(SKALE_DIR) cleanup_dir_content(NFTABLES_CHAIN_FOLDER_PATH) diff --git a/tests/core/core_schains_test.py b/tests/core/core_schains_test.py index c6ce426b..c9281adb 100644 --- a/tests/core/core_schains_test.py +++ b/tests/core/core_schains_test.py @@ -4,7 +4,7 @@ import freezegun -from node_cli.core.schains import cleanup_datadir_for_single_chain_node, toggle_schain_repair_mode +from node_cli.core.schains import cleanup_no_lvm_datadir, toggle_schain_repair_mode from node_cli.utils.helper import read_json from tests.helper import CURRENT_DATETIME, CURRENT_TIMESTAMP @@ -81,5 +81,5 @@ def test_cleanup_sync_datadir(tmp_sync_datadir): hash_path.touch() with mock.patch('node_cli.core.schains.rm_btrfs_subvolume'): - cleanup_datadir_for_single_chain_node(schain_name, base_path=tmp_sync_datadir) + cleanup_no_lvm_datadir(schain_name, base_path=tmp_sync_datadir) assert not os.path.isdir(base_folder) From 0cfefbe6836774bea1917ccbce08eee8e137f4d0 Mon Sep 17 00:00:00 2001 From: badrogger Date: Thu, 17 Jul 2025 19:09:52 +0100 Subject: [PATCH 007/198] Fix files cleanup --- node_cli/core/schains.py | 26 +++++++++++++++++--------- node_cli/fair/fair_node.py | 1 - node_cli/utils/helper.py | 11 ++++++++++- scripts/build.sh | 0 4 files changed, 27 insertions(+), 11 deletions(-) mode change 100644 => 100755 scripts/build.sh diff --git a/node_cli/core/schains.py b/node_cli/core/schains.py index 17bc0c42..fe3aeeb4 100644 --- a/node_cli/core/schains.py +++ b/node_cli/core/schains.py @@ -40,6 +40,7 @@ from node_cli.utils.helper import ( error_exit, get_request, + is_btrfs_subvolume, read_json, run_cmd, safe_load_yml, @@ -255,24 +256,29 @@ def ensure_schain_volume(schain: str, schain_type: str, env_type: str) -> None: def cleanup_datadir_content(datadir_path: str) -> None: regular_folders_pattern = f'{datadir_path}/[!snapshots]*' - logger.info('Removing regular folders') + logger.info('Removing regular folders of %s', datadir_path) for path in glob.glob(regular_folders_pattern): + logger.debug('Removing recursively %s', path) + if os.path.isfile(path): + logger.debug('Deleting file in datadir: %s', path) + os.remove(path) if os.path.isdir(path): - logger.debug('Removing recursively %s', path) + logger.debug('Deleting folder in datadir: %s', path) shutil.rmtree(path) - if os.path.isfile(path): - os.remove(path) - logger.info('Removing subvolumes') + logger.info('Removing subvolumes of %s', datadir_path) subvolumes_pattern = f'{datadir_path}/snapshots/*/*' for path in glob.glob(subvolumes_pattern): - logger.debug('Deleting subvolume %s', path) - if os.path.isdir(path): + if is_btrfs_subvolume(path): + logger.debug('Deleting subvolume %s', path) rm_btrfs_subvolume(path) - else: + if os.path.isfile(path): + logger.debug('Deleting file in snapshots directory: %s', path) os.remove(path) + if os.path.isdir(path): + logger.debug('Deleting folder in snapshots directory %s', path) + shutil.rmtree(path) - logger.info('Removing snapshots folder') shutil.rmtree(os.path.join(datadir_path, 'snapshots'), ignore_errors=True) @@ -291,6 +297,8 @@ def cleanup_no_lvm_datadir( for folder_name in folders: folder_path = os.path.join(base_path, folder_name) if folder_name != 'shared-space': + logger.info('Removing datadir content for %s', folder_path) cleanup_datadir_content(folder_path) + logger.info('Removing datadir content for %s', folder_path) if os.path.isdir(folder_path): shutil.rmtree(folder_path) diff --git a/node_cli/fair/fair_node.py b/node_cli/fair/fair_node.py index e32149e9..0d08fd4c 100644 --- a/node_cli/fair/fair_node.py +++ b/node_cli/fair/fair_node.py @@ -129,7 +129,6 @@ def request_repair(snapshot_from: str = '') -> None: print(TEXTS['fair']['node']['repair']['repair_requested']) -@check_inited @check_user def cleanup() -> None: env = compose_node_env(SKALE_DIR_ENV_FILEPATH, save=False, node_type=NodeType.FAIR) diff --git a/node_cli/utils/helper.py b/node_cli/utils/helper.py index bff97fa6..638b63a3 100644 --- a/node_cli/utils/helper.py +++ b/node_cli/utils/helper.py @@ -326,7 +326,7 @@ def rm_dir(folder: str) -> None: def cleanup_dir_content(folder: str) -> None: if os.path.exists(folder): - logger.info('Removing contents of %s') + logger.info('Removing contents of %s', folder) for filename in os.listdir(folder): file_path = os.path.join(folder, filename) if os.path.isfile(file_path) or os.path.islink(file_path): @@ -413,3 +413,12 @@ def get_ssh_port(ssh_service_name='ssh'): def is_contract_address(value: str) -> bool: return bool(re.fullmatch(r'0x[a-fA-F0-9]{40}', value)) + + +def is_btrfs_subvolume(path: str) -> bool: + """Check if the given path is a Btrfs subvolume.""" + try: + output = run_cmd(['btrfs', 'subvolume', 'show', path], check_code=False) + return output.returncode == 0 + except subprocess.CalledProcessError: + return False diff --git a/scripts/build.sh b/scripts/build.sh old mode 100644 new mode 100755 From 8a2f315363d734b715953931af9f378e5fd72339 Mon Sep 17 00:00:00 2001 From: badrogger Date: Fri, 18 Jul 2025 18:31:12 +0100 Subject: [PATCH 008/198] Add fair repair command --- node_cli/cli/fair_node.py | 13 ++++----- node_cli/core/node.py | 48 ++++++++++++++++----------------- node_cli/fair/fair_node.py | 15 +++++------ node_cli/operations/__init__.py | 5 ++-- node_cli/operations/fair.py | 39 +++++++++++++++++++++++---- node_cli/utils/docker_utils.py | 24 ++++++++--------- 6 files changed, 85 insertions(+), 59 deletions(-) diff --git a/node_cli/cli/fair_node.py b/node_cli/cli/fair_node.py index b6461871..a4e2cbf8 100644 --- a/node_cli/cli/fair_node.py +++ b/node_cli/cli/fair_node.py @@ -21,13 +21,13 @@ from node_cli.core.node import backup from node_cli.fair.fair_node import cleanup as fair_cleanup -from node_cli.fair.fair_node import init as init_fair from node_cli.fair.fair_node import ( + get_node_info, migrate_from_boot, - request_repair, + repair_chain, restore_fair, - get_node_info, ) +from node_cli.fair.fair_node import init as init_fair from node_cli.fair.fair_node import register as register_fair from node_cli.fair.fair_node import update as update_fair from node_cli.utils.helper import IP_TYPE, URL_TYPE, abort_if_false, streamed_cmd @@ -119,7 +119,7 @@ def migrate_node(env_filepath: str) -> None: @click.option( '--snapshot-from', type=URL_TYPE, - default='', + default=None, hidden=True, help=TEXTS['fair']['node']['repair']['snapshot_from'], ) @@ -130,8 +130,9 @@ def migrate_node(env_filepath: str) -> None: expose_value=False, prompt=TEXTS['fair']['node']['repair']['warning'], ) -def repair(snapshot_from: str = '') -> None: - request_repair(snapshot_from=snapshot_from) +@streamed_cmd +def repair(snapshot_from: str | None = None) -> None: + repair_chain(snapshot_from=snapshot_from) @node.command('cleanup', help='Cleanup Fair node.') diff --git a/node_cli/core/node.py b/node_cli/core/node.py index f5e0a16b..d803ef58 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -28,6 +28,7 @@ import docker +from node_cli.cli import __version__ from node_cli.configs import ( BACKUP_ARCHIVE_NAME, CONTAINER_CONFIG_PATH, @@ -41,49 +42,46 @@ SKALE_STATE_DIR, TM_INIT_TIMEOUT, ) -from node_cli.cli import __version__ -from node_cli.configs.user import get_validated_user_config, SKALE_DIR_ENV_FILEPATH from node_cli.configs.cli_logger import LOG_DATA_PATH as CLI_LOG_DATA_PATH - -from node_cli.core.host import is_node_inited, save_env_params, get_flask_secret_key +from node_cli.configs.user import SKALE_DIR_ENV_FILEPATH, get_validated_user_config from node_cli.core.checks import run_checks as run_host_checks +from node_cli.core.host import get_flask_secret_key, is_node_inited, save_env_params from node_cli.core.resources import update_resource_allocation +from node_cli.migrations.focal_to_jammy import migrate as migrate_2_6 from node_cli.operations import ( + cleanup_sync_op, configure_nftables, - update_op, init_op, + init_sync_op, + restore_op, turn_off_op, turn_on_op, - restore_op, - init_sync_op, + update_op, update_sync_op, - cleanup_sync_op, ) -from node_cli.utils.print_formatters import ( - print_failed_requirements_checks, - print_node_cmd_error, - print_node_info, +from node_cli.utils.decorators import check_inited, check_not_inited, check_user +from node_cli.utils.docker_utils import ( + BASE_FAIR_BOOT_COMPOSE_SERVICES, + BASE_FAIR_COMPOSE_SERVICES, + BASE_SKALE_COMPOSE_SERVICES, + BASE_SYNC_COMPOSE_SERVICES, + is_admin_running, + is_api_running, ) +from node_cli.utils.exit_codes import CLIExitCodes from node_cli.utils.helper import ( error_exit, get_request, post_request, ) from node_cli.utils.meta import CliMetaManager -from node_cli.utils.texts import safe_load_texts -from node_cli.utils.exit_codes import CLIExitCodes -from node_cli.utils.decorators import check_not_inited, check_inited, check_user -from node_cli.utils.docker_utils import ( - is_admin_running, - is_api_running, - BASE_SKALE_COMPOSE_SERVICES, - BASE_SYNC_COMPOSE_SERVICES, - BASE_FAIR_COMPOSE_SERVICES, - BASE_FAIR_BOOT_COMPOSE_SERVICES, -) from node_cli.utils.node_type import NodeType -from node_cli.migrations.focal_to_jammy import migrate as migrate_2_6 - +from node_cli.utils.print_formatters import ( + print_failed_requirements_checks, + print_node_cmd_error, + print_node_info, +) +from node_cli.utils.texts import safe_load_texts logger = logging.getLogger(__name__) TEXTS = safe_load_texts() diff --git a/node_cli/fair/fair_node.py b/node_cli/fair/fair_node.py index 0d08fd4c..d2513bf1 100644 --- a/node_cli/fair/fair_node.py +++ b/node_cli/fair/fair_node.py @@ -27,11 +27,11 @@ from node_cli.core.docker_config import cleanup_docker_configuration from node_cli.core.host import is_node_inited, save_env_params from node_cli.core.node import compose_node_env, is_base_containers_alive -from node_cli.fair.record.chain_record import get_fair_chain_record from node_cli.operations import ( FairUpdateType, cleanup_fair_op, init_fair_op, + repair_fair_op, restore_fair_op, update_fair_op, ) @@ -121,14 +121,6 @@ def update(env_filepath: str, pull_config_for_schain: str | None = None) -> None logger.info('Fair update completed successfully') -def request_repair(snapshot_from: str = '') -> None: - env = compose_node_env(SKALE_DIR_ENV_FILEPATH, save=False, node_type=NodeType.FAIR) - record = get_fair_chain_record(env) - record.set_repair_ts(int(time.time())) - record.set_snapshot_from(snapshot_from) - print(TEXTS['fair']['node']['repair']['repair_requested']) - - @check_user def cleanup() -> None: env = compose_node_env(SKALE_DIR_ENV_FILEPATH, save=False, node_type=NodeType.FAIR) @@ -169,3 +161,8 @@ def register(ip: str) -> None: error_msg = payload logger.error(f'Registration error {error_msg}') error_exit(error_msg, exit_code=CLIExitCodes.BAD_API_RESPONSE) + + +def repair_chain(snapshot_from: str | None = None) -> None: + env = compose_node_env(SKALE_DIR_ENV_FILEPATH, save=False, node_type=NodeType.FAIR) + repair_fair_op(env=env, snapshot_from=snapshot_from) diff --git a/node_cli/operations/__init__.py b/node_cli/operations/__init__.py index 4f3aa410..d0f60eed 100644 --- a/node_cli/operations/__init__.py +++ b/node_cli/operations/__init__.py @@ -32,8 +32,9 @@ ) from node_cli.operations.fair import ( # noqa init as init_fair_op, - update_fair as update_fair_op, + update as update_fair_op, FairUpdateType, - restore_fair as restore_fair_op, + restore as restore_fair_op, + repair as repair_fair_op, cleanup as cleanup_fair_op, ) diff --git a/node_cli/operations/fair.py b/node_cli/operations/fair.py index 4bc841d9..1f2df69d 100644 --- a/node_cli/operations/fair.py +++ b/node_cli/operations/fair.py @@ -37,7 +37,7 @@ from node_cli.core.nftables import configure_nftables from node_cli.core.nginx import generate_nginx_config from node_cli.core.schains import cleanup_no_lvm_datadir -from node_cli.fair.record.chain_record import migrate_chain_record +from node_cli.fair.record.chain_record import get_fair_chain_record, migrate_chain_record from node_cli.migrations.fair.from_boot import migrate_nftables_from_boot from node_cli.operations.base import checked_host, turn_off from node_cli.operations.common import configure_filebeat, configure_flask, unpack_backup_archive @@ -53,12 +53,15 @@ compose_rm, compose_up, docker_cleanup, + is_admin_running, remove_dynamic_containers, + start_container_by_name, + stop_container_by_name, wait_for_container, ) from node_cli.utils.helper import cleanup_dir_content, rm_dir, str_to_bool from node_cli.utils.meta import FairCliMetaManager -from node_cli.utils.print_formatters import print_failed_requirements_checks +from node_cli.utils.print_formatters import TEXTS, print_failed_requirements_checks logger = logging.getLogger(__name__) @@ -145,7 +148,7 @@ def update_fair_boot(env_filepath: str, env: dict) -> bool: @checked_host -def update_fair(env_filepath: str, env: dict, update_type: FairUpdateType) -> bool: +def update(env_filepath: str, env: dict, update_type: FairUpdateType) -> bool: compose_rm(node_type=NodeType.FAIR, env=env) if update_type not in (FairUpdateType.INFRA_ONLY, FairUpdateType.FROM_BOOT): remove_dynamic_containers() @@ -193,7 +196,7 @@ def update_fair(env_filepath: str, env: dict, update_type: FairUpdateType) -> bo return True -def restore_fair(env, backup_path, config_only=False): +def restore(env, backup_path, config_only=False): unpack_backup_archive(backup_path) failed_checks = run_host_checks( env['DISK_MOUNTPOINT'], @@ -240,10 +243,36 @@ def restore_fair(env, backup_path, config_only=False): return True -def cleanup(env) -> None: +def cleanup(env: dict) -> None: turn_off(env, node_type=NodeType.FAIR) cleanup_no_lvm_datadir() rm_dir(GLOBAL_SKALE_DIR) rm_dir(SKALE_DIR) cleanup_dir_content(NFTABLES_CHAIN_FOLDER_PATH) cleanup_docker_configuration() + + +def request_repair(env: dict, snapshot_from: str | None = None) -> None: + record = get_fair_chain_record(env) + record.set_repair_ts(int(time.time())) + if not snapshot_from: + snapshot_from = '' + record.set_snapshot_from(snapshot_from) + print(TEXTS['fair']['node']['repair']['repair_requested']) + + +def repair(env: dict, snapshot_from: str | None = None) -> None: + logger.info('Starting fair node repair') + container_name = 'fair_admin' + if is_admin_running(node_type=NodeType.FAIR): + logger.info('Stopping admin container') + stop_container_by_name(container_name=container_name) + logger.info('Removing chain container') + remove_dynamic_containers() + logger.info('Cleaning up datadir') + cleanup_no_lvm_datadir() + logger.info('Requesting fair node repair') + request_repair(env=env, snapshot_from=snapshot_from) + logger.info('Starting admin') + start_container_by_name(container_name=container_name) + logger.info('Fair node repair completed successfully') diff --git a/node_cli/utils/docker_utils.py b/node_cli/utils/docker_utils.py index 6986d44e..6ded9e34 100644 --- a/node_cli/utils/docker_utils.py +++ b/node_cli/utils/docker_utils.py @@ -160,7 +160,7 @@ def safe_rm(container: Container, timeout=DOCKER_DEFAULT_STOP_TIMEOUT, **kwargs) logger.info(f'Container removed: {container_name}') -def stop_container( +def stop_container_by_name( container_name: str, timeout: int = DOCKER_DEFAULT_STOP_TIMEOUT, dclient: Optional[DockerClient] = None, @@ -171,7 +171,7 @@ def stop_container( container.stop(timeout=timeout) -def rm_container( +def remove_container_by_name( container_name: str, timeout: int = DOCKER_DEFAULT_STOP_TIMEOUT, dclient: Optional[DockerClient] = None, @@ -180,19 +180,21 @@ def rm_container( container_names = [container.name for container in get_containers()] if container_name in container_names: container = dc.containers.get(container_name) - safe_rm(container) + safe_rm(container, timeout=timeout) -def start_container(container_name: str, dclient: Optional[DockerClient] = None) -> None: +def start_container_by_name(container_name: str, dclient: Optional[DockerClient] = None) -> None: dc = dclient or docker_client() container = dc.containers.get(container_name) logger.info('Starting container %s', container_name) container.start() -def remove_schain_container(schain_name: str, dclient: Optional[DockerClient] = None) -> None: +def remove_schain_container_by_name( + schain_name: str, dclient: Optional[DockerClient] = None +) -> None: container_name = f'skale_schain_{schain_name}' - rm_container(container_name, timeout=SCHAIN_REMOVE_TIMEOUT, dclient=dclient) + remove_container_by_name(container_name, timeout=SCHAIN_REMOVE_TIMEOUT, dclient=dclient) def backup_container_logs( @@ -401,14 +403,12 @@ def is_api_running(node_type: NodeType, dclient: Optional[DockerClient] = None) def is_admin_running(node_type: NodeType, client: Optional[DockerClient] = None) -> bool: + container_name = 'skale_admin' if node_type == NodeType.FAIR: - result = is_container_running(name='fair_admin', dclient=client) + container_name = 'fair_admin' elif node_type == NodeType.SYNC: - result = is_container_running(name='skale_sync_admin', dclient=client) - else: - result = is_container_running(name='skale_admin', dclient=client) - - return result + container_name = 'skale_sync_admin' + return is_container_running(name=container_name, dclient=client) def system_prune(): From a5523455f09f4e2c7aa6587be714399861c313aa Mon Sep 17 00:00:00 2001 From: badrogger Date: Fri, 18 Jul 2025 19:01:37 +0100 Subject: [PATCH 009/198] Disable test that test nothing --- tests/fair/fair_node_test.py | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/tests/fair/fair_node_test.py b/tests/fair/fair_node_test.py index fab00451..5d0c13bb 100644 --- a/tests/fair/fair_node_test.py +++ b/tests/fair/fair_node_test.py @@ -1,16 +1,14 @@ from unittest import mock -import freezegun import pytest from node_cli.configs import SKALE_DIR from node_cli.configs.user import SKALE_DIR_ENV_FILEPATH from node_cli.fair.fair_boot import init as init_boot from node_cli.fair.fair_boot import update -from node_cli.fair.fair_node import cleanup, migrate_from_boot, request_repair, restore_fair +from node_cli.fair.fair_node import cleanup, migrate_from_boot, restore_fair from node_cli.operations.fair import FairUpdateType from node_cli.utils.node_type import NodeType -from tests.helper import CURRENT_DATETIME, CURRENT_TIMESTAMP @mock.patch('node_cli.fair.fair_node.time.sleep') @@ -131,18 +129,6 @@ def test_migrate_from_boot( ) -@freezegun.freeze_time(CURRENT_DATETIME) -@mock.patch('node_cli.fair.fair_node.compose_node_env', return_value={'ENV_TYPE': 'devnet'}) -@mock.patch('node_cli.fair.record.chain_record.get_fair_chain_name', return_value='test') -def test_fair_repair(compose_node_env_mock, get_static_params_mock, redis_client, inited_node): - request_repair() - assert redis_client.get('test_repair_ts') == f'{CURRENT_TIMESTAMP}'.encode('utf-8') - assert redis_client.get('test_snapshot_from') == b'' - request_repair(snapshot_from='127.0.0.1') - assert redis_client.get('test_repair_ts') == f'{CURRENT_TIMESTAMP}'.encode('utf-8') - assert redis_client.get('test_snapshot_from') == b'127.0.0.1' - - @mock.patch('node_cli.utils.decorators.is_user_valid', return_value=True) @mock.patch('node_cli.fair.fair_node.cleanup_docker_configuration') @mock.patch('node_cli.fair.fair_node.cleanup_fair_op') @@ -203,9 +189,7 @@ def test_cleanup_calls_operations_in_correct_order( @mock.patch('node_cli.utils.decorators.is_user_valid', return_value=True) @mock.patch('node_cli.fair.fair_node.cleanup_docker_configuration') -@mock.patch( - 'node_cli.fair.fair_node.cleanup_fair_op', side_effect=Exception('Cleanup failed') -) +@mock.patch('node_cli.fair.fair_node.cleanup_fair_op', side_effect=Exception('Cleanup failed')) @mock.patch('node_cli.fair.fair_node.compose_node_env') def test_cleanup_continues_after_fair_op_error( mock_compose_env, From 06de31d46cfb29f304a545d5299cf0d3a965823c Mon Sep 17 00:00:00 2001 From: badrogger Date: Mon, 21 Jul 2025 17:53:07 +0100 Subject: [PATCH 010/198] Use only snapshot_from field --- node_cli/cli/fair_node.py | 4 ++-- node_cli/fair/fair_node.py | 2 +- node_cli/operations/fair.py | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/node_cli/cli/fair_node.py b/node_cli/cli/fair_node.py index a4e2cbf8..aa510023 100644 --- a/node_cli/cli/fair_node.py +++ b/node_cli/cli/fair_node.py @@ -119,7 +119,7 @@ def migrate_node(env_filepath: str) -> None: @click.option( '--snapshot-from', type=URL_TYPE, - default=None, + default='any', hidden=True, help=TEXTS['fair']['node']['repair']['snapshot_from'], ) @@ -131,7 +131,7 @@ def migrate_node(env_filepath: str) -> None: prompt=TEXTS['fair']['node']['repair']['warning'], ) @streamed_cmd -def repair(snapshot_from: str | None = None) -> None: +def repair(snapshot_from: str = 'any') -> None: repair_chain(snapshot_from=snapshot_from) diff --git a/node_cli/fair/fair_node.py b/node_cli/fair/fair_node.py index d2513bf1..f16dbdbe 100644 --- a/node_cli/fair/fair_node.py +++ b/node_cli/fair/fair_node.py @@ -163,6 +163,6 @@ def register(ip: str) -> None: error_exit(error_msg, exit_code=CLIExitCodes.BAD_API_RESPONSE) -def repair_chain(snapshot_from: str | None = None) -> None: +def repair_chain(snapshot_from: str = 'any') -> None: env = compose_node_env(SKALE_DIR_ENV_FILEPATH, save=False, node_type=NodeType.FAIR) repair_fair_op(env=env, snapshot_from=snapshot_from) diff --git a/node_cli/operations/fair.py b/node_cli/operations/fair.py index 1f2df69d..9c92667f 100644 --- a/node_cli/operations/fair.py +++ b/node_cli/operations/fair.py @@ -252,16 +252,16 @@ def cleanup(env: dict) -> None: cleanup_docker_configuration() -def request_repair(env: dict, snapshot_from: str | None = None) -> None: +def request_repair(env: dict, snapshot_from: str = 'any') -> None: record = get_fair_chain_record(env) - record.set_repair_ts(int(time.time())) + # record.set_repair_ts(int(time.time())) if not snapshot_from: - snapshot_from = '' + snapshot_from = 'any' record.set_snapshot_from(snapshot_from) print(TEXTS['fair']['node']['repair']['repair_requested']) -def repair(env: dict, snapshot_from: str | None = None) -> None: +def repair(env: dict, snapshot_from: str = 'any') -> None: logger.info('Starting fair node repair') container_name = 'fair_admin' if is_admin_running(node_type=NodeType.FAIR): From aef6cf8623cf89dc8fc8c5b3efdd1abf5ec5196a Mon Sep 17 00:00:00 2001 From: badrogger Date: Mon, 21 Jul 2025 19:57:50 +0100 Subject: [PATCH 011/198] Rename request_repair to trigger_skaled_snapshot_mode --- node_cli/operations/fair.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/node_cli/operations/fair.py b/node_cli/operations/fair.py index 9c92667f..ab1fb3f9 100644 --- a/node_cli/operations/fair.py +++ b/node_cli/operations/fair.py @@ -252,9 +252,8 @@ def cleanup(env: dict) -> None: cleanup_docker_configuration() -def request_repair(env: dict, snapshot_from: str = 'any') -> None: +def trigger_skaled_snapshot_mode(env: dict, snapshot_from: str = 'any') -> None: record = get_fair_chain_record(env) - # record.set_repair_ts(int(time.time())) if not snapshot_from: snapshot_from = 'any' record.set_snapshot_from(snapshot_from) @@ -272,7 +271,7 @@ def repair(env: dict, snapshot_from: str = 'any') -> None: logger.info('Cleaning up datadir') cleanup_no_lvm_datadir() logger.info('Requesting fair node repair') - request_repair(env=env, snapshot_from=snapshot_from) + trigger_skaled_snapshot_mode(env=env, snapshot_from=snapshot_from) logger.info('Starting admin') start_container_by_name(container_name=container_name) logger.info('Fair node repair completed successfully') From 0183dea8814b67c4faa6739d32f9da8b24ca6ae9 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Thu, 24 Jul 2025 20:07:24 +0100 Subject: [PATCH 012/198] Add `change-ip` command --- node_cli/cli/fair_node.py | 11 +++++++++++ node_cli/fair/fair_node.py | 19 +++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/node_cli/cli/fair_node.py b/node_cli/cli/fair_node.py index aa510023..23694adb 100644 --- a/node_cli/cli/fair_node.py +++ b/node_cli/cli/fair_node.py @@ -30,6 +30,7 @@ from node_cli.fair.fair_node import init as init_fair from node_cli.fair.fair_node import register as register_fair from node_cli.fair.fair_node import update as update_fair +from node_cli.fair.fair_node import change_ip as change_ip_fair from node_cli.utils.helper import IP_TYPE, URL_TYPE, abort_if_false, streamed_cmd from node_cli.utils.texts import safe_load_texts @@ -146,3 +147,13 @@ def repair(snapshot_from: str = 'any') -> None: @streamed_cmd def cleanup_node(): fair_cleanup() + + +@node.command('change-ip', help=TEXTS['fair']['node']['change-ip']['help']) +@click.option('--ip', + required=True, + type=IP_TYPE, + help=TEXTS['fair']['node']['change-ip']['ip'] +) +def change_ip(ip: str) -> None: + change_ip_fair(ip=ip) diff --git a/node_cli/fair/fair_node.py b/node_cli/fair/fair_node.py index f16dbdbe..1e33a6d5 100644 --- a/node_cli/fair/fair_node.py +++ b/node_cli/fair/fair_node.py @@ -166,3 +166,22 @@ def register(ip: str) -> None: def repair_chain(snapshot_from: str = 'any') -> None: env = compose_node_env(SKALE_DIR_ENV_FILEPATH, save=False, node_type=NodeType.FAIR) repair_fair_op(env=env, snapshot_from=snapshot_from) + + +@check_inited +@check_user +def change_ip(ip: str) -> None: + if not is_node_inited(): + print(TEXTS['fair']['node']['not_inited']) + return + + json_data = {'ip': ip, 'port': DEFAULT_SKALED_BASE_PORT} + status, payload = post_request(blueprint=BLUEPRINT_NAME, method='change-ip', json=json_data) + if status == 'ok': + msg = TEXTS['fair']['node']['ip_changed'] + logger.info(msg) + print(msg) + else: + error_msg = payload + logger.error(f'Change IP error {error_msg}') + error_exit(error_msg, exit_code=CLIExitCodes.BAD_API_RESPONSE) From 4a719774c0c1efff183279804850ae8dbaee5a3f Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Thu, 24 Jul 2025 20:22:34 +0100 Subject: [PATCH 013/198] Add texts for `change-ip` command --- text.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/text.yml b/text.yml index c93a743e..30315dbe 100644 --- a/text.yml +++ b/text.yml @@ -91,3 +91,7 @@ fair: help: Register node in fair manager name: Name of the node in fair manager ip: IP address of the node in fair manager + ip_changed: Node IP changed in fair manager + change-ip: + help: Change the node IP in fair manager + ip: New IP address of the node in fair manager \ No newline at end of file From c8fe17c074847fdb646b79e7ee9dc033b5489bfc Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Thu, 24 Jul 2025 20:26:44 +0100 Subject: [PATCH 014/198] Improve texts for Fair commands --- text.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/text.yml b/text.yml index 30315dbe..db7617d8 100644 --- a/text.yml +++ b/text.yml @@ -80,18 +80,18 @@ lvmpy: fair: node: repair: - help: Repair fair chain node - warning: Are you sure you want to repair fair chain node? In rare cases may cause data loss and require additional maintenance + help: Repair Fair chain node + warning: Are you sure you want to repair Fair chain node? In rare cases may cause data loss and require additional maintenance snapshot_from: IP of the node to take snapshot from repair_requested: Repair mode is requested not_inited: Node should be initialized to proceed with operation - registered: Node is registered in fair manager. + registered: Node is registered in Fair manager. register: - help: Register node in fair manager - name: Name of the node in fair manager - ip: IP address of the node in fair manager - ip_changed: Node IP changed in fair manager + help: Register node in Fair manager + name: Name of the node in Fair manager + ip: IP address of the node in Fair manager + ip_changed: Node IP changed in Fair manager change-ip: - help: Change the node IP in fair manager - ip: New IP address of the node in fair manager \ No newline at end of file + help: Change the node IP in Fair manager + ip: New IP address of the node in Fair manager \ No newline at end of file From d21fd7aba9a33ac539ed9c4f1491b84f9f56d1ce Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Thu, 24 Jul 2025 20:28:44 +0100 Subject: [PATCH 015/198] Fit code to PEP 8 --- node_cli/cli/fair_node.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/node_cli/cli/fair_node.py b/node_cli/cli/fair_node.py index 23694adb..ae2abe18 100644 --- a/node_cli/cli/fair_node.py +++ b/node_cli/cli/fair_node.py @@ -150,10 +150,11 @@ def cleanup_node(): @node.command('change-ip', help=TEXTS['fair']['node']['change-ip']['help']) -@click.option('--ip', - required=True, - type=IP_TYPE, - help=TEXTS['fair']['node']['change-ip']['ip'] +@click.option( + '--ip', + required=True, + type=IP_TYPE, + help=TEXTS['fair']['node']['change-ip']['ip'] ) def change_ip(ip: str) -> None: change_ip_fair(ip=ip) From f05aafe7e2a1feb29e7decf506264f9d9814394d Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Fri, 25 Jul 2025 19:13:51 +0100 Subject: [PATCH 016/198] Add `change-ip` to ROUTES --- node_cli/configs/routes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node_cli/configs/routes.py b/node_cli/configs/routes.py index 8afc10d9..942cdfd4 100644 --- a/node_cli/configs/routes.py +++ b/node_cli/configs/routes.py @@ -40,7 +40,7 @@ 'schains': ['config', 'list', 'dkg-statuses', 'firewall-rules', 'repair', 'get'], 'ssl': ['status', 'upload'], 'wallet': ['info', 'send-eth'], - 'fair-node': ['info', 'register'], + 'fair-node': ['info', 'register', 'change-ip'], } } From 1251c42b05533d5728860db13bc38bb4ed21ac8b Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Sat, 26 Jul 2025 16:25:38 +0100 Subject: [PATCH 017/198] Replace decorator @click.option for `change-ip` by @click.argument --- node_cli/cli/fair_node.py | 8 +++----- text.yml | 3 +-- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/node_cli/cli/fair_node.py b/node_cli/cli/fair_node.py index ae2abe18..48fbb482 100644 --- a/node_cli/cli/fair_node.py +++ b/node_cli/cli/fair_node.py @@ -150,11 +150,9 @@ def cleanup_node(): @node.command('change-ip', help=TEXTS['fair']['node']['change-ip']['help']) -@click.option( - '--ip', - required=True, - type=IP_TYPE, - help=TEXTS['fair']['node']['change-ip']['ip'] +@click.argument( + 'ip', + type=IP_TYPE ) def change_ip(ip: str) -> None: change_ip_fair(ip=ip) diff --git a/text.yml b/text.yml index db7617d8..9aa99951 100644 --- a/text.yml +++ b/text.yml @@ -93,5 +93,4 @@ fair: ip: IP address of the node in Fair manager ip_changed: Node IP changed in Fair manager change-ip: - help: Change the node IP in Fair manager - ip: New IP address of the node in Fair manager \ No newline at end of file + help: Change the node IP in Fair manager \ No newline at end of file From 300d3c1be99340c0ad6c121eec18f811d5785b4c Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Mon, 28 Jul 2025 12:55:01 +0100 Subject: [PATCH 018/198] Fix tests --- tests/routes_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/routes_test.py b/tests/routes_test.py index 872c53e1..3dd87f84 100644 --- a/tests/routes_test.py +++ b/tests/routes_test.py @@ -33,6 +33,7 @@ '/api/v1/wallet/send-eth', '/api/v1/fair-node/info', '/api/v1/fair-node/register', + '/api/v1/fair-node/change-ip', ] From 9467dad9601a7ac4a0d6b0d163183d2ae14c38da Mon Sep 17 00:00:00 2001 From: badrogger Date: Mon, 28 Jul 2025 19:32:16 +0100 Subject: [PATCH 019/198] Fix firewall chain moving during migrate --- node_cli/migrations/fair/from_boot.py | 20 ++++++++++++-------- node_cli/operations/fair.py | 4 +++- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/node_cli/migrations/fair/from_boot.py b/node_cli/migrations/fair/from_boot.py index 6affdff3..784908f3 100644 --- a/node_cli/migrations/fair/from_boot.py +++ b/node_cli/migrations/fair/from_boot.py @@ -1,4 +1,3 @@ -import glob import logging import os from pathlib import Path @@ -21,6 +20,7 @@ def rename_chain_file(old_filepath: str, new_filepath: str) -> None: new_path = Path(new_filepath) if not old_path.exists(): raise NoLegacyNFTChainConfigError(f'File {old_filepath} does not exists') + old_path.rename(Path(new_path)) @@ -35,24 +35,28 @@ def rename_chain_in_config(config_path: str, old_chain_name: str, new_chain_name f.write(updated_content) -def migrate_nft_chain() -> None: - after_boot_chain_path = glob.glob(os.path.join(NFT_CHAIN_BASE_PATH, '*'))[0] - old_chain_name = Path(after_boot_chain_path).name.removesuffix('.conf') +def migrate_nft_chain(chain_name: str) -> None: + after_boot_chain_path = os.path.join(NFT_CHAIN_BASE_PATH, f'skale-{chain_name}.conf') new_chain_name = NFT_COMMITTEE_SCOPE_CHAIN_NAME - rename_chain_in_config(after_boot_chain_path, old_chain_name, new_chain_name) after_migration_chain_path = os.path.join( NFT_CHAIN_BASE_PATH, f'{NFT_COMMITTEE_SCOPE_CHAIN_NAME}.conf' ) - rename_chain_file(after_boot_chain_path, after_migration_chain_path) + logger.debug('Renaming %s to %s', after_boot_chain_path, after_migration_chain_path) + if os.path.isfile(after_boot_chain_path): + rename_chain_in_config(after_boot_chain_path, f'skale-{chain_name}', new_chain_name) + if os.path.isfile(after_migration_chain_path): + os.remove(after_boot_chain_path) + else: + rename_chain_file(after_boot_chain_path, after_migration_chain_path) def reload_nft(): run_cmd(['nft', '-f', '/etc/nftables.conf']) -def migrate_nftables_from_boot(): +def migrate_nftables_from_boot(chain_name: str): logger.info('Starting nftables migration from boot') - migrate_nft_chain() + migrate_nft_chain(chain_name=chain_name) logger.info('Reloading nftables rules') reload_nft() logger.info('Restart docker service') diff --git a/node_cli/operations/fair.py b/node_cli/operations/fair.py index ab1fb3f9..91e8ca1c 100644 --- a/node_cli/operations/fair.py +++ b/node_cli/operations/fair.py @@ -37,6 +37,7 @@ from node_cli.core.nftables import configure_nftables from node_cli.core.nginx import generate_nginx_config from node_cli.core.schains import cleanup_no_lvm_datadir +from node_cli.core.static_config import get_fair_chain_name from node_cli.fair.record.chain_record import get_fair_chain_record, migrate_chain_record from node_cli.migrations.fair.from_boot import migrate_nftables_from_boot from node_cli.operations.base import checked_host, turn_off @@ -181,8 +182,9 @@ def update(env_filepath: str, env: dict, update_type: FairUpdateType) -> bool: distro.version(), ) + fair_chain_name = get_fair_chain_name(env) if update_type == FairUpdateType.FROM_BOOT: - migrate_nftables_from_boot() + migrate_nftables_from_boot(chain_name=fair_chain_name) update_images(env=env, node_type=NodeType.FAIR) From 2ada60e3104977cf2cdbca41147a7db0e60607ca Mon Sep 17 00:00:00 2001 From: badrogger Date: Tue, 29 Jul 2025 13:48:15 +0100 Subject: [PATCH 020/198] Fix fair repair snapshot-from validation --- node_cli/cli/fair_node.py | 4 ++-- node_cli/utils/helper.py | 10 ++++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/node_cli/cli/fair_node.py b/node_cli/cli/fair_node.py index aa510023..3fd6bc66 100644 --- a/node_cli/cli/fair_node.py +++ b/node_cli/cli/fair_node.py @@ -30,7 +30,7 @@ from node_cli.fair.fair_node import init as init_fair from node_cli.fair.fair_node import register as register_fair from node_cli.fair.fair_node import update as update_fair -from node_cli.utils.helper import IP_TYPE, URL_TYPE, abort_if_false, streamed_cmd +from node_cli.utils.helper import IP_TYPE, URL_OR_ANY_TYPE, abort_if_false, streamed_cmd from node_cli.utils.texts import safe_load_texts TEXTS = safe_load_texts() @@ -118,7 +118,7 @@ def migrate_node(env_filepath: str) -> None: @node.command('repair', help='Toggle fair chain repair mode') @click.option( '--snapshot-from', - type=URL_TYPE, + type=URL_OR_ANY_TYPE, default='any', hidden=True, help=TEXTS['fair']['node']['repair']['snapshot_from'], diff --git a/node_cli/utils/helper.py b/node_cli/utils/helper.py index 638b63a3..2eb778fb 100644 --- a/node_cli/utils/helper.py +++ b/node_cli/utils/helper.py @@ -382,6 +382,15 @@ def convert(self, value, param, ctx): return value +class UrlOrAnyType(click.ParamType): + name = 'url' + + def convert(self, value, param, ctx): + if value == 'any': + return value + super().convert(value, param, ctx) + + class IpType(click.ParamType): name = 'ip' @@ -394,6 +403,7 @@ def convert(self, value, param, ctx): URL_TYPE = UrlType() +URL_OR_ANY_TYPE = UrlOrAnyType() IP_TYPE = IpType() From cf04391ff92a1a34d3e547dbaa72a22a77af6be0 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Thu, 31 Jul 2025 13:16:57 +0100 Subject: [PATCH 021/198] Rename CONTAINER_CONFIGS_STREAM to NODE_VERSION --- README.md | 26 ++++++++++++------------ node_cli/configs/user.py | 2 +- node_cli/fair/record/chain_record.py | 2 +- node_cli/operations/base.py | 30 ++++++++++++++-------------- node_cli/operations/fair.py | 16 +++++++-------- tests/conftest.py | 10 +++++----- 6 files changed, 43 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index 9336f431..8d1e924c 100644 --- a/README.md +++ b/README.md @@ -45,9 +45,9 @@ Ensure that the following packages are installed: **docker**, **docker-compose** This binary (`skale-VERSION-OS`) is used for managing standard SKALE validator nodes. ```shell -# Replace {version} with the desired release version (e.g., 2.6.0) -VERSION_NUM={version} && \ -sudo -E bash -c "curl -L https://github.com/skalenetwork/node-cli/releases/download/$VERSION_NUM/skale-$VERSION_NUM-`uname -s`-`uname -m` > /usr/local/bin/skale" +# Replace {version} with the desired release version (e.g., 3.0.0) +CLI_VERSION={version} && \ +sudo -E bash -c "curl -L https://github.com/skalenetwork/node-cli/releases/download/$CLI_VERSION/skale-$CLI_VERSION-`uname -s`-`uname -m` > /usr/local/bin/skale" ``` ### Sync Node Binary @@ -55,19 +55,19 @@ sudo -E bash -c "curl -L https://github.com/skalenetwork/node-cli/releases/downl This binary (`skale-VERSION-OS-sync`) is used for managing dedicated Sync nodes. **Ensure you download the correct `-sync` suffixed binary for Sync node operations.** ```shell -# Replace {version} with the desired release version (e.g., 2.6.0) -VERSION_NUM={version} && \ -sudo -E bash -c "curl -L https://github.com/skalenetwork/node-cli/releases/download/$VERSION_NUM/skale-$VERSION_NUM-`uname -s`-`uname -m`-sync > /usr/local/bin/skale" +# Replace {version} with the desired release version (e.g., 3.0.0) +CLI_VERSION={version} && \ +sudo -E bash -c "curl -L https://github.com/skalenetwork/node-cli/releases/download/$CLI_VERSION/skale-$CLI_VERSION-`uname -s`-`uname -m`-sync > /usr/local/bin/skale" ``` ### Fair Node Binary -This binary (`skale-VERSION-OS-fair`) is used specifically for managing nodes on the Fair network. It is named `fair`. +This binary (`skale-VERSION-OS-fair`) is used specifically for managing nodes on the Fair network. ```shell -# Replace {version} with the desired release version (e.g., 2.6.0) -VERSION_NUM={version} && \ -sudo -E bash -c "curl -L https://github.com/skalenetwork/node-cli/releases/download/$VERSION_NUM/skale-$VERSION_NUM-`uname -s`-`uname -m`-fair > /usr/local/bin/fair" +# Replace {version} with the desired release version (e.g., 3.0.0) +CLI_VERSION={version} && \ +sudo -E bash -c "curl -L https://github.com/skalenetwork/node-cli/releases/download/$CLI_VERSION/skale-$CLI_VERSION-`uname -s`-`uname -m`-fair > /usr/local/bin/fair" ``` ### Permissions and Testing @@ -155,7 +155,7 @@ Required environment variables in `ENV_FILE`: * `SGX_SERVER_URL` - SGX server URL. * `DISK_MOUNTPOINT` - Mount point for storing sChains data. * `DOCKER_LVMPY_STREAM` - Stream of `docker-lvmpy` to use. -* `CONTAINER_CONFIGS_STREAM` - Stream of `skale-node` to use. +* `NODE_VERSION` - Stream of `skale-node` to use. * `ENDPOINT` - RPC endpoint of the network where SKALE Manager is deployed. * `MANAGER_CONTRACTS` - SKALE Manager `message_proxy_mainnet` contract alias or address. * `IMA_CONTRACTS` - IMA `skale_manager` contract alias or address. @@ -595,7 +595,7 @@ Required environment variables in `ENV_FILE`: * `DISK_MOUNTPOINT` - Mount point for storing sChain data. * `DOCKER_LVMPY_STREAM` - Stream of `docker-lvmpy`. -* `CONTAINER_CONFIGS_STREAM` - Stream of `skale-node`. +* `NODE_VERSION` - Stream of `skale-node`. * `ENDPOINT` - RPC endpoint of the network where SKALE Manager is deployed. * `MANAGER_CONTRACTS` - SKALE Manager alias or address. * `IMA_CONTRACTS` - IMA alias or address. @@ -695,7 +695,7 @@ Required environment variables in `ENV_FILE`: * `SGX_SERVER_URL` - SGX server URL. * `DISK_MOUNTPOINT` - Mount point for storing data (BTRFS recommended). -* `CONTAINER_CONFIGS_STREAM` - Stream of `skale-node` configs. +* `NODE_VERSION` - Stream of `skale-node` configs. * `ENDPOINT` - RPC endpoint of the network where Fair Manager is deployed. * `MANAGER_CONTRACTS` - SKALE Manager alias or address. * `IMA_CONTRACTS` - IMA alias or address (*Note: Required by boot service, may not be used by Fair itself*). diff --git a/node_cli/configs/user.py b/node_cli/configs/user.py index 49f44d58..eb8764ed 100644 --- a/node_cli/configs/user.py +++ b/node_cli/configs/user.py @@ -44,7 +44,7 @@ class ValidationResult(NamedTuple): @dataclass(kw_only=True) class BaseUserConfig(ABC): - container_configs_stream: str + node_version: str env_type: str filebeat_host: str disk_mountpoint: str diff --git a/node_cli/fair/record/chain_record.py b/node_cli/fair/record/chain_record.py index a97ca9fe..3884d197 100644 --- a/node_cli/fair/record/chain_record.py +++ b/node_cli/fair/record/chain_record.py @@ -75,7 +75,7 @@ def get_fair_chain_record(env: dict) -> ChainRecord: def migrate_chain_record(env: dict) -> None: - version = env['CONTAINER_CONFIGS_STREAM'] + version = env['NODE_VERSION'] logger.info('Migrating fair chain record, setting config version to %s', version) record = get_fair_chain_record(env) record.set_config_version(version) diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index cd9a51b2..df7eec45 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -76,7 +76,7 @@ def checked_host(func): @functools.wraps(func) def wrapper(env_filepath: str, env: Dict, *args, **kwargs): - download_skale_node(env.get('CONTAINER_CONFIGS_STREAM'), env.get('CONTAINER_CONFIGS_DIR')) + download_skale_node(env.get('NODE_VERSION'), env.get('CONTAINER_CONFIGS_DIR')) failed_checks = run_host_checks( env['DISK_MOUNTPOINT'], TYPE, @@ -130,17 +130,17 @@ def update(env_filepath: str, env: Dict, node_type: NodeType) -> bool: meta_manager = CliMetaManager() current_stream = meta_manager.get_meta_info().config_stream skip_cleanup = env.get('SKIP_DOCKER_CLEANUP') == 'True' - if not skip_cleanup and current_stream != env['CONTAINER_CONFIGS_STREAM']: + if not skip_cleanup and current_stream != env['NODE_VERSION']: logger.info( 'Stream version was changed from %s to %s', current_stream, - env['CONTAINER_CONFIGS_STREAM'], + env['NODE_VERSION'], ) docker_cleanup() meta_manager.update_meta( VERSION, - env['CONTAINER_CONFIGS_STREAM'], + env['NODE_VERSION'], env['DOCKER_LVMPY_STREAM'], distro.id(), distro.version(), @@ -173,17 +173,17 @@ def update_fair_boot(env_filepath: str, env: Dict) -> bool: meta_manager = FairCliMetaManager() current_stream = meta_manager.get_meta_info().config_stream skip_cleanup = env.get('SKIP_DOCKER_CLEANUP') == 'True' - if not skip_cleanup and current_stream != env['CONTAINER_CONFIGS_STREAM']: + if not skip_cleanup and current_stream != env['NODE_VERSION']: logger.info( 'Stream version was changed from %s to %s', current_stream, - env['CONTAINER_CONFIGS_STREAM'], + env['NODE_VERSION'], ) docker_cleanup() meta_manager.update_meta( VERSION, - env['CONTAINER_CONFIGS_STREAM'], + env['NODE_VERSION'], distro.id(), distro.version(), ) @@ -216,7 +216,7 @@ def init(env_filepath: str, env: dict, node_type: NodeType) -> None: meta_manager = CliMetaManager() meta_manager.update_meta( VERSION, - env['CONTAINER_CONFIGS_STREAM'], + env['NODE_VERSION'], env['DOCKER_LVMPY_STREAM'], distro.id(), distro.version(), @@ -250,7 +250,7 @@ def init_fair_boot(env_filepath: str, env: dict) -> None: meta_manager = FairCliMetaManager() meta_manager.update_meta( VERSION, - env['CONTAINER_CONFIGS_STREAM'], + env['NODE_VERSION'], distro.id(), distro.version(), ) @@ -268,7 +268,7 @@ def init_sync( snapshot_from: Optional[str], ) -> None: cleanup_volume_artifacts(env['DISK_MOUNTPOINT']) - download_skale_node(env.get('CONTAINER_CONFIGS_STREAM'), env.get('CONTAINER_CONFIGS_DIR')) + download_skale_node(env.get('NODE_VERSION'), env.get('CONTAINER_CONFIGS_DIR')) sync_skale_node() if env.get('SKIP_DOCKER_CONFIG') != 'True': @@ -296,7 +296,7 @@ def init_sync( meta_manager = CliMetaManager() meta_manager.update_meta( VERSION, - env['CONTAINER_CONFIGS_STREAM'], + env['NODE_VERSION'], None, distro.id(), distro.version(), @@ -317,7 +317,7 @@ def update_sync(env_filepath: str, env: Dict) -> bool: compose_rm(env=env, node_type=NodeType.SYNC) remove_dynamic_containers() cleanup_volume_artifacts(env['DISK_MOUNTPOINT']) - download_skale_node(env['CONTAINER_CONFIGS_STREAM'], env.get('CONTAINER_CONFIGS_DIR')) + download_skale_node(env['NODE_VERSION'], env.get('CONTAINER_CONFIGS_DIR')) sync_skale_node() if env.get('SKIP_DOCKER_CONFIG') != 'True': @@ -336,7 +336,7 @@ def update_sync(env_filepath: str, env: Dict) -> bool: meta_manager = CliMetaManager() meta_manager.update_meta( VERSION, - env['CONTAINER_CONFIGS_STREAM'], + env['NODE_VERSION'], env['DOCKER_LVMPY_STREAM'], distro.id(), distro.version(), @@ -359,7 +359,7 @@ def turn_on(env: dict, node_type: NodeType) -> None: meta_manager = CliMetaManager() meta_manager.update_meta( VERSION, - env['CONTAINER_CONFIGS_STREAM'], + env['NODE_VERSION'], env['DOCKER_LVMPY_STREAM'], distro.id(), distro.version(), @@ -402,7 +402,7 @@ def restore(env, backup_path, node_type: NodeType, config_only=False): meta_manager = CliMetaManager() meta_manager.update_meta( VERSION, - env['CONTAINER_CONFIGS_STREAM'], + env['NODE_VERSION'], env['DOCKER_LVMPY_STREAM'], distro.id(), distro.version(), diff --git a/node_cli/operations/fair.py b/node_cli/operations/fair.py index 91e8ca1c..f7f708ee 100644 --- a/node_cli/operations/fair.py +++ b/node_cli/operations/fair.py @@ -95,7 +95,7 @@ def init(env_filepath: str, env: dict) -> bool: meta_manager = FairCliMetaManager() meta_manager.update_meta( VERSION, - env['CONTAINER_CONFIGS_STREAM'], + env['NODE_VERSION'], distro.id(), distro.version(), ) @@ -129,17 +129,17 @@ def update_fair_boot(env_filepath: str, env: dict) -> bool: meta_manager = FairCliMetaManager() current_stream = meta_manager.get_meta_info().config_stream skip_cleanup = env.get('SKIP_DOCKER_CLEANUP') == 'True' - if not skip_cleanup and current_stream != env['CONTAINER_CONFIGS_STREAM']: + if not skip_cleanup and current_stream != env['NODE_VERSION']: logger.info( 'Stream version was changed from %s to %s', current_stream, - env['CONTAINER_CONFIGS_STREAM'], + env['NODE_VERSION'], ) docker_cleanup() meta_manager.update_meta( VERSION, - env['CONTAINER_CONFIGS_STREAM'], + env['NODE_VERSION'], distro.id(), distro.version(), ) @@ -167,17 +167,17 @@ def update(env_filepath: str, env: dict, update_type: FairUpdateType) -> bool: meta_manager = FairCliMetaManager() current_stream = meta_manager.get_meta_info().config_stream skip_cleanup = env.get('SKIP_DOCKER_CLEANUP') == 'True' - if not skip_cleanup and current_stream != env['CONTAINER_CONFIGS_STREAM']: + if not skip_cleanup and current_stream != env['NODE_VERSION']: logger.info( 'Stream version was changed from %s to %s', current_stream, - env['CONTAINER_CONFIGS_STREAM'], + env['NODE_VERSION'], ) docker_cleanup() meta_manager.update_meta( VERSION, - env['CONTAINER_CONFIGS_STREAM'], + env['NODE_VERSION'], distro.id(), distro.version(), ) @@ -224,7 +224,7 @@ def restore(env, backup_path, config_only=False): meta_manager = FairCliMetaManager() meta_manager.update_meta( VERSION, - env['CONTAINER_CONFIGS_STREAM'], + env['NODE_VERSION'], distro.id(), distro.version(), ) diff --git a/tests/conftest.py b/tests/conftest.py index 05da0fae..24c85444 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -242,7 +242,7 @@ def valid_env_params(): 'DB_USER': 'user', 'DB_PASSWORD': 'pass', 'DB_PORT': '3307', - 'CONTAINER_CONFIGS_STREAM': 'master', + 'NODE_VERSION': 'master', 'FILEBEAT_HOST': '127.0.0.1:3010', 'SGX_SERVER_URL': 'http://127.0.0.1', 'DISK_MOUNTPOINT': '/dev/sss', @@ -307,7 +307,7 @@ def regular_user_conf(tmp_path): try: test_env = """ ENDPOINT=http://localhost:8545 - CONTAINER_CONFIGS_STREAM='main' + NODE_VERSION='main' FILEBEAT_HOST=127.0.0.1:3010 SGX_SERVER_URL=http://127.0.0.1 DISK_MOUNTPOINT=/dev/sss @@ -329,7 +329,7 @@ def fair_user_conf(tmp_path): try: test_env = """ BOOT_ENDPOINT=http://localhost:8545 - CONTAINER_CONFIGS_STREAM='main' + NODE_VERSION='main' FILEBEAT_HOST=127.0.0.1:3010 SGX_SERVER_URL=http://127.0.0.1 DISK_MOUNTPOINT=/dev/sss @@ -350,7 +350,7 @@ def fair_boot_user_conf(tmp_path): try: test_env = """ ENDPOINT=http://localhost:8545 - CONTAINER_CONFIGS_STREAM='main' + NODE_VERSION='main' FILEBEAT_HOST=127.0.0.1:3010 SGX_SERVER_URL=http://127.0.0.1 DISK_MOUNTPOINT=/dev/sss @@ -371,7 +371,7 @@ def sync_user_conf(tmp_path): try: test_env = """ ENDPOINT=http://localhost:8545 - CONTAINER_CONFIGS_STREAM='main' + NODE_VERSION='main' FILEBEAT_HOST=127.0.0.1:3010 DISK_MOUNTPOINT=/dev/sss ENV_TYPE='devnet' From 2f0e02aaf0fabc9d23764248d517db8fa26b2a8f Mon Sep 17 00:00:00 2001 From: Dmytro Date: Thu, 31 Jul 2025 13:30:54 +0100 Subject: [PATCH 022/198] Update fair node cli readme - add new command, update boot/node commands --- README.md | 312 ++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 280 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 8d1e924c..bdd3d0e4 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,9 @@ SKALE Node CLI, part of the SKALE suite of validator tools, is the command line 1. [Top level commands (Fair)](#top-level-commands-fair) 2. [Fair Boot commands](#fair-boot-commands) 3. [Fair Node commands](#fair-node-commands) + 4. [Fair Wallet commands](#fair-wallet-commands) + 5. [Fair Logs commands](#fair-logs-commands) + 6. [Fair SSL commands](#fair-ssl-commands) 5. [Exit codes](#exit-codes) 6. [Development](#development) @@ -679,17 +682,29 @@ Options: Commands for a Fair node in the Boot phase. +#### Fair Boot Info + +Get information about the Fair node during boot phase. + +```shell +fair boot info [--format FORMAT] +``` + +Options: + +* `--format`/`-f` - Output format (`json` or `text`). + #### Fair Boot Initialization Initialize the Fair node boot phase. ```shell -fair boot init [ENV_FILE] +fair boot init ``` Arguments: -* `ENV_FILE` - path to .env file (required). +* `ENV_FILE` - Path to the environment file containing configuration. Required environment variables in `ENV_FILE`: @@ -714,19 +729,16 @@ Register the Fair node with Fair Manager *during* the boot phase. fair boot register --name --ip --domain [--port ] ``` -Required arguments: - -* `--name`/`-n` - Fair node name. -* `--ip` - Public IP for RPC connections and consensus. -* `--domain`/`-d` - Fair node domain name (e.g., `fair1.example.com`). - -Optional arguments: +Options: -* `--port`/`-p` - Base port for node sChains (default: `10000`). +* `--name`/`-n` - Fair node name (required). +* `--ip` - Public IP for RPC connections & consensus (required). +* `--domain`/`-d` - Fair node domain name (e.g., `fair1.example.com`, required). +* `--port`/`-p` - Base port for node sChains (default: from configuration). #### Fair Boot Signature -Get the node signature for a validator ID *during* the boot phase. +Get the node signature for a validator ID during boot phase. ```shell fair boot signature @@ -736,21 +748,37 @@ Arguments: * `VALIDATOR_ID` - The ID of the validator requesting the signature. -#### Fair Boot Migrate +#### Fair Boot Update -Migrate the Fair node from the boot phase to the main phase (regular operation). +Update the Fair node software during boot phase. ```shell -fair boot migrate [ENV_FILEPATH] [--yes] +fair boot update [--yes] [--pull-config SCHAIN] ``` Arguments: -* `ENV_FILEPATH` - Path to the .env file. +* `ENV_FILE` - Path to the environment file for node configuration. + +Required environment variables in `ENV_FILE`: + +* `SGX_SERVER_URL` - SGX server URL. +* `DISK_MOUNTPOINT` - Mount point for storing data (BTRFS recommended). +* `NODE_VERSION` - Stream of `skale-node` configs. +* `ENDPOINT` - RPC endpoint of the network where Fair Manager is deployed. +* `MANAGER_CONTRACTS` - SKALE Manager alias or address. +* `IMA_CONTRACTS` - IMA alias or address (*Note: Required by boot service, may not be used by Fair itself*). +* `FILEBEAT_HOST` - URL/IP:Port of the Filebeat log server. +* `ENV_TYPE` - Environment type (e.g., 'mainnet', 'devnet'). + +Optional variables: + +* `MONITORING_CONTAINERS` - Enable monitoring containers (`cadvisor`, `node-exporter`). Options: -* `--yes` - Migrate without confirmation. +* `--yes` - Update without confirmation prompt. +* `--pull-config` - Pull configuration for specific sChain (hidden option). ### Fair Node commands @@ -758,47 +786,129 @@ Options: Commands for managing a Fair node during its regular operation (main phase). -#### Fair Node Initialization (Placeholder) +#### Fair Node Info + +Get information about the Fair node. + +```shell +fair node info [--format FORMAT] +``` + +Options: + +* `--format`/`-f` - Output format (`json` or `text`). + +#### Fair Node Initialization Initialize the regular operation phase of the Fair node. ```shell -fair node init +fair node init ``` -> **Note:** This command is currently a placeholder and not implemented. +Arguments: -#### Fair Node Registration (Placeholder) +* `ENV_FILEPATH` - Path to the environment file for node configuration. -Register the node during regular operation. +Required environment variables in `ENV_FILEPATH`: + +* `FAIR_CONTRACTS` - Fair contracts alias or address (e.g., `mainnet`). +* `NODE_VERSION` - Stream of `skale-node` configs (e.g., `fair-main`). +* `BOOT_ENDPOINT` - RPC endpoint of the Fair network (e.g., `https://rpc.fair.cloud/`). +* `SGX_SERVER_URL` - SGX server URL (e.g., `https://127.0.0.1:1026/`). +* `DISK_MOUNTPOINT` - Mount point for storing data (e.g., `/dev/sdc`). +* `ENV_TYPE` - Environment type (e.g., `mainnet`). + +Optional variables: + +* `ENFORCE_BTRFS` - Format existing filesystem on attached disk (`True`/`False`). +* `FILEBEAT_HOST` - URL of the Filebeat log server to send logs. + +#### Fair Node Registration + +Register the Fair node with the specified IP address. ```shell -fair node register +fair node register --ip ``` -> **Note:** This command is currently a placeholder and not implemented. +Options: + +* `--ip` - Public IP address for the Fair node (required). -#### Fair Node Update (Placeholder) +#### Fair Node Update -Update the Fair node during regular operation. +Update the Fair node software. ```shell -fair node update [ENV_FILEPATH] [--yes] [--unsafe] +fair node update [--yes] [--pull-config SCHAIN] ``` -> **Note:** This command is currently a placeholder and not implemented. +Arguments: + +* `ENV_FILEPATH` - Path to the environment file for node configuration. -#### Fair Node Signature +Required environment variables in `ENV_FILEPATH`: -Get the node signature for a validator ID during regular operation. +* `FAIR_CONTRACTS` - Fair contracts alias or address (e.g., `mainnet`). +* `NODE_VERSION` - Stream of `skale-node` configs (e.g., `fair-main`). +* `BOOT_ENDPOINT` - RPC endpoint of the Fair network (e.g., `https://rpc.fair.cloud/`). +* `SGX_SERVER_URL` - SGX server URL (e.g., `https://127.0.0.1:1026/`). +* `DISK_MOUNTPOINT` - Mount point for storing data (e.g., `/dev/sdc`). +* `ENV_TYPE` - Environment type (e.g., `mainnet`). + +Optional variables: + +* `ENFORCE_BTRFS` - Format existing filesystem on attached disk (`True`/`False`). +* `FILEBEAT_HOST` - URL of the Filebeat log server to send logs. + +Options: + +* `--yes` - Update without confirmation prompt. +* `--pull-config` - Pull configuration for specific sChain (hidden option). + +#### Fair Node Migrate + +Switch from boot phase to regular Fair node operation. ```shell -fair node signature +fair node migrate [--yes] ``` Arguments: -* `VALIDATOR_ID` - The ID of the validator requesting the signature. +* `ENV_FILEPATH` - Path to the environment file for node configuration. + +Required environment variables in `ENV_FILEPATH`: + +* `FAIR_CONTRACTS` - Fair contracts alias or address (e.g., `mainnet`). +* `NODE_VERSION` - Stream of `skale-node` configs (e.g., `fair-main`). +* `BOOT_ENDPOINT` - RPC endpoint of the Fair network (e.g., `https://rpc.fair.cloud/`). +* `SGX_SERVER_URL` - SGX server URL (e.g., `https://127.0.0.1:1026/`). +* `DISK_MOUNTPOINT` - Mount point for storing data (e.g., `/dev/sdc`). +* `ENV_TYPE` - Environment type (e.g., `mainnet`). + +Optional variables: + +* `ENFORCE_BTRFS` - Format existing filesystem on attached disk (`True`/`False`). +* `FILEBEAT_HOST` - URL of the Filebeat log server to send logs. + +Options: + +* `--yes` - Migrate without confirmation prompt. + +#### Fair Node Repair + +Toggle fair chain repair mode. + +```shell +fair node repair [--snapshot-from SOURCE] [--yes] +``` + +Options: + +* `--snapshot-from` - Source for snapshots (`any` by default, hidden option). +* `--yes` - Proceed without confirmation prompt. #### Fair Node Backup @@ -822,13 +932,151 @@ fair node restore [--config-only] Arguments: -* `BACKUP_PATH` - Path to the archive. +* `BACKUP_PATH` - Path to the backup archive. * `ENV_FILE` - Path to the .env file for the restored node configuration. Options: * `--config-only` - Only restore configuration files. +#### Fair Node Cleanup + +Cleanup Fair node data and configuration. + +```shell +fair node cleanup [--yes] +``` + +Options: + +* `--yes` - Cleanup without confirmation prompt. + +#### Fair Node Change IP + +Change the IP address of the Fair node. + +```shell +fair node change-ip +``` + +Arguments: + +* `IP_ADDRESS` - New public IP address for the Fair node. + +### Fair Wallet commands + +> Prefix: `fair wallet` + +Commands for managing the node wallet. + +#### Fair Wallet Info + +Get information about the SKALE node wallet. + +```shell +fair wallet info [--format FORMAT] +``` + +Options: + +* `--format`/`-f` - Output format (`json` or `text`). + +#### Fair Wallet Send + +Send ETH from SKALE node wallet to an address. + +```shell +fair wallet send
[--yes] +``` + +Arguments: + +* `ADDRESS` - Destination address for ETH transfer. +* `AMOUNT` - Amount of ETH to send (as float). + +Options: + +* `--yes` - Send without confirmation prompt. + +### Fair Logs commands + +> Prefix: `fair logs` + +Commands for managing and accessing node logs. + +#### Fair CLI Logs + +Fetch the logs of the node-cli. + +```shell +fair logs cli [--debug] +``` + +Options: + +* `--debug` - Show debug logs instead of regular logs. + +#### Fair Logs Dump + +Dump all logs from the connected node. + +```shell +fair logs dump [--container CONTAINER] +``` + +Arguments: + +* `PATH` - Path where the logs dump will be saved. + +Options: + +* `--container`/`-c` - Dump logs only from specified container. + +### Fair SSL commands + +> Prefix: `fair ssl` + +Commands for managing SSL certificates for sChains. + +#### Fair SSL Status + +Check the status of SSL certificates on the node. + +```shell +fair ssl status +``` + +#### Fair SSL Upload + +Upload SSL certificate files to the node. + +```shell +fair ssl upload --cert-path --key-path [--force] +``` + +Options: + +* `--cert-path`/`-c` - Path to the SSL certificate file (required). +* `--key-path`/`-k` - Path to the SSL private key file (required). +* `--force`/`-f` - Overwrite existing certificates. + +#### Fair SSL Check + +Check SSL certificate validity and connectivity. + +```shell +fair ssl check [--cert-path CERT_PATH] [--key-path KEY_PATH] [--port PORT] [--type TYPE] [--no-client] [--no-wss] +``` + +Options: + +* `--cert-path`/`-c` - Path to the certificate file (default: system default). +* `--key-path`/`-k` - Path to the key file (default: system default). +* `--port`/`-p` - Port to start SSL health check server (default: from configuration). +* `--type`/`-t` - Check type: `all`, `openssl`, or `skaled` (default: `all`). +* `--no-client` - Skip client connection for openssl check. +* `--no-wss` - Skip WSS server starting for skaled check. + *** ## Exit codes From 9ea3c1634cc3e8288718c0cc3dc0cc735b99be63 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Thu, 31 Jul 2025 20:02:35 +0100 Subject: [PATCH 023/198] Add force-skaled-start option for fair node update --- node_cli/cli/fair_node.py | 9 +++++++-- node_cli/fair/fair_node.py | 18 +++++++++++++++--- node_cli/fair/record/chain_record.py | 14 ++++++++++++++ node_cli/operations/fair.py | 15 ++++++++++++--- 4 files changed, 48 insertions(+), 8 deletions(-) diff --git a/node_cli/cli/fair_node.py b/node_cli/cli/fair_node.py index 708da591..8c3ebcdf 100644 --- a/node_cli/cli/fair_node.py +++ b/node_cli/cli/fair_node.py @@ -76,9 +76,14 @@ def register(ip: str) -> None: prompt='Are you sure you want to update Fair node software?', ) @click.option('--pull-config', 'pull_config_for_schain', hidden=True, type=str) +@click.option('--force-skaled-start', 'force_skaled_start', hidden=True, type=bool, default=False) @streamed_cmd -def update_node(env_filepath: str, pull_config_for_schain): - update_fair(env_filepath=env_filepath, pull_config_for_schain=pull_config_for_schain) +def update_node(env_filepath: str, pull_config_for_schain, force_skaled_start: bool): + update_fair( + env_filepath=env_filepath, + pull_config_for_schain=pull_config_for_schain, + force_skaled_start=force_skaled_start, + ) @node.command('backup', help='Generate backup file for the Fair node.') diff --git a/node_cli/fair/fair_node.py b/node_cli/fair/fair_node.py index 1e33a6d5..1b69a50c 100644 --- a/node_cli/fair/fair_node.py +++ b/node_cli/fair/fair_node.py @@ -103,8 +103,15 @@ def migrate_from_boot( @check_inited @check_user -def update(env_filepath: str, pull_config_for_schain: str | None = None) -> None: - logger.info('Updating fair node...') +def update( + env_filepath: str, pull_config_for_schain: str | None = None, force_skaled_start: bool = False +) -> None: + logger.info( + 'Updating fair node: %s, pull_config_for_schain: %s, force_skaled_start: %s', + env_filepath, + pull_config_for_schain, + force_skaled_start, + ) env = compose_node_env( env_filepath, inited_node=True, @@ -112,7 +119,12 @@ def update(env_filepath: str, pull_config_for_schain: str | None = None) -> None node_type=NodeType.FAIR, pull_config_for_schain=pull_config_for_schain, ) - update_ok = update_fair_op(env_filepath, env, update_type=FairUpdateType.REGULAR) + update_ok = update_fair_op( + env_filepath, + env, + update_type=FairUpdateType.REGULAR, + force_skaled_start=force_skaled_start, + ) alive = is_base_containers_alive(node_type=NodeType.FAIR) if not update_ok or not alive: print_node_cmd_error() diff --git a/node_cli/fair/record/chain_record.py b/node_cli/fair/record/chain_record.py index 3884d197..d4ac740c 100644 --- a/node_cli/fair/record/chain_record.py +++ b/node_cli/fair/record/chain_record.py @@ -34,6 +34,7 @@ 'repair_date': FieldInfo('repair_date', datetime, datetime.fromtimestamp(0)), 'repair_ts': FieldInfo('repair_ts', int, None), 'snapshot_from': FieldInfo('snapshot_from', str, None), + 'force_skaled_start': FieldInfo('force_skaled_start', bool, False), } @@ -57,6 +58,10 @@ def snapshot_from(self) -> str | None: def repair_ts(self) -> int | None: return cast(int | None, self._get_field('repair_ts')) + @property + def force_skaled_start(self) -> bool: + return cast(bool, self._get_field('force_skaled_start')) + def set_config_version(self, version: str) -> None: self._set_field('config_version', version) @@ -69,6 +74,9 @@ def set_snapshot_from(self, value: str | None) -> None: def set_repair_ts(self, value: int | None) -> None: self._set_field('repair_ts', value) + def set_force_skaled_start(self, value: bool) -> None: + self._set_field('force_skaled_start', value) + def get_fair_chain_record(env: dict) -> ChainRecord: return ChainRecord(get_fair_chain_name(env)) @@ -79,3 +87,9 @@ def migrate_chain_record(env: dict) -> None: logger.info('Migrating fair chain record, setting config version to %s', version) record = get_fair_chain_record(env) record.set_config_version(version) + + +def update_chain_record(env: dict, force_skaled_start: bool) -> None: + record = get_fair_chain_record(env) + record.set_force_skaled_start(force_skaled_start) + logger.info('Updated fair chain record with force_skaled_start=%s', force_skaled_start) diff --git a/node_cli/operations/fair.py b/node_cli/operations/fair.py index f7f708ee..3455c21a 100644 --- a/node_cli/operations/fair.py +++ b/node_cli/operations/fair.py @@ -38,7 +38,11 @@ from node_cli.core.nginx import generate_nginx_config from node_cli.core.schains import cleanup_no_lvm_datadir from node_cli.core.static_config import get_fair_chain_name -from node_cli.fair.record.chain_record import get_fair_chain_record, migrate_chain_record +from node_cli.fair.record.chain_record import ( + get_fair_chain_record, + migrate_chain_record, + update_chain_record, +) from node_cli.migrations.fair.from_boot import migrate_nftables_from_boot from node_cli.operations.base import checked_host, turn_off from node_cli.operations.common import configure_filebeat, configure_flask, unpack_backup_archive @@ -149,7 +153,12 @@ def update_fair_boot(env_filepath: str, env: dict) -> bool: @checked_host -def update(env_filepath: str, env: dict, update_type: FairUpdateType) -> bool: +def update( + env_filepath: str, + env: dict, + update_type: FairUpdateType, + force_skaled_start: bool, +) -> bool: compose_rm(node_type=NodeType.FAIR, env=env) if update_type not in (FairUpdateType.INFRA_ONLY, FairUpdateType.FROM_BOOT): remove_dynamic_containers() @@ -193,7 +202,7 @@ def update(env_filepath: str, env: dict, update_type: FairUpdateType) -> bool: time.sleep(REDIS_START_TIMEOUT) if update_type == FairUpdateType.FROM_BOOT: migrate_chain_record(env) - + update_chain_record(env, force_skaled_start=force_skaled_start) compose_up(env=env, node_type=NodeType.FAIR) return True From 204e6d1f0fcf766a361df7a960a22a084fdd520d Mon Sep 17 00:00:00 2001 From: Dmytro Date: Fri, 1 Aug 2025 13:22:14 +0100 Subject: [PATCH 024/198] Fix force_skaled_start flag --- node_cli/cli/fair_node.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/node_cli/cli/fair_node.py b/node_cli/cli/fair_node.py index 8c3ebcdf..d99f7ddc 100644 --- a/node_cli/cli/fair_node.py +++ b/node_cli/cli/fair_node.py @@ -76,7 +76,14 @@ def register(ip: str) -> None: prompt='Are you sure you want to update Fair node software?', ) @click.option('--pull-config', 'pull_config_for_schain', hidden=True, type=str) -@click.option('--force-skaled-start', 'force_skaled_start', hidden=True, type=bool, default=False) +@click.option( + '--force-skaled-start', + 'force_skaled_start', + hidden=True, + type=bool, + default=False, + is_flag=True, +) @streamed_cmd def update_node(env_filepath: str, pull_config_for_schain, force_skaled_start: bool): update_fair( From b5c293314acf48168329547b8a7c152ce4489365 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Mon, 4 Aug 2025 16:23:21 +0100 Subject: [PATCH 025/198] Add fair node exit command --- node_cli/cli/fair_node.py | 13 ++++++++ node_cli/fair/fair_node.py | 22 ++++++++++++- tests/cli/fair_cli_test.py | 10 ++++++ tests/fair/fair_node_test.py | 62 ++++++++++++++++++++++++++++++++++++ text.yml | 6 +++- 5 files changed, 111 insertions(+), 2 deletions(-) diff --git a/node_cli/cli/fair_node.py b/node_cli/cli/fair_node.py index d99f7ddc..5b303f17 100644 --- a/node_cli/cli/fair_node.py +++ b/node_cli/cli/fair_node.py @@ -22,6 +22,7 @@ from node_cli.core.node import backup from node_cli.fair.fair_node import change_ip as change_ip_fair from node_cli.fair.fair_node import cleanup as fair_cleanup +from node_cli.fair.fair_node import exit as exit_fair from node_cli.fair.fair_node import ( get_node_info, migrate_from_boot, @@ -165,3 +166,15 @@ def cleanup_node(): @click.argument('ip', type=IP_TYPE) def change_ip(ip: str) -> None: change_ip_fair(ip=ip) + + +@node.command('exit', help=TEXTS['fair']['node']['exit']['help']) +@click.option( + '--yes', + is_flag=True, + callback=abort_if_false, + expose_value=False, + prompt=TEXTS['fair']['node']['exit']['prompt'], +) +def exit_node() -> None: + exit_fair() diff --git a/node_cli/fair/fair_node.py b/node_cli/fair/fair_node.py index 1b69a50c..e6f2fb1d 100644 --- a/node_cli/fair/fair_node.py +++ b/node_cli/fair/fair_node.py @@ -92,7 +92,9 @@ def migrate_from_boot( sync_schains=False, node_type=NodeType.FAIR, ) - migrate_ok = update_fair_op(env_filepath, env, update_type=FairUpdateType.FROM_BOOT) + migrate_ok = update_fair_op( + env_filepath, env, update_type=FairUpdateType.FROM_BOOT, force_skaled_start=False + ) alive = is_base_containers_alive(node_type=NodeType.FAIR) if not migrate_ok or not alive: print_node_cmd_error() @@ -197,3 +199,21 @@ def change_ip(ip: str) -> None: error_msg = payload logger.error(f'Change IP error {error_msg}') error_exit(error_msg, exit_code=CLIExitCodes.BAD_API_RESPONSE) + + +@check_inited +@check_user +def exit() -> None: + if not is_node_inited(): + print(TEXTS['fair']['node']['not_inited']) + return + + status, payload = post_request(blueprint=BLUEPRINT_NAME, method='exit', json={}) + if status == 'ok': + msg = TEXTS['fair']['node']['exited'] + logger.info(msg) + print(msg) + else: + error_msg = payload + logger.error(f'Node exit error {error_msg}') + error_exit(error_msg, exit_code=CLIExitCodes.BAD_API_RESPONSE) diff --git a/tests/cli/fair_cli_test.py b/tests/cli/fair_cli_test.py index d29ae357..51fa0a9b 100644 --- a/tests/cli/fair_cli_test.py +++ b/tests/cli/fair_cli_test.py @@ -11,6 +11,7 @@ from node_cli.cli.fair_node import ( backup_node, migrate_node, + exit_node, restore_node, ) @@ -101,3 +102,12 @@ def test_fair_node_migrate(mock_migrate_core, valid_env_file): assert result.exit_code == 0, f'Output: {result.output}\nException: {result.exception}' mock_migrate_core.assert_called_once_with(env_filepath=valid_env_file) + + +@mock.patch('node_cli.cli.fair_node.exit_fair') +def test_fair_node_exit(mock_exit_core): + runner = CliRunner() + result = runner.invoke(exit_node, ['--yes']) + + assert result.exit_code == 0, f'Output: {result.output}\nException: {result.exception}' + mock_exit_core.assert_called_once() diff --git a/tests/fair/fair_node_test.py b/tests/fair/fair_node_test.py index 5d0c13bb..89d1c24b 100644 --- a/tests/fair/fair_node_test.py +++ b/tests/fair/fair_node_test.py @@ -257,3 +257,65 @@ def test_cleanup_logs_success_message( mock_logger.info.assert_called_once_with( 'Fair node was cleaned up, all containers and data removed' ) + + +@mock.patch('node_cli.utils.decorators.is_user_valid', return_value=True) +@mock.patch('node_cli.fair.fair_node.post_request') +@mock.patch('node_cli.fair.fair_node.is_node_inited', return_value=True) +def test_exit_success( + mock_is_inited, + mock_post_request, + mock_is_user_valid, + inited_node, + resource_alloc, + meta_file_v3, +): + from node_cli.fair.fair_node import exit + + mock_post_request.return_value = ('ok', {}) + + exit() + + mock_post_request.assert_called_once_with(blueprint='fair-node', method='exit', json={}) + + +@mock.patch('node_cli.utils.decorators.is_user_valid', return_value=True) +@mock.patch('node_cli.fair.fair_node.error_exit') +@mock.patch('node_cli.fair.fair_node.post_request') +@mock.patch('node_cli.fair.fair_node.is_node_inited', return_value=True) +def test_exit_error( + mock_is_inited, + mock_post_request, + mock_error_exit, + mock_is_user_valid, + inited_node, + resource_alloc, + meta_file_v3, +): + from node_cli.fair.fair_node import exit + + error_msg = 'Exit failed' + mock_post_request.return_value = ('error', error_msg) + + exit() + + mock_post_request.assert_called_once_with(blueprint='fair-node', method='exit', json={}) + mock_error_exit.assert_called_once_with(error_msg, exit_code=mock.ANY) + + +@mock.patch('node_cli.utils.decorators.is_user_valid', return_value=True) +@mock.patch('node_cli.fair.fair_node.is_node_inited', return_value=False) +def test_exit_not_inited( + mock_is_inited, + mock_is_user_valid, + inited_node, + resource_alloc, + meta_file_v3, + capsys, +): + from node_cli.fair.fair_node import exit + + exit() + + captured = capsys.readouterr() + assert 'Node should be initialized to proceed with operation' in captured.out diff --git a/text.yml b/text.yml index 9aa99951..560729a7 100644 --- a/text.yml +++ b/text.yml @@ -93,4 +93,8 @@ fair: ip: IP address of the node in Fair manager ip_changed: Node IP changed in Fair manager change-ip: - help: Change the node IP in Fair manager \ No newline at end of file + help: Change the node IP in Fair manager + exited: Node removed from Fair manager + exit: + help: Remove node from Fair manager + prompt: Are you sure you want to remove the node from Fair manager? From 98089f5ef939a835a9a6e017a271c1297ceb1c93 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Mon, 4 Aug 2025 16:24:50 +0100 Subject: [PATCH 026/198] Add copilot instructions --- .github/copilot-instructions.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 .github/copilot-instructions.md diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 00000000..416cfaf6 --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,16 @@ +- always keep the changes minimal and purposeful +- focus on fixing the exact problem or implementing the exact feature +- keep the code simple, do not write defensive code +- do not describe your changes in details after you made changes, focus on writing code +- do not generate any documentation, the code should be self-explanatory +- do not generate any in-line comments +- for the new files, always add a license header, same format as in the existing files +- no commented out code +- no console logs in production code +- no unused imports +- no redundant code - move repeated logic into helper functions +- use type hints to specify the expected types of function arguments and return values + +- check `ruff.toml` for formatting rules +- always lint changes using `ruff check` +- tests should be placed in `tests/` directory, follow the existing structure and code style \ No newline at end of file From 66e079fe4c85da71a93acacd6a053ff5b6b2b1f2 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Mon, 4 Aug 2025 17:08:23 +0100 Subject: [PATCH 027/198] Fix migrate from boot test --- tests/fair/fair_node_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fair/fair_node_test.py b/tests/fair/fair_node_test.py index 89d1c24b..19e8fe4c 100644 --- a/tests/fair/fair_node_test.py +++ b/tests/fair/fair_node_test.py @@ -125,7 +125,7 @@ def test_migrate_from_boot( node_type=NodeType.FAIR, ) mock_migrate_op.assert_called_once_with( - valid_env_file, mock_env, update_type=FairUpdateType.FROM_BOOT + valid_env_file, mock_env, update_type=FairUpdateType.FROM_BOOT, force_skaled_start=False ) From ec1f26500dad4ba21ef9a2e73981a113c071aa22 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Mon, 4 Aug 2025 19:30:28 +0100 Subject: [PATCH 028/198] Fix node exit route --- node_cli/configs/routes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node_cli/configs/routes.py b/node_cli/configs/routes.py index 942cdfd4..d595d8ea 100644 --- a/node_cli/configs/routes.py +++ b/node_cli/configs/routes.py @@ -40,7 +40,7 @@ 'schains': ['config', 'list', 'dkg-statuses', 'firewall-rules', 'repair', 'get'], 'ssl': ['status', 'upload'], 'wallet': ['info', 'send-eth'], - 'fair-node': ['info', 'register', 'change-ip'], + 'fair-node': ['info', 'register', 'change-ip', 'exit'], } } From 2fe8538ec23793e60da706dc828d498f3222773f Mon Sep 17 00:00:00 2001 From: Dmytro Date: Tue, 5 Aug 2025 12:37:06 +0100 Subject: [PATCH 029/198] Add streamed cmd decorator to node exit, fix api test --- node_cli/cli/fair_node.py | 2 ++ tests/routes_test.py | 1 + 2 files changed, 3 insertions(+) diff --git a/node_cli/cli/fair_node.py b/node_cli/cli/fair_node.py index 5b303f17..19b06c93 100644 --- a/node_cli/cli/fair_node.py +++ b/node_cli/cli/fair_node.py @@ -164,6 +164,7 @@ def cleanup_node(): @node.command('change-ip', help=TEXTS['fair']['node']['change-ip']['help']) @click.argument('ip', type=IP_TYPE) +@streamed_cmd def change_ip(ip: str) -> None: change_ip_fair(ip=ip) @@ -176,5 +177,6 @@ def change_ip(ip: str) -> None: expose_value=False, prompt=TEXTS['fair']['node']['exit']['prompt'], ) +@streamed_cmd def exit_node() -> None: exit_fair() diff --git a/tests/routes_test.py b/tests/routes_test.py index 3dd87f84..6a7918f0 100644 --- a/tests/routes_test.py +++ b/tests/routes_test.py @@ -34,6 +34,7 @@ '/api/v1/fair-node/info', '/api/v1/fair-node/register', '/api/v1/fair-node/change-ip', + '/api/v1/fair-node/exit', ] From e98c82bd89d5c9101f8b7ae2f40c4c2ebaf2aa86 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Tue, 5 Aug 2025 19:35:50 +0100 Subject: [PATCH 030/198] Add fair chain commands - checks and record --- README.md | 41 ++++++++++++++++--- node_cli/cli/chain.py | 44 ++++++++++++++++++++ node_cli/cli/ssl.py | 2 +- node_cli/configs/routes.py | 1 + node_cli/fair/chain.py | 65 ++++++++++++++++++++++++++++++ node_cli/main.py | 2 + node_cli/utils/print_formatters.py | 40 ++++++++++++++++++ 7 files changed, 189 insertions(+), 6 deletions(-) create mode 100644 node_cli/cli/chain.py create mode 100644 node_cli/fair/chain.py diff --git a/README.md b/README.md index bdd3d0e4..b2bad1bc 100644 --- a/README.md +++ b/README.md @@ -29,9 +29,10 @@ SKALE Node CLI, part of the SKALE suite of validator tools, is the command line 1. [Top level commands (Fair)](#top-level-commands-fair) 2. [Fair Boot commands](#fair-boot-commands) 3. [Fair Node commands](#fair-node-commands) - 4. [Fair Wallet commands](#fair-wallet-commands) - 5. [Fair Logs commands](#fair-logs-commands) - 6. [Fair SSL commands](#fair-ssl-commands) + 4. [Fair Chain commands](#fair-chain-commands) + 5. [Fair Wallet commands](#fair-wallet-commands) + 6. [Fair Logs commands](#fair-logs-commands) + 7. [Fair SSL commands](#fair-ssl-commands) 5. [Exit codes](#exit-codes) 6. [Development](#development) @@ -841,7 +842,7 @@ Options: Update the Fair node software. ```shell -fair node update [--yes] [--pull-config SCHAIN] +fair node update [--yes] [--force-skaled-start] ``` Arguments: @@ -865,7 +866,7 @@ Optional variables: Options: * `--yes` - Update without confirmation prompt. -* `--pull-config` - Pull configuration for specific sChain (hidden option). +* `--force-skaled-start` - Force skaled container to start (hidden option). #### Fair Node Migrate @@ -963,6 +964,36 @@ Arguments: * `IP_ADDRESS` - New public IP address for the Fair node. +### Fair Chain commands + +> Prefix: `fair chain` + +Commands for managing and monitoring the Fair chain state and configuration. + +#### Fair Chain Record + +Get information about the Fair chain record, including chain name, configuration status, DKG status, and operational metadata. + +```shell +fair chain record [--json] +``` + +Options: + +* `--json` - Output in JSON format instead of formatted table. + +#### Fair Chain Checks + +Get the status of Fair chain checks, including configuration checks and skaled checks. + +```shell +fair chain checks [--json] +``` + +Options: + +* `--json` - Output in JSON format instead of formatted table. + ### Fair Wallet commands > Prefix: `fair wallet` diff --git a/node_cli/cli/chain.py b/node_cli/cli/chain.py new file mode 100644 index 00000000..10826236 --- /dev/null +++ b/node_cli/cli/chain.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +# +# This file is part of node-cli +# +# Copyright (C) 2025-Present SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import click + +from node_cli.fair.chain import get_chain_record, get_chain_checks + + +@click.group() +def chain_cli(): + pass + + +@chain_cli.group(help='Fair chain commands') +def chain(): + pass + + +@chain.command('record', help='Get Fair chain record information') +@click.option('--json', 'raw', is_flag=True, help='Output in JSON format') +def chain_record(raw: bool) -> None: + get_chain_record(raw=raw) + + +@chain.command('checks', help='Get Fair chain checks status') +@click.option('--json', 'raw', is_flag=True, help='Output in JSON format') +def chain_checks(raw: bool) -> None: + get_chain_checks(raw=raw) diff --git a/node_cli/cli/ssl.py b/node_cli/cli/ssl.py index e371e8ce..c22ac226 100644 --- a/node_cli/cli/ssl.py +++ b/node_cli/cli/ssl.py @@ -82,7 +82,7 @@ def upload(key_path, cert_path, force): @click.option( '--port', '-p', - help='Port to start ssl healtcheck server', + help='Port to start ssl healthcheck server', type=int, default=DEFAULT_SSL_CHECK_PORT, ) diff --git a/node_cli/configs/routes.py b/node_cli/configs/routes.py index d595d8ea..a1a17879 100644 --- a/node_cli/configs/routes.py +++ b/node_cli/configs/routes.py @@ -41,6 +41,7 @@ 'ssl': ['status', 'upload'], 'wallet': ['info', 'send-eth'], 'fair-node': ['info', 'register', 'change-ip', 'exit'], + 'fair-chain': ['record', 'checks'], } } diff --git a/node_cli/fair/chain.py b/node_cli/fair/chain.py new file mode 100644 index 00000000..8ca380ae --- /dev/null +++ b/node_cli/fair/chain.py @@ -0,0 +1,65 @@ +# -*- coding: utf-8 -*- +# +# This file is part of node-cli +# +# Copyright (C) 2025-Present SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import json +from typing import Any, Dict + +from node_cli.utils.exit_codes import CLIExitCodes +from node_cli.utils.helper import error_exit, get_request +from node_cli.utils.print_formatters import print_chain_record, print_chain_checks + +BLUEPRINT_NAME = 'fair-chain' + + +def get_chain_record_plain() -> Dict[str, Any]: + status, payload = get_request(blueprint=BLUEPRINT_NAME, method='record') + if status == 'ok': + if isinstance(payload, dict): + return payload.get('record', {}) + else: + error_exit('Invalid response format', exit_code=CLIExitCodes.BAD_API_RESPONSE) + else: + error_exit(payload, exit_code=CLIExitCodes.BAD_API_RESPONSE) + + +def get_chain_record(raw: bool = False) -> None: + record = get_chain_record_plain() + if raw: + print(json.dumps(record, indent=4)) + else: + print_chain_record(record) + + +def get_chain_checks_plain() -> Dict[str, Any]: + status, payload = get_request(blueprint=BLUEPRINT_NAME, method='checks') + if status == 'ok': + if isinstance(payload, dict): + return payload + else: + error_exit('Invalid response format', exit_code=CLIExitCodes.BAD_API_RESPONSE) + else: + error_exit(payload, exit_code=CLIExitCodes.BAD_API_RESPONSE) + + +def get_chain_checks(raw: bool = False) -> None: + checks = get_chain_checks_plain() + if raw: + print(json.dumps(checks, indent=4)) + else: + print_chain_checks(checks) diff --git a/node_cli/main.py b/node_cli/main.py index 28d87fe0..7fb833c0 100644 --- a/node_cli/main.py +++ b/node_cli/main.py @@ -40,6 +40,7 @@ from node_cli.cli.sync_node import sync_node_cli from node_cli.cli.fair_boot import fair_boot_cli from node_cli.cli.fair_node import fair_node_cli +from node_cli.cli.chain import chain_cli from node_cli.core.host import init_logs_dir from node_cli.utils.node_type import NodeType from node_cli.configs import LONG_LINE @@ -91,6 +92,7 @@ def get_sources_list() -> List[click.MultiCommand]: logs_cli, fair_boot_cli, fair_node_cli, + chain_cli, wallet_cli, ssl_cli, ] diff --git a/node_cli/utils/print_formatters.py b/node_cli/utils/print_formatters.py index 1fd1ca33..81e92be3 100644 --- a/node_cli/utils/print_formatters.py +++ b/node_cli/utils/print_formatters.py @@ -323,3 +323,43 @@ def print_meta_info(meta_info: CliMeta) -> None: {LONG_LINE} """) ) + + +def print_chain_record(record): + print( + inspect.cleandoc(f""" + {LONG_LINE} + Fair Chain Record + Chain Name: {record.get('name', 'N/A')} + Config Version: {record.get('config_version', 'N/A')} + Sync Config Run: {record.get('sync_config_run', 'N/A')} + First Run: {record.get('first_run', 'N/A')} + Backup Run: {record.get('backup_run', 'N/A')} + Restart Count: {record.get('restart_count', 'N/A')} + Failed RPC Count: {record.get('failed_rpc_count', 'N/A')} + Monitor Last Seen: {record.get('monitor_last_seen', 'N/A')} + SSL Change Date: {record.get('ssl_change_date', 'N/A')} + Repair Date: {record.get('repair_date', 'N/A')} + DKG Status: {record.get('dkg_status', 'N/A')} + Repair Timestamp: {record.get('repair_ts', 'N/A')} + Snapshot From: {record.get('snapshot_from', 'N/A')} + Restart Timestamp: {record.get('restart_ts', 'N/A')} + Force Skaled Start: {record.get('force_skaled_start', 'N/A')} + {LONG_LINE} + """) + ) + + +def print_chain_checks(checks): + def format_checks(check_dict, title): + print(f'\n{title}:') + for name, result in check_dict.items(): + status = 'PASS' if result else 'FAIL' + print(f' {name}: {status}') + + print(f'{LONG_LINE}') + print('Fair Chain Checks') + print(f'{LONG_LINE}') + format_checks(checks['config_checks'], 'Config Checks') + format_checks(checks['skaled_checks'], 'Skaled Checks') + print(f'{LONG_LINE}') From 1e2accca331960459797a4dd1d8b42a2d1e3a901 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Tue, 5 Aug 2025 20:33:11 +0100 Subject: [PATCH 031/198] Add string formatters --- node_cli/utils/print_formatters.py | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/node_cli/utils/print_formatters.py b/node_cli/utils/print_formatters.py index 81e92be3..821e51f6 100644 --- a/node_cli/utils/print_formatters.py +++ b/node_cli/utils/print_formatters.py @@ -325,6 +325,23 @@ def print_meta_info(meta_info: CliMeta) -> None: ) +def format_timestamp(value): + if value is None or value == 'N/A' or value == 0 or value == 0.0: + return 'N/A' + try: + timestamp = float(value) + if timestamp == 0: + return 'N/A' + dt = datetime.datetime.fromtimestamp(timestamp) + human_date = dt.strftime('%Y-%m-%d %H:%M:%S') + return f'{human_date} ({timestamp})' + except (ValueError, TypeError): + return str(value) + + +1 + + def print_chain_record(record): print( inspect.cleandoc(f""" @@ -337,13 +354,13 @@ def print_chain_record(record): Backup Run: {record.get('backup_run', 'N/A')} Restart Count: {record.get('restart_count', 'N/A')} Failed RPC Count: {record.get('failed_rpc_count', 'N/A')} - Monitor Last Seen: {record.get('monitor_last_seen', 'N/A')} - SSL Change Date: {record.get('ssl_change_date', 'N/A')} - Repair Date: {record.get('repair_date', 'N/A')} + Monitor Last Seen: {format_timestamp(record.get('monitor_last_seen', 'N/A'))} + SSL Change Date: {format_timestamp(record.get('ssl_change_date', 'N/A'))} + Repair Date: {format_timestamp(record.get('repair_date', 'N/A'))} DKG Status: {record.get('dkg_status', 'N/A')} - Repair Timestamp: {record.get('repair_ts', 'N/A')} + Repair Timestamp: {format_timestamp(record.get('repair_ts', 'N/A'))} Snapshot From: {record.get('snapshot_from', 'N/A')} - Restart Timestamp: {record.get('restart_ts', 'N/A')} + Restart Timestamp: {format_timestamp(record.get('restart_ts', 'N/A'))} Force Skaled Start: {record.get('force_skaled_start', 'N/A')} {LONG_LINE} """) From df49e5b85394275c589922c35abcbd952a0d3b2a Mon Sep 17 00:00:00 2001 From: Dmytro Date: Tue, 5 Aug 2025 21:15:23 +0100 Subject: [PATCH 032/198] Add fair chain routes to tests --- tests/routes_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/routes_test.py b/tests/routes_test.py index 6a7918f0..bcf1bf98 100644 --- a/tests/routes_test.py +++ b/tests/routes_test.py @@ -35,6 +35,8 @@ '/api/v1/fair-node/register', '/api/v1/fair-node/change-ip', '/api/v1/fair-node/exit', + '/api/v1/fair-chain/record', + '/api/v1/fair-chain/checks', ] From 69a2d3748f4c8e04b2d13c350a94205376aada74 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Wed, 6 Aug 2025 13:19:04 +0100 Subject: [PATCH 033/198] Rename sync node to passive --- .github/workflows/publish.yml | 2 +- .github/workflows/test.yml | 8 +-- README.md | 66 +++++++++---------- .../cli/{sync_node.py => passive_node.py} | 28 ++++---- node_cli/configs/__init__.py | 2 +- node_cli/configs/user.py | 6 +- node_cli/core/nginx.py | 2 +- node_cli/core/node.py | 46 ++++++------- node_cli/main.py | 8 +-- node_cli/operations/__init__.py | 6 +- node_cli/operations/base.py | 20 +++--- node_cli/utils/docker_utils.py | 24 +++---- node_cli/utils/node_type.py | 2 +- scripts/build.sh | 6 +- scripts/generate_info.sh | 8 +-- tests/cli/node_test.py | 2 +- ...sync_node_test.py => passive_node_test.py} | 30 ++++----- tests/configs/configs_env_validate_test.py | 6 +- tests/conftest.py | 4 +- tests/core/core_node_test.py | 24 +++---- tests/core/core_schains_test.py | 6 +- tests/core/nginx_test.py | 6 +- text.yml | 8 +-- 23 files changed, 160 insertions(+), 160 deletions(-) rename node_cli/cli/{sync_node.py => passive_node.py} (76%) rename tests/cli/{sync_node_test.py => passive_node_test.py} (84%) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 57852a13..96290f1b 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -72,7 +72,7 @@ jobs: strategy: matrix: os: [ubuntu-22.04] - build_type: [normal, sync, fair] + build_type: [normal, passive, fair] steps: - name: Checkout code uses: actions/checkout@v4 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1a230126..043c1792 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -47,15 +47,15 @@ jobs: - name: Check build - normal run: sudo /home/ubuntu/dist/skale-test-Linux-x86_64 - - name: Build binary - sync + - name: Build binary - passive run: | mkdir -p ./dist docker build . -t node-cli-builder - docker run -v /home/ubuntu/dist:/app/dist node-cli-builder bash scripts/build.sh test test sync + docker run -v /home/ubuntu/dist:/app/dist node-cli-builder bash scripts/build.sh test test passive docker rm -f $(docker ps -aq) - - name: Check build - sync - run: sudo /home/ubuntu/dist/skale-test-Linux-x86_64-sync + - name: Check build - passive + run: sudo /home/ubuntu/dist/skale-test-Linux-x86_64-passive - name: Build binary - fair run: | diff --git a/README.md b/README.md index b2bad1bc..3b8b1b88 100644 --- a/README.md +++ b/README.md @@ -4,13 +4,13 @@ ![Test](https://github.com/skalenetwork/node-cli/workflows/Test/badge.svg) [![Discord](https://img.shields.io/discord/534485763354787851.svg)](https://discord.gg/vvUtWJB) -SKALE Node CLI, part of the SKALE suite of validator tools, is the command line interface to setup, register and maintain your SKALE node. It comes in three distinct build types: Standard (for validator nodes), Sync (for dedicated sChain synchronization), and Fair. +SKALE Node CLI, part of the SKALE suite of validator tools, is the command line interface to setup, register and maintain your SKALE node. It comes in three distinct build types: Standard (for validator nodes), Passive (for dedicated sChain synchronization), and Fair. ## Table of Contents 1. [Installation](#installation) 1. [Standard Node Binary](#standard-node-binary) - 2. [Sync Node Binary](#sync-node-binary) + 2. [Passive Node Binary](#passive-node-binary) 3. [Fair Node Binary](#fair-node-binary) 4. [Permissions and Testing](#permissions-and-testing) 2. [Standard Node Usage (`skale` - Normal Build)](#standard-node-usage-skale---normal-build) @@ -22,9 +22,9 @@ SKALE Node CLI, part of the SKALE suite of validator tools, is the command line 6. [SSL commands (Standard)](#ssl-commands-standard) 7. [Logs commands (Standard)](#logs-commands-standard) 8. [Resources allocation commands (Standard)](#resources-allocation-commands-standard) -3. [Sync Node Usage (`skale` - Sync Build)](#sync-node-usage-skale---sync-build) - 1. [Top level commands (Sync)](#top-level-commands-sync) - 2. [Sync node commands](#sync-node-commands) +3. [Passive Node Usage (`skale` - Passive Build)](#passive-node-usage-skale---passive-build) + 1. [Top level commands (Passive)](#top-level-commands-passive) + 2. [Passive node commands](#passive-node-commands) 4. [Fair Node Usage (`fair`)](#fair-node-usage-fair) 1. [Top level commands (Fair)](#top-level-commands-fair) 2. [Fair Boot commands](#fair-boot-commands) @@ -54,14 +54,14 @@ CLI_VERSION={version} && \ sudo -E bash -c "curl -L https://github.com/skalenetwork/node-cli/releases/download/$CLI_VERSION/skale-$CLI_VERSION-`uname -s`-`uname -m` > /usr/local/bin/skale" ``` -### Sync Node Binary +### Passive Node Binary -This binary (`skale-VERSION-OS-sync`) is used for managing dedicated Sync nodes. **Ensure you download the correct `-sync` suffixed binary for Sync node operations.** +This binary (`skale-VERSION-OS-passive`) is used for managing dedicated Passive nodes. **Ensure you download the correct `-passive` suffixed binary for Passive node operations.** ```shell # Replace {version} with the desired release version (e.g., 3.0.0) CLI_VERSION={version} && \ -sudo -E bash -c "curl -L https://github.com/skalenetwork/node-cli/releases/download/$CLI_VERSION/skale-$CLI_VERSION-`uname -s`-`uname -m`-sync > /usr/local/bin/skale" +sudo -E bash -c "curl -L https://github.com/skalenetwork/node-cli/releases/download/$CLI_VERSION/skale-$CLI_VERSION-`uname -s`-`uname -m`-passive > /usr/local/bin/skale" ``` ### Fair Node Binary @@ -79,7 +79,7 @@ sudo -E bash -c "curl -L https://github.com/skalenetwork/node-cli/releases/downl Apply executable permissions to the downloaded binary (adjust name accordingly): ```shell -# For Standard or Sync binary +# For Standard or Passive binary sudo chmod +x /usr/local/bin/skale # For Fair binary @@ -89,7 +89,7 @@ sudo chmod +x /usr/local/bin/fair Test the installation: ```shell -# Standard or Sync build +# Standard or Passive build skale --help # Fair build @@ -552,24 +552,24 @@ Options: *** -## Sync Node Usage (`skale` - Sync Build) +## Passive Node Usage (`skale` - Passive Build) -Commands available in the **sync `skale` binary** for managing dedicated Sync nodes. +Commands available in the **passive `skale` binary** for managing dedicated Passive nodes. Note that this binary contains a **different set of commands** compared to the standard build. -### Top level commands (Sync) +### Top level commands (Passive) -#### Info (Sync) +#### Info (Passive) -Print build info for the `skale` (sync) binary. +Print build info for the `skale` (passive) binary. ```shell skale info ``` -#### Version (Sync) +#### Version (Passive) -Print version number for the `skale` (sync) binary. +Print version number for the `skale` (passive) binary. ```shell skale version @@ -579,16 +579,16 @@ Options: * `--short` - prints version only, without additional text. -### Sync node commands +### Passive node commands -> Prefix: `skale sync-node` +> Prefix: `skale passive-node` -#### Sync node initialization +#### Passive node initialization -Initialize a dedicated Sync node on the current machine. +Initialize a dedicated Passive node on the current machine. ```shell -skale sync-node init [ENV_FILE] [--indexer | --archive] [--snapshot] [--snapshot-from ] [--yes] +skale passive-node init [ENV_FILE] [--indexer | --archive] [--snapshot] [--snapshot-from ] [--yes] ``` Arguments: @@ -617,12 +617,12 @@ Options: * `--snapshot-from ` - Specify the IP of another node to download a snapshot from. * `--yes` - Initialize without additional confirmation. -#### Sync node update +#### Passive node update -Update the Sync node software and configuration. +Update the Passive node software and configuration. ```shell -skale sync-node update [ENV_FILEPATH] [--yes] +skale passive-node update [ENV_FILEPATH] [--yes] ``` Arguments: @@ -633,21 +633,21 @@ Options: * `--yes` - Update without additionalconfirmation. -> NOTE: You can just update a file with environment variables used during `skale sync-node init`. +> NOTE: You can just update a file with environment variables used during `skale passive-node init`. -#### Sync node cleanup +#### Passive node cleanup -Remove all data and containers for the Sync node. +Remove all data and containers for the Passive node. ```shell -skale sync-node cleanup [--yes] +skale passive-node cleanup [--yes] ``` Options: * `--yes` - Cleanup without confirmation. -> WARNING: This command removes all Sync node data. +> WARNING: This command removes all Passive node data. *** @@ -1166,14 +1166,14 @@ pip install -e ".[dev]" #### Generate info.py locally -Specify the build type (`normal`, `sync`, or `fair`): +Specify the build type (`normal`, `passive`, or `fair`): ```shell # Example for Standard build ./scripts/generate_info.sh 1.0.0 my-branch normal -# Example for Sync build -./scripts/generate_info.sh 1.0.0 my-branch sync +# Example for Passive build +./scripts/generate_info.sh 1.0.0 my-branch passive # Example for Fair build ./scripts/generate_info.sh 1.0.0 my-branch fair diff --git a/node_cli/cli/sync_node.py b/node_cli/cli/passive_node.py similarity index 76% rename from node_cli/cli/sync_node.py rename to node_cli/cli/passive_node.py index 5f0a5217..c24fe0b5 100644 --- a/node_cli/cli/sync_node.py +++ b/node_cli/cli/passive_node.py @@ -21,7 +21,7 @@ import click -from node_cli.core.node import init_sync, update_sync, cleanup_sync +from node_cli.core.node import init_passive, update_passive, cleanup_passive from node_cli.utils.helper import ( abort_if_false, error_exit, @@ -32,20 +32,20 @@ G_TEXTS = safe_load_texts() -TEXTS = G_TEXTS['sync_node'] +TEXTS = G_TEXTS['passive_node'] @click.group() -def sync_node_cli(): +def passive_node_cli(): pass -@sync_node_cli.group(help='SKALE sync node commands') -def sync_node(): +@passive_node_cli.group(help='SKALE passive node commands') +def passive_node(): pass -@sync_node.command('init', help=TEXTS['init']['help']) +@passive_node.command('init', help=TEXTS['init']['help']) @click.argument('env_file') @click.option('--indexer', help=TEXTS['init']['indexer'], is_flag=True) @click.option('--archive', help=TEXTS['init']['archive'], is_flag=True) @@ -54,15 +54,15 @@ def sync_node(): '--snapshot-from', type=URL_TYPE, default=None, hidden=True, help=TEXTS['init']['snapshot_from'] ) @streamed_cmd -def _init_sync( +def _init_passive( env_file, indexer: bool, archive: bool, snapshot: bool, snapshot_from: Optional[str] ) -> None: if indexer and archive: error_exit('Cannot use both --indexer and --archive options') - init_sync(env_file, indexer, archive, snapshot, snapshot_from) + init_passive(env_file, indexer, archive, snapshot, snapshot_from) -@sync_node.command('update', help='Update sync node from .env file') +@passive_node.command('update', help='Update passive node from .env file') @click.option( '--yes', is_flag=True, @@ -73,11 +73,11 @@ def _init_sync( @click.option('--unsafe', 'unsafe_ok', help='Allow unsafe update', hidden=True, is_flag=True) @click.argument('env_file') @streamed_cmd -def _update_sync(env_file, unsafe_ok): - update_sync(env_file) +def _update_passive(env_file, unsafe_ok): + update_passive(env_file) -@sync_node.command('cleanup', help='Remove sync node data and containers') +@passive_node.command('cleanup', help='Remove passive node data and containers') @click.option( '--yes', is_flag=True, @@ -86,5 +86,5 @@ def _update_sync(env_file, unsafe_ok): prompt='Are you sure you want to remove all node containers and data?', ) @streamed_cmd -def _cleanup_sync() -> None: - cleanup_sync() +def _cleanup_passive() -> None: + cleanup_passive() diff --git a/node_cli/configs/__init__.py b/node_cli/configs/__init__.py index 6709ee75..748aa41a 100644 --- a/node_cli/configs/__init__.py +++ b/node_cli/configs/__init__.py @@ -55,7 +55,7 @@ SGX_CERTIFICATES_DIR_NAME = 'sgx_certs' COMPOSE_PATH = os.path.join(CONTAINER_CONFIG_PATH, 'docker-compose.yml') -SYNC_COMPOSE_PATH = os.path.join(CONTAINER_CONFIG_PATH, 'docker-compose-sync.yml') +PASSIVE_COMPOSE_PATH = os.path.join(CONTAINER_CONFIG_PATH, 'docker-compose-passive.yml') FAIR_COMPOSE_PATH = os.path.join(CONTAINER_CONFIG_PATH, 'docker-compose-fair.yml') STATIC_PARAMS_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, 'static_params.yaml') FAIR_STATIC_PARAMS_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, 'fair_static_params.yaml') diff --git a/node_cli/configs/user.py b/node_cli/configs/user.py index eb8764ed..248cdf65 100644 --- a/node_cli/configs/user.py +++ b/node_cli/configs/user.py @@ -116,7 +116,7 @@ class SkaleUserConfig(BaseUserConfig): @dataclass -class SyncUserConfig(BaseUserConfig): +class PassiveUserConfig(BaseUserConfig): endpoint: str manager_contracts: str schain_name: str = '' @@ -176,8 +176,8 @@ def get_user_config_class( user_config_class = FairBootUserConfig elif node_type == NodeType.FAIR: user_config_class = FairUserConfig - elif node_type == NodeType.SYNC: - user_config_class = SyncUserConfig + elif node_type == NodeType.PASSIVE: + user_config_class = PassiveUserConfig else: user_config_class = SkaleUserConfig return user_config_class diff --git a/node_cli/core/nginx.py b/node_cli/core/nginx.py index 97ea4844..c16b0a34 100644 --- a/node_cli/core/nginx.py +++ b/node_cli/core/nginx.py @@ -51,7 +51,7 @@ def check_ssl_certs(): def is_regular_node_nginx() -> bool: - return TYPE in [NodeType.REGULAR, NodeType.SYNC] + return TYPE in [NodeType.REGULAR, NodeType.PASSIVE] def reload_nginx() -> None: diff --git a/node_cli/core/node.py b/node_cli/core/node.py index d803ef58..a47ba10b 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -49,22 +49,22 @@ from node_cli.core.resources import update_resource_allocation from node_cli.migrations.focal_to_jammy import migrate as migrate_2_6 from node_cli.operations import ( - cleanup_sync_op, + cleanup_passive_op, configure_nftables, init_op, - init_sync_op, + init_passive_op, restore_op, turn_off_op, turn_on_op, update_op, - update_sync_op, + update_passive_op, ) from node_cli.utils.decorators import check_inited, check_not_inited, check_user from node_cli.utils.docker_utils import ( BASE_FAIR_BOOT_COMPOSE_SERVICES, BASE_FAIR_COMPOSE_SERVICES, BASE_SKALE_COMPOSE_SERVICES, - BASE_SYNC_COMPOSE_SERVICES, + BASE_PASSIVE_COMPOSE_SERVICES, is_admin_running, is_api_running, ) @@ -103,7 +103,7 @@ class NodeStatuses(Enum): def is_update_safe(node_type: NodeType) -> bool: if not is_admin_running(node_type): - if node_type == NodeType.SYNC: + if node_type == NodeType.PASSIVE: return True elif not is_api_running(node_type): return True @@ -178,33 +178,33 @@ def restore(backup_path, env_filepath, node_type: NodeType, no_snapshot=False, c @check_not_inited -def init_sync( +def init_passive( env_filepath: str, indexer: bool, archive: bool, snapshot: bool, snapshot_from: Optional[str] ) -> None: - env = compose_node_env(env_filepath, node_type=NodeType.SYNC) + env = compose_node_env(env_filepath, node_type=NodeType.PASSIVE) if env is None: return - init_sync_op(env_filepath, env, indexer, archive, snapshot, snapshot_from) + init_passive_op(env_filepath, env, indexer, archive, snapshot, snapshot_from) logger.info('Waiting for containers initialization') time.sleep(TM_INIT_TIMEOUT) - if not is_base_containers_alive(node_type=NodeType.SYNC): + if not is_base_containers_alive(node_type=NodeType.PASSIVE): error_exit('Containers are not running', exit_code=CLIExitCodes.OPERATION_EXECUTION_ERROR) - logger.info('Sync node initialized successfully') + logger.info('Passive node initialized successfully') @check_inited @check_user -def update_sync(env_filepath: str, unsafe_ok: bool = False) -> None: +def update_passive(env_filepath: str, unsafe_ok: bool = False) -> None: logger.info('Node update started') prev_version = CliMetaManager().get_meta_info().version if (__version__ == 'test' or __version__.startswith('2.6')) and prev_version == '2.5.0': migrate_2_6() - env = compose_node_env(env_filepath, node_type=NodeType.SYNC) - update_ok = update_sync_op(env_filepath, env) + env = compose_node_env(env_filepath, node_type=NodeType.PASSIVE) + update_ok = update_passive_op(env_filepath, env) if update_ok: logger.info('Waiting for containers initialization') time.sleep(TM_INIT_TIMEOUT) - alive = is_base_containers_alive(node_type=NodeType.SYNC) + alive = is_base_containers_alive(node_type=NodeType.PASSIVE) if not update_ok or not alive: print_node_cmd_error() return @@ -214,11 +214,11 @@ def update_sync(env_filepath: str, unsafe_ok: bool = False) -> None: @check_inited @check_user -def cleanup_sync() -> None: - env = compose_node_env(SKALE_DIR_ENV_FILEPATH, save=False, node_type=NodeType.SYNC) +def cleanup_passive() -> None: + env = compose_node_env(SKALE_DIR_ENV_FILEPATH, save=False, node_type=NodeType.PASSIVE) schain_name = env['SCHAIN_NAME'] - cleanup_sync_op(env, schain_name) - logger.info('Sync node was cleaned up, all containers and data removed') + cleanup_passive_op(env, schain_name) + logger.info('Passive node was cleaned up, all containers and data removed') def compose_node_env( @@ -245,7 +245,7 @@ def compose_node_env( is_fair_boot=is_fair_boot, ) - if node_type == NodeType.SYNC or node_type == NodeType.FAIR: + if node_type == NodeType.PASSIVE or node_type == NodeType.FAIR: mnt_dir = SCHAINS_MNT_DIR_SINGLE_CHAIN else: mnt_dir = SCHAINS_MNT_DIR_REGULAR @@ -258,10 +258,10 @@ def compose_node_env( **user_config.to_env(), } - if inited_node and not node_type == NodeType.SYNC: + if inited_node and not node_type == NodeType.PASSIVE: env['FLASK_SECRET_KEY'] = get_flask_secret_key() - if sync_schains and not node_type == NodeType.SYNC: + if sync_schains and not node_type == NodeType.PASSIVE: env['BACKUP_RUN'] = 'True' if pull_config_for_schain: @@ -431,8 +431,8 @@ def get_expected_container_names(node_type: NodeType, is_fair_boot: bool) -> lis services = BASE_FAIR_BOOT_COMPOSE_SERVICES elif node_type == NodeType.FAIR and not is_fair_boot: services = BASE_FAIR_COMPOSE_SERVICES - elif node_type == NodeType.SYNC: - services = BASE_SYNC_COMPOSE_SERVICES + elif node_type == NodeType.PASSIVE: + services = BASE_PASSIVE_COMPOSE_SERVICES else: services = BASE_SKALE_COMPOSE_SERVICES diff --git a/node_cli/main.py b/node_cli/main.py index 7fb833c0..0bd7a8de 100644 --- a/node_cli/main.py +++ b/node_cli/main.py @@ -37,7 +37,7 @@ from node_cli.cli.wallet import wallet_cli from node_cli.cli.ssl import ssl_cli from node_cli.cli.resources_allocation import resources_allocation_cli -from node_cli.cli.sync_node import sync_node_cli +from node_cli.cli.passive_node import passive_node_cli from node_cli.cli.fair_boot import fair_boot_cli from node_cli.cli.fair_node import fair_node_cli from node_cli.cli.chain import chain_cli @@ -84,8 +84,8 @@ def info(): def get_sources_list() -> List[click.MultiCommand]: - if TYPE == NodeType.SYNC: - return [cli, sync_node_cli, ssl_cli] + if TYPE == NodeType.PASSIVE: + return [cli, passive_node_cli, ssl_cli] elif TYPE == NodeType.FAIR: return [ cli, @@ -104,7 +104,7 @@ def get_sources_list() -> List[click.MultiCommand]: logs_cli, resources_allocation_cli, node_cli, - sync_node_cli, + passive_node_cli, wallet_cli, ssl_cli, exit_cli, diff --git a/node_cli/operations/__init__.py b/node_cli/operations/__init__.py index d0f60eed..5ae8103f 100644 --- a/node_cli/operations/__init__.py +++ b/node_cli/operations/__init__.py @@ -20,14 +20,14 @@ from node_cli.operations.base import ( # noqa update as update_op, init as init_op, - init_sync as init_sync_op, + init_passive as init_passive_op, init_fair_boot as init_fair_boot_op, update_fair_boot as update_fair_boot_op, - update_sync as update_sync_op, + update_passive as update_passive_op, turn_off as turn_off_op, turn_on as turn_on_op, restore as restore_op, - cleanup_sync as cleanup_sync_op, + cleanup_passive as cleanup_passive_op, configure_nftables, ) from node_cli.operations.fair import ( # noqa diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index df7eec45..bf5b68f9 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -259,7 +259,7 @@ def init_fair_boot(env_filepath: str, env: dict) -> None: compose_up(env=env, node_type=NodeType.FAIR, is_fair_boot=True) -def init_sync( +def init_passive( env_filepath: str, env: dict, indexer: bool, @@ -308,13 +308,13 @@ def init_sync( ts = int(time.time()) update_node_cli_schain_status(schain_name, repair_ts=ts, snapshot_from=snapshot_from) - update_images(env=env, node_type=NodeType.SYNC) + update_images(env=env, node_type=NodeType.PASSIVE) - compose_up(env=env, node_type=NodeType.SYNC) + compose_up(env=env, node_type=NodeType.PASSIVE) -def update_sync(env_filepath: str, env: Dict) -> bool: - compose_rm(env=env, node_type=NodeType.SYNC) +def update_passive(env_filepath: str, env: Dict) -> bool: + compose_rm(env=env, node_type=NodeType.PASSIVE) remove_dynamic_containers() cleanup_volume_artifacts(env['DISK_MOUNTPOINT']) download_skale_node(env['NODE_VERSION'], env.get('CONTAINER_CONFIGS_DIR')) @@ -341,9 +341,9 @@ def update_sync(env_filepath: str, env: Dict) -> bool: distro.id(), distro.version(), ) - update_images(env=env, node_type=NodeType.SYNC) + update_images(env=env, node_type=NodeType.PASSIVE) - compose_up(env=env, node_type=NodeType.SYNC) + compose_up(env=env, node_type=NodeType.PASSIVE) return True @@ -423,8 +423,8 @@ def restore(env, backup_path, node_type: NodeType, config_only=False): return True -def cleanup_sync(env, schain_name: str) -> None: - turn_off(env, node_type=NodeType.SYNC) - cleanup_no_lvm_datadir(schain_name=schain_name) +def cleanup_passive(env, schain_name: str) -> None: + turn_off(env, node_type=NodeType.PASSIVE) + cleanup_no_lvm_datadir(chain_name=schain_name) rm_dir(GLOBAL_SKALE_DIR) rm_dir(SKALE_DIR) diff --git a/node_cli/utils/docker_utils.py b/node_cli/utils/docker_utils.py index 6ded9e34..0c91f30c 100644 --- a/node_cli/utils/docker_utils.py +++ b/node_cli/utils/docker_utils.py @@ -35,7 +35,7 @@ NGINX_CONTAINER_NAME, REMOVED_CONTAINERS_FOLDER_PATH, SGX_CERTIFICATES_DIR_NAME, - SYNC_COMPOSE_PATH, + PASSIVE_COMPOSE_PATH, ) from node_cli.utils.helper import run_cmd, str_to_bool from node_cli.utils.node_type import NodeType @@ -76,8 +76,8 @@ 'fair-boot-api': 'fair_boot_api', } -BASE_SYNC_COMPOSE_SERVICES = { - 'skale-sync-admin': 'skale_sync_admin', +BASE_PASSIVE_COMPOSE_SERVICES = { + 'skale-passive-admin': 'skale_passive_admin', 'nginx': 'skale_nginx', } @@ -285,8 +285,8 @@ def compose_build(env: dict, node_type: NodeType): def get_compose_path(node_type: NodeType) -> str: - if node_type == NodeType.SYNC: - return SYNC_COMPOSE_PATH + if node_type == NodeType.PASSIVE: + return PASSIVE_COMPOSE_PATH elif node_type == NodeType.FAIR: return FAIR_COMPOSE_PATH else: @@ -294,8 +294,8 @@ def get_compose_path(node_type: NodeType) -> str: def get_compose_services(node_type: NodeType) -> list[str]: - if node_type == NodeType.SYNC: - result = list(BASE_SYNC_COMPOSE_SERVICES) + if node_type == NodeType.PASSIVE: + result = list(BASE_PASSIVE_COMPOSE_SERVICES) elif node_type == NodeType.FAIR: result = list(BASE_FAIR_COMPOSE_SERVICES) else: @@ -316,9 +316,9 @@ def get_up_compose_cmd(node_type: NodeType, services: list[str] | None = None) - def compose_up( env, node_type: NodeType, is_fair_boot: bool = False, services: list[str] | None = None ): - if node_type == NodeType.SYNC: - logger.info('Running containers for sync node') - run_cmd(cmd=get_up_compose_cmd(node_type=NodeType.SYNC), env=env) + if node_type == NodeType.PASSIVE: + logger.info('Running containers for passive node') + run_cmd(cmd=get_up_compose_cmd(node_type=NodeType.PASSIVE), env=env) return if 'SGX_CERTIFICATES_DIR_NAME' not in env: @@ -406,8 +406,8 @@ def is_admin_running(node_type: NodeType, client: Optional[DockerClient] = None) container_name = 'skale_admin' if node_type == NodeType.FAIR: container_name = 'fair_admin' - elif node_type == NodeType.SYNC: - container_name = 'skale_sync_admin' + elif node_type == NodeType.PASSIVE: + container_name = 'skale_passive_admin' return is_container_running(name=container_name, dclient=client) diff --git a/node_cli/utils/node_type.py b/node_cli/utils/node_type.py index f35d4640..60a44037 100644 --- a/node_cli/utils/node_type.py +++ b/node_cli/utils/node_type.py @@ -22,5 +22,5 @@ class NodeType(Enum): REGULAR = 0 - SYNC = 1 + PASSIVE = 1 FAIR = 2 diff --git a/scripts/build.sh b/scripts/build.sh index a1adc516..afb06faf 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -24,7 +24,7 @@ fi if [ -z "$3" ] then - (>&2 echo 'You should provide type: normal, sync or fair') + (>&2 echo 'You should provide type: normal, passive or fair') echo $USAGE_MSG exit 1 fi @@ -37,8 +37,8 @@ OS=`uname -s`-`uname -m` # Use the new generate_info.sh script bash "${DIR}/generate_info.sh" "$VERSION" "$BRANCH" "$TYPE" -if [ "$TYPE" = "sync" ]; then - EXECUTABLE_NAME=skale-$VERSION-$OS-sync +if [ "$TYPE" = "passive" ]; then + EXECUTABLE_NAME=skale-$VERSION-$OS-passive elif [ "$TYPE" = "fair" ]; then EXECUTABLE_NAME=skale-$VERSION-$OS-fair else diff --git a/scripts/generate_info.sh b/scripts/generate_info.sh index f4993b7e..3c283497 100644 --- a/scripts/generate_info.sh +++ b/scripts/generate_info.sh @@ -18,7 +18,7 @@ if [ -z "$BRANCH" ]; then exit 1 fi if [ -z "$TYPE_STR" ]; then - (>&2 echo 'You should provide type: normal, sync or fair') + (>&2 echo 'You should provide type: normal, passive or fair') echo $USAGE_MSG exit 1 fi @@ -35,14 +35,14 @@ case "$TYPE_STR" in normal) TYPE_ENUM="NodeType.REGULAR" ;; - sync) - TYPE_ENUM="NodeType.SYNC" + passive) + TYPE_ENUM="NodeType.PASSIVE" ;; fair) TYPE_ENUM="NodeType.FAIR" ;; *) - (>&2 echo "Error: Invalid type '$TYPE_STR'. Must be 'normal', 'sync', or 'fair'") + (>&2 echo "Error: Invalid type '$TYPE_STR'. Must be 'normal', 'passive', or 'fair'") exit 1 ;; esac diff --git a/tests/cli/node_test.py b/tests/cli/node_test.py index 0db1d97a..f1cd3c80 100644 --- a/tests/cli/node_test.py +++ b/tests/cli/node_test.py @@ -326,7 +326,7 @@ def test_backup(): [ (NodeType.REGULAR, 'regular_user_conf'), (NodeType.FAIR, 'fair_user_conf'), - (NodeType.SYNC, 'sync_user_conf'), + (NodeType.PASSIVE, 'passive_user_conf'), ], ) def test_restore(request, node_type, test_user_conf, mocked_g_config, tmp_path): diff --git a/tests/cli/sync_node_test.py b/tests/cli/passive_node_test.py similarity index 84% rename from tests/cli/sync_node_test.py rename to tests/cli/passive_node_test.py index 014510e1..2b5a587d 100644 --- a/tests/cli/sync_node_test.py +++ b/tests/cli/passive_node_test.py @@ -22,7 +22,7 @@ import mock -from node_cli.cli.sync_node import _cleanup_sync, _init_sync, _update_sync +from node_cli.cli.passive_node import _cleanup_passive, _init_passive, _update_passive from node_cli.configs import NODE_DATA_PATH, SKALE_DIR from node_cli.core.node_options import NodeOptions from node_cli.utils.helper import init_default_logger @@ -36,18 +36,18 @@ init_default_logger() -def test_init_sync(mocked_g_config, clean_node_options, sync_user_conf): +def test_init_passive(mocked_g_config, clean_node_options, passive_user_conf): pathlib.Path(SKALE_DIR).mkdir(parents=True, exist_ok=True) with ( mock.patch('subprocess.run', new=subprocess_run_mock), - mock.patch('node_cli.core.node.init_sync_op'), + mock.patch('node_cli.core.node.init_passive_op'), mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), mock.patch('node_cli.operations.base.configure_nftables'), mock.patch('node_cli.utils.decorators.is_node_inited', return_value=False), mock.patch('node_cli.configs.user.validate_alias_or_address'), ): - result = run_command(_init_sync, [sync_user_conf.as_posix()]) + result = run_command(_init_passive, [passive_user_conf.as_posix()]) node_options = NodeOptions() assert not node_options.archive @@ -57,7 +57,7 @@ def test_init_sync(mocked_g_config, clean_node_options, sync_user_conf): assert result.exit_code == 0 -def test_init_sync_archive(mocked_g_config, clean_node_options, sync_user_conf): +def test_init_passive_archive(mocked_g_config, clean_node_options, passive_user_conf): pathlib.Path(NODE_DATA_PATH).mkdir(parents=True, exist_ok=True) with ( mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), @@ -78,9 +78,9 @@ def test_init_sync_archive(mocked_g_config, clean_node_options, sync_user_conf): mock.patch('node_cli.operations.base.configure_nftables'), mock.patch('node_cli.utils.decorators.is_node_inited', return_value=False), mock.patch('node_cli.configs.user.validate_alias_or_address'), - mock.patch('node_cli.cli.node.TYPE', NodeType.SYNC), + mock.patch('node_cli.cli.node.TYPE', NodeType.PASSIVE), ): - result = run_command(_init_sync, [sync_user_conf.as_posix(), '--archive']) + result = run_command(_init_passive, [passive_user_conf.as_posix(), '--archive']) node_options = NodeOptions() assert node_options.archive @@ -94,7 +94,7 @@ def test_init_archive_indexer_fail(mocked_g_config, clean_node_options): pathlib.Path(SKALE_DIR).mkdir(parents=True, exist_ok=True) with ( mock.patch('subprocess.run', new=subprocess_run_mock), - mock.patch('node_cli.core.node.init_sync_op'), + mock.patch('node_cli.core.node.init_passive_op'), mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), mock.patch('node_cli.operations.base.configure_nftables'), @@ -102,17 +102,17 @@ def test_init_archive_indexer_fail(mocked_g_config, clean_node_options): mock.patch('node_cli.core.node.compose_node_env', return_value={}), set_env_var('ENV_TYPE', 'devnet'), ): - result = run_command(_init_sync, ['./tests/test-env', '--archive', '--indexer']) + result = run_command(_init_passive, ['./tests/test-env', '--archive', '--indexer']) assert result.exit_code == 1 assert 'Cannot use both' in result.output -def test_update_sync(sync_user_conf, mocked_g_config): +def test_update_passive(passive_user_conf, mocked_g_config): pathlib.Path(SKALE_DIR).mkdir(parents=True, exist_ok=True) with ( mock.patch('subprocess.run', new=subprocess_run_mock), - mock.patch('node_cli.core.node.update_sync_op'), + mock.patch('node_cli.core.node.update_passive_op'), mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), mock.patch('node_cli.operations.base.configure_nftables'), @@ -123,16 +123,16 @@ def test_update_sync(sync_user_conf, mocked_g_config): ), mock.patch('node_cli.configs.user.validate_alias_or_address'), ): - result = run_command(_update_sync, [sync_user_conf.as_posix(), '--yes']) + result = run_command(_update_passive, [passive_user_conf.as_posix(), '--yes']) assert result.exit_code == 0 -def test_cleanup_sync(mocked_g_config): +def test_cleanup_passive(mocked_g_config): pathlib.Path(SKALE_DIR).mkdir(parents=True, exist_ok=True) with ( mock.patch('subprocess.run', new=subprocess_run_mock), - mock.patch('node_cli.core.node.cleanup_sync_op'), + mock.patch('node_cli.core.node.cleanup_passive_op'), mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), mock.patch('node_cli.operations.base.configure_nftables'), @@ -143,5 +143,5 @@ def test_cleanup_sync(mocked_g_config): return_value=CliMeta(version='2.6.0', config_stream='3.0.2'), ), ): - result = run_command(_cleanup_sync, ['--yes']) + result = run_command(_cleanup_passive, ['--yes']) assert result.exit_code == 0 diff --git a/tests/configs/configs_env_validate_test.py b/tests/configs/configs_env_validate_test.py index be7d1eb8..c9d38cbf 100644 --- a/tests/configs/configs_env_validate_test.py +++ b/tests/configs/configs_env_validate_test.py @@ -17,7 +17,7 @@ FairBootUserConfig, FairUserConfig, SkaleUserConfig, - SyncUserConfig, + PassiveUserConfig, get_user_config_class, get_validated_user_config, validate_env_type, @@ -40,11 +40,11 @@ def json(self): 'node_type, is_fair_boot, expected_type', [ (NodeType.REGULAR, False, SkaleUserConfig), - (NodeType.SYNC, False, SyncUserConfig), + (NodeType.PASSIVE, False, PassiveUserConfig), (NodeType.FAIR, True, FairBootUserConfig), (NodeType.FAIR, False, FairUserConfig), ], - ids=['regular', 'sync', 'fair_boot', 'fair_regular'], + ids=['regular', 'passive', 'fair_boot', 'fair_regular'], ) def test_build_env_params_keys(node_type, is_fair_boot, expected_type): env_type = get_user_config_class(node_type=node_type, is_fair_boot=is_fair_boot) diff --git a/tests/conftest.py b/tests/conftest.py index 24c85444..ba860af9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -226,7 +226,7 @@ def tmp_schains_dir(): @pytest.fixture -def tmp_sync_datadir(): +def tmp_passive_datadir(): os.makedirs(TEST_SCHAINS_MNT_DIR_SINGLE_CHAIN, exist_ok=True) try: yield TEST_SCHAINS_MNT_DIR_SINGLE_CHAIN @@ -366,7 +366,7 @@ def fair_boot_user_conf(tmp_path): @pytest.fixture -def sync_user_conf(tmp_path): +def passive_user_conf(tmp_path): test_env_path = pathlib.Path(tmp_path / 'test-env') try: test_env = """ diff --git a/tests/core/core_node_test.py b/tests/core/core_node_test.py index 894e07b6..dd778a6f 100644 --- a/tests/core/core_node_test.py +++ b/tests/core/core_node_test.py @@ -34,12 +34,12 @@ 'WRONG_CONTAINER_1', 'skale_WRONG_CONTAINER_4', 'fair_WRONG_CONTAINER_6', - 'sync_WRONG_CONTAINER_8', + 'passive_WRONG_CONTAINER_8', ] NODE_TYPE_BOOT_COMBINATIONS: list[tuple[NodeType, bool]] = [ (NodeType.REGULAR, False), - (NodeType.SYNC, False), + (NodeType.PASSIVE, False), (NodeType.FAIR, True), (NodeType.FAIR, False), ] @@ -173,8 +173,8 @@ def test_is_base_containers_alive_empty(node_type, is_boot): True, ), ( - NodeType.SYNC, - 'sync_user_conf', + NodeType.PASSIVE, + 'passive_user_conf', False, False, False, @@ -205,8 +205,8 @@ def test_is_base_containers_alive_empty(node_type, is_boot): ], ids=[ 'regular', - 'regular_sync_flag', - 'sync', + 'regular_passive_flag', + 'passive', 'fair_boot', 'fair_regular', ], @@ -244,7 +244,7 @@ def test_compose_node_env( ) == expect_flask_key if expect_flask_key: assert result_env['FLASK_SECRET_KEY'] == 'mock_secret' - should_have_backup = sync_schains and node_type != NodeType.SYNC + should_have_backup = sync_schains and node_type != NodeType.PASSIVE assert ('BACKUP_RUN' in result_env and result_env['BACKUP_RUN'] == 'True') == should_have_backup @@ -358,7 +358,7 @@ def test_update_node(regular_user_conf, mocked_g_config, resource_file, inited_n assert result is None -@pytest.mark.parametrize('node_type', [NodeType.REGULAR, NodeType.SYNC, NodeType.FAIR]) +@pytest.mark.parametrize('node_type', [NodeType.REGULAR, NodeType.PASSIVE, NodeType.FAIR]) @mock.patch('node_cli.core.node.is_admin_running', return_value=False) @mock.patch('node_cli.core.node.is_api_running', return_value=False) @mock.patch('node_cli.utils.helper.requests.get') @@ -372,14 +372,14 @@ def test_is_update_safe_when_admin_and_api_not_running( @mock.patch('node_cli.core.node.is_admin_running', return_value=False) @mock.patch('node_cli.core.node.is_api_running', return_value=True) @mock.patch('node_cli.utils.helper.requests.get') -def test_is_update_safe_when_admin_not_running_for_sync( +def test_is_update_safe_when_admin_not_running_for_passive( mock_requests_get, mock_is_api_running, mock_is_admin_running ): - assert is_update_safe(node_type=NodeType.SYNC) is True + assert is_update_safe(node_type=NodeType.PASSIVE) is True mock_requests_get.assert_not_called() -@pytest.mark.parametrize('node_type', [NodeType.REGULAR, NodeType.SYNC, NodeType.FAIR]) +@pytest.mark.parametrize('node_type', [NodeType.REGULAR, NodeType.PASSIVE, NodeType.FAIR]) @pytest.mark.parametrize( 'api_is_safe, expected_result', [(True, True), (False, False)], @@ -417,7 +417,7 @@ def test_is_update_safe_when_only_api_running_for_regular( mock_requests_get.assert_called_once() -@pytest.mark.parametrize('node_type', [NodeType.REGULAR, NodeType.SYNC, NodeType.FAIR]) +@pytest.mark.parametrize('node_type', [NodeType.REGULAR, NodeType.PASSIVE, NodeType.FAIR]) @mock.patch('node_cli.core.node.is_admin_running', return_value=True) @mock.patch('node_cli.utils.helper.requests.get') def test_is_update_safe_when_api_call_fails(mock_requests_get, mock_is_admin_running, node_type): diff --git a/tests/core/core_schains_test.py b/tests/core/core_schains_test.py index c9281adb..ee868242 100644 --- a/tests/core/core_schains_test.py +++ b/tests/core/core_schains_test.py @@ -34,9 +34,9 @@ def test_toggle_repair_mode(tmp_schains_dir): @freezegun.freeze_time(CURRENT_DATETIME) -def test_cleanup_sync_datadir(tmp_sync_datadir): +def test_cleanup_passive_datadir(tmp_passive_datadir): schain_name = 'test_schain' - base_folder = Path(tmp_sync_datadir).joinpath(schain_name) + base_folder = Path(tmp_passive_datadir).joinpath(schain_name) base_folder.mkdir() folders = [ '28e07f34', @@ -81,5 +81,5 @@ def test_cleanup_sync_datadir(tmp_sync_datadir): hash_path.touch() with mock.patch('node_cli.core.schains.rm_btrfs_subvolume'): - cleanup_no_lvm_datadir(schain_name, base_path=tmp_sync_datadir) + cleanup_no_lvm_datadir(schain_name, base_path=tmp_passive_datadir) assert not os.path.isdir(base_folder) diff --git a/tests/core/nginx_test.py b/tests/core/nginx_test.py index 56b6eb49..1652a514 100644 --- a/tests/core/nginx_test.py +++ b/tests/core/nginx_test.py @@ -61,8 +61,8 @@ def nginx_template(): [ (NodeType.REGULAR, True, True, True), (NodeType.REGULAR, False, True, False), - (NodeType.SYNC, True, True, True), - (NodeType.SYNC, False, True, False), + (NodeType.PASSIVE, True, True, True), + (NodeType.PASSIVE, False, True, False), (NodeType.FAIR, True, False, True), (NodeType.FAIR, False, False, False), ], @@ -133,7 +133,7 @@ def test_check_ssl_certs_missing_both(ssl_folder): 'node_type, expected_result', [ (NodeType.REGULAR, True), - (NodeType.SYNC, True), + (NodeType.PASSIVE, True), (NodeType.FAIR, False), ], ) diff --git a/text.yml b/text.yml index 560729a7..85ab8328 100644 --- a/text.yml +++ b/text.yml @@ -60,13 +60,13 @@ exit: wait_for_rotations: "Node is waiting to finish rotations" completed: "Node exiting is completed" -sync_node: +passive_node: init: - help: Initialize sync SKALE node - indexer: Run sync node in indexer mode (disable block rotation) + help: Initialize passive SKALE node + indexer: Run passive node in indexer mode (disable block rotation) archive: Enable historic state and disable block rotation snapshot_from: IP of the node to take snapshot from - snapshot: Start sync node from snapshot + snapshot: Start passive node from snapshot lvmpy: help: Lvmpy commands From ea85aa586c826c3fbfa42a36044233d49c130968 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Wed, 6 Aug 2025 21:21:11 +0100 Subject: [PATCH 034/198] Add `set_domain_name` function for FAIR nodes --- node_cli/fair/fair_node.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/node_cli/fair/fair_node.py b/node_cli/fair/fair_node.py index e6f2fb1d..517269fd 100644 --- a/node_cli/fair/fair_node.py +++ b/node_cli/fair/fair_node.py @@ -217,3 +217,23 @@ def exit() -> None: error_msg = payload logger.error(f'Node exit error {error_msg}') error_exit(error_msg, exit_code=CLIExitCodes.BAD_API_RESPONSE) + + +@check_inited +@check_user +def set_domain_name(domain_name): + if not is_node_inited(): + print(TEXTS['fair']['node']['not_inited']) + return + + status, payload = post_request( + blueprint=BLUEPRINT_NAME, method='set-domain-name', json={'domain_name': domain_name} + ) + if status == 'ok': + msg = TEXTS['node']['domain_name_changed'] + logger.info(msg) + print(msg) + else: + error_msg = payload + logger.error(f'Setting domain name error {error_msg}') + error_exit(error_msg, exit_code=CLIExitCodes.BAD_API_RESPONSE) From 24869ec7bb7817d72000ac7f961d63f61e63190f Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Wed, 6 Aug 2025 21:21:55 +0100 Subject: [PATCH 035/198] Add `set_domain_name` command for FAIR nodes --- node_cli/cli/fair_node.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/node_cli/cli/fair_node.py b/node_cli/cli/fair_node.py index 19b06c93..40e0f5f9 100644 --- a/node_cli/cli/fair_node.py +++ b/node_cli/cli/fair_node.py @@ -32,6 +32,7 @@ from node_cli.fair.fair_node import init as init_fair from node_cli.fair.fair_node import register as register_fair from node_cli.fair.fair_node import update as update_fair +from node_cli.fair.fair_node import set_domain_name as set_domain_name_fair from node_cli.utils.helper import IP_TYPE, URL_OR_ANY_TYPE, abort_if_false, streamed_cmd from node_cli.utils.texts import safe_load_texts @@ -180,3 +181,17 @@ def change_ip(ip: str) -> None: @streamed_cmd def exit_node() -> None: exit_fair() + + +@node.command('set-domain', help='Set node domain name') +@click.option('--domain', '-d', prompt='Enter node domain name', type=str, help='Node domain name') +@click.option( + '--yes', + is_flag=True, + callback=abort_if_false, + expose_value=False, + prompt='Are you sure you want to set domain name?', +) +@streamed_cmd +def set_domain_name(domain): + set_domain_name_fair(domain) From d7681a6d0984d15d1e295b3151388bd5b076b1a6 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Wed, 6 Aug 2025 21:33:17 +0100 Subject: [PATCH 036/198] Sort imports --- node_cli/cli/fair_node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node_cli/cli/fair_node.py b/node_cli/cli/fair_node.py index 40e0f5f9..56302429 100644 --- a/node_cli/cli/fair_node.py +++ b/node_cli/cli/fair_node.py @@ -31,8 +31,8 @@ ) from node_cli.fair.fair_node import init as init_fair from node_cli.fair.fair_node import register as register_fair -from node_cli.fair.fair_node import update as update_fair from node_cli.fair.fair_node import set_domain_name as set_domain_name_fair +from node_cli.fair.fair_node import update as update_fair from node_cli.utils.helper import IP_TYPE, URL_OR_ANY_TYPE, abort_if_false, streamed_cmd from node_cli.utils.texts import safe_load_texts From 7acee44374b6e430f17b25ef465b70bba7eeae5b Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Thu, 7 Aug 2025 15:55:59 +0100 Subject: [PATCH 037/198] Add import sort check --- ruff.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ruff.toml b/ruff.toml index 9653b675..7eb49e79 100644 --- a/ruff.toml +++ b/ruff.toml @@ -5,4 +5,4 @@ quote-style = "single" [lint] # Add the `line-too-long` rule to the enforced rule set. -extend-select = ["E501"] \ No newline at end of file +extend-select = ["E501", "I"] From 09594494b6ababbb20e0383ca5328bbfbfca8bde Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Thu, 7 Aug 2025 15:57:45 +0100 Subject: [PATCH 038/198] Add 'set-domain-name' to 'ROUTS' for FAIR node --- node_cli/configs/routes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node_cli/configs/routes.py b/node_cli/configs/routes.py index a1a17879..a0f15ef1 100644 --- a/node_cli/configs/routes.py +++ b/node_cli/configs/routes.py @@ -40,7 +40,7 @@ 'schains': ['config', 'list', 'dkg-statuses', 'firewall-rules', 'repair', 'get'], 'ssl': ['status', 'upload'], 'wallet': ['info', 'send-eth'], - 'fair-node': ['info', 'register', 'change-ip', 'exit'], + 'fair-node': ['info', 'register', 'set-domain-name', 'change-ip', 'exit'], 'fair-chain': ['record', 'checks'], } } From 663b340e5ceb0e44e3d53678d887a0a4b9b55840 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Thu, 7 Aug 2025 17:42:05 +0100 Subject: [PATCH 039/198] Disable import sort check --- ruff.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ruff.toml b/ruff.toml index 7eb49e79..978b38de 100644 --- a/ruff.toml +++ b/ruff.toml @@ -5,4 +5,4 @@ quote-style = "single" [lint] # Add the `line-too-long` rule to the enforced rule set. -extend-select = ["E501", "I"] +extend-select = ["E501"] From a85445d437418b814a08cd25ce7053a3dc995c77 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Thu, 7 Aug 2025 18:13:01 +0100 Subject: [PATCH 040/198] Fix tests --- tests/routes_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/routes_test.py b/tests/routes_test.py index bcf1bf98..d17c3920 100644 --- a/tests/routes_test.py +++ b/tests/routes_test.py @@ -34,6 +34,7 @@ '/api/v1/fair-node/info', '/api/v1/fair-node/register', '/api/v1/fair-node/change-ip', + '/api/v1/fair-node/set-domain-name', '/api/v1/fair-node/exit', '/api/v1/fair-chain/record', '/api/v1/fair-chain/checks', From c9879cf3dcdbcad5ce11d362a98fe27aef2707bc Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Thu, 7 Aug 2025 18:51:29 +0100 Subject: [PATCH 041/198] Fix tests --- tests/routes_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/routes_test.py b/tests/routes_test.py index d17c3920..7a1216a8 100644 --- a/tests/routes_test.py +++ b/tests/routes_test.py @@ -33,8 +33,8 @@ '/api/v1/wallet/send-eth', '/api/v1/fair-node/info', '/api/v1/fair-node/register', - '/api/v1/fair-node/change-ip', '/api/v1/fair-node/set-domain-name', + '/api/v1/fair-node/change-ip', '/api/v1/fair-node/exit', '/api/v1/fair-chain/record', '/api/v1/fair-chain/checks', From e4e313c583010273a2b7872630a568b36b736018 Mon Sep 17 00:00:00 2001 From: badrogger Date: Fri, 8 Aug 2025 16:24:04 +0100 Subject: [PATCH 042/198] Add telegraf related env params to fair --- node_cli/configs/user.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/node_cli/configs/user.py b/node_cli/configs/user.py index eb8764ed..ed9fdeb1 100644 --- a/node_cli/configs/user.py +++ b/node_cli/configs/user.py @@ -86,6 +86,9 @@ class FairUserConfig(BaseUserConfig): boot_endpoint: str sgx_server_url: str enforce_btrfs: str = '' + telegraf: str = '' + influx_token: str = '' + influx_url: str = '' @dataclass From 20a37cd503261b433593fe8617e93aa465ade50c Mon Sep 17 00:00:00 2001 From: badrogger Date: Fri, 8 Aug 2025 17:41:54 +0100 Subject: [PATCH 043/198] Remove influx_db_token --- node_cli/cli/fair_boot.py | 4 +--- node_cli/configs/user.py | 2 -- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/node_cli/cli/fair_boot.py b/node_cli/cli/fair_boot.py index 703bea88..24f3490a 100644 --- a/node_cli/cli/fair_boot.py +++ b/node_cli/cli/fair_boot.py @@ -50,9 +50,7 @@ def init_boot(env_file): @boot.command('register', help='Register Fair node in SKALE Manager (during Boot Phase).') -@click.option( - '--name', '-n', required=True, prompt='Enter fair node name', help='Fair node name' -) +@click.option('--name', '-n', required=True, prompt='Enter fair node name', help='Fair node name') @click.option( '--ip', prompt='Enter node public IP', diff --git a/node_cli/configs/user.py b/node_cli/configs/user.py index ed9fdeb1..64b82b82 100644 --- a/node_cli/configs/user.py +++ b/node_cli/configs/user.py @@ -87,7 +87,6 @@ class FairUserConfig(BaseUserConfig): sgx_server_url: str enforce_btrfs: str = '' telegraf: str = '' - influx_token: str = '' influx_url: str = '' @@ -109,7 +108,6 @@ class SkaleUserConfig(BaseUserConfig): sgx_server_url: str monitoring_containers: str = '' telegraf: str = '' - influx_token: str = '' influx_url: str = '' tg_api_key: str = '' tg_chat_id: str = '' From e6c23e7c65f607d00d185e7a758b85a24dc42618 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Fri, 8 Aug 2025 18:03:49 +0100 Subject: [PATCH 044/198] Update node-cli internal structure - WIP --- .github/workflows/publish.yml | 4 +- .github/workflows/test.yml | 10 ++--- node_cli/cli/resources_allocation.py | 2 +- node_cli/configs/user.py | 24 ++++++++--- node_cli/core/nginx.py | 10 ++--- node_cli/core/node.py | 11 ++--- node_cli/core/node_options.py | 47 ++++++++++++++++++++-- node_cli/main.py | 4 +- node_cli/operations/base.py | 6 ++- node_cli/operations/fair.py | 2 +- node_cli/utils/docker_utils.py | 30 ++++++++------ node_cli/utils/node_type.py | 10 +++-- scripts/build.sh | 7 ++-- scripts/generate_info.sh | 11 ++--- tests/cli/node_test.py | 6 +-- tests/cli/resources_allocation_test.py | 2 +- tests/configs/configs_env_validate_test.py | 10 ++--- tests/core/core_checks_test.py | 4 +- tests/core/core_node_test.py | 18 ++++----- tests/core/nginx_test.py | 14 +++---- 20 files changed, 146 insertions(+), 86 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 96290f1b..660ec07d 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -72,7 +72,7 @@ jobs: strategy: matrix: os: [ubuntu-22.04] - build_type: [normal, passive, fair] + build_type: [skale, fair] steps: - name: Checkout code uses: actions/checkout@v4 @@ -96,7 +96,7 @@ jobs: id: asset_details run: | ASSET_BASE_NAME="skale-${{ needs.create_release.outputs.version }}-Linux-x86_64" - if [[ "${{ matrix.build_type }}" == "normal" ]]; then + if [[ "${{ matrix.build_type }}" == "skale" ]]; then echo "FINAL_ASSET_NAME=${ASSET_BASE_NAME}" >> $GITHUB_OUTPUT else echo "FINAL_ASSET_NAME=${ASSET_BASE_NAME}-${{ matrix.build_type }}" >> $GITHUB_OUTPUT diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 043c1792..ed804b5c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -31,20 +31,20 @@ jobs: pip install -e ".[dev]" - name: Generate info - run: bash ./scripts/generate_info.sh 1.0.0 my-branch normal + run: bash ./scripts/generate_info.sh 1.0.0 my-branch skale - name: Check with ruff run: | ruff check - - name: Build binary - normal + - name: Build binary - skale run: | mkdir -p ./dist docker build . -t node-cli-builder - docker run -v /home/ubuntu/dist:/app/dist node-cli-builder bash scripts/build.sh test test normal + docker run -v /home/ubuntu/dist:/app/dist node-cli-builder bash scripts/build.sh test test skale docker rm -f $(docker ps -aq) - - name: Check build - normal + - name: Check build - skale run: sudo /home/ubuntu/dist/skale-test-Linux-x86_64 - name: Build binary - passive @@ -69,7 +69,7 @@ jobs: - name: Run prepare test build run: | - bash scripts/build.sh test test normal + bash scripts/build.sh test test skale - name: Run redis run: | diff --git a/node_cli/cli/resources_allocation.py b/node_cli/cli/resources_allocation.py index 01825350..a6b2e185 100644 --- a/node_cli/cli/resources_allocation.py +++ b/node_cli/cli/resources_allocation.py @@ -61,4 +61,4 @@ def show(): ) @click.option('--force', '-f', is_flag=True, help='Rewrite if already exists') def generate(env_file, force): - generate_resource_allocation_config(node_type=NodeType.REGULAR, env_file=env_file, force=force) + generate_resource_allocation_config(node_type=NodeType.SKALE, env_file=env_file, force=force) diff --git a/node_cli/configs/user.py b/node_cli/configs/user.py index 248cdf65..6acc6d1a 100644 --- a/node_cli/configs/user.py +++ b/node_cli/configs/user.py @@ -88,6 +88,13 @@ class FairUserConfig(BaseUserConfig): enforce_btrfs: str = '' +@dataclass +class PassiveFairUserConfig(BaseUserConfig): + fair_contracts: str + boot_endpoint: str + enforce_btrfs: str = '' + + @dataclass class FairBootUserConfig(BaseUserConfig): endpoint: str @@ -116,7 +123,7 @@ class SkaleUserConfig(BaseUserConfig): @dataclass -class PassiveUserConfig(BaseUserConfig): +class PassiveSkaleUserConfig(BaseUserConfig): endpoint: str manager_contracts: str schain_name: str = '' @@ -170,16 +177,21 @@ def parse_env_file(env_filepath: str) -> Dict: def get_user_config_class( node_type: NodeType, + is_passive: bool = False, is_fair_boot: bool = False, ) -> type[BaseUserConfig]: if node_type == NodeType.FAIR and is_fair_boot: user_config_class = FairBootUserConfig elif node_type == NodeType.FAIR: - user_config_class = FairUserConfig - elif node_type == NodeType.PASSIVE: - user_config_class = PassiveUserConfig - else: - user_config_class = SkaleUserConfig + if is_passive: + user_config_class = PassiveFairUserConfig + else: + user_config_class = FairUserConfig + elif node_type == NodeType.SKALE: + if is_passive: + user_config_class = PassiveSkaleUserConfig + else: + user_config_class = SkaleUserConfig return user_config_class diff --git a/node_cli/core/nginx.py b/node_cli/core/nginx.py index c16b0a34..340e3299 100644 --- a/node_cli/core/nginx.py +++ b/node_cli/core/nginx.py @@ -35,12 +35,12 @@ def generate_nginx_config() -> None: ssl_on = check_ssl_certs() - regular_node = is_regular_node_nginx() + skale_node = is_skale_node_nginx() template_data = { 'ssl': ssl_on, - 'regular_node': regular_node, + 'skale_node': skale_node, } - logger.info(f'Processing nginx template. ssl: {ssl_on}, regular_node: {regular_node}') + logger.info(f'Processing nginx template. ssl: {ssl_on}, skale_node: {skale_node}') process_template(NGINX_TEMPLATE_FILEPATH, NGINX_CONFIG_FILEPATH, template_data) @@ -50,8 +50,8 @@ def check_ssl_certs(): return os.path.exists(crt_path) and os.path.exists(key_path) -def is_regular_node_nginx() -> bool: - return TYPE in [NodeType.REGULAR, NodeType.PASSIVE] +def is_skale_node_nginx() -> bool: + return TYPE == NodeType.SKALE def reload_nginx() -> None: diff --git a/node_cli/core/node.py b/node_cli/core/node.py index a47ba10b..9b0a8d81 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -46,6 +46,7 @@ from node_cli.configs.user import SKALE_DIR_ENV_FILEPATH, get_validated_user_config from node_cli.core.checks import run_checks as run_host_checks from node_cli.core.host import get_flask_secret_key, is_node_inited, save_env_params +from node_cli.core.node_options import is_passive_node from node_cli.core.resources import update_resource_allocation from node_cli.migrations.focal_to_jammy import migrate as migrate_2_6 from node_cli.operations import ( @@ -101,11 +102,11 @@ class NodeStatuses(Enum): NOT_CREATED = 5 -def is_update_safe(node_type: NodeType) -> bool: - if not is_admin_running(node_type): - if node_type == NodeType.PASSIVE: +def is_update_safe() -> bool: + if not is_admin_running(): + if is_passive_node(): return True - elif not is_api_running(node_type): + elif not is_api_running(): return True status, payload = get_request(BLUEPRINT_NAME, 'update-safe') if status == 'error': @@ -223,7 +224,7 @@ def cleanup_passive() -> None: def compose_node_env( env_filepath: str, - node_type: NodeType, + # node_type: NodeType, inited_node: bool = False, sync_schains: Optional[bool] = None, pull_config_for_schain: Optional[str] = None, diff --git a/node_cli/core/node_options.py b/node_cli/core/node_options.py index 49a0ea05..e6500399 100644 --- a/node_cli/core/node_options.py +++ b/node_cli/core/node_options.py @@ -19,8 +19,11 @@ import logging +from node_cli.utils.node_type import NodeMode, NodeType from node_cli.utils.helper import read_json, write_json, init_file from node_cli.configs.node_options import NODE_OPTIONS_FILEPATH +from node_cli.cli.info import TYPE + logger = logging.getLogger(__name__) @@ -41,7 +44,7 @@ def _set(self, field_name: str, field_value) -> None: @property def archive(self) -> bool: - return self._get('archive') + return self._get('archive') or False @archive.setter def archive(self, archive: bool) -> None: @@ -49,7 +52,7 @@ def archive(self, archive: bool) -> None: @property def catchup(self) -> bool: - return self._get('catchup') + return self._get('catchup') or False @catchup.setter def catchup(self, catchup: bool) -> None: @@ -57,11 +60,49 @@ def catchup(self, catchup: bool) -> None: @property def historic_state(self) -> bool: - return self._get('historic_state') + return self._get('historic_state') or False @historic_state.setter def historic_state(self, historic_state: bool) -> None: return self._set('historic_state', historic_state) + @property + def node_mode(self) -> NodeMode: + return NodeMode(self._get('node_mode')) + + @node_mode.setter + def node_mode(self, node_mode: NodeMode) -> None: + return self._set('node_mode', node_mode.name) + def all(self) -> dict: return read_json(self.filepath) + + +def mark_active_node() -> None: + node_options = NodeOptions() + node_options.node_mode = NodeMode.ACTIVE + logger.info('Node marked as active.') + + +def mark_passive_node() -> None: + node_options = NodeOptions() + node_options.node_mode = NodeMode.PASSIVE + logger.info('Node marked as passive.') + + +def is_active_node() -> bool: + node_options = NodeOptions() + return node_options.node_mode == NodeMode.ACTIVE + + +def is_passive_node() -> bool: + node_options = NodeOptions() + return node_options.node_mode == NodeMode.PASSIVE + + +def is_skale_node() -> bool: + return TYPE == NodeType.SKALE + + +def is_fair_node() -> bool: + return TYPE == NodeType.FAIR diff --git a/node_cli/main.py b/node_cli/main.py index 0bd7a8de..e949f682 100644 --- a/node_cli/main.py +++ b/node_cli/main.py @@ -84,9 +84,7 @@ def info(): def get_sources_list() -> List[click.MultiCommand]: - if TYPE == NodeType.PASSIVE: - return [cli, passive_node_cli, ssl_cli] - elif TYPE == NodeType.FAIR: + if TYPE == NodeType.FAIR: return [ cli, logs_cli, diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index bf5b68f9..47ce58aa 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -67,7 +67,7 @@ ) from node_cli.utils.helper import rm_dir, str_to_bool from node_cli.utils.meta import CliMetaManager, FairCliMetaManager -from node_cli.utils.node_type import NodeType +from node_cli.utils.node_type import NodeType, NodeMode from node_cli.utils.print_formatters import print_failed_requirements_checks logger = logging.getLogger(__name__) @@ -195,7 +195,6 @@ def update_fair_boot(env_filepath: str, env: Dict) -> bool: @checked_host def init(env_filepath: str, env: dict, node_type: NodeType) -> None: sync_skale_node() - ensure_btrfs_kernel_module_autoloaded() if env.get('SKIP_DOCKER_CONFIG') != 'True': configure_docker() @@ -206,6 +205,9 @@ def init(env_filepath: str, env: dict, node_type: NodeType) -> None: prepare_host(env_filepath, env_type=env['ENV_TYPE']) link_env_file() + node_options = NodeOptions() + node_options.node_mode = NodeMode.ACTIVE + configure_filebeat() configure_flask() generate_nginx_config() diff --git a/node_cli/operations/fair.py b/node_cli/operations/fair.py index 3455c21a..9a1d15c6 100644 --- a/node_cli/operations/fair.py +++ b/node_cli/operations/fair.py @@ -274,7 +274,7 @@ def trigger_skaled_snapshot_mode(env: dict, snapshot_from: str = 'any') -> None: def repair(env: dict, snapshot_from: str = 'any') -> None: logger.info('Starting fair node repair') container_name = 'fair_admin' - if is_admin_running(node_type=NodeType.FAIR): + if is_admin_running(): logger.info('Stopping admin container') stop_container_by_name(container_name=container_name) logger.info('Removing chain container') diff --git a/node_cli/utils/docker_utils.py b/node_cli/utils/docker_utils.py index 0c91f30c..ac9a428e 100644 --- a/node_cli/utils/docker_utils.py +++ b/node_cli/utils/docker_utils.py @@ -37,6 +37,7 @@ SGX_CERTIFICATES_DIR_NAME, PASSIVE_COMPOSE_PATH, ) +from node_cli.core.node_options import is_active_node, is_fair_node from node_cli.utils.helper import run_cmd, str_to_bool from node_cli.utils.node_type import NodeType @@ -340,13 +341,13 @@ def compose_up( else: logger.info('Running skale node base set of containers') logger.debug('Launching skale node containers with env %s', env) - run_cmd(cmd=get_up_compose_cmd(node_type=NodeType.REGULAR), env=env) + run_cmd(cmd=get_up_compose_cmd(node_type=NodeType.SKALE), env=env) if 'TG_API_KEY' in env and 'TG_CHAT_ID' in env: logger.info('Running containers for Telegram notifications') run_cmd( cmd=get_up_compose_cmd( - node_type=NodeType.REGULAR, services=list(NOTIFICATION_COMPOSE_SERVICES) + node_type=NodeType.SKALE, services=list(NOTIFICATION_COMPOSE_SERVICES) ), env=env, ) @@ -355,7 +356,7 @@ def compose_up( logger.info('Running monitoring containers') run_cmd( cmd=get_up_compose_cmd( - node_type=NodeType.REGULAR, services=list(MONITORING_COMPOSE_SERVICES) + node_type=NodeType.SKALE, services=list(MONITORING_COMPOSE_SERVICES) ), env=env, ) @@ -395,20 +396,25 @@ def is_container_running(name: str, dclient: Optional[DockerClient] = None) -> b return False -def is_api_running(node_type: NodeType, dclient: Optional[DockerClient] = None) -> bool: - if node_type == NodeType.FAIR: +def is_api_running(dclient: Optional[DockerClient] = None) -> bool: + if is_fair_node(): return is_container_running(name='fair_api', dclient=dclient) else: return is_container_running(name='skale_api', dclient=dclient) -def is_admin_running(node_type: NodeType, client: Optional[DockerClient] = None) -> bool: - container_name = 'skale_admin' - if node_type == NodeType.FAIR: - container_name = 'fair_admin' - elif node_type == NodeType.PASSIVE: - container_name = 'skale_passive_admin' - return is_container_running(name=container_name, dclient=client) +def is_admin_running(dclient: Optional[DockerClient] = None) -> bool: + if is_fair_node(): + if is_active_node(): + container_name = 'fair_admin' + else: + container_name = 'fair_passive_admin' + else: + if is_active_node(): + container_name = 'skale_admin' + else: + container_name = 'skale_passive_admin' + return is_container_running(name=container_name, dclient=dclient) def system_prune(): diff --git a/node_cli/utils/node_type.py b/node_cli/utils/node_type.py index 60a44037..bf3f6d4f 100644 --- a/node_cli/utils/node_type.py +++ b/node_cli/utils/node_type.py @@ -21,6 +21,10 @@ class NodeType(Enum): - REGULAR = 0 - PASSIVE = 1 - FAIR = 2 + SKALE = 0 + FAIR = 1 + + +class NodeMode(str, Enum): + ACTIVE = 'active' + PASSIVE = 'passive' diff --git a/scripts/build.sh b/scripts/build.sh index afb06faf..8e26577a 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -24,7 +24,7 @@ fi if [ -z "$3" ] then - (>&2 echo 'You should provide type: normal, passive or fair') + (>&2 echo 'You should provide type: skale, or fair') echo $USAGE_MSG exit 1 fi @@ -37,9 +37,8 @@ OS=`uname -s`-`uname -m` # Use the new generate_info.sh script bash "${DIR}/generate_info.sh" "$VERSION" "$BRANCH" "$TYPE" -if [ "$TYPE" = "passive" ]; then - EXECUTABLE_NAME=skale-$VERSION-$OS-passive -elif [ "$TYPE" = "fair" ]; then + +if [ "$TYPE" = "fair" ]; then EXECUTABLE_NAME=skale-$VERSION-$OS-fair else EXECUTABLE_NAME=skale-$VERSION-$OS diff --git a/scripts/generate_info.sh b/scripts/generate_info.sh index 3c283497..67ee6aef 100644 --- a/scripts/generate_info.sh +++ b/scripts/generate_info.sh @@ -18,7 +18,7 @@ if [ -z "$BRANCH" ]; then exit 1 fi if [ -z "$TYPE_STR" ]; then - (>&2 echo 'You should provide type: normal, passive or fair') + (>&2 echo 'You should provide type: skale or fair') echo $USAGE_MSG exit 1 fi @@ -32,17 +32,14 @@ CURRENT_DATETIME="$(date "+%Y-%m-%d %H:%M:%S")" OS="$(uname -s)-$(uname -m)" case "$TYPE_STR" in - normal) - TYPE_ENUM="NodeType.REGULAR" - ;; - passive) - TYPE_ENUM="NodeType.PASSIVE" + skale) + TYPE_ENUM="NodeType.SKALE" ;; fair) TYPE_ENUM="NodeType.FAIR" ;; *) - (>&2 echo "Error: Invalid type '$TYPE_STR'. Must be 'normal', 'passive', or 'fair'") + (>&2 echo "Error: Invalid type '$TYPE_STR'. Must be 'skale', or 'fair'") exit 1 ;; esac diff --git a/tests/cli/node_test.py b/tests/cli/node_test.py index f1cd3c80..c20b390c 100644 --- a/tests/cli/node_test.py +++ b/tests/cli/node_test.py @@ -324,7 +324,7 @@ def test_backup(): @pytest.mark.parametrize( 'node_type,test_user_conf', [ - (NodeType.REGULAR, 'regular_user_conf'), + (NodeType.SKALE, 'regular_user_conf'), (NodeType.FAIR, 'fair_user_conf'), (NodeType.PASSIVE, 'passive_user_conf'), ], @@ -394,7 +394,7 @@ def test_turn_off_maintenance_on(mocked_g_config, regular_user_conf): mock.patch('node_cli.core.node.turn_off_op'), mock.patch('node_cli.utils.decorators.is_node_inited', return_value=True), mock.patch('node_cli.configs.user.validate_alias_or_address'), - mock.patch('node_cli.cli.node.TYPE', NodeType.REGULAR), + mock.patch('node_cli.cli.node.TYPE', NodeType.SKALE), ): result = run_command_mock( 'node_cli.utils.helper.requests.post', @@ -427,7 +427,7 @@ def test_turn_on_maintenance_off(mocked_g_config, regular_user_conf): mock.patch('node_cli.core.node.is_base_containers_alive'), mock.patch('node_cli.utils.decorators.is_node_inited', return_value=True), mock.patch('node_cli.configs.user.validate_alias_or_address'), - mock.patch('node_cli.cli.node.TYPE', NodeType.REGULAR), + mock.patch('node_cli.cli.node.TYPE', NodeType.SKALE), ): result = run_command_mock( 'node_cli.utils.helper.requests.post', diff --git a/tests/cli/resources_allocation_test.py b/tests/cli/resources_allocation_test.py index 4f59daea..03b2e73b 100644 --- a/tests/cli/resources_allocation_test.py +++ b/tests/cli/resources_allocation_test.py @@ -72,7 +72,7 @@ def test_generate_already_exists(regular_user_conf, resource_alloc_config): resp_mock = response_mock(requests.codes.created) with ( mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), - mock.patch('node_cli.cli.node.TYPE', NodeType.REGULAR), + mock.patch('node_cli.cli.node.TYPE', NodeType.SKALE), mock.patch('node_cli.configs.user.validate_alias_or_address'), ): result = run_command_mock( diff --git a/tests/configs/configs_env_validate_test.py b/tests/configs/configs_env_validate_test.py index c9d38cbf..4cdfc959 100644 --- a/tests/configs/configs_env_validate_test.py +++ b/tests/configs/configs_env_validate_test.py @@ -17,7 +17,7 @@ FairBootUserConfig, FairUserConfig, SkaleUserConfig, - PassiveUserConfig, + PassiveSkaleUserConfig, get_user_config_class, get_validated_user_config, validate_env_type, @@ -39,8 +39,8 @@ def json(self): @pytest.mark.parametrize( 'node_type, is_fair_boot, expected_type', [ - (NodeType.REGULAR, False, SkaleUserConfig), - (NodeType.PASSIVE, False, PassiveUserConfig), + (NodeType.SKALE, False, SkaleUserConfig), + (NodeType.PASSIVE, False, PassiveSkaleUserConfig), (NodeType.FAIR, True, FairBootUserConfig), (NodeType.FAIR, False, FairUserConfig), ], @@ -166,7 +166,7 @@ def test_validate_env_alias_or_address_with_alias(requests_mock): def test_get_validated_env_config_missing_file(): with pytest.raises(SystemExit): - get_validated_user_config(env_filepath='nonexistent.env', node_type=NodeType.REGULAR) + get_validated_user_config(env_filepath='nonexistent.env', node_type=NodeType.SKALE) def test_get_validated_env_config_unreadable_file(tmp_path): @@ -176,6 +176,6 @@ def test_get_validated_env_config_unreadable_file(tmp_path): try: os.chmod(env_file, 0o000) with pytest.raises(PermissionError): - get_validated_user_config(env_filepath=str(env_file), node_type=NodeType.REGULAR) + get_validated_user_config(env_filepath=str(env_file), node_type=NodeType.SKALE) finally: os.chmod(env_file, original_mode) diff --git a/tests/core/core_checks_test.py b/tests/core/core_checks_test.py index a4f7d437..5206efe4 100644 --- a/tests/core/core_checks_test.py +++ b/tests/core/core_checks_test.py @@ -379,8 +379,8 @@ def test_merge_report(): def test_get_static_params(tmp_config_dir): - params = get_static_params(NodeType.REGULAR) + params = get_static_params(NodeType.SKALE) shutil.copy(STATIC_PARAMS_FILEPATH, tmp_config_dir) - tmp_params = get_static_params(NodeType.REGULAR, config_path=tmp_config_dir) + tmp_params = get_static_params(NodeType.SKALE, config_path=tmp_config_dir) assert params['server']['cpu_total'] == 8 assert params == tmp_params diff --git a/tests/core/core_node_test.py b/tests/core/core_node_test.py index dd778a6f..ba1ab95f 100644 --- a/tests/core/core_node_test.py +++ b/tests/core/core_node_test.py @@ -38,7 +38,7 @@ ] NODE_TYPE_BOOT_COMBINATIONS: list[tuple[NodeType, bool]] = [ - (NodeType.REGULAR, False), + (NodeType.SKALE, False), (NodeType.PASSIVE, False), (NodeType.FAIR, True), (NodeType.FAIR, False), @@ -153,7 +153,7 @@ def test_is_base_containers_alive_empty(node_type, is_boot): ), [ ( - NodeType.REGULAR, + NodeType.SKALE, 'regular_user_conf', False, True, @@ -163,7 +163,7 @@ def test_is_base_containers_alive_empty(node_type, is_boot): False, ), ( - NodeType.REGULAR, + NodeType.SKALE, 'regular_user_conf', False, True, @@ -323,7 +323,7 @@ def test_init_node(regular_user_conf, no_resource_file): # todo: write new init mock.patch('node_cli.utils.helper.post_request', resp_mock), mock.patch('node_cli.configs.user.validate_alias_or_address'), ): - init(env_filepath=regular_user_conf.as_posix(), node_type=NodeType.REGULAR) + init(env_filepath=regular_user_conf.as_posix(), node_type=NodeType.SKALE) assert os.path.isfile(RESOURCE_ALLOCATION_FILEPATH) @@ -353,12 +353,12 @@ def test_update_node(regular_user_conf, mocked_g_config, resource_file, inited_n result = update( regular_user_conf.as_posix(), pull_config_for_schain=None, - node_type=NodeType.REGULAR, + node_type=NodeType.SKALE, ) assert result is None -@pytest.mark.parametrize('node_type', [NodeType.REGULAR, NodeType.PASSIVE, NodeType.FAIR]) +@pytest.mark.parametrize('node_type', [NodeType.SKALE, NodeType.PASSIVE, NodeType.FAIR]) @mock.patch('node_cli.core.node.is_admin_running', return_value=False) @mock.patch('node_cli.core.node.is_api_running', return_value=False) @mock.patch('node_cli.utils.helper.requests.get') @@ -379,7 +379,7 @@ def test_is_update_safe_when_admin_not_running_for_passive( mock_requests_get.assert_not_called() -@pytest.mark.parametrize('node_type', [NodeType.REGULAR, NodeType.PASSIVE, NodeType.FAIR]) +@pytest.mark.parametrize('node_type', [NodeType.SKALE, NodeType.PASSIVE, NodeType.FAIR]) @pytest.mark.parametrize( 'api_is_safe, expected_result', [(True, True), (False, False)], @@ -395,7 +395,7 @@ def test_is_update_safe_when_admin_running( mock_requests_get.assert_called_once() -@pytest.mark.parametrize('node_type', [NodeType.REGULAR, NodeType.FAIR]) +@pytest.mark.parametrize('node_type', [NodeType.SKALE, NodeType.FAIR]) @pytest.mark.parametrize( 'api_is_safe, expected_result', [(True, True), (False, False)], @@ -417,7 +417,7 @@ def test_is_update_safe_when_only_api_running_for_regular( mock_requests_get.assert_called_once() -@pytest.mark.parametrize('node_type', [NodeType.REGULAR, NodeType.PASSIVE, NodeType.FAIR]) +@pytest.mark.parametrize('node_type', [NodeType.SKALE, NodeType.PASSIVE, NodeType.FAIR]) @mock.patch('node_cli.core.node.is_admin_running', return_value=True) @mock.patch('node_cli.utils.helper.requests.get') def test_is_update_safe_when_api_call_fails(mock_requests_get, mock_is_admin_running, node_type): diff --git a/tests/core/nginx_test.py b/tests/core/nginx_test.py index 1652a514..c24645dc 100644 --- a/tests/core/nginx_test.py +++ b/tests/core/nginx_test.py @@ -7,7 +7,7 @@ from node_cli.core.nginx import ( generate_nginx_config, check_ssl_certs, - is_regular_node_nginx, + is_skale_node_nginx, SSL_KEY_NAME, SSL_CRT_NAME, ) @@ -24,7 +24,7 @@ {% endif %} } -{% if regular_node %} +{% if skale_node %} server { listen 80; {% if ssl %} @@ -59,8 +59,8 @@ def nginx_template(): @pytest.mark.parametrize( 'node_type, ssl_exists, expected_regular_flag, expected_ssl_flag', [ - (NodeType.REGULAR, True, True, True), - (NodeType.REGULAR, False, True, False), + (NodeType.SKALE, True, True, True), + (NodeType.SKALE, False, True, False), (NodeType.PASSIVE, True, True, True), (NodeType.PASSIVE, False, True, False), (NodeType.FAIR, True, False, True), @@ -132,14 +132,14 @@ def test_check_ssl_certs_missing_both(ssl_folder): @pytest.mark.parametrize( 'node_type, expected_result', [ - (NodeType.REGULAR, True), + (NodeType.SKALE, True), (NodeType.PASSIVE, True), (NodeType.FAIR, False), ], ) @mock.patch('node_cli.core.nginx.TYPE') -def test_is_regular_node_nginx(mock_type, node_type, expected_result): +def test_is_skale_node_nginx(mock_type, node_type, expected_result): mock_type.__eq__.side_effect = lambda other: node_type == other mock_type.__ne__.side_effect = lambda other: node_type != other - assert is_regular_node_nginx() is expected_result + assert is_skale_node_nginx() is expected_result From 9b5c410020ac037b90842500bf12318862b4e4a1 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Sun, 10 Aug 2025 15:44:22 +0100 Subject: [PATCH 045/198] Restructure node-cli - add NodeType and NodeMode to all modules --- node_cli/cli/fair_boot.py | 4 +- node_cli/configs/__init__.py | 1 + node_cli/configs/user.py | 17 ++- node_cli/core/node.py | 115 ++++++++++++------- node_cli/core/node_options.py | 22 ++-- node_cli/fair/fair_boot.py | 16 ++- node_cli/fair/fair_node.py | 45 +++++--- node_cli/operations/base.py | 62 ++++++----- node_cli/operations/config_repo.py | 8 +- node_cli/operations/fair.py | 35 +++--- node_cli/utils/docker_utils.py | 123 +++++++++++++-------- tests/configs/configs_env_validate_test.py | 30 +++-- 12 files changed, 296 insertions(+), 182 deletions(-) diff --git a/node_cli/cli/fair_boot.py b/node_cli/cli/fair_boot.py index 703bea88..24f3490a 100644 --- a/node_cli/cli/fair_boot.py +++ b/node_cli/cli/fair_boot.py @@ -50,9 +50,7 @@ def init_boot(env_file): @boot.command('register', help='Register Fair node in SKALE Manager (during Boot Phase).') -@click.option( - '--name', '-n', required=True, prompt='Enter fair node name', help='Fair node name' -) +@click.option('--name', '-n', required=True, prompt='Enter fair node name', help='Fair node name') @click.option( '--ip', prompt='Enter node public IP', diff --git a/node_cli/configs/__init__.py b/node_cli/configs/__init__.py index 748aa41a..8fbecb04 100644 --- a/node_cli/configs/__init__.py +++ b/node_cli/configs/__init__.py @@ -57,6 +57,7 @@ COMPOSE_PATH = os.path.join(CONTAINER_CONFIG_PATH, 'docker-compose.yml') PASSIVE_COMPOSE_PATH = os.path.join(CONTAINER_CONFIG_PATH, 'docker-compose-passive.yml') FAIR_COMPOSE_PATH = os.path.join(CONTAINER_CONFIG_PATH, 'docker-compose-fair.yml') +PASSIVE_FAIR_COMPOSE_PATH = os.path.join(CONTAINER_CONFIG_PATH, 'docker-compose-fair-passive.yml') STATIC_PARAMS_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, 'static_params.yaml') FAIR_STATIC_PARAMS_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, 'fair_static_params.yaml') diff --git a/node_cli/configs/user.py b/node_cli/configs/user.py index 6acc6d1a..23e1c7af 100644 --- a/node_cli/configs/user.py +++ b/node_cli/configs/user.py @@ -28,7 +28,7 @@ from node_cli.configs import CONTAINER_CONFIG_PATH, SKALE_DIR from node_cli.configs.alias_address_validation import ContractType, validate_alias_or_address from node_cli.utils.helper import error_exit -from node_cli.utils.node_type import NodeType +from node_cli.utils.node_type import NodeMode, NodeType SKALE_DIR_ENV_FILEPATH = os.path.join(SKALE_DIR, '.env') CONFIGS_ENV_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, '.env') @@ -133,11 +133,16 @@ class PassiveSkaleUserConfig(BaseUserConfig): def get_validated_user_config( node_type: NodeType, + node_mode: NodeMode, env_filepath: str = SKALE_DIR_ENV_FILEPATH, is_fair_boot: bool = False, ) -> BaseUserConfig: params = parse_env_file(env_filepath) - user_config_class = get_user_config_class(node_type, is_fair_boot) + user_config_class = get_user_config_class( + node_type=node_type, + node_mode=node_mode, + is_fair_boot=is_fair_boot, + ) _, missing_params, extra_params = user_config_class.validate_params(params) if len(missing_params) > 0: @@ -177,18 +182,18 @@ def parse_env_file(env_filepath: str) -> Dict: def get_user_config_class( node_type: NodeType, - is_passive: bool = False, - is_fair_boot: bool = False, + node_mode: NodeMode, + is_fair_boot: bool, ) -> type[BaseUserConfig]: if node_type == NodeType.FAIR and is_fair_boot: user_config_class = FairBootUserConfig elif node_type == NodeType.FAIR: - if is_passive: + if node_mode == NodeMode.PASSIVE: user_config_class = PassiveFairUserConfig else: user_config_class = FairUserConfig elif node_type == NodeType.SKALE: - if is_passive: + if node_mode == NodeMode.PASSIVE: user_config_class = PassiveSkaleUserConfig else: user_config_class = SkaleUserConfig diff --git a/node_cli/core/node.py b/node_cli/core/node.py index 9b0a8d81..7577f166 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -46,8 +46,14 @@ from node_cli.configs.user import SKALE_DIR_ENV_FILEPATH, get_validated_user_config from node_cli.core.checks import run_checks as run_host_checks from node_cli.core.host import get_flask_secret_key, is_node_inited, save_env_params -from node_cli.core.node_options import is_passive_node from node_cli.core.resources import update_resource_allocation +from node_cli.core.node_options import ( + active_fair, + active_skale, + get_node_mode, + passive_skale, + passive_fair, +) from node_cli.migrations.focal_to_jammy import migrate as migrate_2_6 from node_cli.operations import ( cleanup_passive_op, @@ -66,6 +72,7 @@ BASE_FAIR_COMPOSE_SERVICES, BASE_SKALE_COMPOSE_SERVICES, BASE_PASSIVE_COMPOSE_SERVICES, + BASE_PASSIVE_FAIR_COMPOSE_SERVICES, is_admin_running, is_api_running, ) @@ -76,7 +83,7 @@ post_request, ) from node_cli.utils.meta import CliMetaManager -from node_cli.utils.node_type import NodeType +from node_cli.utils.node_type import NodeType, NodeMode from node_cli.utils.print_formatters import ( print_failed_requirements_checks, print_node_cmd_error, @@ -102,18 +109,20 @@ class NodeStatuses(Enum): NOT_CREATED = 5 -def is_update_safe() -> bool: - if not is_admin_running(): - if is_passive_node(): +def is_update_safe(node_type: NodeType, node_mode: NodeMode) -> bool: + if not is_admin_running(node_type, node_mode): + if node_mode == NodeMode.PASSIVE: return True - elif not is_api_running(): + elif not is_api_running(node_type): return True status, payload = get_request(BLUEPRINT_NAME, 'update-safe') if status == 'error': return False - safe = payload['update_safe'] + if not isinstance(payload, dict): + return False + safe = bool(payload.get('update_safe')) if not safe: - logger.info('Locked schains: %s', payload['unsafe_chains']) + logger.info('Locked schains: %s', payload.get('unsafe_chains')) return safe @@ -145,12 +154,13 @@ def register_node(name, p2p_ip, public_ip, port, domain_name): @check_not_inited def init(env_filepath: str, node_type: NodeType) -> None: - env = compose_node_env(env_filepath=env_filepath, node_type=node_type) + node_mode = NodeMode.ACTIVE + env = compose_node_env(env_filepath=env_filepath, node_type=node_type, node_mode=node_mode) - init_op(env_filepath=env_filepath, env=env, node_type=node_type) + init_op(env_filepath=env_filepath, env=env, node_type=node_type, node_mode=node_mode) logger.info('Waiting for containers initialization') time.sleep(TM_INIT_TIMEOUT) - if not is_base_containers_alive(node_type=node_type): + if not is_base_containers_alive(node_type=node_type, node_mode=node_mode): error_exit('Containers are not running', exit_code=CLIExitCodes.OPERATION_EXECUTION_ERROR) logger.info('Generating resource allocation file ...') update_resource_allocation(env['ENV_TYPE']) @@ -159,7 +169,8 @@ def init(env_filepath: str, node_type: NodeType) -> None: @check_not_inited def restore(backup_path, env_filepath, node_type: NodeType, no_snapshot=False, config_only=False): - env = compose_node_env(env_filepath=env_filepath, node_type=node_type) + node_mode = NodeMode.ACTIVE + env = compose_node_env(env_filepath=env_filepath, node_type=node_type, node_mode=node_mode) if env is None: return save_env_params(env_filepath) @@ -182,13 +193,14 @@ def restore(backup_path, env_filepath, node_type: NodeType, no_snapshot=False, c def init_passive( env_filepath: str, indexer: bool, archive: bool, snapshot: bool, snapshot_from: Optional[str] ) -> None: - env = compose_node_env(env_filepath, node_type=NodeType.PASSIVE) + node_mode = NodeMode.PASSIVE + env = compose_node_env(env_filepath, node_type=NodeType.SKALE, node_mode=node_mode) if env is None: return init_passive_op(env_filepath, env, indexer, archive, snapshot, snapshot_from) logger.info('Waiting for containers initialization') time.sleep(TM_INIT_TIMEOUT) - if not is_base_containers_alive(node_type=NodeType.PASSIVE): + if not is_base_containers_alive(node_type=NodeType.SKALE, node_mode=node_mode): error_exit('Containers are not running', exit_code=CLIExitCodes.OPERATION_EXECUTION_ERROR) logger.info('Passive node initialized successfully') @@ -200,12 +212,12 @@ def update_passive(env_filepath: str, unsafe_ok: bool = False) -> None: prev_version = CliMetaManager().get_meta_info().version if (__version__ == 'test' or __version__.startswith('2.6')) and prev_version == '2.5.0': migrate_2_6() - env = compose_node_env(env_filepath, node_type=NodeType.PASSIVE) + env = compose_node_env(env_filepath, node_type=NodeType.SKALE, node_mode=NodeMode.PASSIVE) update_ok = update_passive_op(env_filepath, env) if update_ok: logger.info('Waiting for containers initialization') time.sleep(TM_INIT_TIMEOUT) - alive = is_base_containers_alive(node_type=NodeType.PASSIVE) + alive = is_base_containers_alive(node_type=NodeType.SKALE, node_mode=NodeMode.PASSIVE) if not update_ok or not alive: print_node_cmd_error() return @@ -216,7 +228,9 @@ def update_passive(env_filepath: str, unsafe_ok: bool = False) -> None: @check_inited @check_user def cleanup_passive() -> None: - env = compose_node_env(SKALE_DIR_ENV_FILEPATH, save=False, node_type=NodeType.PASSIVE) + env = compose_node_env( + SKALE_DIR_ENV_FILEPATH, save=False, node_type=NodeType.SKALE, node_mode=NodeMode.PASSIVE + ) schain_name = env['SCHAIN_NAME'] cleanup_passive_op(env, schain_name) logger.info('Passive node was cleaned up, all containers and data removed') @@ -224,7 +238,8 @@ def cleanup_passive() -> None: def compose_node_env( env_filepath: str, - # node_type: NodeType, + node_type: NodeType, + node_mode: NodeMode, inited_node: bool = False, sync_schains: Optional[bool] = None, pull_config_for_schain: Optional[str] = None, @@ -234,6 +249,7 @@ def compose_node_env( if env_filepath is not None: user_config = get_validated_user_config( node_type=node_type, + node_mode=node_mode, env_filepath=env_filepath, is_fair_boot=is_fair_boot, ) @@ -246,7 +262,7 @@ def compose_node_env( is_fair_boot=is_fair_boot, ) - if node_type == NodeType.PASSIVE or node_type == NodeType.FAIR: + if node_mode == NodeMode.PASSIVE or node_type == NodeType.FAIR: mnt_dir = SCHAINS_MNT_DIR_SINGLE_CHAIN else: mnt_dir = SCHAINS_MNT_DIR_REGULAR @@ -259,10 +275,10 @@ def compose_node_env( **user_config.to_env(), } - if inited_node and not node_type == NodeType.PASSIVE: + if inited_node and not node_mode == NodeMode.PASSIVE: env['FLASK_SECRET_KEY'] = get_flask_secret_key() - if sync_schains and not node_type == NodeType.PASSIVE: + if sync_schains and not node_mode == NodeMode.PASSIVE: env['BACKUP_RUN'] = 'True' if pull_config_for_schain: @@ -279,7 +295,9 @@ def update( node_type: NodeType, unsafe_ok: bool = False, ) -> None: - if not unsafe_ok and not is_update_safe(node_type=node_type): + node_mode = get_node_mode() + + if not unsafe_ok and not is_update_safe(node_type=node_type, node_mode=node_mode): error_msg = 'Cannot update safely' error_exit(error_msg, exit_code=CLIExitCodes.UNSAFE_UPDATE) @@ -293,12 +311,13 @@ def update( sync_schains=False, pull_config_for_schain=pull_config_for_schain, node_type=node_type, + node_mode=node_mode, ) - update_ok = update_op(env_filepath, env, node_type=node_type) + update_ok = update_op(env_filepath, env, node_type=node_type, node_mode=node_mode) if update_ok: logger.info('Waiting for containers initialization') time.sleep(TM_INIT_TIMEOUT) - alive = is_base_containers_alive(node_type=node_type) + alive = is_base_containers_alive(node_type=node_type, node_mode=node_mode) if not update_ok or not alive: print_node_cmd_error() return @@ -401,25 +420,33 @@ def set_maintenance_mode_off(): @check_inited @check_user def turn_off(node_type: NodeType, maintenance_on: bool = False, unsafe_ok: bool = False) -> None: - if not unsafe_ok and not is_update_safe(node_type=node_type): + node_mode = get_node_mode() + if not unsafe_ok and not is_update_safe(node_type=node_type, node_mode=node_mode): error_msg = 'Cannot turn off safely' error_exit(error_msg, exit_code=CLIExitCodes.UNSAFE_UPDATE) if maintenance_on: set_maintenance_mode_on() - env = compose_node_env(SKALE_DIR_ENV_FILEPATH, save=False, node_type=node_type) - turn_off_op(node_type=node_type, env=env) + env = compose_node_env( + SKALE_DIR_ENV_FILEPATH, save=False, node_type=node_type, node_mode=node_mode + ) + turn_off_op(node_type=node_type, node_mode=node_mode, env=env) @check_inited @check_user def turn_on(maintenance_off, sync_schains, env_file, node_type: NodeType) -> None: + node_mode = get_node_mode() env = compose_node_env( - env_file, inited_node=True, sync_schains=sync_schains, node_type=node_type + env_file, + inited_node=True, + sync_schains=sync_schains, + node_type=node_type, + node_mode=node_mode, ) - turn_on_op(env=env, node_type=node_type) + turn_on_op(env=env, node_type=node_type, node_mode=node_mode) logger.info('Waiting for containers initialization') time.sleep(TM_INIT_TIMEOUT) - if not is_base_containers_alive(node_type=node_type): + if not is_base_containers_alive(node_type=node_type, node_mode=node_mode): print_node_cmd_error() return logger.info('Node turned on') @@ -427,21 +454,30 @@ def turn_on(maintenance_off, sync_schains, env_file, node_type: NodeType) -> Non set_maintenance_mode_off() -def get_expected_container_names(node_type: NodeType, is_fair_boot: bool) -> list[str]: +def get_expected_container_names( + node_type: NodeType, + node_mode: NodeMode, + is_fair_boot: bool, +) -> list[str]: if node_type == NodeType.FAIR and is_fair_boot: services = BASE_FAIR_BOOT_COMPOSE_SERVICES - elif node_type == NodeType.FAIR and not is_fair_boot: + elif active_fair(node_type, node_mode): services = BASE_FAIR_COMPOSE_SERVICES - elif node_type == NodeType.PASSIVE: - services = BASE_PASSIVE_COMPOSE_SERVICES - else: + elif passive_fair(node_type, node_mode): + services = BASE_PASSIVE_FAIR_COMPOSE_SERVICES + elif active_skale(node_type, node_mode): services = BASE_SKALE_COMPOSE_SERVICES - + elif passive_skale(node_type, node_mode): + services = BASE_PASSIVE_COMPOSE_SERVICES return list(services.values()) -def is_base_containers_alive(node_type: NodeType, is_fair_boot: bool = False) -> bool: - base_container_names = get_expected_container_names(node_type, is_fair_boot) +def is_base_containers_alive( + node_type: NodeType, + node_mode: NodeMode, + is_fair_boot: bool = False, +) -> bool: + base_container_names = get_expected_container_names(node_type, node_mode, is_fair_boot) dclient = docker.from_env() running_container_names = set(container.name for container in dclient.containers.list()) @@ -492,6 +528,7 @@ def set_domain_name(domain_name): def run_checks( node_type: NodeType, + node_mode: NodeMode, network: str = 'mainnet', container_config_path: str = CONTAINER_CONFIG_PATH, disk: Optional[str] = None, @@ -501,7 +538,7 @@ def run_checks( return if disk is None: - env_config = get_validated_user_config(node_type=node_type) + env_config = get_validated_user_config(node_type=node_type, node_mode=node_mode) disk = env_config.disk_mountpoint failed_checks = run_host_checks(disk, node_type, network, container_config_path) if not failed_checks: diff --git a/node_cli/core/node_options.py b/node_cli/core/node_options.py index e6500399..889978fe 100644 --- a/node_cli/core/node_options.py +++ b/node_cli/core/node_options.py @@ -22,7 +22,6 @@ from node_cli.utils.node_type import NodeMode, NodeType from node_cli.utils.helper import read_json, write_json, init_file from node_cli.configs.node_options import NODE_OPTIONS_FILEPATH -from node_cli.cli.info import TYPE logger = logging.getLogger(__name__) @@ -90,19 +89,22 @@ def mark_passive_node() -> None: logger.info('Node marked as passive.') -def is_active_node() -> bool: +def get_node_mode() -> NodeMode: node_options = NodeOptions() - return node_options.node_mode == NodeMode.ACTIVE + return node_options.node_mode -def is_passive_node() -> bool: - node_options = NodeOptions() - return node_options.node_mode == NodeMode.PASSIVE +def active_skale(node_type: NodeType, node_mode: NodeMode) -> bool: + return node_mode == NodeMode.ACTIVE and node_type == NodeType.SKALE + + +def active_fair(node_type: NodeType, node_mode: NodeMode) -> bool: + return node_mode == NodeMode.ACTIVE and node_type == NodeType.FAIR -def is_skale_node() -> bool: - return TYPE == NodeType.SKALE +def passive_skale(node_type: NodeType, node_mode: NodeMode) -> bool: + return node_mode == NodeMode.PASSIVE and node_type == NodeType.SKALE -def is_fair_node() -> bool: - return TYPE == NodeType.FAIR +def passive_fair(node_type: NodeType, node_mode: NodeMode) -> bool: + return node_mode == NodeMode.PASSIVE and node_type == NodeType.FAIR diff --git a/node_cli/fair/fair_boot.py b/node_cli/fair/fair_boot.py index 290bbb1e..b8da120c 100644 --- a/node_cli/fair/fair_boot.py +++ b/node_cli/fair/fair_boot.py @@ -23,11 +23,12 @@ from node_cli.configs import TM_INIT_TIMEOUT from node_cli.core.node import compose_node_env, is_base_containers_alive +from node_cli.core.node_options import get_node_mode from node_cli.operations import init_fair_boot_op, update_fair_boot_op from node_cli.utils.decorators import check_inited, check_not_inited, check_user from node_cli.utils.exit_codes import CLIExitCodes from node_cli.utils.helper import error_exit -from node_cli.utils.node_type import NodeType +from node_cli.utils.node_type import NodeMode, NodeType from node_cli.utils.print_formatters import print_node_cmd_error logger = logging.getLogger(__name__) @@ -35,16 +36,19 @@ @check_not_inited def init(env_filepath: str) -> None: + node_mode = NodeMode.ACTIVE + node_type = NodeType.FAIR env = compose_node_env( env_filepath, - node_type=NodeType.FAIR, + node_type=node_type, + node_mode=node_mode, is_fair_boot=True, ) init_fair_boot_op(env_filepath, env) logger.info('Waiting for fair containers initialization') time.sleep(TM_INIT_TIMEOUT) - if not is_base_containers_alive(node_type=NodeType.FAIR, is_fair_boot=True): + if not is_base_containers_alive(node_type=node_type, node_mode=node_mode, is_fair_boot=True): error_exit('Containers are not running', exit_code=CLIExitCodes.OPERATION_EXECUTION_ERROR) logger.info('Init fair procedure finished') @@ -53,19 +57,23 @@ def init(env_filepath: str) -> None: @check_user def update(env_filepath: str, pull_config_for_schain: str) -> None: logger.info('Fair boot node update started') + node_mode = get_node_mode() env = compose_node_env( env_filepath, inited_node=True, sync_schains=False, pull_config_for_schain=pull_config_for_schain, node_type=NodeType.FAIR, + node_mode=node_mode, is_fair_boot=True, ) migrate_ok = update_fair_boot_op(env_filepath, env) if migrate_ok: logger.info('Waiting for containers initialization') time.sleep(TM_INIT_TIMEOUT) - alive = is_base_containers_alive(node_type=NodeType.FAIR, is_fair_boot=True) + alive = is_base_containers_alive( + node_type=NodeType.FAIR, node_mode=node_mode, is_fair_boot=True + ) if not migrate_ok or not alive: print_node_cmd_error() return diff --git a/node_cli/fair/fair_node.py b/node_cli/fair/fair_node.py index e6f2fb1d..a8a29ebb 100644 --- a/node_cli/fair/fair_node.py +++ b/node_cli/fair/fair_node.py @@ -27,6 +27,7 @@ from node_cli.core.docker_config import cleanup_docker_configuration from node_cli.core.host import is_node_inited, save_env_params from node_cli.core.node import compose_node_env, is_base_containers_alive +from node_cli.core.node_options import get_node_mode from node_cli.operations import ( FairUpdateType, cleanup_fair_op, @@ -38,7 +39,7 @@ from node_cli.utils.decorators import check_inited, check_not_inited, check_user from node_cli.utils.exit_codes import CLIExitCodes from node_cli.utils.helper import error_exit, get_request, post_request -from node_cli.utils.node_type import NodeType +from node_cli.utils.node_type import NodeMode, NodeType from node_cli.utils.print_formatters import print_node_cmd_error, print_node_info_fair from node_cli.utils.texts import safe_load_texts @@ -67,13 +68,16 @@ def get_node_info(format): @check_not_inited def restore_fair(backup_path, env_filepath, config_only=False): - env = compose_node_env(env_filepath, node_type=NodeType.FAIR) + node_mode = NodeMode.ACTIVE + env = compose_node_env(env_filepath, node_type=NodeType.FAIR, node_mode=node_mode) if env is None: return save_env_params(env_filepath) env['SKALE_DIR'] = SKALE_DIR - restored_ok = restore_fair_op(env, backup_path, config_only=config_only) + restored_ok = restore_fair_op( + node_mode=node_mode, env=env, backup_path=backup_path, config_only=config_only + ) if not restored_ok: error_exit('Restore operation failed', exit_code=CLIExitCodes.OPERATION_EXECUTION_ERROR) time.sleep(RESTORE_SLEEP_TIMEOUT) @@ -91,11 +95,16 @@ def migrate_from_boot( inited_node=True, sync_schains=False, node_type=NodeType.FAIR, + node_mode=NodeMode.ACTIVE, ) migrate_ok = update_fair_op( - env_filepath, env, update_type=FairUpdateType.FROM_BOOT, force_skaled_start=False + node_mode=NodeMode.ACTIVE, + env_filepath=env_filepath, + env=env, + update_type=FairUpdateType.FROM_BOOT, + force_skaled_start=False, ) - alive = is_base_containers_alive(node_type=NodeType.FAIR) + alive = is_base_containers_alive(node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE) if not migrate_ok or not alive: print_node_cmd_error() return @@ -114,20 +123,23 @@ def update( pull_config_for_schain, force_skaled_start, ) + node_mode = get_node_mode() env = compose_node_env( env_filepath, inited_node=True, sync_schains=False, node_type=NodeType.FAIR, + node_mode=node_mode, pull_config_for_schain=pull_config_for_schain, ) update_ok = update_fair_op( - env_filepath, - env, + node_mode=node_mode, + env_filepath=env_filepath, + env=env, update_type=FairUpdateType.REGULAR, force_skaled_start=force_skaled_start, ) - alive = is_base_containers_alive(node_type=NodeType.FAIR) + alive = is_base_containers_alive(node_type=NodeType.FAIR, node_mode=node_mode) if not update_ok or not alive: print_node_cmd_error() return @@ -137,15 +149,19 @@ def update( @check_user def cleanup() -> None: - env = compose_node_env(SKALE_DIR_ENV_FILEPATH, save=False, node_type=NodeType.FAIR) - cleanup_fair_op(env) + node_mode = get_node_mode() + env = compose_node_env( + SKALE_DIR_ENV_FILEPATH, save=False, node_type=NodeType.FAIR, node_mode=node_mode + ) + cleanup_fair_op(node_mode=node_mode, env=env) logger.info('Fair node was cleaned up, all containers and data removed') cleanup_docker_configuration() @check_not_inited def init(env_filepath: str) -> None: - env = compose_node_env(env_filepath, node_type=NodeType.FAIR) + node_mode = NodeMode.ACTIVE + env = compose_node_env(env_filepath, node_type=NodeType.FAIR, node_mode=node_mode) if env is None: return save_env_params(env_filepath) @@ -178,8 +194,11 @@ def register(ip: str) -> None: def repair_chain(snapshot_from: str = 'any') -> None: - env = compose_node_env(SKALE_DIR_ENV_FILEPATH, save=False, node_type=NodeType.FAIR) - repair_fair_op(env=env, snapshot_from=snapshot_from) + node_mode = get_node_mode() + env = compose_node_env( + SKALE_DIR_ENV_FILEPATH, save=False, node_type=NodeType.FAIR, node_mode=node_mode + ) + repair_fair_op(node_mode=node_mode, env=env, snapshot_from=snapshot_from) @check_inited diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index 47ce58aa..5168fab1 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -41,7 +41,12 @@ ) from node_cli.core.nftables import configure_nftables from node_cli.core.nginx import generate_nginx_config -from node_cli.core.node_options import NodeOptions +from node_cli.core.node_options import ( + NodeOptions, + get_node_mode, + mark_active_node, + mark_passive_node, +) from node_cli.core.resources import init_shared_space_volume, update_resource_allocation from node_cli.core.schains import ( cleanup_no_lvm_datadir, @@ -108,8 +113,8 @@ def wrapper(env_filepath: str, env: Dict, *args, **kwargs): @checked_host -def update(env_filepath: str, env: Dict, node_type: NodeType) -> bool: - compose_rm(node_type=node_type, env=env) +def update(env_filepath: str, env: Dict, node_type: NodeType, node_mode: NodeMode) -> bool: + compose_rm(node_type=node_type, node_mode=node_mode, env=env) remove_dynamic_containers() sync_skale_node() @@ -145,14 +150,14 @@ def update(env_filepath: str, env: Dict, node_type: NodeType) -> bool: distro.id(), distro.version(), ) - update_images(env=env, node_type=node_type) - compose_up(env=env, node_type=node_type) + update_images(env=env, node_type=node_type, node_mode=node_mode) + compose_up(env=env, node_type=node_type, node_mode=node_mode) return True @checked_host def update_fair_boot(env_filepath: str, env: Dict) -> bool: - compose_rm(node_type=NodeType.FAIR, env=env) + compose_rm(node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE, env=env) remove_dynamic_containers() cleanup_volume_artifacts(env['DISK_MOUNTPOINT']) @@ -187,13 +192,13 @@ def update_fair_boot(env_filepath: str, env: Dict) -> bool: distro.id(), distro.version(), ) - update_images(env=env, node_type=NodeType.FAIR) - compose_up(env=env, node_type=NodeType.FAIR, is_fair_boot=True) + update_images(env=env, node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE) + compose_up(env=env, node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE, is_fair_boot=True) return True @checked_host -def init(env_filepath: str, env: dict, node_type: NodeType) -> None: +def init(env_filepath: str, env: dict, node_type: NodeType, node_mode: NodeMode) -> None: sync_skale_node() ensure_btrfs_kernel_module_autoloaded() if env.get('SKIP_DOCKER_CONFIG') != 'True': @@ -205,8 +210,7 @@ def init(env_filepath: str, env: dict, node_type: NodeType) -> None: prepare_host(env_filepath, env_type=env['ENV_TYPE']) link_env_file() - node_options = NodeOptions() - node_options.node_mode = NodeMode.ACTIVE + mark_active_node() configure_filebeat() configure_flask() @@ -224,9 +228,9 @@ def init(env_filepath: str, env: dict, node_type: NodeType) -> None: distro.version(), ) update_resource_allocation(env_type=env['ENV_TYPE']) - update_images(env=env, node_type=node_type) + update_images(env=env, node_type=node_type, node_mode=node_mode) - compose_up(env=env, node_type=node_type) + compose_up(env=env, node_type=node_type, node_mode=node_mode) @checked_host @@ -243,6 +247,7 @@ def init_fair_boot(env_filepath: str, env: dict) -> None: prepare_host(env_filepath, env_type=env['ENV_TYPE']) link_env_file() + mark_active_node() configure_filebeat() configure_flask() @@ -256,9 +261,9 @@ def init_fair_boot(env_filepath: str, env: dict) -> None: distro.id(), distro.version(), ) - update_images(env=env, node_type=NodeType.FAIR) + update_images(env=env, node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE) - compose_up(env=env, node_type=NodeType.FAIR, is_fair_boot=True) + compose_up(env=env, node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE, is_fair_boot=True) def init_passive( @@ -289,6 +294,8 @@ def init_passive( node_options.catchup = archive or indexer node_options.historic_state = archive + mark_passive_node() + ensure_filestorage_mapping() link_env_file() @@ -310,13 +317,12 @@ def init_passive( ts = int(time.time()) update_node_cli_schain_status(schain_name, repair_ts=ts, snapshot_from=snapshot_from) - update_images(env=env, node_type=NodeType.PASSIVE) - - compose_up(env=env, node_type=NodeType.PASSIVE) + update_images(env=env, node_type=NodeType.SKALE, node_mode=NodeMode.PASSIVE) + compose_up(env=env, node_type=NodeType.SKALE, node_mode=NodeMode.PASSIVE) def update_passive(env_filepath: str, env: Dict) -> bool: - compose_rm(env=env, node_type=NodeType.PASSIVE) + compose_rm(env=env, node_type=NodeType.SKALE, node_mode=NodeMode.PASSIVE) remove_dynamic_containers() cleanup_volume_artifacts(env['DISK_MOUNTPOINT']) download_skale_node(env['NODE_VERSION'], env.get('CONTAINER_CONFIGS_DIR')) @@ -343,20 +349,19 @@ def update_passive(env_filepath: str, env: Dict) -> bool: distro.id(), distro.version(), ) - update_images(env=env, node_type=NodeType.PASSIVE) - - compose_up(env=env, node_type=NodeType.PASSIVE) + update_images(env=env, node_type=NodeType.SKALE, node_mode=NodeMode.PASSIVE) + compose_up(env=env, node_type=NodeType.SKALE, node_mode=NodeMode.PASSIVE) return True -def turn_off(env: dict, node_type: NodeType) -> None: +def turn_off(env: dict, node_type: NodeType, node_mode: NodeMode) -> None: logger.info('Turning off the node...') - compose_rm(env=env, node_type=node_type) + compose_rm(env=env, node_type=node_type, node_mode=node_mode) remove_dynamic_containers() logger.info('Node was successfully turned off') -def turn_on(env: dict, node_type: NodeType) -> None: +def turn_on(env: dict, node_type: NodeType, node_mode: NodeMode) -> None: logger.info('Turning on the node...') meta_manager = CliMetaManager() meta_manager.update_meta( @@ -373,10 +378,11 @@ def turn_on(env: dict, node_type: NodeType) -> None: configure_nftables(enable_monitoring=enable_monitoring) logger.info('Launching containers on the node...') - compose_up(env=env, node_type=node_type) + compose_up(env=env, node_type=node_type, node_mode=node_mode) def restore(env, backup_path, node_type: NodeType, config_only=False): + node_mode = get_node_mode() unpack_backup_archive(backup_path) failed_checks = run_host_checks( env['DISK_MOUNTPOINT'], @@ -410,7 +416,7 @@ def restore(env, backup_path, node_type: NodeType, config_only=False): distro.version(), ) if not config_only: - compose_up(env=env, node_type=node_type) + compose_up(env=env, node_type=node_type, node_mode=node_mode) failed_checks = run_host_checks( env['DISK_MOUNTPOINT'], @@ -426,7 +432,7 @@ def restore(env, backup_path, node_type: NodeType, config_only=False): def cleanup_passive(env, schain_name: str) -> None: - turn_off(env, node_type=NodeType.PASSIVE) + turn_off(env, node_type=NodeType.SKALE, node_mode=NodeMode.PASSIVE) cleanup_no_lvm_datadir(chain_name=schain_name) rm_dir(GLOBAL_SKALE_DIR) rm_dir(SKALE_DIR) diff --git a/node_cli/operations/config_repo.py b/node_cli/operations/config_repo.py index 656e4cb1..b8456db6 100644 --- a/node_cli/operations/config_repo.py +++ b/node_cli/operations/config_repo.py @@ -27,18 +27,18 @@ from node_cli.utils.git_utils import clone_repo from node_cli.utils.docker_utils import compose_pull, compose_build from node_cli.configs import CONTAINER_CONFIG_PATH, CONTAINER_CONFIG_TMP_PATH, SKALE_NODE_REPO_URL -from node_cli.utils.node_type import NodeType +from node_cli.utils.node_type import NodeType, NodeMode logger = logging.getLogger(__name__) -def update_images(env: dict, node_type: NodeType) -> None: +def update_images(env: dict, node_type: NodeType, node_mode: NodeMode) -> None: local = env.get('CONTAINER_CONFIGS_DIR') != '' if local: - compose_build(env=env, node_type=node_type) + compose_build(env=env, node_type=node_type, node_mode=node_mode) else: - compose_pull(env=env, node_type=node_type) + compose_pull(env=env, node_type=node_type, node_mode=node_mode) def download_skale_node(stream: Optional[str] = None, src: Optional[str] = None) -> None: diff --git a/node_cli/operations/fair.py b/node_cli/operations/fair.py index 9a1d15c6..d9898a10 100644 --- a/node_cli/operations/fair.py +++ b/node_cli/operations/fair.py @@ -54,7 +54,6 @@ from node_cli.utils.docker_utils import ( REDIS_SERVICE_DICT, REDIS_START_TIMEOUT, - NodeType, compose_rm, compose_up, docker_cleanup, @@ -67,6 +66,7 @@ from node_cli.utils.helper import cleanup_dir_content, rm_dir, str_to_bool from node_cli.utils.meta import FairCliMetaManager from node_cli.utils.print_formatters import TEXTS, print_failed_requirements_checks +from node_cli.utils.node_type import NodeMode, NodeType logger = logging.getLogger(__name__) @@ -103,8 +103,8 @@ def init(env_filepath: str, env: dict) -> bool: distro.id(), distro.version(), ) - update_images(env=env, node_type=NodeType.FAIR) - compose_up(env=env, node_type=NodeType.FAIR) + update_images(env=env, node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE) + compose_up(env=env, node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE) wait_for_container(REDIS_SERVICE_DICT['redis']) time.sleep(REDIS_START_TIMEOUT) return True @@ -112,7 +112,7 @@ def init(env_filepath: str, env: dict) -> bool: @checked_host def update_fair_boot(env_filepath: str, env: dict) -> bool: - compose_rm(node_type=NodeType.FAIR, env=env) + compose_rm(node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE, env=env) remove_dynamic_containers() cleanup_volume_artifacts(env['DISK_MOUNTPOINT']) @@ -147,19 +147,20 @@ def update_fair_boot(env_filepath: str, env: dict) -> bool: distro.id(), distro.version(), ) - update_images(env=env, node_type=NodeType.FAIR) - compose_up(env=env, node_type=NodeType.FAIR, is_fair_boot=True) + update_images(env=env, node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE) + compose_up(env=env, node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE, is_fair_boot=True) return True @checked_host def update( + node_mode: NodeMode, env_filepath: str, env: dict, update_type: FairUpdateType, force_skaled_start: bool, ) -> bool: - compose_rm(node_type=NodeType.FAIR, env=env) + compose_rm(node_type=NodeType.FAIR, node_mode=node_mode, env=env) if update_type not in (FairUpdateType.INFRA_ONLY, FairUpdateType.FROM_BOOT): remove_dynamic_containers() @@ -195,19 +196,21 @@ def update( if update_type == FairUpdateType.FROM_BOOT: migrate_nftables_from_boot(chain_name=fair_chain_name) - update_images(env=env, node_type=NodeType.FAIR) + update_images(env=env, node_type=NodeType.FAIR, node_mode=node_mode) - compose_up(env=env, node_type=NodeType.FAIR, services=list(REDIS_SERVICE_DICT)) + compose_up( + env=env, node_type=NodeType.FAIR, node_mode=node_mode, services=list(REDIS_SERVICE_DICT) + ) wait_for_container(REDIS_SERVICE_DICT['redis']) time.sleep(REDIS_START_TIMEOUT) if update_type == FairUpdateType.FROM_BOOT: migrate_chain_record(env) update_chain_record(env, force_skaled_start=force_skaled_start) - compose_up(env=env, node_type=NodeType.FAIR) + compose_up(env=env, node_type=NodeType.FAIR, node_mode=node_mode) return True -def restore(env, backup_path, config_only=False): +def restore(node_mode: NodeMode, env, backup_path, config_only=False): unpack_backup_archive(backup_path) failed_checks = run_host_checks( env['DISK_MOUNTPOINT'], @@ -239,7 +242,7 @@ def restore(env, backup_path, config_only=False): ) if not config_only: - compose_up(env=env, node_type=NodeType.FAIR) + compose_up(env=env, node_type=NodeType.FAIR, node_mode=node_mode) failed_checks = run_host_checks( env['DISK_MOUNTPOINT'], @@ -254,8 +257,8 @@ def restore(env, backup_path, config_only=False): return True -def cleanup(env: dict) -> None: - turn_off(env, node_type=NodeType.FAIR) +def cleanup(node_mode: NodeMode, env: dict) -> None: + turn_off(env, node_type=NodeType.FAIR, node_mode=node_mode) cleanup_no_lvm_datadir() rm_dir(GLOBAL_SKALE_DIR) rm_dir(SKALE_DIR) @@ -271,10 +274,10 @@ def trigger_skaled_snapshot_mode(env: dict, snapshot_from: str = 'any') -> None: print(TEXTS['fair']['node']['repair']['repair_requested']) -def repair(env: dict, snapshot_from: str = 'any') -> None: +def repair(node_mode: NodeMode, env: dict, snapshot_from: str = 'any') -> None: logger.info('Starting fair node repair') container_name = 'fair_admin' - if is_admin_running(): + if is_admin_running(node_type=NodeType.FAIR, node_mode=node_mode): logger.info('Stopping admin container') stop_container_by_name(container_name=container_name) logger.info('Removing chain container') diff --git a/node_cli/utils/docker_utils.py b/node_cli/utils/docker_utils.py index ac9a428e..83a8dc6f 100644 --- a/node_cli/utils/docker_utils.py +++ b/node_cli/utils/docker_utils.py @@ -33,13 +33,14 @@ COMPOSE_PATH, FAIR_COMPOSE_PATH, NGINX_CONTAINER_NAME, + PASSIVE_FAIR_COMPOSE_PATH, REMOVED_CONTAINERS_FOLDER_PATH, SGX_CERTIFICATES_DIR_NAME, PASSIVE_COMPOSE_PATH, ) -from node_cli.core.node_options import is_active_node, is_fair_node +from node_cli.core.node_options import active_fair, active_skale, passive_fair, passive_skale from node_cli.utils.helper import run_cmd, str_to_bool -from node_cli.utils.node_type import NodeType +from node_cli.utils.node_type import NodeMode, NodeType logger = logging.getLogger(__name__) @@ -82,6 +83,11 @@ 'nginx': 'skale_nginx', } +BASE_PASSIVE_FAIR_COMPOSE_SERVICES = { + 'fair-passive-admin': 'fair_passive_admin', + 'nginx': 'skale_nginx', +} + MONITORING_COMPOSE_SERVICES = { 'node-exporter': 'monitor_node_exporter', 'advisor': 'monitor_cadvisor', @@ -255,9 +261,9 @@ def is_volume_exists(name: str, dutils=None): return True -def compose_rm(node_type: NodeType, env={}): +def compose_rm(node_type: NodeType, node_mode: NodeMode, env={}): logger.info('Removing compose containers') - compose_path = get_compose_path(node_type) + compose_path = get_compose_path(node_type, node_mode) run_cmd( cmd=( 'docker', @@ -273,81 +279,98 @@ def compose_rm(node_type: NodeType, env={}): logger.info('Compose containers removed') -def compose_pull(env: dict, node_type: NodeType): +def compose_pull(env: dict, node_type: NodeType, node_mode: NodeMode): logger.info('Pulling compose containers') - compose_path = get_compose_path(node_type) + compose_path = get_compose_path(node_type, node_mode) run_cmd(cmd=('docker', 'compose', '-f', compose_path, 'pull'), env=env) -def compose_build(env: dict, node_type: NodeType): +def compose_build(env: dict, node_type: NodeType, node_mode: NodeMode): logger.info('Building compose containers') - compose_path = get_compose_path(node_type) + compose_path = get_compose_path(node_type, node_mode) run_cmd(cmd=('docker', 'compose', '-f', compose_path, 'build'), env=env) -def get_compose_path(node_type: NodeType) -> str: - if node_type == NodeType.PASSIVE: +def get_compose_path(node_type: NodeType, node_mode: NodeMode) -> str: + if passive_skale(node_type, node_mode): return PASSIVE_COMPOSE_PATH - elif node_type == NodeType.FAIR: + elif active_fair(node_type, node_mode): return FAIR_COMPOSE_PATH - else: - return COMPOSE_PATH + elif passive_fair(node_type, node_mode): + return PASSIVE_FAIR_COMPOSE_PATH + return COMPOSE_PATH -def get_compose_services(node_type: NodeType) -> list[str]: - if node_type == NodeType.PASSIVE: - result = list(BASE_PASSIVE_COMPOSE_SERVICES) - elif node_type == NodeType.FAIR: - result = list(BASE_FAIR_COMPOSE_SERVICES) - else: - result = list(BASE_SKALE_COMPOSE_SERVICES) +def get_compose_services(node_type: NodeType, node_mode: NodeMode) -> list[str]: + if passive_skale(node_type, node_mode): + return list(BASE_PASSIVE_COMPOSE_SERVICES) + elif active_fair(node_type, node_mode): + return list(BASE_FAIR_COMPOSE_SERVICES) + elif passive_fair(node_type, node_mode): + return list(BASE_PASSIVE_FAIR_COMPOSE_SERVICES) + return list(BASE_SKALE_COMPOSE_SERVICES) - return result - -def get_up_compose_cmd(node_type: NodeType, services: list[str] | None = None) -> tuple: - compose_path = get_compose_path(node_type) +def get_up_compose_cmd( + node_type: NodeType, node_mode: NodeMode, services: list[str] | None = None +) -> tuple: + compose_path = get_compose_path(node_type, node_mode) if services is None: - services = get_compose_services(node_type) + services = get_compose_services(node_type, node_mode) return ('docker', 'compose', '-f', compose_path, 'up', '-d', *services) def compose_up( - env, node_type: NodeType, is_fair_boot: bool = False, services: list[str] | None = None + env, + node_type: NodeType, + node_mode: NodeMode, + is_fair_boot: bool = False, + services: list[str] | None = None, ): - if node_type == NodeType.PASSIVE: + if passive_skale(node_type, node_mode) or passive_fair(node_type, node_mode): logger.info('Running containers for passive node') - run_cmd(cmd=get_up_compose_cmd(node_type=NodeType.PASSIVE), env=env) + run_cmd(cmd=get_up_compose_cmd(node_type=node_type, node_mode=node_mode), env=env) return if 'SGX_CERTIFICATES_DIR_NAME' not in env: env['SGX_CERTIFICATES_DIR_NAME'] = SGX_CERTIFICATES_DIR_NAME - if node_type == NodeType.FAIR: + if active_fair(node_type, node_mode): logger.info('Running fair base set of containers') if is_fair_boot: logger.debug('Launching fair boot containers with env %s', env) run_cmd( cmd=get_up_compose_cmd( - node_type=NodeType.FAIR, services=list(BASE_FAIR_BOOT_COMPOSE_SERVICES) + node_type=node_type, + node_mode=node_mode, + services=list(BASE_FAIR_BOOT_COMPOSE_SERVICES), ), env=env, ) else: logger.debug('Launching fair containers with env %s', env) - run_cmd(cmd=get_up_compose_cmd(node_type=NodeType.FAIR, services=services), env=env) - else: + run_cmd( + cmd=get_up_compose_cmd( + node_type=node_type, + node_mode=node_mode, + services=services, + ), + env=env, + ) + elif active_skale(node_type, node_mode): logger.info('Running skale node base set of containers') logger.debug('Launching skale node containers with env %s', env) - run_cmd(cmd=get_up_compose_cmd(node_type=NodeType.SKALE), env=env) + run_cmd(cmd=get_up_compose_cmd(node_type=node_type, node_mode=node_mode), env=env) if 'TG_API_KEY' in env and 'TG_CHAT_ID' in env: logger.info('Running containers for Telegram notifications') run_cmd( cmd=get_up_compose_cmd( - node_type=NodeType.SKALE, services=list(NOTIFICATION_COMPOSE_SERVICES) + node_type=NodeType.SKALE, + node_mode=node_mode, + services=list(NOTIFICATION_COMPOSE_SERVICES), ), env=env, ) @@ -356,7 +379,9 @@ def compose_up( logger.info('Running monitoring containers') run_cmd( cmd=get_up_compose_cmd( - node_type=NodeType.SKALE, services=list(MONITORING_COMPOSE_SERVICES) + node_type=NodeType.SKALE, + node_mode=node_mode, + services=list(MONITORING_COMPOSE_SERVICES), ), env=env, ) @@ -396,24 +421,26 @@ def is_container_running(name: str, dclient: Optional[DockerClient] = None) -> b return False -def is_api_running(dclient: Optional[DockerClient] = None) -> bool: - if is_fair_node(): +def is_api_running(node_type: NodeType, dclient: Optional[DockerClient] = None) -> bool: + if node_type == NodeType.FAIR: return is_container_running(name='fair_api', dclient=dclient) else: return is_container_running(name='skale_api', dclient=dclient) -def is_admin_running(dclient: Optional[DockerClient] = None) -> bool: - if is_fair_node(): - if is_active_node(): - container_name = 'fair_admin' - else: - container_name = 'fair_passive_admin' - else: - if is_active_node(): - container_name = 'skale_admin' - else: - container_name = 'skale_passive_admin' +def is_admin_running( + node_type: NodeType, + node_mode: NodeMode, + dclient: Optional[DockerClient] = None, +) -> bool: + if active_fair(node_type, node_mode): + container_name = 'fair_admin' + elif passive_fair(node_type, node_mode): + container_name = 'fair_passive_admin' + elif active_skale(node_type, node_mode): + container_name = 'skale_admin' + elif passive_skale(node_type, node_mode): + container_name = 'skale_passive_admin' return is_container_running(name=container_name, dclient=dclient) diff --git a/tests/configs/configs_env_validate_test.py b/tests/configs/configs_env_validate_test.py index 4cdfc959..2db057ff 100644 --- a/tests/configs/configs_env_validate_test.py +++ b/tests/configs/configs_env_validate_test.py @@ -16,13 +16,14 @@ ALLOWED_ENV_TYPES, FairBootUserConfig, FairUserConfig, + PassiveFairUserConfig, SkaleUserConfig, PassiveSkaleUserConfig, get_user_config_class, get_validated_user_config, validate_env_type, ) -from node_cli.utils.node_type import NodeType +from node_cli.utils.node_type import NodeType, NodeMode ENDPOINT = 'http://localhost:8545' @@ -37,17 +38,20 @@ def json(self): @pytest.mark.parametrize( - 'node_type, is_fair_boot, expected_type', + 'node_type, node_mode, is_fair_boot, expected_type', [ - (NodeType.SKALE, False, SkaleUserConfig), - (NodeType.PASSIVE, False, PassiveSkaleUserConfig), - (NodeType.FAIR, True, FairBootUserConfig), - (NodeType.FAIR, False, FairUserConfig), + (NodeType.SKALE, NodeMode.ACTIVE, False, SkaleUserConfig), + (NodeType.SKALE, NodeMode.PASSIVE, False, PassiveSkaleUserConfig), + (NodeType.FAIR, NodeMode.ACTIVE, True, FairBootUserConfig), + (NodeType.FAIR, NodeMode.ACTIVE, False, FairUserConfig), + (NodeType.FAIR, NodeMode.PASSIVE, False, PassiveFairUserConfig), ], - ids=['regular', 'passive', 'fair_boot', 'fair_regular'], + ids=['skale_active', 'skale_passive', 'fair_boot', 'fair_active', 'fair_passive'], ) -def test_build_env_params_keys(node_type, is_fair_boot, expected_type): - env_type = get_user_config_class(node_type=node_type, is_fair_boot=is_fair_boot) +def test_build_env_params_keys(node_type, node_mode, is_fair_boot, expected_type): + env_type = get_user_config_class( + node_type=node_type, node_mode=node_mode, is_fair_boot=is_fair_boot + ) assert env_type == expected_type @@ -166,7 +170,9 @@ def test_validate_env_alias_or_address_with_alias(requests_mock): def test_get_validated_env_config_missing_file(): with pytest.raises(SystemExit): - get_validated_user_config(env_filepath='nonexistent.env', node_type=NodeType.SKALE) + get_validated_user_config( + env_filepath='nonexistent.env', node_type=NodeType.SKALE, node_mode=NodeMode.ACTIVE + ) def test_get_validated_env_config_unreadable_file(tmp_path): @@ -176,6 +182,8 @@ def test_get_validated_env_config_unreadable_file(tmp_path): try: os.chmod(env_file, 0o000) with pytest.raises(PermissionError): - get_validated_user_config(env_filepath=str(env_file), node_type=NodeType.SKALE) + get_validated_user_config( + env_filepath=str(env_file), node_type=NodeType.SKALE, node_mode=NodeMode.ACTIVE + ) finally: os.chmod(env_file, original_mode) From 8e2e70d529cf9ce8216647bb8fb3167ec25e24e2 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Sun, 10 Aug 2025 16:30:16 +0100 Subject: [PATCH 046/198] Add passive fair node cli --- node_cli/cli/fair_boot.py | 2 +- node_cli/cli/fair_node.py | 21 ++-- node_cli/cli/passive_fair_node.py | 86 ++++++++++++++++ node_cli/fair/__init__.py | 15 +++ node_cli/fair/{fair_node.py => active.py} | 112 ++++----------------- node_cli/fair/{fair_boot.py => boot.py} | 0 node_cli/fair/common.py | 113 ++++++++++++++++++++++ node_cli/fair/passive.py | 25 +++++ node_cli/main.py | 2 + node_cli/operations/fair.py | 6 +- tests/fair/fair_node_test.py | 16 +-- 11 files changed, 283 insertions(+), 115 deletions(-) create mode 100644 node_cli/cli/passive_fair_node.py rename node_cli/fair/{fair_node.py => active.py} (67%) rename node_cli/fair/{fair_boot.py => boot.py} (100%) create mode 100644 node_cli/fair/common.py create mode 100644 node_cli/fair/passive.py diff --git a/node_cli/cli/fair_boot.py b/node_cli/cli/fair_boot.py index 24f3490a..f5dce9b1 100644 --- a/node_cli/cli/fair_boot.py +++ b/node_cli/cli/fair_boot.py @@ -22,7 +22,7 @@ from node_cli.configs import DEFAULT_NODE_BASE_PORT from node_cli.core.node import get_node_info, get_node_signature from node_cli.core.node import register_node as register -from node_cli.fair.fair_boot import init, update +from node_cli.fair.boot import init, update from node_cli.utils.helper import IP_TYPE, abort_if_false, error_exit, streamed_cmd diff --git a/node_cli/cli/fair_node.py b/node_cli/cli/fair_node.py index 19b06c93..d76825b1 100644 --- a/node_cli/cli/fair_node.py +++ b/node_cli/cli/fair_node.py @@ -20,19 +20,22 @@ import click from node_cli.core.node import backup -from node_cli.fair.fair_node import change_ip as change_ip_fair -from node_cli.fair.fair_node import cleanup as fair_cleanup -from node_cli.fair.fair_node import exit as exit_fair -from node_cli.fair.fair_node import ( + +from node_cli.fair import change_ip as change_ip_fair +from node_cli.fair import cleanup as fair_cleanup +from node_cli.fair import exit as exit_fair +from node_cli.fair import ( get_node_info, migrate_from_boot, repair_chain, - restore_fair, + restore as restore_fair, ) -from node_cli.fair.fair_node import init as init_fair -from node_cli.fair.fair_node import register as register_fair -from node_cli.fair.fair_node import update as update_fair +from node_cli.fair import init as init_fair +from node_cli.fair import register as register_fair +from node_cli.fair import update as update_fair + from node_cli.utils.helper import IP_TYPE, URL_OR_ANY_TYPE, abort_if_false, streamed_cmd +from node_cli.utils.node_type import NodeMode from node_cli.utils.texts import safe_load_texts TEXTS = safe_load_texts() @@ -58,7 +61,7 @@ def fair_node_info(format): @click.argument('env_filepath') @streamed_cmd def init_node(env_filepath: str): - init_fair(env_filepath=env_filepath) + init_fair(node_mode=NodeMode.ACTIVE, env_filepath=env_filepath) @node.command('register', help=TEXTS['fair']['node']['register']['help']) diff --git a/node_cli/cli/passive_fair_node.py b/node_cli/cli/passive_fair_node.py new file mode 100644 index 00000000..b0fb6c08 --- /dev/null +++ b/node_cli/cli/passive_fair_node.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- +# +# This file is part of node-cli +# +# Copyright (C) 2025-Present SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import click + +from node_cli.fair import init as init_fair +from node_cli.fair import update as update_fair +from node_cli.fair import cleanup as cleanup_fair +from node_cli.utils.helper import abort_if_false, streamed_cmd +from node_cli.utils.node_type import NodeMode +from node_cli.utils.texts import safe_load_texts + +TEXTS = safe_load_texts() + + +@click.group() +def passive_fair_node_cli(): + pass + + +@passive_fair_node_cli.group(help='Commands for passive Fair Node operations.') +def passive_node(): + pass + + +@passive_node.command('init', help='Initialize a passive Fair node') +@click.argument('env_filepath') +@streamed_cmd +def init_passive_node(env_filepath: str): + init_fair(node_mode=NodeMode.PASSIVE, env_filepath=env_filepath) + + +@passive_node.command('update', help='Update Fair node') +@click.argument('env_filepath') +@click.option( + '--yes', + is_flag=True, + callback=abort_if_false, + expose_value=False, + prompt='Are you sure you want to update Fair node software?', +) +@click.option('--pull-config', 'pull_config_for_schain', hidden=True, type=str) +@click.option( + '--force-skaled-start', + 'force_skaled_start', + hidden=True, + type=bool, + default=False, + is_flag=True, +) +@streamed_cmd +def update_node(env_filepath: str, pull_config_for_schain, force_skaled_start: bool): + update_fair( + env_filepath=env_filepath, + pull_config_for_schain=pull_config_for_schain, + force_skaled_start=force_skaled_start, + ) + + +@passive_node.command('cleanup', help='Cleanup Fair node.') +@click.option( + '--yes', + is_flag=True, + callback=abort_if_false, + expose_value=False, + prompt='Are you sure you want to cleanup Fair node?', +) +@streamed_cmd +def cleanup_node(): + cleanup_fair() diff --git a/node_cli/fair/__init__.py b/node_cli/fair/__init__.py index e69de29b..acac7093 100644 --- a/node_cli/fair/__init__.py +++ b/node_cli/fair/__init__.py @@ -0,0 +1,15 @@ +from node_cli.fair.common import ( + init as init, + update as update, + cleanup as cleanup, + repair_chain as repair_chain, +) +from node_cli.fair.active import ( + get_node_info_plain as get_node_info_plain, + get_node_info as get_node_info, + migrate_from_boot as migrate_from_boot, + register as register, + change_ip as change_ip, + exit as exit, + restore as restore, +) diff --git a/node_cli/fair/fair_node.py b/node_cli/fair/active.py similarity index 67% rename from node_cli/fair/fair_node.py rename to node_cli/fair/active.py index a8a29ebb..44ea1961 100644 --- a/node_cli/fair/fair_node.py +++ b/node_cli/fair/active.py @@ -23,16 +23,10 @@ from typing import cast from node_cli.configs import DEFAULT_SKALED_BASE_PORT, RESTORE_SLEEP_TIMEOUT, SKALE_DIR -from node_cli.configs.user import SKALE_DIR_ENV_FILEPATH -from node_cli.core.docker_config import cleanup_docker_configuration from node_cli.core.host import is_node_inited, save_env_params from node_cli.core.node import compose_node_env, is_base_containers_alive -from node_cli.core.node_options import get_node_mode from node_cli.operations import ( FairUpdateType, - cleanup_fair_op, - init_fair_op, - repair_fair_op, restore_fair_op, update_fair_op, ) @@ -66,24 +60,6 @@ def get_node_info(format): print_node_info_fair(node_info) -@check_not_inited -def restore_fair(backup_path, env_filepath, config_only=False): - node_mode = NodeMode.ACTIVE - env = compose_node_env(env_filepath, node_type=NodeType.FAIR, node_mode=node_mode) - if env is None: - return - save_env_params(env_filepath) - env['SKALE_DIR'] = SKALE_DIR - - restored_ok = restore_fair_op( - node_mode=node_mode, env=env, backup_path=backup_path, config_only=config_only - ) - if not restored_ok: - error_exit('Restore operation failed', exit_code=CLIExitCodes.OPERATION_EXECUTION_ERROR) - time.sleep(RESTORE_SLEEP_TIMEOUT) - print('Fair node is restored from backup') - - @check_inited @check_user def migrate_from_boot( @@ -112,68 +88,6 @@ def migrate_from_boot( logger.info('Migration from boot to fair completed successfully') -@check_inited -@check_user -def update( - env_filepath: str, pull_config_for_schain: str | None = None, force_skaled_start: bool = False -) -> None: - logger.info( - 'Updating fair node: %s, pull_config_for_schain: %s, force_skaled_start: %s', - env_filepath, - pull_config_for_schain, - force_skaled_start, - ) - node_mode = get_node_mode() - env = compose_node_env( - env_filepath, - inited_node=True, - sync_schains=False, - node_type=NodeType.FAIR, - node_mode=node_mode, - pull_config_for_schain=pull_config_for_schain, - ) - update_ok = update_fair_op( - node_mode=node_mode, - env_filepath=env_filepath, - env=env, - update_type=FairUpdateType.REGULAR, - force_skaled_start=force_skaled_start, - ) - alive = is_base_containers_alive(node_type=NodeType.FAIR, node_mode=node_mode) - if not update_ok or not alive: - print_node_cmd_error() - return - else: - logger.info('Fair update completed successfully') - - -@check_user -def cleanup() -> None: - node_mode = get_node_mode() - env = compose_node_env( - SKALE_DIR_ENV_FILEPATH, save=False, node_type=NodeType.FAIR, node_mode=node_mode - ) - cleanup_fair_op(node_mode=node_mode, env=env) - logger.info('Fair node was cleaned up, all containers and data removed') - cleanup_docker_configuration() - - -@check_not_inited -def init(env_filepath: str) -> None: - node_mode = NodeMode.ACTIVE - env = compose_node_env(env_filepath, node_type=NodeType.FAIR, node_mode=node_mode) - if env is None: - return - save_env_params(env_filepath) - env['SKALE_DIR'] = SKALE_DIR - - init_ok = init_fair_op(env_filepath, env) - if not init_ok: - error_exit('Init operation failed', exit_code=CLIExitCodes.OPERATION_EXECUTION_ERROR) - time.sleep(RESTORE_SLEEP_TIMEOUT) - print('Fair node is initialized') - - @check_inited @check_user def register(ip: str) -> None: @@ -193,14 +107,6 @@ def register(ip: str) -> None: error_exit(error_msg, exit_code=CLIExitCodes.BAD_API_RESPONSE) -def repair_chain(snapshot_from: str = 'any') -> None: - node_mode = get_node_mode() - env = compose_node_env( - SKALE_DIR_ENV_FILEPATH, save=False, node_type=NodeType.FAIR, node_mode=node_mode - ) - repair_fair_op(node_mode=node_mode, env=env, snapshot_from=snapshot_from) - - @check_inited @check_user def change_ip(ip: str) -> None: @@ -236,3 +142,21 @@ def exit() -> None: error_msg = payload logger.error(f'Node exit error {error_msg}') error_exit(error_msg, exit_code=CLIExitCodes.BAD_API_RESPONSE) + + +@check_not_inited +def restore(backup_path, env_filepath, config_only=False): + node_mode = NodeMode.ACTIVE + env = compose_node_env(env_filepath, node_type=NodeType.FAIR, node_mode=node_mode) + if env is None: + return + save_env_params(env_filepath) + env['SKALE_DIR'] = SKALE_DIR + + restored_ok = restore_fair_op( + node_mode=node_mode, env=env, backup_path=backup_path, config_only=config_only + ) + if not restored_ok: + error_exit('Restore operation failed', exit_code=CLIExitCodes.OPERATION_EXECUTION_ERROR) + time.sleep(RESTORE_SLEEP_TIMEOUT) + print('Fair node is restored from backup') diff --git a/node_cli/fair/fair_boot.py b/node_cli/fair/boot.py similarity index 100% rename from node_cli/fair/fair_boot.py rename to node_cli/fair/boot.py diff --git a/node_cli/fair/common.py b/node_cli/fair/common.py new file mode 100644 index 00000000..9e0163f6 --- /dev/null +++ b/node_cli/fair/common.py @@ -0,0 +1,113 @@ +# -*- coding: utf-8 -*- +# +# This file is part of node-cli +# +# Copyright (C) 2025-Present SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import time +import logging + +from node_cli.configs import RESTORE_SLEEP_TIMEOUT, SKALE_DIR +from node_cli.configs.user import SKALE_DIR_ENV_FILEPATH +from node_cli.core.docker_config import cleanup_docker_configuration +from node_cli.core.node import compose_node_env, is_base_containers_alive +from node_cli.core.node_options import get_node_mode +from node_cli.operations import ( + FairUpdateType, + cleanup_fair_op, + repair_fair_op, + update_fair_op, + init_fair_op, +) +from node_cli.core.host import save_env_params +from node_cli.utils.decorators import check_inited, check_not_inited, check_user +from node_cli.utils.node_type import NodeMode, NodeType +from node_cli.utils.print_formatters import print_node_cmd_error +from node_cli.utils.texts import safe_load_texts +from node_cli.utils.exit_codes import CLIExitCodes +from node_cli.utils.helper import error_exit + +logger = logging.getLogger(__name__) +TEXTS = safe_load_texts() + + +@check_not_inited +def init(node_mode: NodeMode, env_filepath: str) -> None: + env = compose_node_env(env_filepath, node_type=NodeType.FAIR, node_mode=node_mode) + if env is None: + return + save_env_params(env_filepath) + env['SKALE_DIR'] = SKALE_DIR + + init_ok = init_fair_op(node_mode, env_filepath, env) + if not init_ok: + error_exit('Init operation failed', exit_code=CLIExitCodes.OPERATION_EXECUTION_ERROR) + time.sleep(RESTORE_SLEEP_TIMEOUT) + print('Fair node is initialized') + + +@check_user +def cleanup() -> None: + node_mode = get_node_mode() + env = compose_node_env( + SKALE_DIR_ENV_FILEPATH, save=False, node_type=NodeType.FAIR, node_mode=node_mode + ) + cleanup_fair_op(node_mode=node_mode, env=env) + logger.info('Fair node was cleaned up, all containers and data removed') + cleanup_docker_configuration() + + +@check_inited +@check_user +def update( + env_filepath: str, pull_config_for_schain: str | None = None, force_skaled_start: bool = False +) -> None: + logger.info( + 'Updating fair node: %s, pull_config_for_schain: %s, force_skaled_start: %s', + env_filepath, + pull_config_for_schain, + force_skaled_start, + ) + node_mode = get_node_mode() + env = compose_node_env( + env_filepath, + inited_node=True, + sync_schains=False, + node_type=NodeType.FAIR, + node_mode=node_mode, + pull_config_for_schain=pull_config_for_schain, + ) + update_ok = update_fair_op( + node_mode=node_mode, + env_filepath=env_filepath, + env=env, + update_type=FairUpdateType.REGULAR, + force_skaled_start=force_skaled_start, + ) + alive = is_base_containers_alive(node_type=NodeType.FAIR, node_mode=node_mode) + if not update_ok or not alive: + print_node_cmd_error() + return + else: + logger.info('Fair update completed successfully') + + +def repair_chain(snapshot_from: str = 'any') -> None: + node_mode = get_node_mode() + env = compose_node_env( + SKALE_DIR_ENV_FILEPATH, save=False, node_type=NodeType.FAIR, node_mode=node_mode + ) + repair_fair_op(node_mode=node_mode, env=env, snapshot_from=snapshot_from) diff --git a/node_cli/fair/passive.py b/node_cli/fair/passive.py new file mode 100644 index 00000000..be047233 --- /dev/null +++ b/node_cli/fair/passive.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# +# This file is part of node-cli +# +# Copyright (C) 2025-Present SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import logging + +from node_cli.utils.texts import safe_load_texts + +logger = logging.getLogger(__name__) +TEXTS = safe_load_texts() diff --git a/node_cli/main.py b/node_cli/main.py index e949f682..a2d3aba9 100644 --- a/node_cli/main.py +++ b/node_cli/main.py @@ -40,6 +40,7 @@ from node_cli.cli.passive_node import passive_node_cli from node_cli.cli.fair_boot import fair_boot_cli from node_cli.cli.fair_node import fair_node_cli +from node_cli.cli.passive_fair_node import passive_fair_node_cli from node_cli.cli.chain import chain_cli from node_cli.core.host import init_logs_dir from node_cli.utils.node_type import NodeType @@ -90,6 +91,7 @@ def get_sources_list() -> List[click.MultiCommand]: logs_cli, fair_boot_cli, fair_node_cli, + passive_fair_node_cli, chain_cli, wallet_cli, ssl_cli, diff --git a/node_cli/operations/fair.py b/node_cli/operations/fair.py index d9898a10..a35854a3 100644 --- a/node_cli/operations/fair.py +++ b/node_cli/operations/fair.py @@ -78,7 +78,7 @@ class FairUpdateType(Enum): @checked_host -def init(env_filepath: str, env: dict) -> bool: +def init(node_mode: NodeMode, env_filepath: str, env: dict) -> bool: sync_skale_node() ensure_btrfs_kernel_module_autoloaded() cleanup_volume_artifacts(env['DISK_MOUNTPOINT']) @@ -103,8 +103,8 @@ def init(env_filepath: str, env: dict) -> bool: distro.id(), distro.version(), ) - update_images(env=env, node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE) - compose_up(env=env, node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE) + update_images(env=env, node_type=NodeType.FAIR, node_mode=node_mode) + compose_up(env=env, node_type=NodeType.FAIR, node_mode=node_mode) wait_for_container(REDIS_SERVICE_DICT['redis']) time.sleep(REDIS_START_TIMEOUT) return True diff --git a/tests/fair/fair_node_test.py b/tests/fair/fair_node_test.py index 19e8fe4c..a4f1423c 100644 --- a/tests/fair/fair_node_test.py +++ b/tests/fair/fair_node_test.py @@ -4,9 +4,9 @@ from node_cli.configs import SKALE_DIR from node_cli.configs.user import SKALE_DIR_ENV_FILEPATH -from node_cli.fair.fair_boot import init as init_boot -from node_cli.fair.fair_boot import update -from node_cli.fair.fair_node import cleanup, migrate_from_boot, restore_fair +from node_cli.fair.boot import init as init_boot +from node_cli.fair.boot import update +from node_cli.fair import cleanup, migrate_from_boot, restore from node_cli.operations.fair import FairUpdateType from node_cli.utils.node_type import NodeType @@ -28,7 +28,7 @@ def test_restore_fair( mock_restore_op.return_value = True backup_path = '/fake/backup' - restore_fair(backup_path, valid_env_file) + restore(backup_path, valid_env_file) mock_compose_env.assert_called_once_with(valid_env_file, node_type=NodeType.FAIR) mock_save_env.assert_called_once_with(valid_env_file) @@ -221,7 +221,7 @@ def test_cleanup_fails_when_user_invalid( """Test that cleanup fails when user validation fails""" import pytest - from node_cli.fair.fair_node import cleanup + from node_cli.fair import cleanup with pytest.raises(SystemExit): cleanup() @@ -270,7 +270,7 @@ def test_exit_success( resource_alloc, meta_file_v3, ): - from node_cli.fair.fair_node import exit + from node_cli.fair import exit mock_post_request.return_value = ('ok', {}) @@ -292,7 +292,7 @@ def test_exit_error( resource_alloc, meta_file_v3, ): - from node_cli.fair.fair_node import exit + from node_cli.fair import exit error_msg = 'Exit failed' mock_post_request.return_value = ('error', error_msg) @@ -313,7 +313,7 @@ def test_exit_not_inited( meta_file_v3, capsys, ): - from node_cli.fair.fair_node import exit + from node_cli.fair import exit exit() From a82671ebd5c2bd0b2eefacfd8ba20799fc04caec Mon Sep 17 00:00:00 2001 From: Dmytro Date: Sun, 10 Aug 2025 16:48:06 +0100 Subject: [PATCH 047/198] Update imports structure --- .github/workflows/test.yml | 10 ---------- node_cli/cli/fair_node.py | 18 +++++++++--------- node_cli/cli/passive_fair_node.py | 6 +++--- node_cli/fair/__init__.py | 16 ---------------- 4 files changed, 12 insertions(+), 38 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ed804b5c..42173eeb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -47,16 +47,6 @@ jobs: - name: Check build - skale run: sudo /home/ubuntu/dist/skale-test-Linux-x86_64 - - name: Build binary - passive - run: | - mkdir -p ./dist - docker build . -t node-cli-builder - docker run -v /home/ubuntu/dist:/app/dist node-cli-builder bash scripts/build.sh test test passive - docker rm -f $(docker ps -aq) - - - name: Check build - passive - run: sudo /home/ubuntu/dist/skale-test-Linux-x86_64-passive - - name: Build binary - fair run: | mkdir -p ./dist diff --git a/node_cli/cli/fair_node.py b/node_cli/cli/fair_node.py index dad84db2..6013b44e 100644 --- a/node_cli/cli/fair_node.py +++ b/node_cli/cli/fair_node.py @@ -21,19 +21,19 @@ from node_cli.core.node import backup -from node_cli.fair import change_ip as change_ip_fair -from node_cli.fair import cleanup as fair_cleanup -from node_cli.fair import exit as exit_fair -from node_cli.fair import ( +from node_cli.fair.active import change_ip as change_ip_fair +from node_cli.fair.common import cleanup as fair_cleanup +from node_cli.fair.active import exit as exit_fair +from node_cli.fair.active import ( get_node_info, migrate_from_boot, - repair_chain, restore as restore_fair, ) -from node_cli.fair import init as init_fair -from node_cli.fair import register as register_fair -from node_cli.fair import update as update_fair -from node_cli.fair import set_domain_name as set_domain_name_fair +from node_cli.fair.common import init as init_fair +from node_cli.fair.active import register as register_fair +from node_cli.fair.common import update as update_fair +from node_cli.fair.active import set_domain_name as set_domain_name_fair +from node_cli.fair.common import repair_chain from node_cli.utils.helper import IP_TYPE, URL_OR_ANY_TYPE, abort_if_false, streamed_cmd from node_cli.utils.node_type import NodeMode diff --git a/node_cli/cli/passive_fair_node.py b/node_cli/cli/passive_fair_node.py index b0fb6c08..6c63bff7 100644 --- a/node_cli/cli/passive_fair_node.py +++ b/node_cli/cli/passive_fair_node.py @@ -19,9 +19,9 @@ import click -from node_cli.fair import init as init_fair -from node_cli.fair import update as update_fair -from node_cli.fair import cleanup as cleanup_fair +from node_cli.fair.common import init as init_fair +from node_cli.fair.common import update as update_fair +from node_cli.fair.common import cleanup as cleanup_fair from node_cli.utils.helper import abort_if_false, streamed_cmd from node_cli.utils.node_type import NodeMode from node_cli.utils.texts import safe_load_texts diff --git a/node_cli/fair/__init__.py b/node_cli/fair/__init__.py index eb92c927..e69de29b 100644 --- a/node_cli/fair/__init__.py +++ b/node_cli/fair/__init__.py @@ -1,16 +0,0 @@ -from node_cli.fair.common import ( - init as init, - update as update, - cleanup as cleanup, - repair_chain as repair_chain, -) -from node_cli.fair.active import ( - get_node_info_plain as get_node_info_plain, - get_node_info as get_node_info, - migrate_from_boot as migrate_from_boot, - register as register, - change_ip as change_ip, - set_domain_name as set_domain_name, - exit as exit, - restore as restore, -) From d3b9387725c7f4709cdb96af17bd2f8c555acd2b Mon Sep 17 00:00:00 2001 From: Dmytro Date: Mon, 11 Aug 2025 12:23:04 +0100 Subject: [PATCH 048/198] Update node-cli tests --- node_cli/configs/user.py | 24 +++--- tests/cli/node_test.py | 12 +-- tests/cli/passive_node_test.py | 2 +- tests/core/core_node_test.py | 130 ++++++++++++++++++++++----------- tests/core/nginx_test.py | 27 +++---- tests/fair/fair_node_test.py | 79 ++++++++++---------- 6 files changed, 163 insertions(+), 111 deletions(-) diff --git a/node_cli/configs/user.py b/node_cli/configs/user.py index a4c70d5b..b094150d 100644 --- a/node_cli/configs/user.py +++ b/node_cli/configs/user.py @@ -29,6 +29,12 @@ from node_cli.configs.alias_address_validation import ContractType, validate_alias_or_address from node_cli.utils.helper import error_exit from node_cli.utils.node_type import NodeMode, NodeType +from node_cli.core.node_options import ( + active_fair, + active_skale, + passive_skale, + passive_fair, +) SKALE_DIR_ENV_FILEPATH = os.path.join(SKALE_DIR, '.env') CONFIGS_ENV_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, '.env') @@ -188,16 +194,14 @@ def get_user_config_class( ) -> type[BaseUserConfig]: if node_type == NodeType.FAIR and is_fair_boot: user_config_class = FairBootUserConfig - elif node_type == NodeType.FAIR: - if node_mode == NodeMode.PASSIVE: - user_config_class = PassiveFairUserConfig - else: - user_config_class = FairUserConfig - elif node_type == NodeType.SKALE: - if node_mode == NodeMode.PASSIVE: - user_config_class = PassiveSkaleUserConfig - else: - user_config_class = SkaleUserConfig + elif passive_fair(node_type, node_mode): + user_config_class = PassiveFairUserConfig + elif active_fair(node_type, node_mode): + user_config_class = FairUserConfig + elif passive_skale(node_type, node_mode): + user_config_class = PassiveSkaleUserConfig + elif active_skale(node_type, node_mode): + user_config_class = SkaleUserConfig return user_config_class diff --git a/tests/cli/node_test.py b/tests/cli/node_test.py index c20b390c..13d3d0c5 100644 --- a/tests/cli/node_test.py +++ b/tests/cli/node_test.py @@ -42,7 +42,7 @@ from node_cli.utils.exit_codes import CLIExitCodes from node_cli.utils.helper import init_default_logger from node_cli.utils.meta import CliMeta -from node_cli.utils.node_type import NodeType +from node_cli.utils.node_type import NodeType, NodeMode from tests.helper import ( response_mock, run_command, @@ -322,14 +322,14 @@ def test_backup(): @pytest.mark.parametrize( - 'node_type,test_user_conf', + 'node_type,node_mode,test_user_conf', [ - (NodeType.SKALE, 'regular_user_conf'), - (NodeType.FAIR, 'fair_user_conf'), - (NodeType.PASSIVE, 'passive_user_conf'), + (NodeType.SKALE, NodeMode.ACTIVE, 'regular_user_conf'), + (NodeType.SKALE, NodeMode.PASSIVE, 'passive_user_conf'), + (NodeType.FAIR, NodeMode.ACTIVE, 'fair_user_conf'), ], ) -def test_restore(request, node_type, test_user_conf, mocked_g_config, tmp_path): +def test_restore(request, node_type, node_mode, test_user_conf, mocked_g_config, tmp_path): pathlib.Path(SKALE_DIR).mkdir(parents=True, exist_ok=True) result = run_command(backup_node, [tmp_path]) backup_path = result.output.replace('Backup archive successfully created: ', '').replace( diff --git a/tests/cli/passive_node_test.py b/tests/cli/passive_node_test.py index 2b5a587d..94cde559 100644 --- a/tests/cli/passive_node_test.py +++ b/tests/cli/passive_node_test.py @@ -78,7 +78,7 @@ def test_init_passive_archive(mocked_g_config, clean_node_options, passive_user_ mock.patch('node_cli.operations.base.configure_nftables'), mock.patch('node_cli.utils.decorators.is_node_inited', return_value=False), mock.patch('node_cli.configs.user.validate_alias_or_address'), - mock.patch('node_cli.cli.node.TYPE', NodeType.PASSIVE), + mock.patch('node_cli.cli.node.TYPE', NodeType.SKALE), ): result = run_command(_init_passive, [passive_user_conf.as_posix(), '--archive']) node_options = NodeOptions() diff --git a/tests/core/core_node_test.py b/tests/core/core_node_test.py index ba1ab95f..b35f7b64 100644 --- a/tests/core/core_node_test.py +++ b/tests/core/core_node_test.py @@ -5,6 +5,7 @@ from pathlib import Path import docker +from docker import errors as docker_errors import mock import pytest import requests @@ -21,7 +22,7 @@ update, ) from node_cli.utils.meta import CliMeta -from node_cli.utils.node_type import NodeType +from node_cli.utils.node_type import NodeType, NodeMode from tests.helper import response_mock, safe_update_api_response, subprocess_run_mock from tests.resources_test import BIG_DISK_SIZE @@ -37,43 +38,46 @@ 'passive_WRONG_CONTAINER_8', ] -NODE_TYPE_BOOT_COMBINATIONS: list[tuple[NodeType, bool]] = [ - (NodeType.SKALE, False), - (NodeType.PASSIVE, False), - (NodeType.FAIR, True), - (NodeType.FAIR, False), +NODE_TYPE_MODE_BOOT_COMBINATIONS: list[tuple[NodeType, NodeMode, bool]] = [ + (NodeType.SKALE, NodeMode.ACTIVE, False), + (NodeType.SKALE, NodeMode.PASSIVE, False), + (NodeType.FAIR, NodeMode.ACTIVE, True), + (NodeType.FAIR, NodeMode.ACTIVE, False), ] alive_test_params = [ pytest.param( node_type, + node_mode, is_boot, - get_expected_container_names(node_type, is_boot), - id=f'{node_type.name}-boot_{is_boot}-correct_containers', + get_expected_container_names(node_type, node_mode, is_boot), + id=f'{node_type.name}-{node_mode.name}-boot_{is_boot}-correct_containers', ) - for node_type, is_boot in NODE_TYPE_BOOT_COMBINATIONS + for node_type, node_mode, is_boot in NODE_TYPE_MODE_BOOT_COMBINATIONS ] wrong_test_params = [ pytest.param( node_type, + node_mode, is_boot, WRONG_CONTAINERS, - id=f'{node_type.name}-boot_{is_boot}-wrong_containers', + id=f'{node_type.name}-{node_mode.name}-boot_{is_boot}-wrong_containers', ) - for node_type, is_boot in NODE_TYPE_BOOT_COMBINATIONS + for node_type, node_mode, is_boot in NODE_TYPE_MODE_BOOT_COMBINATIONS ] missing_test_params = [] -for node_type, is_boot in NODE_TYPE_BOOT_COMBINATIONS: - expected_names = get_expected_container_names(node_type, is_boot) +for node_type, node_mode, is_boot in NODE_TYPE_MODE_BOOT_COMBINATIONS: + expected_names = get_expected_container_names(node_type, node_mode, is_boot) containers_to_create = expected_names[1:] missing_test_params.append( pytest.param( node_type, + node_mode, is_boot, containers_to_create, - id=f'{node_type.name}-boot_{is_boot}-missing_containers', + id=f'{node_type.name}-{node_mode.name}-boot_{is_boot}-missing_containers', ) ) @@ -87,7 +91,7 @@ def manage_node_containers(request): try: existing_container = dclient.containers.get(name) existing_container.remove(force=True) - except docker.errors.NotFound: + except docker_errors.NotFound: pass container = dclient.containers.run( ALPINE_IMAGE_NAME, @@ -110,50 +114,63 @@ def manage_node_containers(request): try: container_obj.remove(force=True) cleaned_count += 1 - except docker.errors.NotFound: + except docker_errors.NotFound: pass @pytest.mark.parametrize( - 'node_type, is_boot, manage_node_containers', + 'node_type, node_mode, is_boot, manage_node_containers', alive_test_params, indirect=['manage_node_containers'], ) -def test_is_base_containers_alive(manage_node_containers, node_type, is_boot): - assert is_base_containers_alive(node_type=node_type, is_fair_boot=is_boot) is True +def test_is_base_containers_alive(manage_node_containers, node_type, node_mode, is_boot): + assert ( + is_base_containers_alive(node_type=node_type, node_mode=node_mode, is_fair_boot=is_boot) + is True + ) @pytest.mark.parametrize( - 'node_type, is_boot, manage_node_containers', + 'node_type, node_mode, is_boot, manage_node_containers', wrong_test_params, indirect=['manage_node_containers'], ) -def test_is_base_containers_alive_wrong(manage_node_containers, node_type, is_boot): - assert is_base_containers_alive(node_type=node_type, is_fair_boot=is_boot) is False +def test_is_base_containers_alive_wrong(manage_node_containers, node_type, node_mode, is_boot): + assert ( + is_base_containers_alive(node_type=node_type, node_mode=node_mode, is_fair_boot=is_boot) + is False + ) @pytest.mark.parametrize( - 'node_type, is_boot, manage_node_containers', + 'node_type, node_mode, is_boot, manage_node_containers', missing_test_params, indirect=['manage_node_containers'], ) -def test_is_base_containers_alive_missing(manage_node_containers, node_type, is_boot): - assert is_base_containers_alive(node_type=node_type, is_fair_boot=is_boot) is False +def test_is_base_containers_alive_missing(manage_node_containers, node_type, node_mode, is_boot): + assert ( + is_base_containers_alive(node_type=node_type, node_mode=node_mode, is_fair_boot=is_boot) + is False + ) -@pytest.mark.parametrize('node_type, is_boot', NODE_TYPE_BOOT_COMBINATIONS) -def test_is_base_containers_alive_empty(node_type, is_boot): - assert is_base_containers_alive(node_type=node_type, is_fair_boot=is_boot) is False +@pytest.mark.parametrize('node_type, node_mode, is_boot', NODE_TYPE_MODE_BOOT_COMBINATIONS) +def test_is_base_containers_alive_empty(node_type, node_mode, is_boot): + assert ( + is_base_containers_alive(node_type=node_type, node_mode=node_mode, is_fair_boot=is_boot) + is False + ) @pytest.mark.parametrize( ( - 'node_type, test_user_conf, is_boot, inited_node, sync_schains, expected_mnt_dir,' + 'node_type, node_mode, test_user_conf, is_boot, inited_node, sync_schains, expected_mnt_dir,' 'expect_flask_key, expect_backup_run' ), [ ( NodeType.SKALE, + NodeMode.ACTIVE, 'regular_user_conf', False, True, @@ -164,6 +181,7 @@ def test_is_base_containers_alive_empty(node_type, is_boot): ), ( NodeType.SKALE, + NodeMode.ACTIVE, 'regular_user_conf', False, True, @@ -173,7 +191,8 @@ def test_is_base_containers_alive_empty(node_type, is_boot): True, ), ( - NodeType.PASSIVE, + NodeType.SKALE, + NodeMode.PASSIVE, 'passive_user_conf', False, False, @@ -184,6 +203,7 @@ def test_is_base_containers_alive_empty(node_type, is_boot): ), ( NodeType.FAIR, + NodeMode.ACTIVE, 'fair_boot_user_conf', True, True, @@ -194,6 +214,7 @@ def test_is_base_containers_alive_empty(node_type, is_boot): ), ( NodeType.FAIR, + NodeMode.ACTIVE, 'fair_user_conf', False, True, @@ -214,6 +235,7 @@ def test_is_base_containers_alive_empty(node_type, is_boot): def test_compose_node_env( request, node_type, + node_mode, test_user_conf, is_boot, inited_node, @@ -234,6 +256,7 @@ def test_compose_node_env( inited_node=inited_node, sync_schains=sync_schains, node_type=node_type, + node_mode=node_mode, is_fair_boot=is_boot, save=True, ) @@ -244,7 +267,7 @@ def test_compose_node_env( ) == expect_flask_key if expect_flask_key: assert result_env['FLASK_SECRET_KEY'] == 'mock_secret' - should_have_backup = sync_schains and node_type != NodeType.PASSIVE + should_have_backup = sync_schains and node_mode != NodeMode.PASSIVE assert ('BACKUP_RUN' in result_env and result_env['BACKUP_RUN'] == 'True') == should_have_backup @@ -358,14 +381,21 @@ def test_update_node(regular_user_conf, mocked_g_config, resource_file, inited_n assert result is None -@pytest.mark.parametrize('node_type', [NodeType.SKALE, NodeType.PASSIVE, NodeType.FAIR]) +@pytest.mark.parametrize( + 'node_type,node_mode', + [ + (NodeType.SKALE, NodeMode.ACTIVE), + (NodeType.SKALE, NodeMode.PASSIVE), + (NodeType.FAIR, NodeMode.ACTIVE), + ], +) @mock.patch('node_cli.core.node.is_admin_running', return_value=False) @mock.patch('node_cli.core.node.is_api_running', return_value=False) @mock.patch('node_cli.utils.helper.requests.get') def test_is_update_safe_when_admin_and_api_not_running( - mock_requests_get, mock_is_api_running, mock_is_admin_running, node_type + mock_requests_get, mock_is_api_running, mock_is_admin_running, node_type, node_mode ): - assert is_update_safe(node_type=node_type) is True + assert is_update_safe(node_type=node_type, node_mode=node_mode) is True mock_requests_get.assert_not_called() @@ -375,11 +405,18 @@ def test_is_update_safe_when_admin_and_api_not_running( def test_is_update_safe_when_admin_not_running_for_passive( mock_requests_get, mock_is_api_running, mock_is_admin_running ): - assert is_update_safe(node_type=NodeType.PASSIVE) is True + assert is_update_safe(node_type=NodeType.SKALE, node_mode=NodeMode.PASSIVE) is True mock_requests_get.assert_not_called() -@pytest.mark.parametrize('node_type', [NodeType.SKALE, NodeType.PASSIVE, NodeType.FAIR]) +@pytest.mark.parametrize( + 'node_type,node_mode', + [ + (NodeType.SKALE, NodeMode.ACTIVE), + (NodeType.SKALE, NodeMode.PASSIVE), + (NodeType.FAIR, NodeMode.ACTIVE), + ], +) @pytest.mark.parametrize( 'api_is_safe, expected_result', [(True, True), (False, False)], @@ -388,10 +425,10 @@ def test_is_update_safe_when_admin_not_running_for_passive( @mock.patch('node_cli.core.node.is_admin_running', return_value=True) @mock.patch('node_cli.utils.helper.requests.get') def test_is_update_safe_when_admin_running( - mock_requests_get, mock_is_admin_running, api_is_safe, expected_result, node_type + mock_requests_get, mock_is_admin_running, api_is_safe, expected_result, node_type, node_mode ): mock_requests_get.return_value = safe_update_api_response(safe=api_is_safe) - assert is_update_safe(node_type=node_type) is expected_result + assert is_update_safe(node_type=node_type, node_mode=node_mode) is expected_result mock_requests_get.assert_called_once() @@ -413,14 +450,23 @@ def test_is_update_safe_when_only_api_running_for_regular( node_type, ): mock_requests_get.return_value = safe_update_api_response(safe=api_is_safe) - assert is_update_safe(node_type=node_type) is expected_result + assert is_update_safe(node_type=node_type, node_mode=NodeMode.ACTIVE) is expected_result mock_requests_get.assert_called_once() -@pytest.mark.parametrize('node_type', [NodeType.SKALE, NodeType.PASSIVE, NodeType.FAIR]) +@pytest.mark.parametrize( + 'node_type,node_mode', + [ + (NodeType.SKALE, NodeMode.ACTIVE), + (NodeType.SKALE, NodeMode.PASSIVE), + (NodeType.FAIR, NodeMode.ACTIVE), + ], +) @mock.patch('node_cli.core.node.is_admin_running', return_value=True) @mock.patch('node_cli.utils.helper.requests.get') -def test_is_update_safe_when_api_call_fails(mock_requests_get, mock_is_admin_running, node_type): +def test_is_update_safe_when_api_call_fails( + mock_requests_get, mock_is_admin_running, node_type, node_mode +): mock_requests_get.side_effect = requests.exceptions.ConnectionError('Test connection error') - assert is_update_safe(node_type=node_type) is False + assert is_update_safe(node_type=node_type, node_mode=node_mode) is False mock_requests_get.assert_called_once() diff --git a/tests/core/nginx_test.py b/tests/core/nginx_test.py index c24645dc..a867b0b0 100644 --- a/tests/core/nginx_test.py +++ b/tests/core/nginx_test.py @@ -11,7 +11,7 @@ SSL_KEY_NAME, SSL_CRT_NAME, ) -from node_cli.utils.node_type import NodeType +from node_cli.utils.node_type import NodeType, NodeMode from node_cli.configs import NGINX_TEMPLATE_FILEPATH, NGINX_CONFIG_FILEPATH, NODE_CERTS_PATH TEST_NGINX_TEMPLATE = """ @@ -57,14 +57,14 @@ def nginx_template(): @pytest.mark.parametrize( - 'node_type, ssl_exists, expected_regular_flag, expected_ssl_flag', + 'node_type, node_mode, ssl_exists, expected_regular_flag, expected_ssl_flag', [ - (NodeType.SKALE, True, True, True), - (NodeType.SKALE, False, True, False), - (NodeType.PASSIVE, True, True, True), - (NodeType.PASSIVE, False, True, False), - (NodeType.FAIR, True, False, True), - (NodeType.FAIR, False, False, False), + (NodeType.SKALE, NodeMode.ACTIVE, True, True, True), + (NodeType.SKALE, NodeMode.ACTIVE, False, True, False), + (NodeType.SKALE, NodeMode.PASSIVE, True, True, True), + (NodeType.SKALE, NodeMode.PASSIVE, False, True, False), + (NodeType.FAIR, NodeMode.ACTIVE, True, False, True), + (NodeType.FAIR, NodeMode.ACTIVE, False, False, False), ], ids=[ 'regular_ssl_on', @@ -81,6 +81,7 @@ def test_generate_nginx_config( mock_type, mock_check_ssl, node_type, + node_mode, ssl_exists, expected_regular_flag, expected_ssl_flag, @@ -130,15 +131,15 @@ def test_check_ssl_certs_missing_both(ssl_folder): @pytest.mark.parametrize( - 'node_type, expected_result', + 'node_type, node_mode, expected_result', [ - (NodeType.SKALE, True), - (NodeType.PASSIVE, True), - (NodeType.FAIR, False), + (NodeType.SKALE, NodeMode.ACTIVE, True), + (NodeType.SKALE, NodeMode.PASSIVE, True), + (NodeType.FAIR, NodeMode.ACTIVE, False), ], ) @mock.patch('node_cli.core.nginx.TYPE') -def test_is_skale_node_nginx(mock_type, node_type, expected_result): +def test_is_skale_node_nginx(mock_type, node_type, node_mode, expected_result): mock_type.__eq__.side_effect = lambda other: node_type == other mock_type.__ne__.side_effect = lambda other: node_type != other diff --git a/tests/fair/fair_node_test.py b/tests/fair/fair_node_test.py index a4f1423c..897a642c 100644 --- a/tests/fair/fair_node_test.py +++ b/tests/fair/fair_node_test.py @@ -6,15 +6,16 @@ from node_cli.configs.user import SKALE_DIR_ENV_FILEPATH from node_cli.fair.boot import init as init_boot from node_cli.fair.boot import update -from node_cli.fair import cleanup, migrate_from_boot, restore +from node_cli.fair.common import cleanup +from node_cli.fair.active import migrate_from_boot, restore from node_cli.operations.fair import FairUpdateType from node_cli.utils.node_type import NodeType -@mock.patch('node_cli.fair.fair_node.time.sleep') -@mock.patch('node_cli.fair.fair_node.restore_fair_op') -@mock.patch('node_cli.fair.fair_node.save_env_params') -@mock.patch('node_cli.fair.fair_node.compose_node_env') +@mock.patch('node_cli.fair.active.time.sleep') +@mock.patch('node_cli.fair.active.restore_fair_op') +@mock.patch('node_cli.fair.active.save_env_params') +@mock.patch('node_cli.fair.active.compose_node_env') def test_restore_fair( mock_compose_env, mock_save_env, @@ -37,10 +38,10 @@ def test_restore_fair( mock_sleep.assert_called_once() -@mock.patch('node_cli.fair.fair_boot.is_base_containers_alive', return_value=True) -@mock.patch('node_cli.fair.fair_boot.time.sleep') -@mock.patch('node_cli.fair.fair_boot.init_fair_boot_op') -@mock.patch('node_cli.fair.fair_boot.compose_node_env') +@mock.patch('node_cli.fair.boot.is_base_containers_alive', return_value=True) +@mock.patch('node_cli.fair.boot.time.sleep') +@mock.patch('node_cli.fair.boot.init_fair_boot_op') +@mock.patch('node_cli.fair.boot.compose_node_env') def test_init_fair_boot( mock_compose_env, mock_init_op, @@ -65,10 +66,10 @@ def test_init_fair_boot( @mock.patch('node_cli.utils.decorators.is_user_valid', return_value=True) -@mock.patch('node_cli.fair.fair_boot.is_base_containers_alive', return_value=True) -@mock.patch('node_cli.fair.fair_boot.time.sleep') -@mock.patch('node_cli.fair.fair_boot.update_fair_boot_op') -@mock.patch('node_cli.fair.fair_boot.compose_node_env') +@mock.patch('node_cli.fair.boot.is_base_containers_alive', return_value=True) +@mock.patch('node_cli.fair.boot.time.sleep') +@mock.patch('node_cli.fair.boot.update_fair_boot_op') +@mock.patch('node_cli.fair.boot.compose_node_env') def test_update_fair_boot( mock_compose_env, mock_update_op, @@ -100,8 +101,8 @@ def test_update_fair_boot( mock_is_alive.assert_called_once_with(node_type=NodeType.FAIR, is_fair_boot=True) -@mock.patch('node_cli.fair.fair_node.update_fair_op') -@mock.patch('node_cli.fair.fair_node.compose_node_env') +@mock.patch('node_cli.fair.active.update_fair_op') +@mock.patch('node_cli.fair.active.compose_node_env') @mock.patch('node_cli.utils.decorators.is_user_valid', return_value=True) def test_migrate_from_boot( mock_is_user_valid, @@ -130,9 +131,9 @@ def test_migrate_from_boot( @mock.patch('node_cli.utils.decorators.is_user_valid', return_value=True) -@mock.patch('node_cli.fair.fair_node.cleanup_docker_configuration') -@mock.patch('node_cli.fair.fair_node.cleanup_fair_op') -@mock.patch('node_cli.fair.fair_node.compose_node_env') +@mock.patch('node_cli.fair.common.cleanup_docker_configuration') +@mock.patch('node_cli.fair.common.cleanup_fair_op') +@mock.patch('node_cli.fair.common.compose_node_env') def test_cleanup_success( mock_compose_env, mock_cleanup_fair_op, @@ -155,9 +156,9 @@ def test_cleanup_success( @mock.patch('node_cli.utils.decorators.is_user_valid', return_value=True) -@mock.patch('node_cli.fair.fair_node.cleanup_docker_configuration') -@mock.patch('node_cli.fair.fair_node.cleanup_fair_op') -@mock.patch('node_cli.fair.fair_node.compose_node_env') +@mock.patch('node_cli.fair.common.cleanup_docker_configuration') +@mock.patch('node_cli.fair.common.cleanup_fair_op') +@mock.patch('node_cli.fair.common.compose_node_env') def test_cleanup_calls_operations_in_correct_order( mock_compose_env, mock_cleanup_fair_op, @@ -167,7 +168,7 @@ def test_cleanup_calls_operations_in_correct_order( resource_alloc, meta_file_v3, ): - from node_cli.fair.fair_node import cleanup + from node_cli.fair.common import cleanup mock_env = {'ENV_TYPE': 'devnet'} mock_compose_env.return_value = mock_env @@ -188,9 +189,9 @@ def test_cleanup_calls_operations_in_correct_order( @mock.patch('node_cli.utils.decorators.is_user_valid', return_value=True) -@mock.patch('node_cli.fair.fair_node.cleanup_docker_configuration') -@mock.patch('node_cli.fair.fair_node.cleanup_fair_op', side_effect=Exception('Cleanup failed')) -@mock.patch('node_cli.fair.fair_node.compose_node_env') +@mock.patch('node_cli.fair.common.cleanup_docker_configuration') +@mock.patch('node_cli.fair.common.cleanup_fair_op', side_effect=Exception('Cleanup failed')) +@mock.patch('node_cli.fair.common.compose_node_env') def test_cleanup_continues_after_fair_op_error( mock_compose_env, mock_cleanup_fair_op, @@ -221,7 +222,7 @@ def test_cleanup_fails_when_user_invalid( """Test that cleanup fails when user validation fails""" import pytest - from node_cli.fair import cleanup + from node_cli.fair.common import cleanup with pytest.raises(SystemExit): cleanup() @@ -235,10 +236,10 @@ def test_cleanup_fails_when_not_inited(ensure_meta_removed): @mock.patch('node_cli.utils.decorators.is_user_valid', return_value=True) -@mock.patch('node_cli.fair.fair_node.cleanup_docker_configuration') -@mock.patch('node_cli.fair.fair_node.cleanup_fair_op') -@mock.patch('node_cli.fair.fair_node.compose_node_env') -@mock.patch('node_cli.fair.fair_node.logger') +@mock.patch('node_cli.fair.common.cleanup_docker_configuration') +@mock.patch('node_cli.fair.common.cleanup_fair_op') +@mock.patch('node_cli.fair.common.compose_node_env') +@mock.patch('node_cli.fair.common.logger') def test_cleanup_logs_success_message( mock_logger, mock_compose_env, @@ -260,8 +261,8 @@ def test_cleanup_logs_success_message( @mock.patch('node_cli.utils.decorators.is_user_valid', return_value=True) -@mock.patch('node_cli.fair.fair_node.post_request') -@mock.patch('node_cli.fair.fair_node.is_node_inited', return_value=True) +@mock.patch('node_cli.fair.active.post_request') +@mock.patch('node_cli.fair.active.is_node_inited', return_value=True) def test_exit_success( mock_is_inited, mock_post_request, @@ -270,7 +271,7 @@ def test_exit_success( resource_alloc, meta_file_v3, ): - from node_cli.fair import exit + from node_cli.fair.active import exit mock_post_request.return_value = ('ok', {}) @@ -280,9 +281,9 @@ def test_exit_success( @mock.patch('node_cli.utils.decorators.is_user_valid', return_value=True) -@mock.patch('node_cli.fair.fair_node.error_exit') -@mock.patch('node_cli.fair.fair_node.post_request') -@mock.patch('node_cli.fair.fair_node.is_node_inited', return_value=True) +@mock.patch('node_cli.fair.active.error_exit') +@mock.patch('node_cli.fair.active.post_request') +@mock.patch('node_cli.fair.active.is_node_inited', return_value=True) def test_exit_error( mock_is_inited, mock_post_request, @@ -292,7 +293,7 @@ def test_exit_error( resource_alloc, meta_file_v3, ): - from node_cli.fair import exit + from node_cli.fair.active import exit error_msg = 'Exit failed' mock_post_request.return_value = ('error', error_msg) @@ -304,7 +305,7 @@ def test_exit_error( @mock.patch('node_cli.utils.decorators.is_user_valid', return_value=True) -@mock.patch('node_cli.fair.fair_node.is_node_inited', return_value=False) +@mock.patch('node_cli.fair.active.is_node_inited', return_value=False) def test_exit_not_inited( mock_is_inited, mock_is_user_valid, @@ -313,7 +314,7 @@ def test_exit_not_inited( meta_file_v3, capsys, ): - from node_cli.fair import exit + from node_cli.fair.active import exit exit() From 64e8db7d289776a05f95a9950e3b014e5d74c763 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Mon, 11 Aug 2025 13:25:02 +0100 Subject: [PATCH 049/198] Replace get_node_mode with upsert_node_mode, update fair and skale node modules --- node_cli/cli/fair_node.py | 1 + node_cli/cli/node.py | 6 +++++- node_cli/cli/passive_fair_node.py | 1 + node_cli/core/node.py | 9 +++++---- node_cli/core/node_options.py | 19 +++++++++++++++++-- node_cli/fair/boot.py | 4 ++-- node_cli/fair/common.py | 14 +++++++++----- node_cli/operations/fair.py | 2 ++ tests/core/core_node_test.py | 4 ++-- 9 files changed, 44 insertions(+), 16 deletions(-) diff --git a/node_cli/cli/fair_node.py b/node_cli/cli/fair_node.py index 6013b44e..07533802 100644 --- a/node_cli/cli/fair_node.py +++ b/node_cli/cli/fair_node.py @@ -92,6 +92,7 @@ def register(ip: str) -> None: @streamed_cmd def update_node(env_filepath: str, pull_config_for_schain, force_skaled_start: bool): update_fair( + node_mode=NodeMode.ACTIVE, env_filepath=env_filepath, pull_config_for_schain=pull_config_for_schain, force_skaled_start=force_skaled_start, diff --git a/node_cli/cli/node.py b/node_cli/cli/node.py index 61d71af7..20e2d701 100644 --- a/node_cli/cli/node.py +++ b/node_cli/cli/node.py @@ -38,8 +38,10 @@ ) from node_cli.configs import DEFAULT_NODE_BASE_PORT from node_cli.configs.user import ALLOWED_ENV_TYPES +from node_cli.core.node_options import upsert_node_mode from node_cli.utils.decorators import check_inited from node_cli.utils.helper import abort_if_false, streamed_cmd, IP_TYPE +from node_cli.utils.node_type import NodeMode from node_cli.utils.texts import safe_load_texts from node_cli.utils.meta import CliMetaManager from node_cli.utils.print_formatters import print_meta_info @@ -102,6 +104,7 @@ def init_node(env_file): @streamed_cmd def update_node(env_file, pull_config_for_schain, unsafe_ok): update( + node_mode=NodeMode.ACTIVE, env_filepath=env_file, pull_config_for_schain=pull_config_for_schain, node_type=TYPE, @@ -228,7 +231,8 @@ def _set_domain_name(domain): help='Network to check', ) def check(network): - run_checks(node_type=TYPE, network=network) + node_mode = upsert_node_mode() + run_checks(node_type=TYPE, node_mode=node_mode, network=network) @node.command(help='Reconfigure nftables rules') diff --git a/node_cli/cli/passive_fair_node.py b/node_cli/cli/passive_fair_node.py index 6c63bff7..f1b79085 100644 --- a/node_cli/cli/passive_fair_node.py +++ b/node_cli/cli/passive_fair_node.py @@ -67,6 +67,7 @@ def init_passive_node(env_filepath: str): @streamed_cmd def update_node(env_filepath: str, pull_config_for_schain, force_skaled_start: bool): update_fair( + node_mode=NodeMode.PASSIVE, env_filepath=env_filepath, pull_config_for_schain=pull_config_for_schain, force_skaled_start=force_skaled_start, diff --git a/node_cli/core/node.py b/node_cli/core/node.py index 7577f166..79ac63c4 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -50,7 +50,7 @@ from node_cli.core.node_options import ( active_fair, active_skale, - get_node_mode, + upsert_node_mode, passive_skale, passive_fair, ) @@ -293,9 +293,10 @@ def update( env_filepath: str, pull_config_for_schain: Optional[str], node_type: NodeType, + node_mode: NodeMode, unsafe_ok: bool = False, ) -> None: - node_mode = get_node_mode() + node_mode = upsert_node_mode(node_mode=node_mode) if not unsafe_ok and not is_update_safe(node_type=node_type, node_mode=node_mode): error_msg = 'Cannot update safely' @@ -420,7 +421,7 @@ def set_maintenance_mode_off(): @check_inited @check_user def turn_off(node_type: NodeType, maintenance_on: bool = False, unsafe_ok: bool = False) -> None: - node_mode = get_node_mode() + node_mode = upsert_node_mode() if not unsafe_ok and not is_update_safe(node_type=node_type, node_mode=node_mode): error_msg = 'Cannot turn off safely' error_exit(error_msg, exit_code=CLIExitCodes.UNSAFE_UPDATE) @@ -435,7 +436,7 @@ def turn_off(node_type: NodeType, maintenance_on: bool = False, unsafe_ok: bool @check_inited @check_user def turn_on(maintenance_off, sync_schains, env_file, node_type: NodeType) -> None: - node_mode = get_node_mode() + node_mode = upsert_node_mode() env = compose_node_env( env_file, inited_node=True, diff --git a/node_cli/core/node_options.py b/node_cli/core/node_options.py index 889978fe..49b49a4e 100644 --- a/node_cli/core/node_options.py +++ b/node_cli/core/node_options.py @@ -89,9 +89,24 @@ def mark_passive_node() -> None: logger.info('Node marked as passive.') -def get_node_mode() -> NodeMode: +class NodeModeMismatchError(Exception): + pass + + +def upsert_node_mode(node_mode: NodeMode | None = None) -> NodeMode: node_options = NodeOptions() - return node_options.node_mode + try: + options_mode = node_options.node_mode + if options_mode != node_mode: + raise NodeModeMismatchError( + f'Cannot change node mode from {options_mode} to {node_mode}' + ) + return options_mode + except ValueError: + if node_mode is None: + raise NodeModeMismatchError('Node mode is not set') + node_options.node_mode = node_mode + return node_mode def active_skale(node_type: NodeType, node_mode: NodeMode) -> bool: diff --git a/node_cli/fair/boot.py b/node_cli/fair/boot.py index b8da120c..9acf9b63 100644 --- a/node_cli/fair/boot.py +++ b/node_cli/fair/boot.py @@ -23,7 +23,7 @@ from node_cli.configs import TM_INIT_TIMEOUT from node_cli.core.node import compose_node_env, is_base_containers_alive -from node_cli.core.node_options import get_node_mode +from node_cli.core.node_options import upsert_node_mode from node_cli.operations import init_fair_boot_op, update_fair_boot_op from node_cli.utils.decorators import check_inited, check_not_inited, check_user from node_cli.utils.exit_codes import CLIExitCodes @@ -57,7 +57,7 @@ def init(env_filepath: str) -> None: @check_user def update(env_filepath: str, pull_config_for_schain: str) -> None: logger.info('Fair boot node update started') - node_mode = get_node_mode() + node_mode = upsert_node_mode(node_mode=NodeMode.ACTIVE) env = compose_node_env( env_filepath, inited_node=True, diff --git a/node_cli/fair/common.py b/node_cli/fair/common.py index 9e0163f6..8af1f017 100644 --- a/node_cli/fair/common.py +++ b/node_cli/fair/common.py @@ -24,7 +24,7 @@ from node_cli.configs.user import SKALE_DIR_ENV_FILEPATH from node_cli.core.docker_config import cleanup_docker_configuration from node_cli.core.node import compose_node_env, is_base_containers_alive -from node_cli.core.node_options import get_node_mode +from node_cli.core.node_options import upsert_node_mode from node_cli.operations import ( FairUpdateType, cleanup_fair_op, @@ -61,7 +61,7 @@ def init(node_mode: NodeMode, env_filepath: str) -> None: @check_user def cleanup() -> None: - node_mode = get_node_mode() + node_mode = upsert_node_mode() env = compose_node_env( SKALE_DIR_ENV_FILEPATH, save=False, node_type=NodeType.FAIR, node_mode=node_mode ) @@ -73,7 +73,10 @@ def cleanup() -> None: @check_inited @check_user def update( - env_filepath: str, pull_config_for_schain: str | None = None, force_skaled_start: bool = False + node_mode: NodeMode, + env_filepath: str, + pull_config_for_schain: str | None = None, + force_skaled_start: bool = False, ) -> None: logger.info( 'Updating fair node: %s, pull_config_for_schain: %s, force_skaled_start: %s', @@ -81,7 +84,8 @@ def update( pull_config_for_schain, force_skaled_start, ) - node_mode = get_node_mode() + node_mode = upsert_node_mode(node_mode=node_mode) + env = compose_node_env( env_filepath, inited_node=True, @@ -106,7 +110,7 @@ def update( def repair_chain(snapshot_from: str = 'any') -> None: - node_mode = get_node_mode() + node_mode = upsert_node_mode() env = compose_node_env( SKALE_DIR_ENV_FILEPATH, save=False, node_type=NodeType.FAIR, node_mode=node_mode ) diff --git a/node_cli/operations/fair.py b/node_cli/operations/fair.py index a35854a3..20f70374 100644 --- a/node_cli/operations/fair.py +++ b/node_cli/operations/fair.py @@ -38,6 +38,7 @@ from node_cli.core.nginx import generate_nginx_config from node_cli.core.schains import cleanup_no_lvm_datadir from node_cli.core.static_config import get_fair_chain_name +from node_cli.core.node_options import upsert_node_mode from node_cli.fair.record.chain_record import ( get_fair_chain_record, migrate_chain_record, @@ -93,6 +94,7 @@ def init(node_mode: NodeMode, env_filepath: str, env: dict) -> bool: prepare_host(env_filepath, env_type=env['ENV_TYPE']) link_env_file() + upsert_node_mode(node_mode=node_mode) prepare_block_device(env['DISK_MOUNTPOINT'], force=env['ENFORCE_BTRFS'] == 'True') diff --git a/tests/core/core_node_test.py b/tests/core/core_node_test.py index b35f7b64..6535562e 100644 --- a/tests/core/core_node_test.py +++ b/tests/core/core_node_test.py @@ -164,8 +164,8 @@ def test_is_base_containers_alive_empty(node_type, node_mode, is_boot): @pytest.mark.parametrize( ( - 'node_type, node_mode, test_user_conf, is_boot, inited_node, sync_schains, expected_mnt_dir,' - 'expect_flask_key, expect_backup_run' + 'node_type, node_mode, test_user_conf, is_boot, inited_node, sync_schains,' + 'expected_mnt_dir, expect_flask_key, expect_backup_run' ), [ ( From 8426a49d44221efb7a7630a16344d5bd5a8d55dc Mon Sep 17 00:00:00 2001 From: Dmytro Date: Mon, 11 Aug 2025 13:49:35 +0100 Subject: [PATCH 050/198] Bump version, update node tests --- node_cli/cli/__init__.py | 2 +- node_cli/operations/base.py | 4 ++-- tests/core/core_node_test.py | 1 + tests/fair/fair_node_test.py | 5 +++-- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/node_cli/cli/__init__.py b/node_cli/cli/__init__.py index b0105764..e2eb22f4 100644 --- a/node_cli/cli/__init__.py +++ b/node_cli/cli/__init__.py @@ -1,4 +1,4 @@ -__version__ = '3.0.0' +__version__ = '3.1.0' if __name__ == '__main__': print(__version__) diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index 5168fab1..b9d2cc09 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -43,9 +43,9 @@ from node_cli.core.nginx import generate_nginx_config from node_cli.core.node_options import ( NodeOptions, - get_node_mode, mark_active_node, mark_passive_node, + upsert_node_mode, ) from node_cli.core.resources import init_shared_space_volume, update_resource_allocation from node_cli.core.schains import ( @@ -382,7 +382,7 @@ def turn_on(env: dict, node_type: NodeType, node_mode: NodeMode) -> None: def restore(env, backup_path, node_type: NodeType, config_only=False): - node_mode = get_node_mode() + node_mode = upsert_node_mode(node_mode=NodeMode.ACTIVE) unpack_backup_archive(backup_path) failed_checks = run_host_checks( env['DISK_MOUNTPOINT'], diff --git a/tests/core/core_node_test.py b/tests/core/core_node_test.py index 6535562e..d84ddc37 100644 --- a/tests/core/core_node_test.py +++ b/tests/core/core_node_test.py @@ -374,6 +374,7 @@ def test_update_node(regular_user_conf, mocked_g_config, resource_file, inited_n 'node_cli.utils.helper.requests.get', return_value=safe_update_api_response() ): # noqa result = update( + node_mode=NodeMode.ACTIVE, regular_user_conf.as_posix(), pull_config_for_schain=None, node_type=NodeType.SKALE, diff --git a/tests/fair/fair_node_test.py b/tests/fair/fair_node_test.py index 897a642c..685bbbf1 100644 --- a/tests/fair/fair_node_test.py +++ b/tests/fair/fair_node_test.py @@ -9,7 +9,7 @@ from node_cli.fair.common import cleanup from node_cli.fair.active import migrate_from_boot, restore from node_cli.operations.fair import FairUpdateType -from node_cli.utils.node_type import NodeType +from node_cli.utils.node_type import NodeMode, NodeType @mock.patch('node_cli.fair.active.time.sleep') @@ -34,7 +34,8 @@ def test_restore_fair( mock_compose_env.assert_called_once_with(valid_env_file, node_type=NodeType.FAIR) mock_save_env.assert_called_once_with(valid_env_file) expected_env = {**mock_env, 'SKALE_DIR': SKALE_DIR} - mock_restore_op.assert_called_once_with(expected_env, backup_path, config_only=False) + mock_restore_op.assert_called_once_with( + node_mode=NodeMode.ACTIVE, expected_env, backup_path, config_only=False) mock_sleep.assert_called_once() From 7e96e12dc906b714eb489d2d9f2e692f44093ed8 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Mon, 11 Aug 2025 16:24:16 +0100 Subject: [PATCH 051/198] Add active_node_option fixture, update tests --- tests/conftest.py | 22 ++++++++++++++++++++++ tests/core/core_node_test.py | 2 +- tests/fair/fair_node_test.py | 7 +++++-- 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index ba860af9..97e7c674 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -43,8 +43,10 @@ from node_cli.configs.node_options import NODE_OPTIONS_FILEPATH from node_cli.configs.resource_allocation import RESOURCE_ALLOCATION_FILEPATH from node_cli.configs.ssl import SSL_FOLDER_PATH +from node_cli.core.node_options import NodeOptions from node_cli.utils.docker_utils import docker_client from node_cli.utils.global_config import generate_g_config_file +from node_cli.utils.node_type import NodeMode from tests.helper import TEST_META_V1, TEST_META_V2, TEST_META_V3, TEST_SCHAINS_MNT_DIR_SINGLE_CHAIN @@ -144,6 +146,26 @@ def ssl_folder(): shutil.rmtree(SSL_FOLDER_PATH) +@pytest.fixture +def active_node_option(): + node_options = NodeOptions() + node_options.node_mode = NodeMode.ACTIVE + try: + yield + finally: + shutil.rmtree(NODE_OPTIONS_FILEPATH) + + +@pytest.fixture +def passive_node_option(): + node_options = NodeOptions() + node_options.node_mode = NodeMode.PASSIVE + try: + yield + finally: + shutil.rmtree(NODE_OPTIONS_FILEPATH) + + @pytest.fixture def dutils(): return docker_client() diff --git a/tests/core/core_node_test.py b/tests/core/core_node_test.py index d84ddc37..70671418 100644 --- a/tests/core/core_node_test.py +++ b/tests/core/core_node_test.py @@ -374,10 +374,10 @@ def test_update_node(regular_user_conf, mocked_g_config, resource_file, inited_n 'node_cli.utils.helper.requests.get', return_value=safe_update_api_response() ): # noqa result = update( - node_mode=NodeMode.ACTIVE, regular_user_conf.as_posix(), pull_config_for_schain=None, node_type=NodeType.SKALE, + node_mode=NodeMode.ACTIVE, ) assert result is None diff --git a/tests/fair/fair_node_test.py b/tests/fair/fair_node_test.py index 685bbbf1..861267ca 100644 --- a/tests/fair/fair_node_test.py +++ b/tests/fair/fair_node_test.py @@ -35,7 +35,8 @@ def test_restore_fair( mock_save_env.assert_called_once_with(valid_env_file) expected_env = {**mock_env, 'SKALE_DIR': SKALE_DIR} mock_restore_op.assert_called_once_with( - node_mode=NodeMode.ACTIVE, expected_env, backup_path, config_only=False) + NodeMode.ACTIVE, expected_env, backup_path, config_only=False + ) mock_sleep.assert_called_once() @@ -168,6 +169,7 @@ def test_cleanup_calls_operations_in_correct_order( inited_node, resource_alloc, meta_file_v3, + active_node_option, ): from node_cli.fair.common import cleanup @@ -229,7 +231,7 @@ def test_cleanup_fails_when_user_invalid( cleanup() -def test_cleanup_fails_when_not_inited(ensure_meta_removed): +def test_cleanup_fails_when_not_inited(ensure_meta_removed, active_node_option): import pytest with pytest.raises(SystemExit): @@ -250,6 +252,7 @@ def test_cleanup_logs_success_message( inited_node, resource_alloc, meta_file_v3, + active_node_option, ): mock_env = {'ENV_TYPE': 'devnet'} mock_compose_env.return_value = mock_env From 6a1bd8b1f551f96dbf313bdd6cd538813ee398d7 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Mon, 11 Aug 2025 18:36:48 +0100 Subject: [PATCH 052/198] Fix fair node commands --- node_cli/fair/active.py | 4 ++-- node_cli/fair/common.py | 6 +++--- node_cli/operations/fair.py | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/node_cli/fair/active.py b/node_cli/fair/active.py index 5ab1acfe..963686fe 100644 --- a/node_cli/fair/active.py +++ b/node_cli/fair/active.py @@ -74,9 +74,9 @@ def migrate_from_boot( node_mode=NodeMode.ACTIVE, ) migrate_ok = update_fair_op( + env_filepath, + env, node_mode=NodeMode.ACTIVE, - env_filepath=env_filepath, - env=env, update_type=FairUpdateType.FROM_BOOT, force_skaled_start=False, ) diff --git a/node_cli/fair/common.py b/node_cli/fair/common.py index 8af1f017..52524421 100644 --- a/node_cli/fair/common.py +++ b/node_cli/fair/common.py @@ -52,7 +52,7 @@ def init(node_mode: NodeMode, env_filepath: str) -> None: save_env_params(env_filepath) env['SKALE_DIR'] = SKALE_DIR - init_ok = init_fair_op(node_mode, env_filepath, env) + init_ok = init_fair_op(env_filepath, env, node_mode=node_mode) if not init_ok: error_exit('Init operation failed', exit_code=CLIExitCodes.OPERATION_EXECUTION_ERROR) time.sleep(RESTORE_SLEEP_TIMEOUT) @@ -95,9 +95,9 @@ def update( pull_config_for_schain=pull_config_for_schain, ) update_ok = update_fair_op( + env_filepath, + env, node_mode=node_mode, - env_filepath=env_filepath, - env=env, update_type=FairUpdateType.REGULAR, force_skaled_start=force_skaled_start, ) diff --git a/node_cli/operations/fair.py b/node_cli/operations/fair.py index 20f70374..d3a04e20 100644 --- a/node_cli/operations/fair.py +++ b/node_cli/operations/fair.py @@ -79,7 +79,7 @@ class FairUpdateType(Enum): @checked_host -def init(node_mode: NodeMode, env_filepath: str, env: dict) -> bool: +def init(env_filepath: str, env: dict, node_mode: NodeMode) -> bool: sync_skale_node() ensure_btrfs_kernel_module_autoloaded() cleanup_volume_artifacts(env['DISK_MOUNTPOINT']) @@ -156,9 +156,9 @@ def update_fair_boot(env_filepath: str, env: dict) -> bool: @checked_host def update( - node_mode: NodeMode, env_filepath: str, env: dict, + node_mode: NodeMode, update_type: FairUpdateType, force_skaled_start: bool, ) -> bool: From 1bb430eb22bc12fac23681796b5462a14562033e Mon Sep 17 00:00:00 2001 From: Dmytro Date: Mon, 11 Aug 2025 21:42:06 +0100 Subject: [PATCH 053/198] Remove resource allocation cli, add passive node setup command --- README.md | 32 --------- node_cli/cli/passive_node.py | 14 ++-- node_cli/cli/resources_allocation.py | 64 ----------------- node_cli/configs/user.py | 4 +- node_cli/core/node_config.py | 35 ---------- node_cli/core/node_options.py | 4 +- node_cli/core/resources.py | 7 +- node_cli/fair/passive.py | 21 ++++++ node_cli/main.py | 2 - tests/cli/resources_allocation_test.py | 96 -------------------------- tests/conftest.py | 9 +++ text.yml | 5 +- 12 files changed, 52 insertions(+), 241 deletions(-) delete mode 100644 node_cli/cli/resources_allocation.py delete mode 100644 node_cli/core/node_config.py delete mode 100644 tests/cli/resources_allocation_test.py diff --git a/README.md b/README.md index 3b8b1b88..7484ad2b 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,6 @@ SKALE Node CLI, part of the SKALE suite of validator tools, is the command line 5. [Health commands (Standard)](#health-commands-standard) 6. [SSL commands (Standard)](#ssl-commands-standard) 7. [Logs commands (Standard)](#logs-commands-standard) - 8. [Resources allocation commands (Standard)](#resources-allocation-commands-standard) 3. [Passive Node Usage (`skale` - Passive Build)](#passive-node-usage-skale---passive-build) 1. [Top level commands (Passive)](#top-level-commands-passive) 2. [Passive node commands](#passive-node-commands) @@ -519,37 +518,6 @@ Options: * `--container`, `-c` - Dump logs only from specified container. -### Resources allocation commands (Standard) - -> Prefix: `skale resources-allocation` - -Manage the resources allocation file for the standard node. - -#### Show allocation file - -Show resources allocation file: - -```shell -skale resources-allocation show -``` - -#### Generate/update allocation file - -Generate/update allocation file: - -```shell -skale resources-allocation generate [ENV_FILE] [--yes] [-f/--force] -``` - -Arguments: - -* `ENV_FILE` - path to .env file (required parameters are listed in the `skale node init` command). - -Options: - -* `--yes` - generate without additional confirmation. -* `-f/--force` - rewrite allocation file if it exists. - *** ## Passive Node Usage (`skale` - Passive Build) diff --git a/node_cli/cli/passive_node.py b/node_cli/cli/passive_node.py index c24fe0b5..4e9583f1 100644 --- a/node_cli/cli/passive_node.py +++ b/node_cli/cli/passive_node.py @@ -22,12 +22,8 @@ import click from node_cli.core.node import init_passive, update_passive, cleanup_passive -from node_cli.utils.helper import ( - abort_if_false, - error_exit, - streamed_cmd, - URL_TYPE, -) +from node_cli.fair.passive import setup_fair_passive +from node_cli.utils.helper import abort_if_false, error_exit, streamed_cmd, URL_TYPE, IP_TYPE from node_cli.utils.texts import safe_load_texts @@ -88,3 +84,9 @@ def _update_passive(env_file, unsafe_ok): @streamed_cmd def _cleanup_passive() -> None: cleanup_passive() + + +@passive_node.command('setup', help=TEXTS['setup']['help']) +@click.option('--id', required=True, type=int, help=TEXTS['setup']['id']) +def _setup(id: int) -> None: + setup_fair_passive(node_id=id) diff --git a/node_cli/cli/resources_allocation.py b/node_cli/cli/resources_allocation.py deleted file mode 100644 index a6b2e185..00000000 --- a/node_cli/cli/resources_allocation.py +++ /dev/null @@ -1,64 +0,0 @@ -# -*- coding: utf-8 -*- -# -# This file is part of node-cli -# -# Copyright (C) 2019 SKALE Labs -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -import json -import click - -from node_cli.core.resources import ( - get_resource_allocation_info, - generate_resource_allocation_config, -) -from node_cli.utils.helper import abort_if_false -from node_cli.utils.texts import safe_load_texts -from node_cli.utils.node_type import NodeType - -TEXTS = safe_load_texts() - - -@click.group() -def resources_allocation_cli(): - pass - - -@resources_allocation_cli.group(help='Resources allocation commands') -def resources_allocation(): - pass - - -@resources_allocation.command('show', help='Show resources allocation file') -def show(): - resource_allocation_info = get_resource_allocation_info() - if resource_allocation_info: - print(json.dumps(resource_allocation_info, indent=4)) - else: - print('No resources allocation file on this machine') - - -@resources_allocation.command('generate', help='Generate/update resources allocation file') -@click.argument('env_file') -@click.option( - '--yes', - is_flag=True, - callback=abort_if_false, - expose_value=False, - prompt='Are you sure you want to generate/update resource allocation file?', -) -@click.option('--force', '-f', is_flag=True, help='Rewrite if already exists') -def generate(env_file, force): - generate_resource_allocation_config(node_type=NodeType.SKALE, env_file=env_file, force=force) diff --git a/node_cli/configs/user.py b/node_cli/configs/user.py index b094150d..2fd8145e 100644 --- a/node_cli/configs/user.py +++ b/node_cli/configs/user.py @@ -168,7 +168,9 @@ def get_validated_user_config( def validate_user_config(user_config: BaseUserConfig) -> None: validate_env_type(env_type=user_config.env_type) - if not isinstance(user_config, FairUserConfig): + if not isinstance(user_config, FairUserConfig) and not isinstance( + user_config, PassiveFairUserConfig + ): validate_alias_or_address( user_config.manager_contracts, ContractType.MANAGER, user_config.endpoint ) diff --git a/node_cli/core/node_config.py b/node_cli/core/node_config.py deleted file mode 100644 index c7050918..00000000 --- a/node_cli/core/node_config.py +++ /dev/null @@ -1,35 +0,0 @@ -# -*- coding: utf-8 -*- -# -# This file is part of node-cli -# -# Copyright (C) 2021 SKALE Labs -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - - -class NodeConfig: - def __init__(self, config_filepath, env_filepath=None): - pass - - def load_env(self): - pass - - def validate_env(self): - pass - - def load_config(self): - pass - - def validate_config(self): - pass diff --git a/node_cli/core/node_options.py b/node_cli/core/node_options.py index 49b49a4e..fc766f3a 100644 --- a/node_cli/core/node_options.py +++ b/node_cli/core/node_options.py @@ -71,7 +71,7 @@ def node_mode(self) -> NodeMode: @node_mode.setter def node_mode(self, node_mode: NodeMode) -> None: - return self._set('node_mode', node_mode.name) + return self._set('node_mode', node_mode.value) def all(self) -> dict: return read_json(self.filepath) @@ -97,7 +97,7 @@ def upsert_node_mode(node_mode: NodeMode | None = None) -> NodeMode: node_options = NodeOptions() try: options_mode = node_options.node_mode - if options_mode != node_mode: + if node_mode is not None and options_mode != node_mode: raise NodeModeMismatchError( f'Cannot change node mode from {options_mode} to {node_mode}' ) diff --git a/node_cli/core/resources.py b/node_cli/core/resources.py index 216aa657..e62a0f6f 100644 --- a/node_cli/core/resources.py +++ b/node_cli/core/resources.py @@ -28,7 +28,7 @@ from node_cli.utils.docker_utils import ensure_volume from node_cli.utils.schain_types import SchainTypes from node_cli.utils.helper import write_json, read_json, run_cmd, safe_load_yml -from node_cli.utils.node_type import NodeType +from node_cli.utils.node_type import NodeType, NodeMode from node_cli.configs import ALLOCATION_FILEPATH, STATIC_PARAMS_FILEPATH, SNAPSHOTS_SHARED_VOLUME from node_cli.configs.resource_allocation import ( RESOURCE_ALLOCATION_FILEPATH, @@ -95,6 +95,7 @@ def compose_resource_allocation_config(env_type: str, params_by_env_type: Dict = def generate_resource_allocation_config( env_file, node_type: NodeType, + node_mode: NodeMode, force=False, ) -> None: if not force and os.path.isfile(RESOURCE_ALLOCATION_FILEPATH): @@ -102,7 +103,9 @@ def generate_resource_allocation_config( logger.debug(msg) print(msg) return - user_config = get_validated_user_config(node_type=node_type, env_filepath=env_file) + user_config = get_validated_user_config( + node_type=node_type, node_mode=node_mode, env_filepath=env_file + ) logger.info('Generating resource allocation file ...') try: update_resource_allocation(user_config.env_type) diff --git a/node_cli/fair/passive.py b/node_cli/fair/passive.py index be047233..dcd175d9 100644 --- a/node_cli/fair/passive.py +++ b/node_cli/fair/passive.py @@ -19,7 +19,28 @@ import logging +from node_cli.core.host import is_node_inited from node_cli.utils.texts import safe_load_texts +from node_cli.utils.helper import error_exit, post_request +from node_cli.utils.exit_codes import CLIExitCodes logger = logging.getLogger(__name__) TEXTS = safe_load_texts() +BLUEPRINT_NAME = 'fair-node-passive' + + +def setup_fair_passive(node_id: int) -> None: + if not is_node_inited(): + print(TEXTS['fair']['node']['not_inited']) + return + + json_data = {'id': node_id} + status, payload = post_request(blueprint=BLUEPRINT_NAME, method='setup', json=json_data) + if status == 'ok': + msg = TEXTS['fair']['node']['setup_complete'] + logger.info(msg) + print(msg) + else: + error_msg = payload + logger.error(f'Setup error {error_msg}') + error_exit(error_msg, exit_code=CLIExitCodes.BAD_API_RESPONSE) diff --git a/node_cli/main.py b/node_cli/main.py index a2d3aba9..8e734d17 100644 --- a/node_cli/main.py +++ b/node_cli/main.py @@ -36,7 +36,6 @@ from node_cli.cli.schains import schains_cli from node_cli.cli.wallet import wallet_cli from node_cli.cli.ssl import ssl_cli -from node_cli.cli.resources_allocation import resources_allocation_cli from node_cli.cli.passive_node import passive_node_cli from node_cli.cli.fair_boot import fair_boot_cli from node_cli.cli.fair_node import fair_node_cli @@ -102,7 +101,6 @@ def get_sources_list() -> List[click.MultiCommand]: health_cli, schains_cli, logs_cli, - resources_allocation_cli, node_cli, passive_node_cli, wallet_cli, diff --git a/tests/cli/resources_allocation_test.py b/tests/cli/resources_allocation_test.py deleted file mode 100644 index 03b2e73b..00000000 --- a/tests/cli/resources_allocation_test.py +++ /dev/null @@ -1,96 +0,0 @@ -# -*- coding: utf-8 -*- -# -# This file is part of node-cli -# -# Copyright (C) 2019 SKALE Labs -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -import json -import os - -import mock -import pytest -import requests - -from node_cli.cli.resources_allocation import generate, show -from node_cli.configs.resource_allocation import NODE_DATA_PATH, RESOURCE_ALLOCATION_FILEPATH -from node_cli.utils.helper import safe_mkdir, write_json -from node_cli.utils.node_type import NodeType -from tests.helper import response_mock, run_command_mock -from tests.resources_test import BIG_DISK_SIZE - -TEST_CONFIG = {'test': 1} - - -@pytest.fixture -def resource_alloc_config(): - write_json(RESOURCE_ALLOCATION_FILEPATH, TEST_CONFIG) - yield RESOURCE_ALLOCATION_FILEPATH - os.remove(RESOURCE_ALLOCATION_FILEPATH) - - -def test_show(resource_alloc_config): - resp_mock = response_mock(requests.codes.created) - write_json(RESOURCE_ALLOCATION_FILEPATH, TEST_CONFIG) - result = run_command_mock('node_cli.utils.helper.post_request', resp_mock, show) - assert result.output == json.dumps(TEST_CONFIG, indent=4) + '\n' - assert result.exit_code == 0 - - -def test_generate(regular_user_conf): - safe_mkdir(NODE_DATA_PATH) - resp_mock = response_mock(requests.codes.created) - with ( - mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), - mock.patch('node_cli.configs.user.validate_alias_or_address'), - ): - result = run_command_mock( - 'node_cli.utils.helper.post_request', - resp_mock, - generate, - [regular_user_conf.as_posix(), '--yes'], - ) - assert result.output == ( - f'Resource allocation file generated: {RESOURCE_ALLOCATION_FILEPATH}\n' - ) - assert result.exit_code == 0 - - -def test_generate_already_exists(regular_user_conf, resource_alloc_config): - resp_mock = response_mock(requests.codes.created) - with ( - mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), - mock.patch('node_cli.cli.node.TYPE', NodeType.SKALE), - mock.patch('node_cli.configs.user.validate_alias_or_address'), - ): - result = run_command_mock( - 'node_cli.utils.helper.post_request', - resp_mock, - generate, - [regular_user_conf.as_posix(), '--yes'], - ) - assert result.output == 'Resource allocation file already exists\n' - assert result.exit_code == 0 - - result = run_command_mock( - 'node_cli.utils.helper.post_request', - resp_mock, - generate, - [regular_user_conf.as_posix(), '--yes', '--force'], - ) - assert result.output == ( - f'Resource allocation file generated: {RESOURCE_ALLOCATION_FILEPATH}\n' - ) - assert result.exit_code == 0 diff --git a/tests/conftest.py b/tests/conftest.py index 97e7c674..2e10a9a4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -36,6 +36,7 @@ META_FILEPATH, NGINX_CONFIG_FILEPATH, NGINX_CONTAINER_NAME, + NODE_DATA_PATH, REDIS_URI, REMOVED_CONTAINERS_FOLDER_PATH, SCHAIN_NODE_DATA_PATH, @@ -148,6 +149,10 @@ def ssl_folder(): @pytest.fixture def active_node_option(): + if os.path.isdir(NODE_DATA_PATH): + shutil.rmtree(NODE_DATA_PATH) + path = pathlib.Path(NODE_DATA_PATH) + path.mkdir(parents=True, exist_ok=True) node_options = NodeOptions() node_options.node_mode = NodeMode.ACTIVE try: @@ -158,6 +163,10 @@ def active_node_option(): @pytest.fixture def passive_node_option(): + if os.path.isdir(NODE_DATA_PATH): + shutil.rmtree(NODE_DATA_PATH) + path = pathlib.Path(NODE_DATA_PATH) + path.mkdir(parents=True, exist_ok=True) node_options = NodeOptions() node_options.node_mode = NodeMode.PASSIVE try: diff --git a/text.yml b/text.yml index 85ab8328..bde70b7f 100644 --- a/text.yml +++ b/text.yml @@ -67,6 +67,9 @@ passive_node: archive: Enable historic state and disable block rotation snapshot_from: IP of the node to take snapshot from snapshot: Start passive node from snapshot + setup: + help: Setup passive Fair node + id: ID of the node in Fair manager lvmpy: help: Lvmpy commands @@ -87,9 +90,9 @@ fair: not_inited: Node should be initialized to proceed with operation registered: Node is registered in Fair manager. + setup_complete: Passive node setup complete. register: help: Register node in Fair manager - name: Name of the node in Fair manager ip: IP address of the node in Fair manager ip_changed: Node IP changed in Fair manager change-ip: From 9dc3dd21087872a956ba87301cbfa3903b88ac1a Mon Sep 17 00:00:00 2001 From: Dmytro Date: Tue, 12 Aug 2025 11:36:33 +0100 Subject: [PATCH 054/198] Fix ruff checks --- node_cli/cli/passive_node.py | 2 +- node_cli/utils/docker_utils.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/node_cli/cli/passive_node.py b/node_cli/cli/passive_node.py index 4e9583f1..c53521cf 100644 --- a/node_cli/cli/passive_node.py +++ b/node_cli/cli/passive_node.py @@ -23,7 +23,7 @@ from node_cli.core.node import init_passive, update_passive, cleanup_passive from node_cli.fair.passive import setup_fair_passive -from node_cli.utils.helper import abort_if_false, error_exit, streamed_cmd, URL_TYPE, IP_TYPE +from node_cli.utils.helper import abort_if_false, error_exit, streamed_cmd, URL_TYPE from node_cli.utils.texts import safe_load_texts diff --git a/node_cli/utils/docker_utils.py b/node_cli/utils/docker_utils.py index 83a8dc6f..71400b67 100644 --- a/node_cli/utils/docker_utils.py +++ b/node_cli/utils/docker_utils.py @@ -84,8 +84,10 @@ } BASE_PASSIVE_FAIR_COMPOSE_SERVICES = { - 'fair-passive-admin': 'fair_passive_admin', + 'fair-admin': 'fair_admin', + 'fair-api': 'fair_api', 'nginx': 'skale_nginx', + **REDIS_SERVICE_DICT, } MONITORING_COMPOSE_SERVICES = { From 45a92545275d62078e6894fe65733ef750dfef1c Mon Sep 17 00:00:00 2001 From: Dmytro Date: Tue, 12 Aug 2025 23:53:12 +0100 Subject: [PATCH 055/198] Fix setup cmd for skale node, add watchdog and filebeat to the passive node --- node_cli/cli/passive_fair_node.py | 7 +++++++ node_cli/cli/passive_node.py | 7 ------- node_cli/configs/routes.py | 1 + node_cli/utils/docker_utils.py | 2 ++ tests/.skale/node_data/.gitkeep | 0 tests/routes_test.py | 1 + text.yml | 7 ++++--- 7 files changed, 15 insertions(+), 10 deletions(-) delete mode 100644 tests/.skale/node_data/.gitkeep diff --git a/node_cli/cli/passive_fair_node.py b/node_cli/cli/passive_fair_node.py index f1b79085..046ff41f 100644 --- a/node_cli/cli/passive_fair_node.py +++ b/node_cli/cli/passive_fair_node.py @@ -22,6 +22,7 @@ from node_cli.fair.common import init as init_fair from node_cli.fair.common import update as update_fair from node_cli.fair.common import cleanup as cleanup_fair +from node_cli.fair.passive import setup_fair_passive from node_cli.utils.helper import abort_if_false, streamed_cmd from node_cli.utils.node_type import NodeMode from node_cli.utils.texts import safe_load_texts @@ -85,3 +86,9 @@ def update_node(env_filepath: str, pull_config_for_schain, force_skaled_start: b @streamed_cmd def cleanup_node(): cleanup_fair() + + +@passive_node.command('setup', help=TEXTS['fair']['node']['setup']['help']) +@click.option('--id', required=True, type=int, help=TEXTS['fair']['node']['setup']['id']) +def _setup(id: int) -> None: + setup_fair_passive(node_id=id) diff --git a/node_cli/cli/passive_node.py b/node_cli/cli/passive_node.py index c53521cf..6be74b28 100644 --- a/node_cli/cli/passive_node.py +++ b/node_cli/cli/passive_node.py @@ -22,7 +22,6 @@ import click from node_cli.core.node import init_passive, update_passive, cleanup_passive -from node_cli.fair.passive import setup_fair_passive from node_cli.utils.helper import abort_if_false, error_exit, streamed_cmd, URL_TYPE from node_cli.utils.texts import safe_load_texts @@ -84,9 +83,3 @@ def _update_passive(env_file, unsafe_ok): @streamed_cmd def _cleanup_passive() -> None: cleanup_passive() - - -@passive_node.command('setup', help=TEXTS['setup']['help']) -@click.option('--id', required=True, type=int, help=TEXTS['setup']['id']) -def _setup(id: int) -> None: - setup_fair_passive(node_id=id) diff --git a/node_cli/configs/routes.py b/node_cli/configs/routes.py index a0f15ef1..7d7d9e9c 100644 --- a/node_cli/configs/routes.py +++ b/node_cli/configs/routes.py @@ -42,6 +42,7 @@ 'wallet': ['info', 'send-eth'], 'fair-node': ['info', 'register', 'set-domain-name', 'change-ip', 'exit'], 'fair-chain': ['record', 'checks'], + 'fair-node-passive': ['setup'], } } diff --git a/node_cli/utils/docker_utils.py b/node_cli/utils/docker_utils.py index 71400b67..06427a20 100644 --- a/node_cli/utils/docker_utils.py +++ b/node_cli/utils/docker_utils.py @@ -87,6 +87,8 @@ 'fair-admin': 'fair_admin', 'fair-api': 'fair_api', 'nginx': 'skale_nginx', + 'watchdog': 'skale_watchdog', + 'filebeat': 'skale_filebeat', **REDIS_SERVICE_DICT, } diff --git a/tests/.skale/node_data/.gitkeep b/tests/.skale/node_data/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/routes_test.py b/tests/routes_test.py index 7a1216a8..19845fba 100644 --- a/tests/routes_test.py +++ b/tests/routes_test.py @@ -38,6 +38,7 @@ '/api/v1/fair-node/exit', '/api/v1/fair-chain/record', '/api/v1/fair-chain/checks', + '/api/v1/fair-node-passive/setup', ] diff --git a/text.yml b/text.yml index bde70b7f..4291e475 100644 --- a/text.yml +++ b/text.yml @@ -67,9 +67,6 @@ passive_node: archive: Enable historic state and disable block rotation snapshot_from: IP of the node to take snapshot from snapshot: Start passive node from snapshot - setup: - help: Setup passive Fair node - id: ID of the node in Fair manager lvmpy: help: Lvmpy commands @@ -89,6 +86,10 @@ fair: repair_requested: Repair mode is requested not_inited: Node should be initialized to proceed with operation + setup: + help: Setup passive Fair node + id: ID of the node in Fair manager + registered: Node is registered in Fair manager. setup_complete: Passive node setup complete. register: From 34762f143001499122b25f13f0ffb0293d913815 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Thu, 14 Aug 2025 19:47:30 +0100 Subject: [PATCH 056/198] Add passive node options FAIR, add id to init command --- node_cli/cli/fair_node.py | 2 +- node_cli/cli/passive_fair_node.py | 33 ++++++++++++++++++++++++++++--- node_cli/configs/__init__.py | 2 +- node_cli/core/node_options.py | 12 +++++++++++ node_cli/fair/common.py | 27 +++++++++++++++++++++---- node_cli/operations/base.py | 10 ++-------- node_cli/operations/fair.py | 29 +++++++++++++++++++++------ node_cli/utils/docker_utils.py | 6 ++---- text.yml | 2 +- 9 files changed, 95 insertions(+), 28 deletions(-) diff --git a/node_cli/cli/fair_node.py b/node_cli/cli/fair_node.py index 07533802..4a289a28 100644 --- a/node_cli/cli/fair_node.py +++ b/node_cli/cli/fair_node.py @@ -136,7 +136,7 @@ def migrate_node(env_filepath: str) -> None: @node.command('repair', help='Toggle fair chain repair mode') @click.option( - '--snapshot-from', + '--snapshot', type=URL_OR_ANY_TYPE, default='any', hidden=True, diff --git a/node_cli/cli/passive_fair_node.py b/node_cli/cli/passive_fair_node.py index 046ff41f..dc32c394 100644 --- a/node_cli/cli/passive_fair_node.py +++ b/node_cli/cli/passive_fair_node.py @@ -23,7 +23,12 @@ from node_cli.fair.common import update as update_fair from node_cli.fair.common import cleanup as cleanup_fair from node_cli.fair.passive import setup_fair_passive -from node_cli.utils.helper import abort_if_false, streamed_cmd +from node_cli.utils.helper import ( + URL_OR_ANY_TYPE, + abort_if_false, + error_exit, + streamed_cmd, +) from node_cli.utils.node_type import NodeMode from node_cli.utils.texts import safe_load_texts @@ -42,9 +47,31 @@ def passive_node(): @passive_node.command('init', help='Initialize a passive Fair node') @click.argument('env_filepath') +@click.option('--id', required=True, type=int, help=TEXTS['fair']['node']['setup']['id']) +@click.option('--indexer', help=TEXTS['passive_node']['init']['indexer'], is_flag=True) +@click.option('--archive', help=TEXTS['passive_node']['init']['archive'], is_flag=True) +@click.option( + '--snapshot', + type=URL_OR_ANY_TYPE, + default=None, + help=TEXTS['passive_node']['init']['snapshot_from'], +) @streamed_cmd -def init_passive_node(env_filepath: str): - init_fair(node_mode=NodeMode.PASSIVE, env_filepath=env_filepath) +def init_passive_node( + env_filepath: str, id: int, indexer: bool, archive: bool, snapshot: str | None +): + if indexer and archive: + error_exit('Cannot use both --indexer and --archive options') + if (indexer or archive) and snapshot == 'any': + error_exit('Cannot use any for indexer/archive node') + init_fair( + node_mode=NodeMode.PASSIVE, + env_filepath=env_filepath, + node_id=id, + indexer=indexer, + archive=archive, + snapshot=snapshot, + ) @passive_node.command('update', help='Update Fair node') diff --git a/node_cli/configs/__init__.py b/node_cli/configs/__init__.py index 8fbecb04..5a09dd83 100644 --- a/node_cli/configs/__init__.py +++ b/node_cli/configs/__init__.py @@ -57,7 +57,6 @@ COMPOSE_PATH = os.path.join(CONTAINER_CONFIG_PATH, 'docker-compose.yml') PASSIVE_COMPOSE_PATH = os.path.join(CONTAINER_CONFIG_PATH, 'docker-compose-passive.yml') FAIR_COMPOSE_PATH = os.path.join(CONTAINER_CONFIG_PATH, 'docker-compose-fair.yml') -PASSIVE_FAIR_COMPOSE_PATH = os.path.join(CONTAINER_CONFIG_PATH, 'docker-compose-fair-passive.yml') STATIC_PARAMS_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, 'static_params.yaml') FAIR_STATIC_PARAMS_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, 'fair_static_params.yaml') @@ -141,6 +140,7 @@ def _get_env(): TM_INIT_TIMEOUT = 20 RESTORE_SLEEP_TIMEOUT = 20 +INIT_TIMEOUT = 20 META_FILEPATH = os.path.join(NODE_DATA_PATH, 'meta.json') diff --git a/node_cli/core/node_options.py b/node_cli/core/node_options.py index fc766f3a..764e3b5f 100644 --- a/node_cli/core/node_options.py +++ b/node_cli/core/node_options.py @@ -89,6 +89,18 @@ def mark_passive_node() -> None: logger.info('Node marked as passive.') +def set_passive_node_options( + archive: bool, + indexer: bool, +) -> None: + node_options = NodeOptions() + node_options.node_mode = NodeMode.PASSIVE + node_options.archive = archive or indexer + node_options.catchup = archive or indexer + node_options.historic_state = archive + logger.info('Node options set for passive mode.') + + class NodeModeMismatchError(Exception): pass diff --git a/node_cli/fair/common.py b/node_cli/fair/common.py index 52524421..0a67f3e0 100644 --- a/node_cli/fair/common.py +++ b/node_cli/fair/common.py @@ -20,11 +20,12 @@ import time import logging -from node_cli.configs import RESTORE_SLEEP_TIMEOUT, SKALE_DIR +from node_cli.configs import INIT_TIMEOUT, SKALE_DIR from node_cli.configs.user import SKALE_DIR_ENV_FILEPATH from node_cli.core.docker_config import cleanup_docker_configuration from node_cli.core.node import compose_node_env, is_base_containers_alive from node_cli.core.node_options import upsert_node_mode +from node_cli.fair.passive import setup_fair_passive from node_cli.operations import ( FairUpdateType, cleanup_fair_op, @@ -45,17 +46,35 @@ @check_not_inited -def init(node_mode: NodeMode, env_filepath: str) -> None: +def init( + node_mode: NodeMode, + env_filepath: str, + node_id: int | None = None, + indexer: bool = False, + archive: bool = False, + snapshot: str | None = None, +) -> None: env = compose_node_env(env_filepath, node_type=NodeType.FAIR, node_mode=node_mode) if env is None: return save_env_params(env_filepath) env['SKALE_DIR'] = SKALE_DIR - init_ok = init_fair_op(env_filepath, env, node_mode=node_mode) + init_ok = init_fair_op( + env_filepath, + env, + node_mode=node_mode, + indexer=indexer, + archive=archive, + snapshot=snapshot, + ) if not init_ok: error_exit('Init operation failed', exit_code=CLIExitCodes.OPERATION_EXECUTION_ERROR) - time.sleep(RESTORE_SLEEP_TIMEOUT) + time.sleep(INIT_TIMEOUT) + + if node_mode == NodeMode.PASSIVE and node_id is not None: + setup_fair_passive(node_id) + print('Fair node is initialized') diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index b9d2cc09..c60222a3 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -42,9 +42,8 @@ from node_cli.core.nftables import configure_nftables from node_cli.core.nginx import generate_nginx_config from node_cli.core.node_options import ( - NodeOptions, mark_active_node, - mark_passive_node, + set_passive_node_options, upsert_node_mode, ) from node_cli.core.resources import init_shared_space_volume, update_resource_allocation @@ -289,12 +288,7 @@ def init_passive( env_type=env['ENV_TYPE'], ) - node_options = NodeOptions() - node_options.archive = archive or indexer - node_options.catchup = archive or indexer - node_options.historic_state = archive - - mark_passive_node() + set_passive_node_options(archive=archive, indexer=indexer) ensure_filestorage_mapping() link_env_file() diff --git a/node_cli/operations/fair.py b/node_cli/operations/fair.py index d3a04e20..89fef47c 100644 --- a/node_cli/operations/fair.py +++ b/node_cli/operations/fair.py @@ -38,7 +38,7 @@ from node_cli.core.nginx import generate_nginx_config from node_cli.core.schains import cleanup_no_lvm_datadir from node_cli.core.static_config import get_fair_chain_name -from node_cli.core.node_options import upsert_node_mode +from node_cli.core.node_options import set_passive_node_options, upsert_node_mode from node_cli.fair.record.chain_record import ( get_fair_chain_record, migrate_chain_record, @@ -53,6 +53,7 @@ ) from node_cli.operations.volume import cleanup_volume_artifacts, prepare_block_device from node_cli.utils.docker_utils import ( + BASE_PASSIVE_FAIR_COMPOSE_SERVICES, REDIS_SERVICE_DICT, REDIS_START_TIMEOUT, compose_rm, @@ -79,7 +80,14 @@ class FairUpdateType(Enum): @checked_host -def init(env_filepath: str, env: dict, node_mode: NodeMode) -> bool: +def init( + env_filepath: str, + env: dict, + node_mode: NodeMode, + indexer: bool, + archive: bool, + snapshot: str | None, +) -> bool: sync_skale_node() ensure_btrfs_kernel_module_autoloaded() cleanup_volume_artifacts(env['DISK_MOUNTPOINT']) @@ -94,8 +102,18 @@ def init(env_filepath: str, env: dict, node_mode: NodeMode) -> bool: prepare_host(env_filepath, env_type=env['ENV_TYPE']) link_env_file() - upsert_node_mode(node_mode=node_mode) + update_images(env=env, node_type=NodeType.FAIR, node_mode=node_mode) + compose_up( + env=env, node_type=NodeType.FAIR, node_mode=node_mode, services=list(REDIS_SERVICE_DICT) + ) + + upsert_node_mode(node_mode=node_mode) + if node_mode == NodeMode.PASSIVE: + set_passive_node_options(archive=archive, indexer=indexer) + if snapshot: + time.sleep(REDIS_START_TIMEOUT) + trigger_skaled_snapshot_mode(env=env, snapshot_from=snapshot) prepare_block_device(env['DISK_MOUNTPOINT'], force=env['ENFORCE_BTRFS'] == 'True') meta_manager = FairCliMetaManager() @@ -105,9 +123,9 @@ def init(env_filepath: str, env: dict, node_mode: NodeMode) -> bool: distro.id(), distro.version(), ) - update_images(env=env, node_type=NodeType.FAIR, node_mode=node_mode) + compose_up(env=env, node_type=NodeType.FAIR, node_mode=node_mode) - wait_for_container(REDIS_SERVICE_DICT['redis']) + wait_for_container(BASE_PASSIVE_FAIR_COMPOSE_SERVICES['fair-api']) time.sleep(REDIS_START_TIMEOUT) return True @@ -273,7 +291,6 @@ def trigger_skaled_snapshot_mode(env: dict, snapshot_from: str = 'any') -> None: if not snapshot_from: snapshot_from = 'any' record.set_snapshot_from(snapshot_from) - print(TEXTS['fair']['node']['repair']['repair_requested']) def repair(node_mode: NodeMode, env: dict, snapshot_from: str = 'any') -> None: diff --git a/node_cli/utils/docker_utils.py b/node_cli/utils/docker_utils.py index 06427a20..3aa46e56 100644 --- a/node_cli/utils/docker_utils.py +++ b/node_cli/utils/docker_utils.py @@ -33,7 +33,6 @@ COMPOSE_PATH, FAIR_COMPOSE_PATH, NGINX_CONTAINER_NAME, - PASSIVE_FAIR_COMPOSE_PATH, REMOVED_CONTAINERS_FOLDER_PATH, SGX_CERTIFICATES_DIR_NAME, PASSIVE_COMPOSE_PATH, @@ -298,10 +297,8 @@ def compose_build(env: dict, node_type: NodeType, node_mode: NodeMode): def get_compose_path(node_type: NodeType, node_mode: NodeMode) -> str: if passive_skale(node_type, node_mode): return PASSIVE_COMPOSE_PATH - elif active_fair(node_type, node_mode): + elif active_fair(node_type, node_mode) or passive_fair(node_type, node_mode): return FAIR_COMPOSE_PATH - elif passive_fair(node_type, node_mode): - return PASSIVE_FAIR_COMPOSE_PATH return COMPOSE_PATH @@ -333,6 +330,7 @@ def compose_up( is_fair_boot: bool = False, services: list[str] | None = None, ): + env['PASSIVE_NODE'] = str(node_mode == NodeMode.PASSIVE) if passive_skale(node_type, node_mode) or passive_fair(node_type, node_mode): logger.info('Running containers for passive node') run_cmd(cmd=get_up_compose_cmd(node_type=node_type, node_mode=node_mode), env=env) diff --git a/text.yml b/text.yml index 4291e475..c0cbe23a 100644 --- a/text.yml +++ b/text.yml @@ -82,7 +82,7 @@ fair: repair: help: Repair Fair chain node warning: Are you sure you want to repair Fair chain node? In rare cases may cause data loss and require additional maintenance - snapshot_from: IP of the node to take snapshot from + snapshot_from: IP of the node to take snapshot from (put "any" to use any available node) repair_requested: Repair mode is requested not_inited: Node should be initialized to proceed with operation From 46291315d9968f42b7c1733606446bc06eaf5fa3 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Thu, 14 Aug 2025 19:51:38 +0100 Subject: [PATCH 057/198] Fix ruff check --- node_cli/operations/fair.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node_cli/operations/fair.py b/node_cli/operations/fair.py index 89fef47c..40720a91 100644 --- a/node_cli/operations/fair.py +++ b/node_cli/operations/fair.py @@ -67,7 +67,7 @@ ) from node_cli.utils.helper import cleanup_dir_content, rm_dir, str_to_bool from node_cli.utils.meta import FairCliMetaManager -from node_cli.utils.print_formatters import TEXTS, print_failed_requirements_checks +from node_cli.utils.print_formatters import print_failed_requirements_checks from node_cli.utils.node_type import NodeMode, NodeType logger = logging.getLogger(__name__) From d623dc35a5100826ec64cd9a402051012d02d988 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Fri, 15 Aug 2025 01:12:19 +0100 Subject: [PATCH 058/198] Fix fair node tests --- tests/conftest.py | 23 +++++++++++++++++++---- tests/fair/fair_node_test.py | 13 ++++++++----- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 2e10a9a4..a877c32b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -121,7 +121,10 @@ def resource_alloc(): with open(RESOURCE_ALLOCATION_FILEPATH, 'w') as alloc_file: json.dump({}, alloc_file) yield RESOURCE_ALLOCATION_FILEPATH - os.remove(RESOURCE_ALLOCATION_FILEPATH) + try: + os.remove(RESOURCE_ALLOCATION_FILEPATH) + except FileNotFoundError: + pass @pytest.fixture @@ -132,7 +135,10 @@ def inited_node(): try: yield finally: - os.remove(NGINX_CONFIG_FILEPATH) + try: + os.remove(NGINX_CONFIG_FILEPATH) + except FileNotFoundError: + pass @pytest.fixture @@ -158,7 +164,13 @@ def active_node_option(): try: yield finally: - shutil.rmtree(NODE_OPTIONS_FILEPATH) + try: + if os.path.isdir(NODE_OPTIONS_FILEPATH): + shutil.rmtree(NODE_OPTIONS_FILEPATH) + elif os.path.isfile(NODE_OPTIONS_FILEPATH): + os.remove(NODE_OPTIONS_FILEPATH) + except FileNotFoundError: + pass @pytest.fixture @@ -226,7 +238,10 @@ def meta_file_v3(): try: yield META_FILEPATH finally: - os.remove(META_FILEPATH) + try: + os.remove(META_FILEPATH) + except FileNotFoundError: + pass @pytest.fixture diff --git a/tests/fair/fair_node_test.py b/tests/fair/fair_node_test.py index 861267ca..9dc14983 100644 --- a/tests/fair/fair_node_test.py +++ b/tests/fair/fair_node_test.py @@ -151,9 +151,12 @@ def test_cleanup_success( cleanup() mock_compose_env.assert_called_once_with( - SKALE_DIR_ENV_FILEPATH, save=False, node_type=NodeType.FAIR + SKALE_DIR_ENV_FILEPATH, + save=False, + node_type=NodeType.FAIR, + node_mode=NodeMode.ACTIVE, ) - mock_cleanup_fair_op.assert_called_once_with(mock_env) + mock_cleanup_fair_op.assert_called_once_with(node_mode=NodeMode.ACTIVE, env=mock_env) mock_cleanup_docker_config.assert_called_once() @@ -184,8 +187,8 @@ def test_cleanup_calls_operations_in_correct_order( cleanup() expected_calls = [ - mock.call.compose_env(mock.ANY, save=False, node_type=mock.ANY), - mock.call.cleanup_fair_op(mock_env), + mock.call.compose_env(mock.ANY, save=False, node_type=mock.ANY, node_mode=NodeMode.ACTIVE), + mock.call.cleanup_fair_op(node_mode=NodeMode.ACTIVE, env=mock_env), mock.call.cleanup_docker_config(), ] manager.assert_has_calls(expected_calls, any_order=False) @@ -211,7 +214,7 @@ def test_cleanup_continues_after_fair_op_error( cleanup() mock_compose_env.assert_called_once() - mock_cleanup_fair_op.assert_called_once_with(mock_env) + mock_cleanup_fair_op.assert_called_once_with(node_mode=NodeMode.ACTIVE, env=mock_env) mock_cleanup_docker_config.assert_not_called() From f5befb7cb798dc0e63e9c87bae166104b02b4ee1 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Mon, 18 Aug 2025 11:23:02 +0100 Subject: [PATCH 059/198] Fix disk mounting in fair node init --- README.md | 56 ++++++++++++++++++++++++++++++++++++ node_cli/operations/fair.py | 3 +- tests/cli/node_test.py | 7 ++--- tests/fair/fair_node_test.py | 13 +++++++-- 4 files changed, 72 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 7484ad2b..558bc61a 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,7 @@ SKALE Node CLI, part of the SKALE suite of validator tools, is the command line 5. [Fair Wallet commands](#fair-wallet-commands) 6. [Fair Logs commands](#fair-logs-commands) 7. [Fair SSL commands](#fair-ssl-commands) + 8. [Passive Fair Node commands](#passive-fair-node-commands) 5. [Exit codes](#exit-codes) 6. [Development](#development) @@ -1076,6 +1077,61 @@ Options: * `--no-client` - Skip client connection for openssl check. * `--no-wss` - Skip WSS server starting for skaled check. +### Passive Fair Node commands + +> Prefix: `fair passive-node` (passive Fair build) + +Commands for operating a passive Fair node (sync/indexer/archive). + +#### Passive Fair Node Initialization + +Initialize a passive Fair node. + +```shell +fair passive-node init --id [--indexer | --archive] [--snapshot ] +``` + +Arguments: + +* `ENV_FILEPATH` - Path to the environment file with configuration. + +Required environment variables in `ENV_FILEPATH`: + +* `FAIR_CONTRACTS` - Fair Manager contracts alias or address. +* `NODE_VERSION` - Stream of `skale-node` configs. +* `BOOT_ENDPOINT` - RPC endpoint of Fair network. +* `DISK_MOUNTPOINT` - Mount point for storing chain data. +* `ENV_TYPE` - Environment type (e.g., `mainnet`, `devnet`). + +Options: + +* `--id` - Numerical node identifier (required). +* `--indexer` - Run in indexer mode (no block rotation). +* `--archive` - Run in archive mode (historical state kept; disables block rotation). Mutually exclusive with `--indexer`. +* `--snapshot ` - Start from provided snapshot URL or from any available source (not allowed together with `--indexer` or `--archive`). + +By default runs a regular sync node. + +#### Passive Fair Node Update + +Update software / configs for passive Fair node. + +```shell +fair passive-node update [--yes] +``` + +#### Passive Fair Node Cleanup + +Remove all passive Fair node data and containers. + +```shell +fair passive-node cleanup [--yes] +``` + +Options: + +* `--yes` - Proceed without confirmation. + *** ## Exit codes diff --git a/node_cli/operations/fair.py b/node_cli/operations/fair.py index 40720a91..772791f3 100644 --- a/node_cli/operations/fair.py +++ b/node_cli/operations/fair.py @@ -103,6 +103,8 @@ def init( prepare_host(env_filepath, env_type=env['ENV_TYPE']) link_env_file() + prepare_block_device(env['DISK_MOUNTPOINT'], force=env['ENFORCE_BTRFS'] == 'True') + update_images(env=env, node_type=NodeType.FAIR, node_mode=node_mode) compose_up( env=env, node_type=NodeType.FAIR, node_mode=node_mode, services=list(REDIS_SERVICE_DICT) @@ -114,7 +116,6 @@ def init( if snapshot: time.sleep(REDIS_START_TIMEOUT) trigger_skaled_snapshot_mode(env=env, snapshot_from=snapshot) - prepare_block_device(env['DISK_MOUNTPOINT'], force=env['ENFORCE_BTRFS'] == 'True') meta_manager = FairCliMetaManager() meta_manager.update_meta( diff --git a/tests/cli/node_test.py b/tests/cli/node_test.py index 13d3d0c5..3361e083 100644 --- a/tests/cli/node_test.py +++ b/tests/cli/node_test.py @@ -325,7 +325,6 @@ def test_backup(): 'node_type,node_mode,test_user_conf', [ (NodeType.SKALE, NodeMode.ACTIVE, 'regular_user_conf'), - (NodeType.SKALE, NodeMode.PASSIVE, 'passive_user_conf'), (NodeType.FAIR, NodeMode.ACTIVE, 'fair_user_conf'), ], ) @@ -350,7 +349,6 @@ def test_restore(request, node_type, node_mode, test_user_conf, mocked_g_config, patch('node_cli.configs.user.validate_alias_or_address'), ): user_conf_path = request.getfixturevalue(test_user_conf).as_posix() - result = run_command(restore_node, [backup_path, user_conf_path]) assert result.exit_code == 0 assert 'Node is restored from backup\n' in result.output # noqa @@ -386,7 +384,7 @@ def test_maintenance_off(mocked_g_config): ) -def test_turn_off_maintenance_on(mocked_g_config, regular_user_conf): +def test_turn_off_maintenance_on(mocked_g_config, regular_user_conf, active_node_option): resp_mock = response_mock(requests.codes.ok, {'status': 'ok', 'payload': None}) with ( mock.patch('subprocess.run', new=subprocess_run_mock), @@ -402,6 +400,7 @@ def test_turn_off_maintenance_on(mocked_g_config, regular_user_conf): _turn_off, ['--maintenance-on', '--yes'], ) + assert ( result.output == 'Setting maintenance mode on...\nNode is successfully set in maintenance mode\n' @@ -418,7 +417,7 @@ def test_turn_off_maintenance_on(mocked_g_config, regular_user_conf): assert result.exit_code == CLIExitCodes.UNSAFE_UPDATE -def test_turn_on_maintenance_off(mocked_g_config, regular_user_conf): +def test_turn_on_maintenance_off(mocked_g_config, regular_user_conf, active_node_option): resp_mock = response_mock(requests.codes.ok, {'status': 'ok', 'payload': None}) with ( mock.patch('subprocess.run', new=subprocess_run_mock), diff --git a/tests/fair/fair_node_test.py b/tests/fair/fair_node_test.py index 9dc14983..7b621499 100644 --- a/tests/fair/fair_node_test.py +++ b/tests/fair/fair_node_test.py @@ -23,6 +23,7 @@ def test_restore_fair( mock_sleep, valid_env_file, ensure_meta_removed, + active_node_option, ): mock_env = {'ENV_TYPE': 'devnet'} mock_compose_env.return_value = mock_env @@ -31,11 +32,16 @@ def test_restore_fair( restore(backup_path, valid_env_file) - mock_compose_env.assert_called_once_with(valid_env_file, node_type=NodeType.FAIR) + mock_compose_env.assert_called_once_with( + valid_env_file, node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE + ) mock_save_env.assert_called_once_with(valid_env_file) expected_env = {**mock_env, 'SKALE_DIR': SKALE_DIR} mock_restore_op.assert_called_once_with( - NodeMode.ACTIVE, expected_env, backup_path, config_only=False + node_mode=NodeMode.ACTIVE, + env=expected_env, + backup_path=backup_path, + config_only=False, ) mock_sleep.assert_called_once() @@ -60,6 +66,7 @@ def test_init_fair_boot( mock_compose_env.assert_called_once_with( valid_env_file, node_type=NodeType.FAIR, + node_mode=NodeMode.ACTIVE, is_fair_boot=True, ) mock_init_op.assert_called_once_with(valid_env_file, mock_env) @@ -96,6 +103,7 @@ def test_update_fair_boot( sync_schains=False, pull_config_for_schain=pull_config_for_schain, node_type=NodeType.FAIR, + node_mode=NodeMode.ACTIVE, is_fair_boot=True, ) mock_update_op.assert_called_once_with(valid_env_file, mock_env) @@ -126,6 +134,7 @@ def test_migrate_from_boot( inited_node=True, sync_schains=False, node_type=NodeType.FAIR, + node_mode=NodeMode.ACTIVE, ) mock_migrate_op.assert_called_once_with( valid_env_file, mock_env, update_type=FairUpdateType.FROM_BOOT, force_skaled_start=False From 48d587c928e7dd8dbba465752ad3cdbae5eb04e9 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Mon, 18 Aug 2025 11:48:09 +0100 Subject: [PATCH 060/198] Update fair node tests --- tests/fair/fair_node_test.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/fair/fair_node_test.py b/tests/fair/fair_node_test.py index 7b621499..be487b8f 100644 --- a/tests/fair/fair_node_test.py +++ b/tests/fair/fair_node_test.py @@ -71,7 +71,9 @@ def test_init_fair_boot( ) mock_init_op.assert_called_once_with(valid_env_file, mock_env) mock_sleep.assert_called_once() - mock_is_alive.assert_called_once_with(node_type=NodeType.FAIR, is_fair_boot=True) + mock_is_alive.assert_called_once_with( + node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE, is_fair_boot=True + ) @mock.patch('node_cli.utils.decorators.is_user_valid', return_value=True) @@ -108,7 +110,9 @@ def test_update_fair_boot( ) mock_update_op.assert_called_once_with(valid_env_file, mock_env) mock_sleep.assert_called_once() - mock_is_alive.assert_called_once_with(node_type=NodeType.FAIR, is_fair_boot=True) + mock_is_alive.assert_called_once_with( + node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE, is_fair_boot=True + ) @mock.patch('node_cli.fair.active.update_fair_op') @@ -137,7 +141,11 @@ def test_migrate_from_boot( node_mode=NodeMode.ACTIVE, ) mock_migrate_op.assert_called_once_with( - valid_env_file, mock_env, update_type=FairUpdateType.FROM_BOOT, force_skaled_start=False + valid_env_file, + mock_env, + node_mode=NodeMode.ACTIVE, + update_type=FairUpdateType.FROM_BOOT, + force_skaled_start=False, ) From 20aa725fee8a15f26843bca95e1fcdc3b0e10442 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Mon, 18 Aug 2025 12:09:19 +0100 Subject: [PATCH 061/198] Update fair node tests --- tests/fair/fair_node_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/fair/fair_node_test.py b/tests/fair/fair_node_test.py index be487b8f..0fb907d8 100644 --- a/tests/fair/fair_node_test.py +++ b/tests/fair/fair_node_test.py @@ -223,6 +223,7 @@ def test_cleanup_continues_after_fair_op_error( inited_node, resource_alloc, meta_file_v3, + active_node_option, ): mock_env = {'ENV_TYPE': 'devnet'} mock_compose_env.return_value = mock_env From d26de21f6cc545363ebff2269153a23af5de5aba Mon Sep 17 00:00:00 2001 From: Dmytro Date: Tue, 19 Aug 2025 11:43:13 +0100 Subject: [PATCH 062/198] Skip host requirements for fair passive nodes, update tests --- node_cli/cli/fair_node.py | 4 ++-- node_cli/cli/passive_fair_node.py | 2 +- node_cli/core/checks.py | 16 +++++++++++----- node_cli/fair/common.py | 4 ++-- tests/core/core_checks_test.py | 14 +++++++------- 5 files changed, 23 insertions(+), 17 deletions(-) diff --git a/node_cli/cli/fair_node.py b/node_cli/cli/fair_node.py index 4a289a28..7022f51d 100644 --- a/node_cli/cli/fair_node.py +++ b/node_cli/cli/fair_node.py @@ -22,7 +22,7 @@ from node_cli.core.node import backup from node_cli.fair.active import change_ip as change_ip_fair -from node_cli.fair.common import cleanup as fair_cleanup +from node_cli.fair.common import cleanup as cleanup_fair from node_cli.fair.active import exit as exit_fair from node_cli.fair.active import ( get_node_info, @@ -164,7 +164,7 @@ def repair(snapshot_from: str = 'any') -> None: ) @streamed_cmd def cleanup_node(): - fair_cleanup() + cleanup_fair(node_mode=NodeMode.ACTIVE) @node.command('change-ip', help=TEXTS['fair']['node']['change-ip']['help']) diff --git a/node_cli/cli/passive_fair_node.py b/node_cli/cli/passive_fair_node.py index dc32c394..487ded44 100644 --- a/node_cli/cli/passive_fair_node.py +++ b/node_cli/cli/passive_fair_node.py @@ -112,7 +112,7 @@ def update_node(env_filepath: str, pull_config_for_schain, force_skaled_start: b ) @streamed_cmd def cleanup_node(): - cleanup_fair() + cleanup_fair(node_mode=NodeMode.PASSIVE) @passive_node.command('setup', help=TEXTS['fair']['node']['setup']['help']) diff --git a/node_cli/core/checks.py b/node_cli/core/checks.py index 929cc60b..d72c4153 100644 --- a/node_cli/core/checks.py +++ b/node_cli/core/checks.py @@ -56,10 +56,12 @@ REPORTS_PATH, ) from node_cli.core.host import is_ufw_ipv6_chain_exists, is_ufw_ipv6_option_enabled +from node_cli.core.node_options import upsert_node_mode from node_cli.core.resources import get_disk_size from node_cli.core.static_config import get_static_params from node_cli.utils.docker_utils import NodeType from node_cli.utils.helper import run_cmd, safe_mkdir +from node_cli.utils.node_type import NodeMode logger = logging.getLogger(__name__) @@ -272,7 +274,7 @@ class PackageChecker(BaseChecker): def __init__(self, requirements: Dict) -> None: super().__init__(requirements=requirements) - def _check_apt_package(self, package_name: str, version: str = None) -> CheckResult: + def _check_apt_package(self, package_name: str, version: str | None = None) -> CheckResult: # TODO: check versions dpkg_cmd_result = run_cmd(['dpkg', '-s', package_name], check_code=False) output = dpkg_cmd_result.stdout.decode('utf-8').strip() @@ -457,12 +459,14 @@ def get_checks(checkers: List[BaseChecker], check_type: CheckType = CheckType.AL ) -def get_all_checkers(disk: str, requirements: Dict) -> List[BaseChecker]: - return [ - MachineChecker(requirements['server'], disk), +def get_all_checkers(disk: str, requirements: Dict, node_mode: NodeMode) -> List[BaseChecker]: + checkers = [ PackageChecker(requirements['package']), DockerChecker(requirements['docker']), ] + if node_mode == NodeMode.ACTIVE: + checkers.append(MachineChecker(requirements['server'], disk)) + return checkers def run_checks( @@ -474,7 +478,9 @@ def run_checks( ) -> ResultList: logger.info('Executing checks. Type: %s', check_type) requirements = get_static_params(node_type, env_type, config_path) - checkers = get_all_checkers(disk, requirements) + node_mode = upsert_node_mode() + + checkers = get_all_checkers(disk, requirements, node_mode) checks = get_checks(checkers, check_type) results = [check() for check in checks] diff --git a/node_cli/fair/common.py b/node_cli/fair/common.py index 0a67f3e0..ce24ad45 100644 --- a/node_cli/fair/common.py +++ b/node_cli/fair/common.py @@ -79,8 +79,8 @@ def init( @check_user -def cleanup() -> None: - node_mode = upsert_node_mode() +def cleanup(node_mode: NodeMode) -> None: + node_mode = upsert_node_mode(node_mode=node_mode) env = compose_node_env( SKALE_DIR_ENV_FILEPATH, save=False, node_type=NodeType.FAIR, node_mode=node_mode ) diff --git a/tests/core/core_checks_test.py b/tests/core/core_checks_test.py index 5206efe4..819116d7 100644 --- a/tests/core/core_checks_test.py +++ b/tests/core/core_checks_test.py @@ -21,7 +21,7 @@ save_report, ) -from node_cli.utils.node_type import NodeType +from node_cli.utils.node_type import NodeMode, NodeType @pytest.fixture @@ -318,18 +318,18 @@ def run_cmd_mock(*args, **kwargs): assert r.status == 'ok' -def test_get_all_checkers(requirements_data): +def test_get_all_checkers(requirements_data, active_node_option): disk = 'test-disk' - checkers = get_all_checkers(disk, requirements_data) + checkers = get_all_checkers(disk, requirements_data, node_mode=NodeMode.ACTIVE) assert len(checkers) == 3 assert isinstance(checkers[0], MachineChecker) assert isinstance(checkers[1], PackageChecker) assert isinstance(checkers[2], DockerChecker) -def test_get_checks(requirements_data): +def test_get_checks(requirements_data, active_node_option): disk = 'test-disk' - checkers = get_all_checkers(disk, requirements_data) + checkers = get_all_checkers(disk, requirements_data, node_mode=NodeMode.ACTIVE) checks = get_checks(checkers) assert len(checks) == 16 checks = get_checks(checkers, check_type=CheckType.PREINSTALL) @@ -338,9 +338,9 @@ def test_get_checks(requirements_data): assert len(checks) == 2 -def test_get_checks_fair(fair_requirements_data): +def test_get_checks_fair(fair_requirements_data, active_node_option): disk = 'test-disk' - fair_checkers = get_all_checkers(disk, fair_requirements_data) + fair_checkers = get_all_checkers(disk, fair_requirements_data, node_mode=NodeMode.ACTIVE) fair_all_checks = get_checks(fair_checkers, CheckType.ALL) fair_all_names = {f.func.__name__ for f in fair_all_checks} From bf871da6691751e549634606aeff4b8d71ec4381 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Tue, 19 Aug 2025 13:33:22 +0100 Subject: [PATCH 063/198] Fix cleanup and checkers tests --- tests/core/core_checks_test.py | 6 +++--- tests/fair/fair_node_test.py | 13 +++++++------ 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/tests/core/core_checks_test.py b/tests/core/core_checks_test.py index 819116d7..f06efb9e 100644 --- a/tests/core/core_checks_test.py +++ b/tests/core/core_checks_test.py @@ -322,9 +322,9 @@ def test_get_all_checkers(requirements_data, active_node_option): disk = 'test-disk' checkers = get_all_checkers(disk, requirements_data, node_mode=NodeMode.ACTIVE) assert len(checkers) == 3 - assert isinstance(checkers[0], MachineChecker) - assert isinstance(checkers[1], PackageChecker) - assert isinstance(checkers[2], DockerChecker) + assert isinstance(checkers[0], PackageChecker) + assert isinstance(checkers[1], DockerChecker) + assert isinstance(checkers[2], MachineChecker) def test_get_checks(requirements_data, active_node_option): diff --git a/tests/fair/fair_node_test.py b/tests/fair/fair_node_test.py index 0fb907d8..e42766ff 100644 --- a/tests/fair/fair_node_test.py +++ b/tests/fair/fair_node_test.py @@ -161,11 +161,12 @@ def test_cleanup_success( inited_node, resource_alloc, meta_file_v3, + active_node_option, ): mock_env = {'ENV_TYPE': 'devnet'} mock_compose_env.return_value = mock_env - cleanup() + cleanup(node_mode=NodeMode.ACTIVE) mock_compose_env.assert_called_once_with( SKALE_DIR_ENV_FILEPATH, @@ -201,7 +202,7 @@ def test_cleanup_calls_operations_in_correct_order( manager.attach_mock(mock_cleanup_fair_op, 'cleanup_fair_op') manager.attach_mock(mock_cleanup_docker_config, 'cleanup_docker_config') - cleanup() + cleanup(node_mode=NodeMode.ACTIVE) expected_calls = [ mock.call.compose_env(mock.ANY, save=False, node_type=mock.ANY, node_mode=NodeMode.ACTIVE), @@ -229,7 +230,7 @@ def test_cleanup_continues_after_fair_op_error( mock_compose_env.return_value = mock_env with pytest.raises(Exception, match='Cleanup failed'): - cleanup() + cleanup(node_mode=NodeMode.ACTIVE) mock_compose_env.assert_called_once() mock_cleanup_fair_op.assert_called_once_with(node_mode=NodeMode.ACTIVE, env=mock_env) @@ -249,14 +250,14 @@ def test_cleanup_fails_when_user_invalid( from node_cli.fair.common import cleanup with pytest.raises(SystemExit): - cleanup() + cleanup(node_mode=NodeMode.ACTIVE) def test_cleanup_fails_when_not_inited(ensure_meta_removed, active_node_option): import pytest with pytest.raises(SystemExit): - cleanup() + cleanup(node_mode=NodeMode.ACTIVE) @mock.patch('node_cli.utils.decorators.is_user_valid', return_value=True) @@ -278,7 +279,7 @@ def test_cleanup_logs_success_message( mock_env = {'ENV_TYPE': 'devnet'} mock_compose_env.return_value = mock_env - cleanup() + cleanup(node_mode=NodeMode.ACTIVE) mock_logger.info.assert_called_once_with( 'Fair node was cleaned up, all containers and data removed' From 569fe0cbd258cb9f431d90a472f4036058dd4090 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Tue, 19 Aug 2025 13:34:51 +0100 Subject: [PATCH 064/198] Add test for passive node checker --- tests/conftest.py | 8 +++++++- tests/core/core_checks_test.py | 8 ++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index a877c32b..047e35a0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -184,7 +184,13 @@ def passive_node_option(): try: yield finally: - shutil.rmtree(NODE_OPTIONS_FILEPATH) + try: + if os.path.isdir(NODE_OPTIONS_FILEPATH): + shutil.rmtree(NODE_OPTIONS_FILEPATH) + elif os.path.isfile(NODE_OPTIONS_FILEPATH): + os.remove(NODE_OPTIONS_FILEPATH) + except FileNotFoundError: + pass @pytest.fixture diff --git a/tests/core/core_checks_test.py b/tests/core/core_checks_test.py index f06efb9e..aa986cc7 100644 --- a/tests/core/core_checks_test.py +++ b/tests/core/core_checks_test.py @@ -327,6 +327,14 @@ def test_get_all_checkers(requirements_data, active_node_option): assert isinstance(checkers[2], MachineChecker) +def test_get_all_checkers_passive(requirements_data, passive_node_option): + disk = 'test-disk' + checkers = get_all_checkers(disk, requirements_data, node_mode=NodeMode.PASSIVE) + assert len(checkers) == 2 + assert isinstance(checkers[0], PackageChecker) + assert isinstance(checkers[1], DockerChecker) + + def test_get_checks(requirements_data, active_node_option): disk = 'test-disk' checkers = get_all_checkers(disk, requirements_data, node_mode=NodeMode.ACTIVE) From e957437a067c45cbf14daed5dd34e9da260843ec Mon Sep 17 00:00:00 2001 From: Dmytro Date: Tue, 19 Aug 2025 16:28:29 +0100 Subject: [PATCH 065/198] Add node mode to checked host --- node_cli/core/checks.py | 2 +- node_cli/core/node.py | 2 +- node_cli/operations/base.py | 12 ++++++++---- node_cli/operations/fair.py | 6 ++++-- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/node_cli/core/checks.py b/node_cli/core/checks.py index d72c4153..2a25319e 100644 --- a/node_cli/core/checks.py +++ b/node_cli/core/checks.py @@ -472,13 +472,13 @@ def get_all_checkers(disk: str, requirements: Dict, node_mode: NodeMode) -> List def run_checks( disk: str, node_type: NodeType, + node_mode: NodeMode, env_type: str = 'mainnet', config_path: str = CONTAINER_CONFIG_PATH, check_type: CheckType = CheckType.ALL, ) -> ResultList: logger.info('Executing checks. Type: %s', check_type) requirements = get_static_params(node_type, env_type, config_path) - node_mode = upsert_node_mode() checkers = get_all_checkers(disk, requirements, node_mode) checks = get_checks(checkers, check_type) diff --git a/node_cli/core/node.py b/node_cli/core/node.py index 79ac63c4..1437c86a 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -541,7 +541,7 @@ def run_checks( if disk is None: env_config = get_validated_user_config(node_type=node_type, node_mode=node_mode) disk = env_config.disk_mountpoint - failed_checks = run_host_checks(disk, node_type, network, container_config_path) + failed_checks = run_host_checks(disk, node_type, node_mode, network, container_config_path) if not failed_checks: print('Requirements checking successfully finished!') else: diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index c60222a3..01017d59 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -79,11 +79,12 @@ def checked_host(func): @functools.wraps(func) - def wrapper(env_filepath: str, env: Dict, *args, **kwargs): + def wrapper(env_filepath: str, env: Dict, node_mode: NodeMode, *args, **kwargs): download_skale_node(env.get('NODE_VERSION'), env.get('CONTAINER_CONFIGS_DIR')) failed_checks = run_host_checks( env['DISK_MOUNTPOINT'], TYPE, + node_mode, env['ENV_TYPE'], CONTAINER_CONFIG_TMP_PATH, check_type=CheckType.PREINSTALL, @@ -99,6 +100,7 @@ def wrapper(env_filepath: str, env: Dict, *args, **kwargs): failed_checks = run_host_checks( env['DISK_MOUNTPOINT'], TYPE, + node_mode, env['ENV_TYPE'], CONTAINER_CONFIG_PATH, check_type=CheckType.POSTINSTALL, @@ -155,8 +157,8 @@ def update(env_filepath: str, env: Dict, node_type: NodeType, node_mode: NodeMod @checked_host -def update_fair_boot(env_filepath: str, env: Dict) -> bool: - compose_rm(node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE, env=env) +def update_fair_boot(env_filepath: str, env: Dict, node_mode: NodeMode = NodeMode.ACTIVE) -> bool: + compose_rm(node_type=NodeType.FAIR, node_mode=node_mode, env=env) remove_dynamic_containers() cleanup_volume_artifacts(env['DISK_MOUNTPOINT']) @@ -233,7 +235,7 @@ def init(env_filepath: str, env: dict, node_type: NodeType, node_mode: NodeMode) @checked_host -def init_fair_boot(env_filepath: str, env: dict) -> None: +def init_fair_boot(env_filepath: str, env: dict, node_mode: NodeMode = NodeMode.ACTIVE) -> None: sync_skale_node() cleanup_volume_artifacts(env['DISK_MOUNTPOINT']) @@ -381,6 +383,7 @@ def restore(env, backup_path, node_type: NodeType, config_only=False): failed_checks = run_host_checks( env['DISK_MOUNTPOINT'], TYPE, + node_mode, env['ENV_TYPE'], CONTAINER_CONFIG_PATH, check_type=CheckType.PREINSTALL, @@ -415,6 +418,7 @@ def restore(env, backup_path, node_type: NodeType, config_only=False): failed_checks = run_host_checks( env['DISK_MOUNTPOINT'], TYPE, + node_mode, env['ENV_TYPE'], CONTAINER_CONFIG_PATH, check_type=CheckType.POSTINSTALL, diff --git a/node_cli/operations/fair.py b/node_cli/operations/fair.py index 772791f3..37274d79 100644 --- a/node_cli/operations/fair.py +++ b/node_cli/operations/fair.py @@ -132,8 +132,8 @@ def init( @checked_host -def update_fair_boot(env_filepath: str, env: dict) -> bool: - compose_rm(node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE, env=env) +def update_fair_boot(env_filepath: str, env: dict, node_mode: NodeMode = NodeMode.ACTIVE) -> bool: + compose_rm(node_type=NodeType.FAIR, node_mode=node_mode, env=env) remove_dynamic_containers() cleanup_volume_artifacts(env['DISK_MOUNTPOINT']) @@ -236,6 +236,7 @@ def restore(node_mode: NodeMode, env, backup_path, config_only=False): failed_checks = run_host_checks( env['DISK_MOUNTPOINT'], TYPE, + node_mode, env['ENV_TYPE'], CONTAINER_CONFIG_PATH, check_type=CheckType.PREINSTALL, @@ -268,6 +269,7 @@ def restore(node_mode: NodeMode, env, backup_path, config_only=False): failed_checks = run_host_checks( env['DISK_MOUNTPOINT'], TYPE, + node_mode, env['ENV_TYPE'], CONTAINER_CONFIG_PATH, check_type=CheckType.POSTINSTALL, From 70e56e4ec8555949ba7c00a3f72a351e5d5abe9b Mon Sep 17 00:00:00 2001 From: Dmytro Date: Tue, 19 Aug 2025 16:32:06 +0100 Subject: [PATCH 066/198] Fix ruff check --- node_cli/core/checks.py | 1 - 1 file changed, 1 deletion(-) diff --git a/node_cli/core/checks.py b/node_cli/core/checks.py index 2a25319e..9c312476 100644 --- a/node_cli/core/checks.py +++ b/node_cli/core/checks.py @@ -56,7 +56,6 @@ REPORTS_PATH, ) from node_cli.core.host import is_ufw_ipv6_chain_exists, is_ufw_ipv6_option_enabled -from node_cli.core.node_options import upsert_node_mode from node_cli.core.resources import get_disk_size from node_cli.core.static_config import get_static_params from node_cli.utils.docker_utils import NodeType From 3291fa8271c5c4fc3efb013f01a1bf3ca8c9abdb Mon Sep 17 00:00:00 2001 From: Dmytro Date: Tue, 19 Aug 2025 17:55:49 +0100 Subject: [PATCH 067/198] Fix fair boot op --- node_cli/fair/boot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node_cli/fair/boot.py b/node_cli/fair/boot.py index 9acf9b63..114bb59a 100644 --- a/node_cli/fair/boot.py +++ b/node_cli/fair/boot.py @@ -45,7 +45,7 @@ def init(env_filepath: str) -> None: is_fair_boot=True, ) - init_fair_boot_op(env_filepath, env) + init_fair_boot_op(env_filepath, env, node_mode) logger.info('Waiting for fair containers initialization') time.sleep(TM_INIT_TIMEOUT) if not is_base_containers_alive(node_type=node_type, node_mode=node_mode, is_fair_boot=True): From 432b5091093e54fc2abdf58b30f5d8a3c40b420b Mon Sep 17 00:00:00 2001 From: Dmytro Date: Tue, 19 Aug 2025 19:18:02 +0100 Subject: [PATCH 068/198] Fix fair boot commands --- node_cli/fair/boot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node_cli/fair/boot.py b/node_cli/fair/boot.py index 114bb59a..a6415f75 100644 --- a/node_cli/fair/boot.py +++ b/node_cli/fair/boot.py @@ -67,7 +67,7 @@ def update(env_filepath: str, pull_config_for_schain: str) -> None: node_mode=node_mode, is_fair_boot=True, ) - migrate_ok = update_fair_boot_op(env_filepath, env) + migrate_ok = update_fair_boot_op(env_filepath, env, node_mode=NodeMode.ACTIVE) if migrate_ok: logger.info('Waiting for containers initialization') time.sleep(TM_INIT_TIMEOUT) From 8425e4022c38b8fe82280574e7470122845cdac5 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Tue, 19 Aug 2025 19:41:31 +0100 Subject: [PATCH 069/198] Fix fair boot tests --- tests/fair/fair_node_test.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/fair/fair_node_test.py b/tests/fair/fair_node_test.py index e42766ff..c3789ff1 100644 --- a/tests/fair/fair_node_test.py +++ b/tests/fair/fair_node_test.py @@ -69,7 +69,11 @@ def test_init_fair_boot( node_mode=NodeMode.ACTIVE, is_fair_boot=True, ) - mock_init_op.assert_called_once_with(valid_env_file, mock_env) + mock_init_op.assert_called_once_with( + valid_env_file, + mock_env, + NodeMode.ACTIVE, + ) mock_sleep.assert_called_once() mock_is_alive.assert_called_once_with( node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE, is_fair_boot=True @@ -108,7 +112,11 @@ def test_update_fair_boot( node_mode=NodeMode.ACTIVE, is_fair_boot=True, ) - mock_update_op.assert_called_once_with(valid_env_file, mock_env) + mock_update_op.assert_called_once_with( + valid_env_file, + mock_env, + node_mode=NodeMode.ACTIVE, + ) mock_sleep.assert_called_once() mock_is_alive.assert_called_once_with( node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE, is_fair_boot=True From 6f24d6acc39bbd51262cdf1f4a914092b629061c Mon Sep 17 00:00:00 2001 From: Dmytro Date: Tue, 19 Aug 2025 20:17:35 +0100 Subject: [PATCH 070/198] Fix checked host wrapper --- node_cli/operations/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index 01017d59..a856ef6f 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -93,7 +93,7 @@ def wrapper(env_filepath: str, env: Dict, node_mode: NodeMode, *args, **kwargs): print_failed_requirements_checks(failed_checks) return False - result = func(env_filepath, env, *args, **kwargs) + result = func(env_filepath, env, node_mode, *args, **kwargs) if not result: return result From 03a9eac54c753a9df6562cc3cf4f3dd1519d30cc Mon Sep 17 00:00:00 2001 From: badrogger Date: Wed, 20 Aug 2025 19:45:30 +0100 Subject: [PATCH 071/198] Add 311 port to ServicePort --- node_cli/core/nftables.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/node_cli/core/nftables.py b/node_cli/core/nftables.py index 50afff82..042ff2e1 100644 --- a/node_cli/core/nftables.py +++ b/node_cli/core/nftables.py @@ -22,9 +22,9 @@ import os import shutil import sys +from dataclasses import dataclass from pathlib import Path from typing import Optional -from dataclasses import dataclass from node_cli.configs import ( ENV, @@ -44,7 +44,8 @@ class ServicePort: DNS: int = 53 CADVISOR: int = 9100 EXPORTER: int = 8080 - WATCHDOG: int = 3009 + WATCHDOG_HTTP: int = 3009 + WATCHDOG_HTTPS: int = 311 HTTPS: int = 443 HTTP: int = 80 @@ -561,7 +562,8 @@ def setup_firewall(self, enable_monitoring: bool = False) -> None: ServicePort.DNS, ServicePort.HTTPS, ServicePort.HTTP, - ServicePort.WATCHDOG, + ServicePort.WATCHDOG_HTTP, + ServicePort.WATCHDOG_HTTPS, ] if enable_monitoring: tcp_ports.extend([ServicePort.EXPORTER, ServicePort.CADVISOR]) @@ -601,7 +603,8 @@ def cleanup_legacy_rules(self, ssh: bool = False, dns: bool = False) -> None: self.remove_drop_rule('udp') tcp_ports = [ ServicePort.HTTPS, - ServicePort.WATCHDOG, + ServicePort.WATCHDOG_HTTP, + ServicePort.WATCHDOG_HTTPS, ServicePort.EXPORTER, ServicePort.CADVISOR, ServicePort.DNS, # tcp is redundant, making sure it's removed From c65ec3616159118531ee5eb3ae22767d5f622b54 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Thu, 21 Aug 2025 17:50:47 +0100 Subject: [PATCH 072/198] Rename disk mountpoint to block device --- .gitignore | 2 +- README.md | 16 ++++++++-------- node_cli/cli/__init__.py | 2 +- node_cli/configs/user.py | 2 +- node_cli/core/node.py | 2 +- node_cli/operations/base.py | 24 ++++++++++++------------ node_cli/operations/docker_lvmpy.py | 4 ++-- node_cli/operations/fair.py | 12 ++++++------ node_cli/operations/volume.py | 2 +- tests/conftest.py | 10 +++++----- 10 files changed, 38 insertions(+), 38 deletions(-) diff --git a/.gitignore b/.gitignore index c1b8fee2..87a0d8f8 100644 --- a/.gitignore +++ b/.gitignore @@ -113,7 +113,7 @@ node_cli/cli/info.py meta.json -disk_mountpoint.txt +block_device.txt sgx_server_url.txt resource_allocation.json conf.json diff --git a/README.md b/README.md index 558bc61a..0c811957 100644 --- a/README.md +++ b/README.md @@ -157,7 +157,7 @@ Arguments: Required environment variables in `ENV_FILE`: * `SGX_SERVER_URL` - SGX server URL. -* `DISK_MOUNTPOINT` - Mount point for storing sChains data. +* `BLOCK_DEVICE` - Absolute path to a dedicated raw block device (e.g. /dev/sdc) * `DOCKER_LVMPY_STREAM` - Stream of `docker-lvmpy` to use. * `NODE_VERSION` - Stream of `skale-node` to use. * `ENDPOINT` - RPC endpoint of the network where SKALE Manager is deployed. @@ -566,7 +566,7 @@ Arguments: Required environment variables in `ENV_FILE`: -* `DISK_MOUNTPOINT` - Mount point for storing sChain data. +* `BLOCK_DEVICE` - Absolute path to a dedicated raw block device (e.g. /dev/sdc). * `DOCKER_LVMPY_STREAM` - Stream of `docker-lvmpy`. * `NODE_VERSION` - Stream of `skale-node`. * `ENDPOINT` - RPC endpoint of the network where SKALE Manager is deployed. @@ -679,7 +679,7 @@ Arguments: Required environment variables in `ENV_FILE`: * `SGX_SERVER_URL` - SGX server URL. -* `DISK_MOUNTPOINT` - Mount point for storing data (BTRFS recommended). +* `BLOCK_DEVICE` - Absolute path to a dedicated raw block device (e.g. /dev/sdc). * `NODE_VERSION` - Stream of `skale-node` configs. * `ENDPOINT` - RPC endpoint of the network where Fair Manager is deployed. * `MANAGER_CONTRACTS` - SKALE Manager alias or address. @@ -733,7 +733,7 @@ Arguments: Required environment variables in `ENV_FILE`: * `SGX_SERVER_URL` - SGX server URL. -* `DISK_MOUNTPOINT` - Mount point for storing data (BTRFS recommended). +* `BLOCK_DEVICE` - Absolute path to a dedicated raw block device (e.g. /dev/sdc). * `NODE_VERSION` - Stream of `skale-node` configs. * `ENDPOINT` - RPC endpoint of the network where Fair Manager is deployed. * `MANAGER_CONTRACTS` - SKALE Manager alias or address. @@ -786,7 +786,7 @@ Required environment variables in `ENV_FILEPATH`: * `NODE_VERSION` - Stream of `skale-node` configs (e.g., `fair-main`). * `BOOT_ENDPOINT` - RPC endpoint of the Fair network (e.g., `https://rpc.fair.cloud/`). * `SGX_SERVER_URL` - SGX server URL (e.g., `https://127.0.0.1:1026/`). -* `DISK_MOUNTPOINT` - Mount point for storing data (e.g., `/dev/sdc`). +* `BLOCK_DEVICE` - Absolute path to a dedicated raw block device (e.g., `/dev/sdc`). * `ENV_TYPE` - Environment type (e.g., `mainnet`). Optional variables: @@ -824,7 +824,7 @@ Required environment variables in `ENV_FILEPATH`: * `NODE_VERSION` - Stream of `skale-node` configs (e.g., `fair-main`). * `BOOT_ENDPOINT` - RPC endpoint of the Fair network (e.g., `https://rpc.fair.cloud/`). * `SGX_SERVER_URL` - SGX server URL (e.g., `https://127.0.0.1:1026/`). -* `DISK_MOUNTPOINT` - Mount point for storing data (e.g., `/dev/sdc`). +* `BLOCK_DEVICE` - Absolute path to a dedicated raw block device (e.g., `/dev/sdc`). * `ENV_TYPE` - Environment type (e.g., `mainnet`). Optional variables: @@ -855,7 +855,7 @@ Required environment variables in `ENV_FILEPATH`: * `NODE_VERSION` - Stream of `skale-node` configs (e.g., `fair-main`). * `BOOT_ENDPOINT` - RPC endpoint of the Fair network (e.g., `https://rpc.fair.cloud/`). * `SGX_SERVER_URL` - SGX server URL (e.g., `https://127.0.0.1:1026/`). -* `DISK_MOUNTPOINT` - Mount point for storing data (e.g., `/dev/sdc`). +* `BLOCK_DEVICE` - Absolute path to a dedicated raw block device (e.g., `/dev/sdc`). * `ENV_TYPE` - Environment type (e.g., `mainnet`). Optional variables: @@ -1100,7 +1100,7 @@ Required environment variables in `ENV_FILEPATH`: * `FAIR_CONTRACTS` - Fair Manager contracts alias or address. * `NODE_VERSION` - Stream of `skale-node` configs. * `BOOT_ENDPOINT` - RPC endpoint of Fair network. -* `DISK_MOUNTPOINT` - Mount point for storing chain data. +* `BLOCK_DEVICE` - Absolute path to a dedicated raw block device (e.g., `/dev/sdc`). * `ENV_TYPE` - Environment type (e.g., `mainnet`, `devnet`). Options: diff --git a/node_cli/cli/__init__.py b/node_cli/cli/__init__.py index e2eb22f4..46419e54 100644 --- a/node_cli/cli/__init__.py +++ b/node_cli/cli/__init__.py @@ -1,4 +1,4 @@ -__version__ = '3.1.0' +__version__ = '3.2.0' if __name__ == '__main__': print(__version__) diff --git a/node_cli/configs/user.py b/node_cli/configs/user.py index 2fd8145e..9243a5fb 100644 --- a/node_cli/configs/user.py +++ b/node_cli/configs/user.py @@ -53,7 +53,7 @@ class BaseUserConfig(ABC): node_version: str env_type: str filebeat_host: str - disk_mountpoint: str + block_device: str container_configs_dir: str = '' skip_docker_config: str = '' diff --git a/node_cli/core/node.py b/node_cli/core/node.py index 1437c86a..3253f6c5 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -540,7 +540,7 @@ def run_checks( if disk is None: env_config = get_validated_user_config(node_type=node_type, node_mode=node_mode) - disk = env_config.disk_mountpoint + disk = env_config.block_device failed_checks = run_host_checks(disk, node_type, node_mode, network, container_config_path) if not failed_checks: print('Requirements checking successfully finished!') diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index a856ef6f..f99fbc6b 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -82,7 +82,7 @@ def checked_host(func): def wrapper(env_filepath: str, env: Dict, node_mode: NodeMode, *args, **kwargs): download_skale_node(env.get('NODE_VERSION'), env.get('CONTAINER_CONFIGS_DIR')) failed_checks = run_host_checks( - env['DISK_MOUNTPOINT'], + env['BLOCK_DEVICE'], TYPE, node_mode, env['ENV_TYPE'], @@ -98,7 +98,7 @@ def wrapper(env_filepath: str, env: Dict, node_mode: NodeMode, *args, **kwargs): return result failed_checks = run_host_checks( - env['DISK_MOUNTPOINT'], + env['BLOCK_DEVICE'], TYPE, node_mode, env['ENV_TYPE'], @@ -160,7 +160,7 @@ def update(env_filepath: str, env: Dict, node_type: NodeType, node_mode: NodeMod def update_fair_boot(env_filepath: str, env: Dict, node_mode: NodeMode = NodeMode.ACTIVE) -> bool: compose_rm(node_type=NodeType.FAIR, node_mode=node_mode, env=env) remove_dynamic_containers() - cleanup_volume_artifacts(env['DISK_MOUNTPOINT']) + cleanup_volume_artifacts(env['BLOCK_DEVICE']) sync_skale_node() ensure_btrfs_kernel_module_autoloaded() @@ -172,7 +172,7 @@ def update_fair_boot(env_filepath: str, env: Dict, node_mode: NodeMode = NodeMod configure_nftables(enable_monitoring=enable_monitoring) generate_nginx_config() - prepare_block_device(env['DISK_MOUNTPOINT'], force=env['ENFORCE_BTRFS'] == 'True') + prepare_block_device(env['BLOCK_DEVICE'], force=env['ENFORCE_BTRFS'] == 'True') prepare_host(env_filepath, env['ENV_TYPE']) @@ -237,7 +237,7 @@ def init(env_filepath: str, env: dict, node_type: NodeType, node_mode: NodeMode) @checked_host def init_fair_boot(env_filepath: str, env: dict, node_mode: NodeMode = NodeMode.ACTIVE) -> None: sync_skale_node() - cleanup_volume_artifacts(env['DISK_MOUNTPOINT']) + cleanup_volume_artifacts(env['BLOCK_DEVICE']) ensure_btrfs_kernel_module_autoloaded() if env.get('SKIP_DOCKER_CONFIG') != 'True': @@ -253,7 +253,7 @@ def init_fair_boot(env_filepath: str, env: dict, node_mode: NodeMode = NodeMode. configure_filebeat() configure_flask() generate_nginx_config() - prepare_block_device(env['DISK_MOUNTPOINT'], force=env['ENFORCE_BTRFS'] == 'True') + prepare_block_device(env['BLOCK_DEVICE'], force=env['ENFORCE_BTRFS'] == 'True') meta_manager = FairCliMetaManager() meta_manager.update_meta( @@ -275,7 +275,7 @@ def init_passive( snapshot: bool, snapshot_from: Optional[str], ) -> None: - cleanup_volume_artifacts(env['DISK_MOUNTPOINT']) + cleanup_volume_artifacts(env['BLOCK_DEVICE']) download_skale_node(env.get('NODE_VERSION'), env.get('CONTAINER_CONFIGS_DIR')) sync_skale_node() @@ -296,7 +296,7 @@ def init_passive( link_env_file() generate_nginx_config() - prepare_block_device(env['DISK_MOUNTPOINT'], force=env['ENFORCE_BTRFS'] == 'True') + prepare_block_device(env['BLOCK_DEVICE'], force=env['ENFORCE_BTRFS'] == 'True') meta_manager = CliMetaManager() meta_manager.update_meta( @@ -320,7 +320,7 @@ def init_passive( def update_passive(env_filepath: str, env: Dict) -> bool: compose_rm(env=env, node_type=NodeType.SKALE, node_mode=NodeMode.PASSIVE) remove_dynamic_containers() - cleanup_volume_artifacts(env['DISK_MOUNTPOINT']) + cleanup_volume_artifacts(env['BLOCK_DEVICE']) download_skale_node(env['NODE_VERSION'], env.get('CONTAINER_CONFIGS_DIR')) sync_skale_node() @@ -332,7 +332,7 @@ def update_passive(env_filepath: str, env: Dict) -> bool: ensure_filestorage_mapping() - prepare_block_device(env['DISK_MOUNTPOINT'], force=env['ENFORCE_BTRFS'] == 'True') + prepare_block_device(env['BLOCK_DEVICE'], force=env['ENFORCE_BTRFS'] == 'True') generate_nginx_config() prepare_host(env_filepath, env['ENV_TYPE'], allocation=True) @@ -381,7 +381,7 @@ def restore(env, backup_path, node_type: NodeType, config_only=False): node_mode = upsert_node_mode(node_mode=NodeMode.ACTIVE) unpack_backup_archive(backup_path) failed_checks = run_host_checks( - env['DISK_MOUNTPOINT'], + env['BLOCK_DEVICE'], TYPE, node_mode, env['ENV_TYPE'], @@ -416,7 +416,7 @@ def restore(env, backup_path, node_type: NodeType, config_only=False): compose_up(env=env, node_type=node_type, node_mode=node_mode) failed_checks = run_host_checks( - env['DISK_MOUNTPOINT'], + env['BLOCK_DEVICE'], TYPE, node_mode, env['ENV_TYPE'], diff --git a/node_cli/operations/docker_lvmpy.py b/node_cli/operations/docker_lvmpy.py index 6e28d58c..0b01c249 100644 --- a/node_cli/operations/docker_lvmpy.py +++ b/node_cli/operations/docker_lvmpy.py @@ -41,7 +41,7 @@ def update_docker_lvmpy_env(env): - env['PHYSICAL_VOLUME'] = env['DISK_MOUNTPOINT'] + env['PHYSICAL_VOLUME'] = env['BLOCK_DEVICE'] env['VOLUME_GROUP'] = 'schains' env['FILESTORAGE_MAPPING'] = FILESTORAGE_MAPPING env['MNT_DIR'] = SCHAINS_MNT_DIR_REGULAR @@ -64,7 +64,7 @@ def lvmpy_install(env): ensure_filestorage_mapping() logging.info('Configuring and starting lvmpy') setup_lvmpy( - block_device=env['DISK_MOUNTPOINT'], volume_group=VOLUME_GROUP, exec_start=LVMPY_RUN_CMD + block_device=env['BLOCK_DEVICE'], volume_group=VOLUME_GROUP, exec_start=LVMPY_RUN_CMD ) init_healing_cron() logger.info('docker-lvmpy is configured and started') diff --git a/node_cli/operations/fair.py b/node_cli/operations/fair.py index 37274d79..50f87c1d 100644 --- a/node_cli/operations/fair.py +++ b/node_cli/operations/fair.py @@ -90,7 +90,7 @@ def init( ) -> bool: sync_skale_node() ensure_btrfs_kernel_module_autoloaded() - cleanup_volume_artifacts(env['DISK_MOUNTPOINT']) + cleanup_volume_artifacts(env['BLOCK_DEVICE']) if env.get('SKIP_DOCKER_CONFIG') != 'True': configure_docker() @@ -103,7 +103,7 @@ def init( prepare_host(env_filepath, env_type=env['ENV_TYPE']) link_env_file() - prepare_block_device(env['DISK_MOUNTPOINT'], force=env['ENFORCE_BTRFS'] == 'True') + prepare_block_device(env['BLOCK_DEVICE'], force=env['ENFORCE_BTRFS'] == 'True') update_images(env=env, node_type=NodeType.FAIR, node_mode=node_mode) compose_up( @@ -135,7 +135,7 @@ def init( def update_fair_boot(env_filepath: str, env: dict, node_mode: NodeMode = NodeMode.ACTIVE) -> bool: compose_rm(node_type=NodeType.FAIR, node_mode=node_mode, env=env) remove_dynamic_containers() - cleanup_volume_artifacts(env['DISK_MOUNTPOINT']) + cleanup_volume_artifacts(env['BLOCK_DEVICE']) sync_skale_node() ensure_btrfs_kernel_module_autoloaded() @@ -147,7 +147,7 @@ def update_fair_boot(env_filepath: str, env: dict, node_mode: NodeMode = NodeMod configure_nftables(enable_monitoring=enable_monitoring) generate_nginx_config() - prepare_block_device(env['DISK_MOUNTPOINT'], force=env['ENFORCE_BTRFS'] == 'True') + prepare_block_device(env['BLOCK_DEVICE'], force=env['ENFORCE_BTRFS'] == 'True') prepare_host(env_filepath, env['ENV_TYPE']) @@ -234,7 +234,7 @@ def update( def restore(node_mode: NodeMode, env, backup_path, config_only=False): unpack_backup_archive(backup_path) failed_checks = run_host_checks( - env['DISK_MOUNTPOINT'], + env['BLOCK_DEVICE'], TYPE, node_mode, env['ENV_TYPE'], @@ -267,7 +267,7 @@ def restore(node_mode: NodeMode, env, backup_path, config_only=False): compose_up(env=env, node_type=NodeType.FAIR, node_mode=node_mode) failed_checks = run_host_checks( - env['DISK_MOUNTPOINT'], + env['BLOCK_DEVICE'], TYPE, node_mode, env['ENV_TYPE'], diff --git a/node_cli/operations/volume.py b/node_cli/operations/volume.py index 1595a442..49c1898f 100644 --- a/node_cli/operations/volume.py +++ b/node_cli/operations/volume.py @@ -42,7 +42,7 @@ class FilesystemExistsError(Exception): def update_docker_lvmpy_env(env): - env['PHYSICAL_VOLUME'] = env['DISK_MOUNTPOINT'] + env['PHYSICAL_VOLUME'] = env['BLOCK_DEVICE'] env['VOLUME_GROUP'] = 'schains' env['FILESTORAGE_MAPPING'] = FILESTORAGE_MAPPING env['SCHAINS_MNT_DIR'] = SCHAINS_MNT_DIR_REGULAR diff --git a/tests/conftest.py b/tests/conftest.py index 047e35a0..c6733957 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -297,7 +297,7 @@ def valid_env_params(): 'NODE_VERSION': 'master', 'FILEBEAT_HOST': '127.0.0.1:3010', 'SGX_SERVER_URL': 'http://127.0.0.1', - 'DISK_MOUNTPOINT': '/dev/sss', + 'BLOCK_DEVICE': '/dev/sss', 'DOCKER_LVMPY_STREAM': 'master', 'ENV_TYPE': 'devnet', 'SCHAIN_NAME': 'test', @@ -362,7 +362,7 @@ def regular_user_conf(tmp_path): NODE_VERSION='main' FILEBEAT_HOST=127.0.0.1:3010 SGX_SERVER_URL=http://127.0.0.1 - DISK_MOUNTPOINT=/dev/sss + BLOCK_DEVICE=/dev/sss DOCKER_LVMPY_STREAM='master' ENV_TYPE='devnet' MANAGER_CONTRACTS='test-manager' @@ -384,7 +384,7 @@ def fair_user_conf(tmp_path): NODE_VERSION='main' FILEBEAT_HOST=127.0.0.1:3010 SGX_SERVER_URL=http://127.0.0.1 - DISK_MOUNTPOINT=/dev/sss + BLOCK_DEVICE=/dev/sss ENV_TYPE='devnet' ENFORCE_BTRFS=False FAIR_CONTRACTS='test-fair' @@ -405,7 +405,7 @@ def fair_boot_user_conf(tmp_path): NODE_VERSION='main' FILEBEAT_HOST=127.0.0.1:3010 SGX_SERVER_URL=http://127.0.0.1 - DISK_MOUNTPOINT=/dev/sss + BLOCK_DEVICE=/dev/sss ENV_TYPE='devnet' MANAGER_CONTRACTS='test-manager' IMA_CONTRACTS='test-ima' @@ -425,7 +425,7 @@ def passive_user_conf(tmp_path): ENDPOINT=http://localhost:8545 NODE_VERSION='main' FILEBEAT_HOST=127.0.0.1:3010 - DISK_MOUNTPOINT=/dev/sss + BLOCK_DEVICE=/dev/sss ENV_TYPE='devnet' SCHAIN_NAME='test-schain' ENFORCE_BTRFS=False From 16f8e8e50c59b7062c82ab4333c308626b69a868 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Thu, 21 Aug 2025 18:03:26 +0100 Subject: [PATCH 073/198] rename docker lvmpy version, update readme --- README.md | 36 ++++++++++----------------- node_cli/configs/user.py | 2 +- node_cli/operations/base.py | 10 ++++---- node_cli/operations/docker_lvmpy.py | 2 +- node_cli/operations/volume.py | 2 +- node_cli/utils/meta.py | 12 ++++----- node_cli/utils/print_formatters.py | 2 +- tests/cli/node_test.py | 2 +- tests/conftest.py | 4 +-- tests/helper.py | 4 +-- tests/tools_meta_test.py | 38 ++++++++++++++--------------- 11 files changed, 52 insertions(+), 62 deletions(-) diff --git a/README.md b/README.md index 0c811957..808e1b02 100644 --- a/README.md +++ b/README.md @@ -44,9 +44,9 @@ SKALE Node CLI, part of the SKALE suite of validator tools, is the command line Ensure that the following packages are installed: **docker**, **docker-compose** (1.27.4+) -### Standard Node Binary +### SKALE Node Binary -This binary (`skale-VERSION-OS`) is used for managing standard SKALE validator nodes. +This binary (`skale-VERSION-OS`) is used for managing SKALE validator nodes. ```shell # Replace {version} with the desired release version (e.g., 3.0.0) @@ -54,19 +54,9 @@ CLI_VERSION={version} && \ sudo -E bash -c "curl -L https://github.com/skalenetwork/node-cli/releases/download/$CLI_VERSION/skale-$CLI_VERSION-`uname -s`-`uname -m` > /usr/local/bin/skale" ``` -### Passive Node Binary - -This binary (`skale-VERSION-OS-passive`) is used for managing dedicated Passive nodes. **Ensure you download the correct `-passive` suffixed binary for Passive node operations.** - -```shell -# Replace {version} with the desired release version (e.g., 3.0.0) -CLI_VERSION={version} && \ -sudo -E bash -c "curl -L https://github.com/skalenetwork/node-cli/releases/download/$CLI_VERSION/skale-$CLI_VERSION-`uname -s`-`uname -m`-passive > /usr/local/bin/skale" -``` - ### Fair Node Binary -This binary (`skale-VERSION-OS-fair`) is used specifically for managing nodes on the Fair network. +This binary (`skale-VERSION-OS-fair`) is used for managing nodes on the Fair network. ```shell # Replace {version} with the desired release version (e.g., 3.0.0) @@ -158,8 +148,8 @@ Required environment variables in `ENV_FILE`: * `SGX_SERVER_URL` - SGX server URL. * `BLOCK_DEVICE` - Absolute path to a dedicated raw block device (e.g. /dev/sdc) -* `DOCKER_LVMPY_STREAM` - Stream of `docker-lvmpy` to use. -* `NODE_VERSION` - Stream of `skale-node` to use. +* `DOCKER_LVMPY_VERSION` - Version of `docker-lvmpy`. +* `NODE_VERSION` - Version of `skale-node`. * `ENDPOINT` - RPC endpoint of the network where SKALE Manager is deployed. * `MANAGER_CONTRACTS` - SKALE Manager `message_proxy_mainnet` contract alias or address. * `IMA_CONTRACTS` - IMA `skale_manager` contract alias or address. @@ -567,8 +557,8 @@ Arguments: Required environment variables in `ENV_FILE`: * `BLOCK_DEVICE` - Absolute path to a dedicated raw block device (e.g. /dev/sdc). -* `DOCKER_LVMPY_STREAM` - Stream of `docker-lvmpy`. -* `NODE_VERSION` - Stream of `skale-node`. +* `DOCKER_LVMPY_VERSION` - Version of `docker-lvmpy`. +* `NODE_VERSION` - Version of `skale-node`. * `ENDPOINT` - RPC endpoint of the network where SKALE Manager is deployed. * `MANAGER_CONTRACTS` - SKALE Manager alias or address. * `IMA_CONTRACTS` - IMA alias or address. @@ -680,7 +670,7 @@ Required environment variables in `ENV_FILE`: * `SGX_SERVER_URL` - SGX server URL. * `BLOCK_DEVICE` - Absolute path to a dedicated raw block device (e.g. /dev/sdc). -* `NODE_VERSION` - Stream of `skale-node` configs. +* `NODE_VERSION` - Version of `skale-node`. * `ENDPOINT` - RPC endpoint of the network where Fair Manager is deployed. * `MANAGER_CONTRACTS` - SKALE Manager alias or address. * `IMA_CONTRACTS` - IMA alias or address (*Note: Required by boot service, may not be used by Fair itself*). @@ -734,7 +724,7 @@ Required environment variables in `ENV_FILE`: * `SGX_SERVER_URL` - SGX server URL. * `BLOCK_DEVICE` - Absolute path to a dedicated raw block device (e.g. /dev/sdc). -* `NODE_VERSION` - Stream of `skale-node` configs. +* `NODE_VERSION` - Version of `skale-node`. * `ENDPOINT` - RPC endpoint of the network where Fair Manager is deployed. * `MANAGER_CONTRACTS` - SKALE Manager alias or address. * `IMA_CONTRACTS` - IMA alias or address (*Note: Required by boot service, may not be used by Fair itself*). @@ -783,7 +773,7 @@ Arguments: Required environment variables in `ENV_FILEPATH`: * `FAIR_CONTRACTS` - Fair contracts alias or address (e.g., `mainnet`). -* `NODE_VERSION` - Stream of `skale-node` configs (e.g., `fair-main`). +* `NODE_VERSION` - Version of `skale-node`. * `BOOT_ENDPOINT` - RPC endpoint of the Fair network (e.g., `https://rpc.fair.cloud/`). * `SGX_SERVER_URL` - SGX server URL (e.g., `https://127.0.0.1:1026/`). * `BLOCK_DEVICE` - Absolute path to a dedicated raw block device (e.g., `/dev/sdc`). @@ -821,7 +811,7 @@ Arguments: Required environment variables in `ENV_FILEPATH`: * `FAIR_CONTRACTS` - Fair contracts alias or address (e.g., `mainnet`). -* `NODE_VERSION` - Stream of `skale-node` configs (e.g., `fair-main`). +* `NODE_VERSION` - Version of `skale-node`. * `BOOT_ENDPOINT` - RPC endpoint of the Fair network (e.g., `https://rpc.fair.cloud/`). * `SGX_SERVER_URL` - SGX server URL (e.g., `https://127.0.0.1:1026/`). * `BLOCK_DEVICE` - Absolute path to a dedicated raw block device (e.g., `/dev/sdc`). @@ -852,7 +842,7 @@ Arguments: Required environment variables in `ENV_FILEPATH`: * `FAIR_CONTRACTS` - Fair contracts alias or address (e.g., `mainnet`). -* `NODE_VERSION` - Stream of `skale-node` configs (e.g., `fair-main`). +* `NODE_VERSION` - Version of `skale-node`. * `BOOT_ENDPOINT` - RPC endpoint of the Fair network (e.g., `https://rpc.fair.cloud/`). * `SGX_SERVER_URL` - SGX server URL (e.g., `https://127.0.0.1:1026/`). * `BLOCK_DEVICE` - Absolute path to a dedicated raw block device (e.g., `/dev/sdc`). @@ -1098,7 +1088,7 @@ Arguments: Required environment variables in `ENV_FILEPATH`: * `FAIR_CONTRACTS` - Fair Manager contracts alias or address. -* `NODE_VERSION` - Stream of `skale-node` configs. +* `NODE_VERSION` - Version of `skale-node`. * `BOOT_ENDPOINT` - RPC endpoint of Fair network. * `BLOCK_DEVICE` - Absolute path to a dedicated raw block device (e.g., `/dev/sdc`). * `ENV_TYPE` - Environment type (e.g., `mainnet`, `devnet`). diff --git a/node_cli/configs/user.py b/node_cli/configs/user.py index 9243a5fb..3aa7b831 100644 --- a/node_cli/configs/user.py +++ b/node_cli/configs/user.py @@ -117,7 +117,7 @@ class SkaleUserConfig(BaseUserConfig): endpoint: str manager_contracts: str ima_contracts: str - docker_lvmpy_stream: str + docker_lvmpy_version: str sgx_server_url: str monitoring_containers: str = '' telegraf: str = '' diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index f99fbc6b..2380d91c 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -147,7 +147,7 @@ def update(env_filepath: str, env: Dict, node_type: NodeType, node_mode: NodeMod meta_manager.update_meta( VERSION, env['NODE_VERSION'], - env['DOCKER_LVMPY_STREAM'], + env['DOCKER_LVMPY_VERSION'], distro.id(), distro.version(), ) @@ -224,7 +224,7 @@ def init(env_filepath: str, env: dict, node_type: NodeType, node_mode: NodeMode) meta_manager.update_meta( VERSION, env['NODE_VERSION'], - env['DOCKER_LVMPY_STREAM'], + env['DOCKER_LVMPY_VERSION'], distro.id(), distro.version(), ) @@ -341,7 +341,7 @@ def update_passive(env_filepath: str, env: Dict) -> bool: meta_manager.update_meta( VERSION, env['NODE_VERSION'], - env['DOCKER_LVMPY_STREAM'], + env['DOCKER_LVMPY_VERSION'], distro.id(), distro.version(), ) @@ -363,7 +363,7 @@ def turn_on(env: dict, node_type: NodeType, node_mode: NodeMode) -> None: meta_manager.update_meta( VERSION, env['NODE_VERSION'], - env['DOCKER_LVMPY_STREAM'], + env['DOCKER_LVMPY_VERSION'], distro.id(), distro.version(), ) @@ -408,7 +408,7 @@ def restore(env, backup_path, node_type: NodeType, config_only=False): meta_manager.update_meta( VERSION, env['NODE_VERSION'], - env['DOCKER_LVMPY_STREAM'], + env['DOCKER_LVMPY_VERSION'], distro.id(), distro.version(), ) diff --git a/node_cli/operations/docker_lvmpy.py b/node_cli/operations/docker_lvmpy.py index 0b01c249..fb70e480 100644 --- a/node_cli/operations/docker_lvmpy.py +++ b/node_cli/operations/docker_lvmpy.py @@ -57,7 +57,7 @@ def ensure_filestorage_mapping(mapping_dir=FILESTORAGE_MAPPING): def sync_docker_lvmpy_repo(env): if os.path.isdir(DOCKER_LVMPY_PATH): shutil.rmtree(DOCKER_LVMPY_PATH) - sync_repo(DOCKER_LVMPY_REPO_URL, DOCKER_LVMPY_PATH, env['DOCKER_LVMPY_STREAM']) + sync_repo(DOCKER_LVMPY_REPO_URL, DOCKER_LVMPY_PATH, env['DOCKER_LVMPY_VERSION']) def lvmpy_install(env): diff --git a/node_cli/operations/volume.py b/node_cli/operations/volume.py index 49c1898f..e944547a 100644 --- a/node_cli/operations/volume.py +++ b/node_cli/operations/volume.py @@ -58,7 +58,7 @@ def ensure_filestorage_mapping(mapping_dir=FILESTORAGE_MAPPING): def sync_docker_lvmpy_repo(env): if os.path.isdir(DOCKER_LVMPY_PATH): shutil.rmtree(DOCKER_LVMPY_PATH) - sync_repo(DOCKER_LVMPY_REPO_URL, DOCKER_LVMPY_PATH, env['DOCKER_LVMPY_STREAM']) + sync_repo(DOCKER_LVMPY_REPO_URL, DOCKER_LVMPY_PATH, env['DOCKER_LVMPY_VERSION']) def docker_lvmpy_update(env): diff --git a/node_cli/utils/meta.py b/node_cli/utils/meta.py index 0dafd54d..bb8ad8af 100644 --- a/node_cli/utils/meta.py +++ b/node_cli/utils/meta.py @@ -7,7 +7,7 @@ DEFAULT_VERSION = '1.0.0' DEFAULT_CONFIG_STREAM = '1.1.0' -DEFAULT_DOCKER_LVMPY_STREAM = '1.0.0' +DEFAULT_DOCKER_LVMPY_VERSION = '1.0.0' DEFAULT_OS_ID = 'ubuntu' DEFAULT_OS_VERSION = '18.04' @@ -26,13 +26,13 @@ def asdict(self) -> dict: @dataclass class CliMeta(CliMetaBase): - docker_lvmpy_stream: str = DEFAULT_DOCKER_LVMPY_STREAM + docker_lvmpy_version: str = DEFAULT_DOCKER_LVMPY_VERSION def asdict(self) -> dict: return { 'version': self.version, 'config_stream': self.config_stream, - 'docker_lvmpy_stream': self.docker_lvmpy_stream, + 'docker_lvmpy_version': self.docker_lvmpy_version, 'os_id': self.os_id, 'os_version': self.os_version, } @@ -96,7 +96,7 @@ def get_meta_info(self, raw: bool = False) -> CliMeta | dict | None: def compose_default_meta(self) -> CliMeta: return CliMeta( version=DEFAULT_VERSION, - docker_lvmpy_stream=DEFAULT_DOCKER_LVMPY_STREAM, + docker_lvmpy_version=DEFAULT_DOCKER_LVMPY_VERSION, config_stream=DEFAULT_CONFIG_STREAM, os_id=DEFAULT_OS_ID, os_version=DEFAULT_OS_VERSION, @@ -106,7 +106,7 @@ def update_meta( self, version: str, config_stream: str, - docker_lvmpy_stream: str | None, + docker_lvmpy_version: str | None, os_id: str, os_version: str, ) -> None: @@ -116,7 +116,7 @@ def update_meta( config_stream, os_id, os_version, - docker_lvmpy_stream, + docker_lvmpy_version, ) self.save_meta(meta) diff --git a/node_cli/utils/print_formatters.py b/node_cli/utils/print_formatters.py index 821e51f6..7edec113 100644 --- a/node_cli/utils/print_formatters.py +++ b/node_cli/utils/print_formatters.py @@ -319,7 +319,7 @@ def print_meta_info(meta_info: CliMeta) -> None: {LONG_LINE} Version: {meta_info.version} Config Stream: {meta_info.config_stream} - Lvmpy stream: {meta_info.docker_lvmpy_stream} + Lvmpy stream: {meta_info.docker_lvmpy_version} {LONG_LINE} """) ) diff --git a/tests/cli/node_test.py b/tests/cli/node_test.py index 3361e083..bfd5263b 100644 --- a/tests/cli/node_test.py +++ b/tests/cli/node_test.py @@ -471,5 +471,5 @@ def test_node_version(meta_file_v2): assert result.exit_code == 0 assert ( result.output - == "{'version': '0.1.1', 'config_stream': 'develop', 'docker_lvmpy_stream': '1.1.2'}\n" + == "{'version': '0.1.1', 'config_stream': 'develop', 'docker_lvmpy_version': '1.1.2'}\n" ) diff --git a/tests/conftest.py b/tests/conftest.py index c6733957..5434e697 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -298,7 +298,7 @@ def valid_env_params(): 'FILEBEAT_HOST': '127.0.0.1:3010', 'SGX_SERVER_URL': 'http://127.0.0.1', 'BLOCK_DEVICE': '/dev/sss', - 'DOCKER_LVMPY_STREAM': 'master', + 'DOCKER_LVMPY_VERSION': 'master', 'ENV_TYPE': 'devnet', 'SCHAIN_NAME': 'test', 'ENFORCE_BTRFS': 'False', @@ -363,7 +363,7 @@ def regular_user_conf(tmp_path): FILEBEAT_HOST=127.0.0.1:3010 SGX_SERVER_URL=http://127.0.0.1 BLOCK_DEVICE=/dev/sss - DOCKER_LVMPY_STREAM='master' + DOCKER_LVMPY_VERSION='master' ENV_TYPE='devnet' MANAGER_CONTRACTS='test-manager' IMA_CONTRACTS='test-ima' diff --git a/tests/helper.py b/tests/helper.py index 7bf18a98..6af0af14 100644 --- a/tests/helper.py +++ b/tests/helper.py @@ -34,12 +34,12 @@ TEST_META_V1 = {'version': '0.1.1', 'config_stream': 'develop'} -TEST_META_V2 = {'version': '0.1.1', 'config_stream': 'develop', 'docker_lvmpy_stream': '1.1.2'} +TEST_META_V2 = {'version': '0.1.1', 'config_stream': 'develop', 'docker_lvmpy_version': '1.1.2'} TEST_META_V3 = { 'version': '0.1.1', 'config_stream': 'develop', - 'docker_lvmpy_stream': '1.1.2', + 'docker_lvmpy_version': '1.1.2', 'os_id': 'ubuntu', 'os_version': '18.04', } diff --git a/tests/tools_meta_test.py b/tests/tools_meta_test.py index a2a6a2fc..c82e435c 100644 --- a/tests/tools_meta_test.py +++ b/tests/tools_meta_test.py @@ -17,21 +17,21 @@ def test_get_meta_info_v1(meta_file_v1): meta = CliMetaManager().get_meta_info() assert meta.version == TEST_META_V1['version'] assert meta.config_stream == TEST_META_V1['config_stream'] - assert meta.docker_lvmpy_stream == '1.0.0' + assert meta.docker_lvmpy_version == '1.0.0' def test_get_meta_info_v2(meta_file_v2): meta = CliMetaManager().get_meta_info() assert meta.version == TEST_META_V2['version'] assert meta.config_stream == TEST_META_V2['config_stream'] - assert meta.docker_lvmpy_stream == TEST_META_V2['docker_lvmpy_stream'] + assert meta.docker_lvmpy_version == TEST_META_V2['docker_lvmpy_version'] def test_get_meta_info_v3(meta_file_v3): meta = CliMetaManager().get_meta_info() assert meta.version == TEST_META_V3['version'] assert meta.config_stream == TEST_META_V3['config_stream'] - assert meta.docker_lvmpy_stream == TEST_META_V3['docker_lvmpy_stream'] + assert meta.docker_lvmpy_version == TEST_META_V3['docker_lvmpy_version'] assert meta.os_id == TEST_META_V3['os_id'] assert meta.os_version == TEST_META_V3['os_version'] @@ -45,7 +45,7 @@ def test_compose_default_meta(): meta = CliMetaManager().compose_default_meta() assert meta.version == '1.0.0' assert meta.config_stream == '1.1.0' - assert meta.docker_lvmpy_stream == '1.0.0' + assert meta.docker_lvmpy_version == '1.0.0' assert meta.os_id == 'ubuntu' assert meta.os_version == '18.04' @@ -58,7 +58,7 @@ def test_save_meta(meta_file_v2): assert saved_json == { 'version': '1.1.2', 'config_stream': '2.2.2', - 'docker_lvmpy_stream': '1.0.0', + 'docker_lvmpy_version': '1.0.0', 'os_id': 'ubuntu', 'os_version': '18.04', } @@ -69,14 +69,14 @@ def test_update_meta_from_v2_to_v3(meta_file_v2): CliMetaManager().update_meta( version='3.3.3', config_stream='1.1.1', - docker_lvmpy_stream='1.2.2', + docker_lvmpy_version='1.2.2', os_id='debian', os_version='11', ) meta = CliMetaManager().get_meta_info() assert meta.version == '3.3.3' assert meta.config_stream == '1.1.1' - assert meta.docker_lvmpy_stream == '1.2.2' + assert meta.docker_lvmpy_version == '1.2.2' assert meta.os_id == 'debian' assert meta.os_version == '11' assert meta != old_meta @@ -86,14 +86,14 @@ def test_update_meta_from_v1(meta_file_v1): CliMetaManager().update_meta( version='4.4.4', config_stream='beta', - docker_lvmpy_stream='1.3.3', + docker_lvmpy_version='1.3.3', os_id='debian', os_version='11', ) meta = CliMetaManager().get_meta_info() assert meta.version == '4.4.4' assert meta.config_stream == 'beta' - assert meta.docker_lvmpy_stream == '1.3.3' + assert meta.docker_lvmpy_version == '1.3.3' assert meta.os_id == 'debian' assert meta.os_version == '11' @@ -102,14 +102,14 @@ def test_update_meta_from_v3(meta_file_v3): CliMetaManager().update_meta( version='5.5.5', config_stream='stable', - docker_lvmpy_stream='1.2.3', + docker_lvmpy_version='1.2.3', os_id='ubuntu', os_version='20.04', ) meta = CliMetaManager().get_meta_info() assert meta.version == '5.5.5' assert meta.config_stream == 'stable' - assert meta.docker_lvmpy_stream == '1.2.3' + assert meta.docker_lvmpy_version == '1.2.3' assert meta.os_id == 'ubuntu' assert meta.os_version == '20.04' @@ -156,7 +156,7 @@ def test_fair_compose_default_meta(): assert meta.config_stream == '1.1.0' assert meta.os_id == 'ubuntu' assert meta.os_version == '18.04' - assert not hasattr(meta, 'docker_lvmpy_stream') + assert not hasattr(meta, 'docker_lvmpy_version') def test_fair_save_meta(meta_file_v2): @@ -172,7 +172,7 @@ def test_fair_save_meta(meta_file_v2): 'os_id': 'debian', 'os_version': '11', } - assert 'docker_lvmpy_stream' not in saved_json + assert 'docker_lvmpy_version' not in saved_json def test_fair_update_meta_from_v2_to_v3(meta_file_v2): @@ -237,7 +237,7 @@ def test_fair_get_meta_info_raw(meta_file_v3): assert raw_meta['config_stream'] == TEST_META_V3['config_stream'] assert raw_meta['os_id'] == TEST_META_V3['os_id'] assert raw_meta['os_version'] == TEST_META_V3['os_version'] - assert 'docker_lvmpy_stream' not in raw_meta + assert 'docker_lvmpy_version' not in raw_meta def test_fair_get_meta_info_raw_empty(): @@ -257,7 +257,7 @@ def test_fair_asdict(): 'os_version': '35', } assert meta_dict == expected - assert 'docker_lvmpy_stream' not in meta_dict + assert 'docker_lvmpy_version' not in meta_dict def test_fair_meta_compatibility_with_cli_meta_file(meta_file_v3): @@ -266,21 +266,21 @@ def test_fair_meta_compatibility_with_cli_meta_file(meta_file_v3): assert meta.config_stream == TEST_META_V3['config_stream'] assert meta.os_id == TEST_META_V3['os_id'] assert meta.os_version == TEST_META_V3['os_version'] - # Should not have docker_lvmpy_stream even though it's in the file - assert not hasattr(meta, 'docker_lvmpy_stream') + # Should not have docker_lvmpy_version even though it's in the file + assert not hasattr(meta, 'docker_lvmpy_version') def test_fair_save_meta_overwrites_cli_meta(meta_file_v3): with open(META_FILEPATH) as f: original_data = json.load(f) - assert 'docker_lvmpy_stream' in original_data + assert 'docker_lvmpy_version' in original_data fair_meta = FairCliMeta(version='2.0.0', config_stream='fair-new') FairCliMetaManager().save_meta(fair_meta) with open(META_FILEPATH) as f: saved_data = json.load(f) - assert 'docker_lvmpy_stream' not in saved_data + assert 'docker_lvmpy_version' not in saved_data assert saved_data['version'] == '2.0.0' assert saved_data['config_stream'] == 'fair-new' From 21426c260282df69c0c178a704671b5f71cf38b7 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Tue, 26 Aug 2025 17:07:15 +0100 Subject: [PATCH 074/198] Add staking commands, update README --- README.md | 98 ++++++++++++++++++++++++++++++++- node_cli/cli/staking.py | 88 ++++++++++++++++++++++++++++++ node_cli/configs/routes.py | 10 ++++ node_cli/fair/staking.py | 109 +++++++++++++++++++++++++++++++++++++ node_cli/main.py | 2 + tests/routes_test.py | 8 +++ 6 files changed, 314 insertions(+), 1 deletion(-) create mode 100644 node_cli/cli/staking.py create mode 100644 node_cli/fair/staking.py diff --git a/README.md b/README.md index 808e1b02..e06bd98e 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,8 @@ SKALE Node CLI, part of the SKALE suite of validator tools, is the command line 5. [Fair Wallet commands](#fair-wallet-commands) 6. [Fair Logs commands](#fair-logs-commands) 7. [Fair SSL commands](#fair-ssl-commands) - 8. [Passive Fair Node commands](#passive-fair-node-commands) + 8. [Fair Staking commands](#fair-staking-commands) + 9. [Passive Fair Node commands](#passive-fair-node-commands) 5. [Exit codes](#exit-codes) 6. [Development](#development) @@ -1067,6 +1068,101 @@ Options: * `--no-client` - Skip client connection for openssl check. * `--no-wss` - Skip WSS server starting for skaled check. +### Fair Staking commands + +> Prefix: `fair staking` + +Commands for interacting with the Fair staking functionality. + +#### Add allowed receiver + +Allow an address to receive staking fees. + +```shell +fair staking add-allowed-receiver +``` + +Arguments: + +* `RECEIVER_ADDRESS` - Address to add to the allowed receivers list. + +#### Remove allowed receiver + +Remove an address from the allowed receivers list. + +```shell +fair staking remove-allowed-receiver +``` + +Arguments: + +* `RECEIVER_ADDRESS` - Address to remove from the allowed receivers list. + +#### Send all fees + +Send all accumulated fees to the specified address. + +```shell +fair staking send-all-fees +``` + +Arguments: + +* `TO_ADDRESS` - Destination address to receive all fees. + +#### Claim all fees + +Claim all accumulated fees to the node wallet. + +```shell +fair staking claim-all-fees +``` + +#### Set fee rate + +Set the fee rate (uint16 value) used by the staking logic. + +```shell +fair staking set-fee-rate +``` + +Arguments: + +* `FEE_RATE` - Fee rate value as integer (uint16). + +#### Claim fees + +Claim a specific amount of fees to the node wallet. + +```shell +fair staking claim-fees +``` + +Arguments: + +* `AMOUNT` - Amount of fees to claim (FAIR). + +#### Send fees + +Send a specific amount of fees to an address. + +```shell +fair staking send-fees +``` + +Arguments: + +* `TO_ADDRESS` - Destination address for the fee transfer. +* `AMOUNT` - Amount of fees to send (FAIR). + +#### Get earned fee amount + +Get the currently earned fee amount. + +```shell +fair staking get-earned-fee-amount +``` + ### Passive Fair Node commands > Prefix: `fair passive-node` (passive Fair build) diff --git a/node_cli/cli/staking.py b/node_cli/cli/staking.py new file mode 100644 index 00000000..5286fa38 --- /dev/null +++ b/node_cli/cli/staking.py @@ -0,0 +1,88 @@ +# -*- coding: utf-8 -*- +# +# This file is part of node-cli +# +# Copyright (C) 2025-Present SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import click + +from node_cli.fair.staking import ( + add_allowed_receiver, + remove_allowed_receiver, + send_all_fees, + claim_all_fees, + set_fee_rate, + claim_fees, + send_fees, + get_earned_fee_amount, +) + + +@click.group() +def staking_cli(): + pass + + +@staking_cli.group(help='Staking commands') +def staking(): + pass + + +@staking.command('add-allowed-receiver', help='Add allowed receiver') +@click.argument('receiver') +def _add_allowed_receiver(receiver: str) -> None: + add_allowed_receiver(receiver) + + +@staking.command('remove-allowed-receiver', help='Remove allowed receiver') +@click.argument('receiver') +def _remove_allowed_receiver(receiver: str) -> None: + remove_allowed_receiver(receiver) + + +@staking.command('send-all-fees', help='Send all fees to address') +@click.argument('to') +def _send_all_fees(to: str) -> None: + send_all_fees(to) + + +@staking.command('claim-all-fees', help='Claim all fees') +def _claim_all_fees() -> None: + claim_all_fees() + + +@staking.command('set-fee-rate', help='Set fee rate (uint16)') +@click.argument('fee_rate', type=int) +def _set_fee_rate(fee_rate: int) -> None: + set_fee_rate(fee_rate) + + +@staking.command('claim-fees', help='Claim fees amount (FAIR)') +@click.argument('amount', type=float) +def _claim_fees(amount: float) -> None: + claim_fees(amount) + + +@staking.command('send-fees', help='Send fees to address') +@click.argument('to') +@click.argument('amount', type=float) +def _send_fees(to: str, amount: float) -> None: + send_fees(to, amount) + + +@staking.command('get-earned-fee-amount', help='Get earned fee amount') +def _get_earned_fee_amount() -> None: + get_earned_fee_amount() diff --git a/node_cli/configs/routes.py b/node_cli/configs/routes.py index 7d7d9e9c..6f0829b3 100644 --- a/node_cli/configs/routes.py +++ b/node_cli/configs/routes.py @@ -43,6 +43,16 @@ 'fair-node': ['info', 'register', 'set-domain-name', 'change-ip', 'exit'], 'fair-chain': ['record', 'checks'], 'fair-node-passive': ['setup'], + 'fair-staking': [ + 'add-allowed-receiver', + 'remove-allowed-receiver', + 'send-all-fees', + 'claim-all-fees', + 'set-fee-rate', + 'claim-fees', + 'send-fees', + 'get-earned-fee-amount', + ], } } diff --git a/node_cli/fair/staking.py b/node_cli/fair/staking.py new file mode 100644 index 00000000..771d2b99 --- /dev/null +++ b/node_cli/fair/staking.py @@ -0,0 +1,109 @@ +# -*- coding: utf-8 -*- +# +# This file is part of node-cli +# +# Copyright (C) 2025-Present SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from typing import Any, Optional + +from node_cli.core.host import is_node_inited +from node_cli.utils.exit_codes import CLIExitCodes +from node_cli.utils.helper import error_exit, post_request + +BLUEPRINT_NAME = 'fair-staking' + + +def _handle_response(status: str, payload: Any, success: Optional[str] = None) -> None: + if status == 'ok': + print(success if success is not None else 'OK') + else: + error_exit(payload, exit_code=CLIExitCodes.BAD_API_RESPONSE) + + +def add_allowed_receiver(receiver: str) -> None: + if not is_node_inited(): + print('Node is not initialized') + return + status, payload = post_request( + blueprint=BLUEPRINT_NAME, method='add-allowed-receiver', json={'receiver': receiver} + ) + _handle_response(status, payload, success=f'Allowed receiver added: {receiver}') + + +def remove_allowed_receiver(receiver: str) -> None: + if not is_node_inited(): + print('Node is not initialized') + return + status, payload = post_request( + blueprint=BLUEPRINT_NAME, method='remove-allowed-receiver', json={'receiver': receiver} + ) + _handle_response(status, payload, success=f'Allowed receiver removed: {receiver}') + + +def send_all_fees(to: str) -> None: + if not is_node_inited(): + print('Node is not initialized') + return + status, payload = post_request( + blueprint=BLUEPRINT_NAME, method='send-all-fees', json={'to': to} + ) + _handle_response(status, payload, success=f'All fees sent to {to}') + + +def claim_all_fees() -> None: + if not is_node_inited(): + print('Node is not initialized') + return + status, payload = post_request(blueprint=BLUEPRINT_NAME, method='claim-all-fees') + _handle_response(status, payload, success='All fees claimed') + + +def set_fee_rate(fee_rate: int) -> None: + if not is_node_inited(): + print('Node is not initialized') + return + status, payload = post_request( + blueprint=BLUEPRINT_NAME, method='set-fee-rate', json={'feeRate': fee_rate} + ) + _handle_response(status, payload, success=f'Fee rate set to {fee_rate}') + + +def claim_fees(amount: float) -> None: + if not is_node_inited(): + print('Node is not initialized') + return + status, payload = post_request( + blueprint=BLUEPRINT_NAME, method='claim-fees', json={'amount': amount} + ) + _handle_response(status, payload, success=f'Fees claimed: {amount}') + + +def send_fees(to: str, amount: float) -> None: + if not is_node_inited(): + print('Node is not initialized') + return + status, payload = post_request( + blueprint=BLUEPRINT_NAME, method='send-fees', json={'to': to, 'amount': amount} + ) + _handle_response(status, payload, success=f'Fees sent: {amount} to {to}') + + +def get_earned_fee_amount() -> None: + if not is_node_inited(): + print('Node is not initialized') + return + status, payload = post_request(blueprint=BLUEPRINT_NAME, method='get-earned-fee-amount') + _handle_response(status, payload, success=f'Earned fee amount: {payload}') diff --git a/node_cli/main.py b/node_cli/main.py index 8e734d17..a18b4d67 100644 --- a/node_cli/main.py +++ b/node_cli/main.py @@ -41,6 +41,7 @@ from node_cli.cli.fair_node import fair_node_cli from node_cli.cli.passive_fair_node import passive_fair_node_cli from node_cli.cli.chain import chain_cli +from node_cli.cli.staking import staking_cli from node_cli.core.host import init_logs_dir from node_cli.utils.node_type import NodeType from node_cli.configs import LONG_LINE @@ -92,6 +93,7 @@ def get_sources_list() -> List[click.MultiCommand]: fair_node_cli, passive_fair_node_cli, chain_cli, + staking_cli, wallet_cli, ssl_cli, ] diff --git a/tests/routes_test.py b/tests/routes_test.py index 19845fba..958a4683 100644 --- a/tests/routes_test.py +++ b/tests/routes_test.py @@ -39,6 +39,14 @@ '/api/v1/fair-chain/record', '/api/v1/fair-chain/checks', '/api/v1/fair-node-passive/setup', + '/api/v1/staking/add-allowed-receiver', + '/api/v1/staking/remove-allowed-receiver', + '/api/v1/staking/send-all-fees', + '/api/v1/staking/claim-all-fees', + '/api/v1/staking/set-fee-rate', + '/api/v1/staking/claim-fees', + '/api/v1/staking/send-fees', + '/api/v1/staking/get-earned-fee-amount', ] From e5a4995ed790b863ea5714812f49e13963ef2724 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Tue, 26 Aug 2025 19:08:08 +0100 Subject: [PATCH 075/198] Fix routes test --- node_cli/utils/helper.py | 15 ++++++++++++++- tests/routes_test.py | 16 ++++++++-------- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/node_cli/utils/helper.py b/node_cli/utils/helper.py index 2eb778fb..b0bff97e 100644 --- a/node_cli/utils/helper.py +++ b/node_cli/utils/helper.py @@ -197,13 +197,26 @@ def abort_if_false(ctx, param, value): def post_request(blueprint, method, json=None, files=None): + print('in post request') route = get_route(blueprint, method) url = construct_url(route) + logger.info('url: ') + logger.info(url) try: + logger.info('json: ') + print(json) response = requests.post(url, json=json, files=files) + + logger.info('--------') + logger.info(response) + logger.info(response.status_code) + logger.info(response.reason) + logger.info(response.raw) + logger.info('=========') + data = response.json() except Exception as err: - logger.error('Request failed', exc_info=err) + logger.exception('Request failed', exc_info=err) data = DEFAULT_ERROR_DATA status = data['status'] payload = data['payload'] diff --git a/tests/routes_test.py b/tests/routes_test.py index 958a4683..bfb44d4c 100644 --- a/tests/routes_test.py +++ b/tests/routes_test.py @@ -39,14 +39,14 @@ '/api/v1/fair-chain/record', '/api/v1/fair-chain/checks', '/api/v1/fair-node-passive/setup', - '/api/v1/staking/add-allowed-receiver', - '/api/v1/staking/remove-allowed-receiver', - '/api/v1/staking/send-all-fees', - '/api/v1/staking/claim-all-fees', - '/api/v1/staking/set-fee-rate', - '/api/v1/staking/claim-fees', - '/api/v1/staking/send-fees', - '/api/v1/staking/get-earned-fee-amount', + '/api/v1/fair-staking/add-allowed-receiver', + '/api/v1/fair-staking/remove-allowed-receiver', + '/api/v1/fair-staking/send-all-fees', + '/api/v1/fair-staking/claim-all-fees', + '/api/v1/fair-staking/set-fee-rate', + '/api/v1/fair-staking/claim-fees', + '/api/v1/fair-staking/send-fees', + '/api/v1/fair-staking/get-earned-fee-amount', ] From b3e3e5e81bd042eb79b65c6bdb9cad31e417cf66 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Tue, 26 Aug 2025 20:05:44 +0100 Subject: [PATCH 076/198] Add confirmation to all staking commands --- node_cli/cli/staking.py | 52 +++++++++++++++++++++++++++++++++++++++- node_cli/fair/staking.py | 7 +++++- node_cli/utils/helper.py | 15 +----------- 3 files changed, 58 insertions(+), 16 deletions(-) diff --git a/node_cli/cli/staking.py b/node_cli/cli/staking.py index 5286fa38..c23ad75a 100644 --- a/node_cli/cli/staking.py +++ b/node_cli/cli/staking.py @@ -29,6 +29,7 @@ send_fees, get_earned_fee_amount, ) +from node_cli.utils.helper import abort_if_false @click.group() @@ -43,35 +44,77 @@ def staking(): @staking.command('add-allowed-receiver', help='Add allowed receiver') @click.argument('receiver') +@click.option( + '--yes', + is_flag=True, + callback=abort_if_false, + expose_value=False, + prompt='Are you sure you want to add allowed receiver?', +) def _add_allowed_receiver(receiver: str) -> None: add_allowed_receiver(receiver) @staking.command('remove-allowed-receiver', help='Remove allowed receiver') @click.argument('receiver') +@click.option( + '--yes', + is_flag=True, + callback=abort_if_false, + expose_value=False, + prompt='Are you sure you want to remove allowed receiver?', +) def _remove_allowed_receiver(receiver: str) -> None: remove_allowed_receiver(receiver) @staking.command('send-all-fees', help='Send all fees to address') @click.argument('to') +@click.option( + '--yes', + is_flag=True, + callback=abort_if_false, + expose_value=False, + prompt='Are you sure you want to send all fees?', +) def _send_all_fees(to: str) -> None: send_all_fees(to) @staking.command('claim-all-fees', help='Claim all fees') +@click.option( + '--yes', + is_flag=True, + callback=abort_if_false, + expose_value=False, + prompt='Are you sure you want to claim all fees?', +) def _claim_all_fees() -> None: claim_all_fees() -@staking.command('set-fee-rate', help='Set fee rate (uint16)') +@staking.command('set-fee-rate', help='Set fee rate (uint16, basis points; 25 = 2.5%)') @click.argument('fee_rate', type=int) +@click.option( + '--yes', + is_flag=True, + callback=abort_if_false, + expose_value=False, + prompt='Are you sure you want to set fee rate?', +) def _set_fee_rate(fee_rate: int) -> None: set_fee_rate(fee_rate) @staking.command('claim-fees', help='Claim fees amount (FAIR)') @click.argument('amount', type=float) +@click.option( + '--yes', + is_flag=True, + callback=abort_if_false, + expose_value=False, + prompt='Are you sure you want to claim fees?', +) def _claim_fees(amount: float) -> None: claim_fees(amount) @@ -79,6 +122,13 @@ def _claim_fees(amount: float) -> None: @staking.command('send-fees', help='Send fees to address') @click.argument('to') @click.argument('amount', type=float) +@click.option( + '--yes', + is_flag=True, + callback=abort_if_false, + expose_value=False, + prompt='Are you sure you want to send fees?', +) def _send_fees(to: str, amount: float) -> None: send_fees(to, amount) diff --git a/node_cli/fair/staking.py b/node_cli/fair/staking.py index 771d2b99..9c21dd7c 100644 --- a/node_cli/fair/staking.py +++ b/node_cli/fair/staking.py @@ -106,4 +106,9 @@ def get_earned_fee_amount() -> None: print('Node is not initialized') return status, payload = post_request(blueprint=BLUEPRINT_NAME, method='get-earned-fee-amount') - _handle_response(status, payload, success=f'Earned fee amount: {payload}') + if status == 'ok' and isinstance(payload, dict): + amount_wei = payload.get('amount_wei') + amount_ether = payload.get('amount_ether') + print(f'Earned fee amount: {amount_wei} wei ({amount_ether} FAIR)') + return + error_exit(payload, exit_code=CLIExitCodes.BAD_API_RESPONSE) diff --git a/node_cli/utils/helper.py b/node_cli/utils/helper.py index b0bff97e..d299fbf6 100644 --- a/node_cli/utils/helper.py +++ b/node_cli/utils/helper.py @@ -69,7 +69,7 @@ DEFAULT_ERROR_DATA = { 'status': 'error', - 'payload': 'Request failed. Check skale_api container logs', + 'payload': 'Request failed. Check API container logs', } @@ -197,23 +197,10 @@ def abort_if_false(ctx, param, value): def post_request(blueprint, method, json=None, files=None): - print('in post request') route = get_route(blueprint, method) url = construct_url(route) - logger.info('url: ') - logger.info(url) try: - logger.info('json: ') - print(json) response = requests.post(url, json=json, files=files) - - logger.info('--------') - logger.info(response) - logger.info(response.status_code) - logger.info(response.reason) - logger.info(response.raw) - logger.info('=========') - data = response.json() except Exception as err: logger.exception('Request failed', exc_info=err) From f119f4f837cb0031a28445be2473b4bb42845478 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Wed, 27 Aug 2025 18:12:28 +0100 Subject: [PATCH 077/198] Merge send fees commands, update api routes --- .github/copilot-instructions.md | 3 +- README.md | 20 +++-------- node_cli/cli/staking.py | 29 +++++----------- node_cli/configs/routes.py | 5 ++- node_cli/fair/staking.py | 59 ++++++++++----------------------- tests/routes_test.py | 5 ++- 6 files changed, 37 insertions(+), 84 deletions(-) diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 416cfaf6..a5c8ebdf 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -13,4 +13,5 @@ - check `ruff.toml` for formatting rules - always lint changes using `ruff check` -- tests should be placed in `tests/` directory, follow the existing structure and code style \ No newline at end of file +- tests should be placed in `tests/` directory, follow the existing structure and code style +- to run a test always use `bash scripts/run_tests.sh tests/path_to_test.py -k [TEST_NAME]` command \ No newline at end of file diff --git a/README.md b/README.md index e06bd98e..eab1c67a 100644 --- a/README.md +++ b/README.md @@ -1079,7 +1079,7 @@ Commands for interacting with the Fair staking functionality. Allow an address to receive staking fees. ```shell -fair staking add-allowed-receiver +fair staking add-receiver ``` Arguments: @@ -1091,25 +1091,13 @@ Arguments: Remove an address from the allowed receivers list. ```shell -fair staking remove-allowed-receiver +fair staking remove-receiver ``` Arguments: * `RECEIVER_ADDRESS` - Address to remove from the allowed receivers list. -#### Send all fees - -Send all accumulated fees to the specified address. - -```shell -fair staking send-all-fees -``` - -Arguments: - -* `TO_ADDRESS` - Destination address to receive all fees. - #### Claim all fees Claim all accumulated fees to the node wallet. @@ -1144,7 +1132,7 @@ Arguments: #### Send fees -Send a specific amount of fees to an address. +Send a specific amount of fees to the default allowed receiver. ```shell fair staking send-fees @@ -1153,7 +1141,7 @@ fair staking send-fees Arguments: * `TO_ADDRESS` - Destination address for the fee transfer. -* `AMOUNT` - Amount of fees to send (FAIR). +* `AMOUNT` - Amount of fees to send (FAIR). Use `--all` to send all. #### Get earned fee amount diff --git a/node_cli/cli/staking.py b/node_cli/cli/staking.py index c23ad75a..ddd8a2ab 100644 --- a/node_cli/cli/staking.py +++ b/node_cli/cli/staking.py @@ -22,7 +22,6 @@ from node_cli.fair.staking import ( add_allowed_receiver, remove_allowed_receiver, - send_all_fees, claim_all_fees, set_fee_rate, claim_fees, @@ -42,7 +41,7 @@ def staking(): pass -@staking.command('add-allowed-receiver', help='Add allowed receiver') +@staking.command('add-receiver', help='Add allowed receiver') @click.argument('receiver') @click.option( '--yes', @@ -55,7 +54,7 @@ def _add_allowed_receiver(receiver: str) -> None: add_allowed_receiver(receiver) -@staking.command('remove-allowed-receiver', help='Remove allowed receiver') +@staking.command('remove-receiver', help='Remove allowed receiver') @click.argument('receiver') @click.option( '--yes', @@ -68,19 +67,6 @@ def _remove_allowed_receiver(receiver: str) -> None: remove_allowed_receiver(receiver) -@staking.command('send-all-fees', help='Send all fees to address') -@click.argument('to') -@click.option( - '--yes', - is_flag=True, - callback=abort_if_false, - expose_value=False, - prompt='Are you sure you want to send all fees?', -) -def _send_all_fees(to: str) -> None: - send_all_fees(to) - - @staking.command('claim-all-fees', help='Claim all fees') @click.option( '--yes', @@ -119,9 +105,10 @@ def _claim_fees(amount: float) -> None: claim_fees(amount) -@staking.command('send-fees', help='Send fees to address') +@staking.command('send-fees', help='Send fees to address (or all with --all)') @click.argument('to') -@click.argument('amount', type=float) +@click.argument('value', type=float, required=False) +@click.option('--all', 'send_all', is_flag=True, help='Send all fees to address') @click.option( '--yes', is_flag=True, @@ -129,8 +116,10 @@ def _claim_fees(amount: float) -> None: expose_value=False, prompt='Are you sure you want to send fees?', ) -def _send_fees(to: str, amount: float) -> None: - send_fees(to, amount) +def _send_fees(to: str, value: float | None, send_all: bool) -> None: + if value is None and not send_all: + raise click.UsageError('Provide or use --all') + send_fees(to, None if send_all else value) @staking.command('get-earned-fee-amount', help='Get earned fee amount') diff --git a/node_cli/configs/routes.py b/node_cli/configs/routes.py index 6f0829b3..5eba041f 100644 --- a/node_cli/configs/routes.py +++ b/node_cli/configs/routes.py @@ -44,9 +44,8 @@ 'fair-chain': ['record', 'checks'], 'fair-node-passive': ['setup'], 'fair-staking': [ - 'add-allowed-receiver', - 'remove-allowed-receiver', - 'send-all-fees', + 'add-receiver', + 'remove-receiver', 'claim-all-fees', 'set-fee-rate', 'claim-fees', diff --git a/node_cli/fair/staking.py b/node_cli/fair/staking.py index 9c21dd7c..d9469638 100644 --- a/node_cli/fair/staking.py +++ b/node_cli/fair/staking.py @@ -17,94 +17,71 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from typing import Any, Optional +from typing import Any -from node_cli.core.host import is_node_inited +from node_cli.utils.decorators import check_inited from node_cli.utils.exit_codes import CLIExitCodes from node_cli.utils.helper import error_exit, post_request BLUEPRINT_NAME = 'fair-staking' -def _handle_response(status: str, payload: Any, success: Optional[str] = None) -> None: +def _handle_response(status: str, payload: Any, success: str | None = None) -> None: if status == 'ok': print(success if success is not None else 'OK') else: error_exit(payload, exit_code=CLIExitCodes.BAD_API_RESPONSE) +@check_inited def add_allowed_receiver(receiver: str) -> None: - if not is_node_inited(): - print('Node is not initialized') - return status, payload = post_request( - blueprint=BLUEPRINT_NAME, method='add-allowed-receiver', json={'receiver': receiver} + blueprint=BLUEPRINT_NAME, method='add-receiver', json={'receiver': receiver} ) _handle_response(status, payload, success=f'Allowed receiver added: {receiver}') +@check_inited def remove_allowed_receiver(receiver: str) -> None: - if not is_node_inited(): - print('Node is not initialized') - return status, payload = post_request( - blueprint=BLUEPRINT_NAME, method='remove-allowed-receiver', json={'receiver': receiver} + blueprint=BLUEPRINT_NAME, method='remove-receiver', json={'receiver': receiver} ) _handle_response(status, payload, success=f'Allowed receiver removed: {receiver}') -def send_all_fees(to: str) -> None: - if not is_node_inited(): - print('Node is not initialized') - return - status, payload = post_request( - blueprint=BLUEPRINT_NAME, method='send-all-fees', json={'to': to} - ) - _handle_response(status, payload, success=f'All fees sent to {to}') +@check_inited +def send_fees(to: str, value: float | None) -> None: + json_data: dict[str, Any] = {'to': to} + if value is not None: + json_data['value'] = value + status, payload = post_request(blueprint=BLUEPRINT_NAME, method='send-fees', json=json_data) + _handle_response(status, payload, success=f'Fees sent to {to}') +@check_inited def claim_all_fees() -> None: - if not is_node_inited(): - print('Node is not initialized') - return status, payload = post_request(blueprint=BLUEPRINT_NAME, method='claim-all-fees') _handle_response(status, payload, success='All fees claimed') +@check_inited def set_fee_rate(fee_rate: int) -> None: - if not is_node_inited(): - print('Node is not initialized') - return status, payload = post_request( blueprint=BLUEPRINT_NAME, method='set-fee-rate', json={'feeRate': fee_rate} ) _handle_response(status, payload, success=f'Fee rate set to {fee_rate}') +@check_inited def claim_fees(amount: float) -> None: - if not is_node_inited(): - print('Node is not initialized') - return status, payload = post_request( blueprint=BLUEPRINT_NAME, method='claim-fees', json={'amount': amount} ) _handle_response(status, payload, success=f'Fees claimed: {amount}') -def send_fees(to: str, amount: float) -> None: - if not is_node_inited(): - print('Node is not initialized') - return - status, payload = post_request( - blueprint=BLUEPRINT_NAME, method='send-fees', json={'to': to, 'amount': amount} - ) - _handle_response(status, payload, success=f'Fees sent: {amount} to {to}') - - +@check_inited def get_earned_fee_amount() -> None: - if not is_node_inited(): - print('Node is not initialized') - return status, payload = post_request(blueprint=BLUEPRINT_NAME, method='get-earned-fee-amount') if status == 'ok' and isinstance(payload, dict): amount_wei = payload.get('amount_wei') diff --git a/tests/routes_test.py b/tests/routes_test.py index bfb44d4c..6b18e4e1 100644 --- a/tests/routes_test.py +++ b/tests/routes_test.py @@ -39,9 +39,8 @@ '/api/v1/fair-chain/record', '/api/v1/fair-chain/checks', '/api/v1/fair-node-passive/setup', - '/api/v1/fair-staking/add-allowed-receiver', - '/api/v1/fair-staking/remove-allowed-receiver', - '/api/v1/fair-staking/send-all-fees', + '/api/v1/fair-staking/add-receiver', + '/api/v1/fair-staking/remove-receiver', '/api/v1/fair-staking/claim-all-fees', '/api/v1/fair-staking/set-fee-rate', '/api/v1/fair-staking/claim-fees', From af5cc41e831ad78a33ebd5ad6fdd83535781c913 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Wed, 27 Aug 2025 19:10:31 +0100 Subject: [PATCH 078/198] Rename value to amount in send fees --- node_cli/cli/staking.py | 10 +++++----- node_cli/fair/staking.py | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/node_cli/cli/staking.py b/node_cli/cli/staking.py index ddd8a2ab..2fce0f6b 100644 --- a/node_cli/cli/staking.py +++ b/node_cli/cli/staking.py @@ -107,7 +107,7 @@ def _claim_fees(amount: float) -> None: @staking.command('send-fees', help='Send fees to address (or all with --all)') @click.argument('to') -@click.argument('value', type=float, required=False) +@click.argument('amount', type=float, required=False) @click.option('--all', 'send_all', is_flag=True, help='Send all fees to address') @click.option( '--yes', @@ -116,10 +116,10 @@ def _claim_fees(amount: float) -> None: expose_value=False, prompt='Are you sure you want to send fees?', ) -def _send_fees(to: str, value: float | None, send_all: bool) -> None: - if value is None and not send_all: - raise click.UsageError('Provide or use --all') - send_fees(to, None if send_all else value) +def _send_fees(to: str, amount: float | None, send_all: bool) -> None: + if amount is None and not send_all: + raise click.UsageError('Provide or use --all') + send_fees(to, None if send_all else amount) @staking.command('get-earned-fee-amount', help='Get earned fee amount') diff --git a/node_cli/fair/staking.py b/node_cli/fair/staking.py index d9469638..ca3a8f80 100644 --- a/node_cli/fair/staking.py +++ b/node_cli/fair/staking.py @@ -50,10 +50,10 @@ def remove_allowed_receiver(receiver: str) -> None: @check_inited -def send_fees(to: str, value: float | None) -> None: +def send_fees(to: str, amount: float | None) -> None: json_data: dict[str, Any] = {'to': to} - if value is not None: - json_data['value'] = value + if amount is not None: + json_data['amount'] = amount status, payload = post_request(blueprint=BLUEPRINT_NAME, method='send-fees', json=json_data) _handle_response(status, payload, success=f'Fees sent to {to}') From 7a960299a8126720747e915445a65b640ade95ea Mon Sep 17 00:00:00 2001 From: Dmytro Date: Wed, 27 Aug 2025 19:32:22 +0100 Subject: [PATCH 079/198] Merge claim fees commands --- README.md | 19 ++++--------------- node_cli/cli/staking.py | 24 +++++++----------------- node_cli/configs/routes.py | 1 - node_cli/fair/staking.py | 19 +++++++++---------- tests/routes_test.py | 1 - 5 files changed, 20 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index eab1c67a..a50d8a3e 100644 --- a/README.md +++ b/README.md @@ -1098,12 +1098,13 @@ Arguments: * `RECEIVER_ADDRESS` - Address to remove from the allowed receivers list. -#### Claim all fees +#### Claim fees -Claim all accumulated fees to the node wallet. +Claim a specific amount of fees or all fees to the node wallet. ```shell -fair staking claim-all-fees +fair staking claim-fees +fair staking claim-fees --all ``` #### Set fee rate @@ -1118,18 +1119,6 @@ Arguments: * `FEE_RATE` - Fee rate value as integer (uint16). -#### Claim fees - -Claim a specific amount of fees to the node wallet. - -```shell -fair staking claim-fees -``` - -Arguments: - -* `AMOUNT` - Amount of fees to claim (FAIR). - #### Send fees Send a specific amount of fees to the default allowed receiver. diff --git a/node_cli/cli/staking.py b/node_cli/cli/staking.py index 2fce0f6b..8cde234c 100644 --- a/node_cli/cli/staking.py +++ b/node_cli/cli/staking.py @@ -22,7 +22,6 @@ from node_cli.fair.staking import ( add_allowed_receiver, remove_allowed_receiver, - claim_all_fees, set_fee_rate, claim_fees, send_fees, @@ -67,18 +66,6 @@ def _remove_allowed_receiver(receiver: str) -> None: remove_allowed_receiver(receiver) -@staking.command('claim-all-fees', help='Claim all fees') -@click.option( - '--yes', - is_flag=True, - callback=abort_if_false, - expose_value=False, - prompt='Are you sure you want to claim all fees?', -) -def _claim_all_fees() -> None: - claim_all_fees() - - @staking.command('set-fee-rate', help='Set fee rate (uint16, basis points; 25 = 2.5%)') @click.argument('fee_rate', type=int) @click.option( @@ -92,8 +79,9 @@ def _set_fee_rate(fee_rate: int) -> None: set_fee_rate(fee_rate) -@staking.command('claim-fees', help='Claim fees amount (FAIR)') -@click.argument('amount', type=float) +@staking.command('claim-fees', help='Claim fees amount (FAIR) or all with --all') +@click.argument('amount', type=float, required=False) +@click.option('--all', 'claim_all', is_flag=True, help='Claim all fees') @click.option( '--yes', is_flag=True, @@ -101,8 +89,10 @@ def _set_fee_rate(fee_rate: int) -> None: expose_value=False, prompt='Are you sure you want to claim fees?', ) -def _claim_fees(amount: float) -> None: - claim_fees(amount) +def _claim_fees(amount: float | None, claim_all: bool) -> None: + if amount is None and not claim_all: + raise click.UsageError('Provide or use --all') + claim_fees(None if claim_all else amount) @staking.command('send-fees', help='Send fees to address (or all with --all)') diff --git a/node_cli/configs/routes.py b/node_cli/configs/routes.py index 5eba041f..83bb2866 100644 --- a/node_cli/configs/routes.py +++ b/node_cli/configs/routes.py @@ -46,7 +46,6 @@ 'fair-staking': [ 'add-receiver', 'remove-receiver', - 'claim-all-fees', 'set-fee-rate', 'claim-fees', 'send-fees', diff --git a/node_cli/fair/staking.py b/node_cli/fair/staking.py index ca3a8f80..9e81e69f 100644 --- a/node_cli/fair/staking.py +++ b/node_cli/fair/staking.py @@ -58,12 +58,6 @@ def send_fees(to: str, amount: float | None) -> None: _handle_response(status, payload, success=f'Fees sent to {to}') -@check_inited -def claim_all_fees() -> None: - status, payload = post_request(blueprint=BLUEPRINT_NAME, method='claim-all-fees') - _handle_response(status, payload, success='All fees claimed') - - @check_inited def set_fee_rate(fee_rate: int) -> None: status, payload = post_request( @@ -73,11 +67,16 @@ def set_fee_rate(fee_rate: int) -> None: @check_inited -def claim_fees(amount: float) -> None: - status, payload = post_request( - blueprint=BLUEPRINT_NAME, method='claim-fees', json={'amount': amount} +def claim_fees(amount: float | None) -> None: + json_data: dict[str, Any] = {} + if amount is not None: + json_data['amount'] = amount + status, payload = post_request(blueprint=BLUEPRINT_NAME, method='claim-fees', json=json_data) + _handle_response( + status, + payload, + success='All fees claimed' if amount is None else f'Fees claimed: {amount}', ) - _handle_response(status, payload, success=f'Fees claimed: {amount}') @check_inited diff --git a/tests/routes_test.py b/tests/routes_test.py index 6b18e4e1..b64e9e0a 100644 --- a/tests/routes_test.py +++ b/tests/routes_test.py @@ -41,7 +41,6 @@ '/api/v1/fair-node-passive/setup', '/api/v1/fair-staking/add-receiver', '/api/v1/fair-staking/remove-receiver', - '/api/v1/fair-staking/claim-all-fees', '/api/v1/fair-staking/set-fee-rate', '/api/v1/fair-staking/claim-fees', '/api/v1/fair-staking/send-fees', From c3af5509b1ba11a9d5011afdfcce530435e47ad2 Mon Sep 17 00:00:00 2001 From: Mikolaj Kucharski Date: Tue, 17 Jun 2025 18:04:36 +0000 Subject: [PATCH 080/198] Fix check_ssl_connection() function Function doesn't wait for `openssl s_client ...` to finish. It assumes that when the command is still running that is the successful condition. However the function should wait for exit code from the binary. We saw in production intermittent and very often `skale ssl upload` failures. This change should fix this problem and underlying race condition. --- node_cli/core/ssl/check.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/node_cli/core/ssl/check.py b/node_cli/core/ssl/check.py index 289bc77e..a82ea11b 100644 --- a/node_cli/core/ssl/check.py +++ b/node_cli/core/ssl/check.py @@ -196,8 +196,15 @@ def check_ssl_connection(host, port, silent=False): ] expose_output = not silent with detached_subprocess(ssl_check_cmd, expose_output=expose_output) as dp: - time.sleep(1) - code = dp.poll() - if code is not None: + for _ in range(10): + code = dp.poll() + if code is None: + logger.info('Healthcheck process still running...') + time.sleep(2) + continue + elif code == 0: + return logger.error('Healthcheck connection failed') raise SSLHealthcheckError('OpenSSL connection verification failed') + logger.error('Healthcheck timed-out') + raise SSLHealthcheckError('OpenSSL connection verification timed-out') From a5c70aa7f8e210e951e9a836c20566e3596cef12 Mon Sep 17 00:00:00 2001 From: Mikolaj Kucharski Date: Mon, 23 Jun 2025 18:55:53 +0000 Subject: [PATCH 081/198] Move to dp.wait() in check_ssl_connection() Replace for loop and dp.poll() with more straightforward dp.wait() with a timeout, as requested during diff review. --- node_cli/core/ssl/check.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/node_cli/core/ssl/check.py b/node_cli/core/ssl/check.py index a82ea11b..327f72a6 100644 --- a/node_cli/core/ssl/check.py +++ b/node_cli/core/ssl/check.py @@ -20,6 +20,7 @@ import time import socket import logging +import subprocess from contextlib import contextmanager from node_cli.core.ssl.utils import detached_subprocess @@ -196,15 +197,15 @@ def check_ssl_connection(host, port, silent=False): ] expose_output = not silent with detached_subprocess(ssl_check_cmd, expose_output=expose_output) as dp: - for _ in range(10): - code = dp.poll() - if code is None: - logger.info('Healthcheck process still running...') - time.sleep(2) - continue - elif code == 0: - return - logger.error('Healthcheck connection failed') - raise SSLHealthcheckError('OpenSSL connection verification failed') - logger.error('Healthcheck timed-out') - raise SSLHealthcheckError('OpenSSL connection verification timed-out') + timeout = 20 + try: + dp.wait(timeout=timeout) + except subprocess.TimeoutExpired: + logger.error('Healthcheck timed-out after %s s', timeout) + raise SSLHealthcheckError('OpenSSL connection verification timed-out') + + if dp.returncode == 0: # success + return + + logger.error('Healthcheck connection failed (code %s)', dp.returncode) + raise SSLHealthcheckError('OpenSSL connection verification failed') From 54ab23ae6a8d405b5891f93db1ad6ed51d24f568 Mon Sep 17 00:00:00 2001 From: Mikolaj Kucharski Date: Mon, 23 Jun 2025 20:25:04 +0000 Subject: [PATCH 082/198] Read from /dev/null in detached_subprocess() Redirect the child's standard input to subprocess.DEVNULL, so it starts with no stdin attached. This prevents the OpenSSL health-check process from reading from, or blocking on, the parent's terminal or execution environment stdin stream. --- node_cli/core/ssl/utils.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/node_cli/core/ssl/utils.py b/node_cli/core/ssl/utils.py index a5a329dd..c80b3a8a 100644 --- a/node_cli/core/ssl/utils.py +++ b/node_cli/core/ssl/utils.py @@ -17,13 +17,13 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import logging import os import shutil -import logging import subprocess from contextlib import contextmanager -from node_cli.configs.ssl import SSL_CERT_FILEPATH, SSL_KEY_FILEPATH, SSL_FOLDER_PATH +from node_cli.configs.ssl import SSL_CERT_FILEPATH, SSL_FOLDER_PATH, SSL_KEY_FILEPATH logger = logging.getLogger(__name__) @@ -48,7 +48,13 @@ def is_ssl_folder_empty(ssl_path=SSL_FOLDER_PATH): @contextmanager def detached_subprocess(cmd, expose_output=False): logger.debug(f'Starting detached subprocess: {cmd}') - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding='utf-8') + p = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + stdin=subprocess.DEVNULL, + encoding='utf-8', + ) try: yield p finally: From 2272e94affdd21986793c2bc930c5139010e8ec6 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Wed, 3 Sep 2025 13:46:27 +0100 Subject: [PATCH 083/198] Rename compose services, update container names --- node_cli/core/node.py | 2 +- node_cli/fair/common.py | 2 +- node_cli/operations/fair.py | 6 ++-- node_cli/utils/docker_utils.py | 62 ++++++++++++++-------------------- tests/cli/health_test.py | 4 +-- 5 files changed, 32 insertions(+), 44 deletions(-) diff --git a/node_cli/core/node.py b/node_cli/core/node.py index 3253f6c5..8586f3f9 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -110,7 +110,7 @@ class NodeStatuses(Enum): def is_update_safe(node_type: NodeType, node_mode: NodeMode) -> bool: - if not is_admin_running(node_type, node_mode): + if not is_admin_running(): if node_mode == NodeMode.PASSIVE: return True elif not is_api_running(node_type): diff --git a/node_cli/fair/common.py b/node_cli/fair/common.py index ce24ad45..9d24fa29 100644 --- a/node_cli/fair/common.py +++ b/node_cli/fair/common.py @@ -133,4 +133,4 @@ def repair_chain(snapshot_from: str = 'any') -> None: env = compose_node_env( SKALE_DIR_ENV_FILEPATH, save=False, node_type=NodeType.FAIR, node_mode=node_mode ) - repair_fair_op(node_mode=node_mode, env=env, snapshot_from=snapshot_from) + repair_fair_op(env=env, snapshot_from=snapshot_from) diff --git a/node_cli/operations/fair.py b/node_cli/operations/fair.py index 50f87c1d..d6d4334b 100644 --- a/node_cli/operations/fair.py +++ b/node_cli/operations/fair.py @@ -296,10 +296,10 @@ def trigger_skaled_snapshot_mode(env: dict, snapshot_from: str = 'any') -> None: record.set_snapshot_from(snapshot_from) -def repair(node_mode: NodeMode, env: dict, snapshot_from: str = 'any') -> None: +def repair(env: dict, snapshot_from: str = 'any') -> None: logger.info('Starting fair node repair') - container_name = 'fair_admin' - if is_admin_running(node_type=NodeType.FAIR, node_mode=node_mode): + container_name = 'sk_admin' + if is_admin_running(): logger.info('Stopping admin container') stop_container_by_name(container_name=container_name) logger.info('Removing chain container') diff --git a/node_cli/utils/docker_utils.py b/node_cli/utils/docker_utils.py index 3aa46e56..60514cdb 100644 --- a/node_cli/utils/docker_utils.py +++ b/node_cli/utils/docker_utils.py @@ -48,46 +48,46 @@ TELEGRAF_REMOVE_TIMEOUT = 20 REDIS_START_TIMEOUT = 10 -REDIS_SERVICE_DICT = {'redis': 'skale_redis'} +REDIS_SERVICE_DICT = {'redis': 'sk_redis'} CORE_COMMON_COMPOSE_SERVICES = { - 'transaction-manager': 'skale_transaction-manager', - 'redis': 'skale_redis', - 'watchdog': 'skale_watchdog', - 'nginx': 'skale_nginx', - 'filebeat': 'skale_filebeat', + 'transaction-manager': 'sk_tm', + 'redis': 'sk_redis', + 'watchdog': 'sk_watchdog', + 'nginx': 'sk_nginx', + 'filebeat': 'sk_filebeat', } BASE_SKALE_COMPOSE_SERVICES = { **CORE_COMMON_COMPOSE_SERVICES, - 'skale-admin': 'skale_admin', - 'skale-api': 'skale_api', - 'bounty': 'skale_bounty', + 'admin': 'sk_admin', + 'api': 'sk_api', + 'bounty': 'sk_bounty', } BASE_FAIR_COMPOSE_SERVICES = { **CORE_COMMON_COMPOSE_SERVICES, - 'fair-admin': 'fair_admin', - 'fair-api': 'fair_api', + 'admin': 'sk_admin', + 'api': 'sk_api', } BASE_FAIR_BOOT_COMPOSE_SERVICES = { **CORE_COMMON_COMPOSE_SERVICES, - 'fair-boot': 'fair_boot_admin', - 'fair-boot-api': 'fair_boot_api', + 'boot-admin': 'sk_boot_admin', + 'boot-api': 'sk_boot_api', } BASE_PASSIVE_COMPOSE_SERVICES = { - 'skale-passive-admin': 'skale_passive_admin', - 'nginx': 'skale_nginx', + 'admin': 'sk_admin', + 'nginx': 'sk_nginx', } BASE_PASSIVE_FAIR_COMPOSE_SERVICES = { - 'fair-admin': 'fair_admin', - 'fair-api': 'fair_api', - 'nginx': 'skale_nginx', - 'watchdog': 'skale_watchdog', - 'filebeat': 'skale_filebeat', + 'admin': 'sk_admin', + 'api': 'sk_api', + 'nginx': 'sk_nginx', + 'watchdog': 'sk_watchdog', + 'filebeat': 'sk_filebeat', **REDIS_SERVICE_DICT, } @@ -120,11 +120,11 @@ def get_containers(container_name_filter=None, _all=True) -> list: def get_all_schain_containers(_all=True) -> list: - return docker_client().containers.list(all=_all, filters={'name': 'skale_schain_*'}) + return docker_client().containers.list(all=_all, filters={'name': 'sk_chain_*'}) def get_all_ima_containers(_all=True) -> list: - return docker_client().containers.list(all=_all, filters={'name': 'skale_ima_*'}) + return docker_client().containers.list(all=_all, filters={'name': 'sk_ima_*'}) def remove_dynamic_containers() -> None: @@ -203,7 +203,7 @@ def start_container_by_name(container_name: str, dclient: Optional[DockerClient] def remove_schain_container_by_name( schain_name: str, dclient: Optional[DockerClient] = None ) -> None: - container_name = f'skale_schain_{schain_name}' + container_name = f'sk_chain_{schain_name}' remove_container_by_name(container_name, timeout=SCHAIN_REMOVE_TIMEOUT, dclient=dclient) @@ -430,20 +430,8 @@ def is_api_running(node_type: NodeType, dclient: Optional[DockerClient] = None) return is_container_running(name='skale_api', dclient=dclient) -def is_admin_running( - node_type: NodeType, - node_mode: NodeMode, - dclient: Optional[DockerClient] = None, -) -> bool: - if active_fair(node_type, node_mode): - container_name = 'fair_admin' - elif passive_fair(node_type, node_mode): - container_name = 'fair_passive_admin' - elif active_skale(node_type, node_mode): - container_name = 'skale_admin' - elif passive_skale(node_type, node_mode): - container_name = 'skale_passive_admin' - return is_container_running(name=container_name, dclient=dclient) +def is_admin_running(dclient: Optional[DockerClient] = None) -> bool: + return is_container_running(name='sk_admin', dclient=dclient) def system_prune(): diff --git a/tests/cli/health_test.py b/tests/cli/health_test.py index de14c7c2..c6ea16df 100644 --- a/tests/cli/health_test.py +++ b/tests/cli/health_test.py @@ -9,7 +9,7 @@ 'payload': [ { 'image': 'skalenetwork/schain:1.46-develop.21', - 'name': 'skale_schain_shapely-alfecca-meridiana', + 'name': 'sk_chain_shapely-alfecca-meridiana', 'state': { 'Status': 'running', 'Running': True, @@ -51,7 +51,7 @@ def test_containers(): assert result.exit_code == 0 assert ( result.output - == ' Name Status Started At Image \n-------------------------------------------------------------------------------------------------------------\nskale_schain_shapely-alfecca-meridiana Running Jul 31 2020 11:56:35 skalenetwork/schain:1.46-develop.21\nskale_api Running Jul 31 2020 11:55:17 skale-admin:latest \n' # noqa + == ' Name Status Started At Image \n-------------------------------------------------------------------------------------------------------------\nsk_chain_shapely-alfecca-meridiana Running Jul 31 2020 11:56:35 skalenetwork/schain:1.46-develop.21\nskale_api Running Jul 31 2020 11:55:17 skale-admin:latest \n' # noqa ) From 4d37dbc7f5c6927c246f103cd70e80fe91167b07 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Wed, 3 Sep 2025 13:56:56 +0100 Subject: [PATCH 084/198] Revert to sk_schain prefix --- node_cli/utils/docker_utils.py | 4 ++-- tests/cli/health_test.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/node_cli/utils/docker_utils.py b/node_cli/utils/docker_utils.py index 60514cdb..f17644fe 100644 --- a/node_cli/utils/docker_utils.py +++ b/node_cli/utils/docker_utils.py @@ -120,7 +120,7 @@ def get_containers(container_name_filter=None, _all=True) -> list: def get_all_schain_containers(_all=True) -> list: - return docker_client().containers.list(all=_all, filters={'name': 'sk_chain_*'}) + return docker_client().containers.list(all=_all, filters={'name': 'sk_schain_*'}) def get_all_ima_containers(_all=True) -> list: @@ -203,7 +203,7 @@ def start_container_by_name(container_name: str, dclient: Optional[DockerClient] def remove_schain_container_by_name( schain_name: str, dclient: Optional[DockerClient] = None ) -> None: - container_name = f'sk_chain_{schain_name}' + container_name = f'sk_schain_{schain_name}' remove_container_by_name(container_name, timeout=SCHAIN_REMOVE_TIMEOUT, dclient=dclient) diff --git a/tests/cli/health_test.py b/tests/cli/health_test.py index c6ea16df..951b9f0d 100644 --- a/tests/cli/health_test.py +++ b/tests/cli/health_test.py @@ -9,7 +9,7 @@ 'payload': [ { 'image': 'skalenetwork/schain:1.46-develop.21', - 'name': 'sk_chain_shapely-alfecca-meridiana', + 'name': 'sk_schain_shapely-alfecca-meridiana', 'state': { 'Status': 'running', 'Running': True, @@ -51,7 +51,7 @@ def test_containers(): assert result.exit_code == 0 assert ( result.output - == ' Name Status Started At Image \n-------------------------------------------------------------------------------------------------------------\nsk_chain_shapely-alfecca-meridiana Running Jul 31 2020 11:56:35 skalenetwork/schain:1.46-develop.21\nskale_api Running Jul 31 2020 11:55:17 skale-admin:latest \n' # noqa + == ' Name Status Started At Image \n-------------------------------------------------------------------------------------------------------------\nsk_schain_shapely-alfecca-meridiana Running Jul 31 2020 11:56:35 skalenetwork/schain:1.46-develop.21\nskale_api Running Jul 31 2020 11:55:17 skale-admin:latest \n' # noqa ) From 99290e60a7759a55aff964199d3b48fb3e4ec715 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Wed, 3 Sep 2025 16:00:32 +0100 Subject: [PATCH 085/198] Update is api running helper, update containers test --- node_cli/core/node.py | 8 ++++---- node_cli/utils/docker_utils.py | 7 ++----- tests/cli/health_test.py | 4 ++-- tests/core/core_node_test.py | 10 +++++----- 4 files changed, 13 insertions(+), 16 deletions(-) diff --git a/node_cli/core/node.py b/node_cli/core/node.py index 8586f3f9..3a704ddc 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -109,11 +109,11 @@ class NodeStatuses(Enum): NOT_CREATED = 5 -def is_update_safe(node_type: NodeType, node_mode: NodeMode) -> bool: +def is_update_safe(node_mode: NodeMode) -> bool: if not is_admin_running(): if node_mode == NodeMode.PASSIVE: return True - elif not is_api_running(node_type): + elif not is_api_running(): return True status, payload = get_request(BLUEPRINT_NAME, 'update-safe') if status == 'error': @@ -298,7 +298,7 @@ def update( ) -> None: node_mode = upsert_node_mode(node_mode=node_mode) - if not unsafe_ok and not is_update_safe(node_type=node_type, node_mode=node_mode): + if not unsafe_ok and not is_update_safe(node_mode=node_mode): error_msg = 'Cannot update safely' error_exit(error_msg, exit_code=CLIExitCodes.UNSAFE_UPDATE) @@ -422,7 +422,7 @@ def set_maintenance_mode_off(): @check_user def turn_off(node_type: NodeType, maintenance_on: bool = False, unsafe_ok: bool = False) -> None: node_mode = upsert_node_mode() - if not unsafe_ok and not is_update_safe(node_type=node_type, node_mode=node_mode): + if not unsafe_ok and not is_update_safe(node_mode=node_mode): error_msg = 'Cannot turn off safely' error_exit(error_msg, exit_code=CLIExitCodes.UNSAFE_UPDATE) if maintenance_on: diff --git a/node_cli/utils/docker_utils.py b/node_cli/utils/docker_utils.py index f17644fe..26d16bd7 100644 --- a/node_cli/utils/docker_utils.py +++ b/node_cli/utils/docker_utils.py @@ -423,11 +423,8 @@ def is_container_running(name: str, dclient: Optional[DockerClient] = None) -> b return False -def is_api_running(node_type: NodeType, dclient: Optional[DockerClient] = None) -> bool: - if node_type == NodeType.FAIR: - return is_container_running(name='fair_api', dclient=dclient) - else: - return is_container_running(name='skale_api', dclient=dclient) +def is_api_running(dclient: Optional[DockerClient] = None) -> bool: + return is_container_running(name='sk_api', dclient=dclient) def is_admin_running(dclient: Optional[DockerClient] = None) -> bool: diff --git a/tests/cli/health_test.py b/tests/cli/health_test.py index 951b9f0d..6f3fc022 100644 --- a/tests/cli/health_test.py +++ b/tests/cli/health_test.py @@ -26,7 +26,7 @@ }, { 'image': 'skale-admin:latest', - 'name': 'skale_api', + 'name': 'sk_api', 'state': { 'Status': 'running', 'Running': True, @@ -51,7 +51,7 @@ def test_containers(): assert result.exit_code == 0 assert ( result.output - == ' Name Status Started At Image \n-------------------------------------------------------------------------------------------------------------\nsk_schain_shapely-alfecca-meridiana Running Jul 31 2020 11:56:35 skalenetwork/schain:1.46-develop.21\nskale_api Running Jul 31 2020 11:55:17 skale-admin:latest \n' # noqa + == ' Name Status Started At Image \n----------------------------------------------------------------------------------------------------------\nsk_schain_shapely-alfecca-meridiana Running Jul 31 2020 11:56:35 skalenetwork/schain:1.46-develop.21\nsk_api Running Jul 31 2020 11:55:17 skale-admin:latest \n' # noqa ) diff --git a/tests/core/core_node_test.py b/tests/core/core_node_test.py index 70671418..7188ffae 100644 --- a/tests/core/core_node_test.py +++ b/tests/core/core_node_test.py @@ -396,7 +396,7 @@ def test_update_node(regular_user_conf, mocked_g_config, resource_file, inited_n def test_is_update_safe_when_admin_and_api_not_running( mock_requests_get, mock_is_api_running, mock_is_admin_running, node_type, node_mode ): - assert is_update_safe(node_type=node_type, node_mode=node_mode) is True + assert is_update_safe(node_mode=node_mode) is True mock_requests_get.assert_not_called() @@ -406,7 +406,7 @@ def test_is_update_safe_when_admin_and_api_not_running( def test_is_update_safe_when_admin_not_running_for_passive( mock_requests_get, mock_is_api_running, mock_is_admin_running ): - assert is_update_safe(node_type=NodeType.SKALE, node_mode=NodeMode.PASSIVE) is True + assert is_update_safe(node_mode=NodeMode.PASSIVE) is True mock_requests_get.assert_not_called() @@ -429,7 +429,7 @@ def test_is_update_safe_when_admin_running( mock_requests_get, mock_is_admin_running, api_is_safe, expected_result, node_type, node_mode ): mock_requests_get.return_value = safe_update_api_response(safe=api_is_safe) - assert is_update_safe(node_type=node_type, node_mode=node_mode) is expected_result + assert is_update_safe(node_mode=node_mode) is expected_result mock_requests_get.assert_called_once() @@ -451,7 +451,7 @@ def test_is_update_safe_when_only_api_running_for_regular( node_type, ): mock_requests_get.return_value = safe_update_api_response(safe=api_is_safe) - assert is_update_safe(node_type=node_type, node_mode=NodeMode.ACTIVE) is expected_result + assert is_update_safe(node_mode=NodeMode.ACTIVE) is expected_result mock_requests_get.assert_called_once() @@ -469,5 +469,5 @@ def test_is_update_safe_when_api_call_fails( mock_requests_get, mock_is_admin_running, node_type, node_mode ): mock_requests_get.side_effect = requests.exceptions.ConnectionError('Test connection error') - assert is_update_safe(node_type=node_type, node_mode=node_mode) is False + assert is_update_safe(node_mode=node_mode) is False mock_requests_get.assert_called_once() From 62167f85d6506b33a2a3c491e8156138b3259fa7 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Thu, 4 Sep 2025 15:33:37 +0100 Subject: [PATCH 086/198] Update resource alloc generation --- node_cli/core/resources.py | 2 +- tests/resources_test.py | 38 +++++++++++++++++++------------------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/node_cli/core/resources.py b/node_cli/core/resources.py index e62a0f6f..68686738 100644 --- a/node_cli/core/resources.py +++ b/node_cli/core/resources.py @@ -81,7 +81,7 @@ def compose_resource_allocation_config(env_type: str, params_by_env_type: Dict = schain_allocation_data = safe_load_yml(ALLOCATION_FILEPATH) return { - 'schain': { + 'skaled': { 'cpu_shares': schain_cpu_alloc.dict(), 'mem': schain_mem_alloc.dict(), 'disk': schain_allocation_data[env_type]['disk'], diff --git a/tests/resources_test.py b/tests/resources_test.py index 39effaf7..1f545080 100644 --- a/tests/resources_test.py +++ b/tests/resources_test.py @@ -81,23 +81,23 @@ def test_generate_resource_allocation_config(): with mock.patch('node_cli.core.resources.get_disk_size', return_value=NORMAL_DISK_SIZE): resource_allocation_config = compose_resource_allocation_config(DEFAULT_ENV_TYPE) - assert resource_allocation_config['schain']['cpu_shares']['test4'] == 102 - assert resource_allocation_config['schain']['cpu_shares']['test'] == 102 - assert resource_allocation_config['schain']['cpu_shares']['small'] == 6 - assert resource_allocation_config['schain']['cpu_shares']['medium'] == 102 - assert resource_allocation_config['schain']['cpu_shares']['large'] == 819 - - assert isinstance(resource_allocation_config['schain']['mem']['test4'], int) - assert isinstance(resource_allocation_config['schain']['mem']['test'], int) - assert isinstance(resource_allocation_config['schain']['mem']['small'], int) - assert isinstance(resource_allocation_config['schain']['mem']['medium'], int) - assert isinstance(resource_allocation_config['schain']['mem']['large'], int) - - assert resource_allocation_config['schain']['disk']['test4'] == 8879996928 - assert resource_allocation_config['schain']['disk']['test'] == 8879996928 - assert resource_allocation_config['schain']['disk']['small'] == 554999808 - assert resource_allocation_config['schain']['disk']['medium'] == 8879996928 - assert resource_allocation_config['schain']['disk']['large'] == 71039975424 + assert resource_allocation_config['skaled']['cpu_shares']['test4'] == 102 + assert resource_allocation_config['skaled']['cpu_shares']['test'] == 102 + assert resource_allocation_config['skaled']['cpu_shares']['small'] == 6 + assert resource_allocation_config['skaled']['cpu_shares']['medium'] == 102 + assert resource_allocation_config['skaled']['cpu_shares']['large'] == 819 + + assert isinstance(resource_allocation_config['skaled']['mem']['test4'], int) + assert isinstance(resource_allocation_config['skaled']['mem']['test'], int) + assert isinstance(resource_allocation_config['skaled']['mem']['small'], int) + assert isinstance(resource_allocation_config['skaled']['mem']['medium'], int) + assert isinstance(resource_allocation_config['skaled']['mem']['large'], int) + + assert resource_allocation_config['skaled']['disk']['test4'] == 8879996928 + assert resource_allocation_config['skaled']['disk']['test'] == 8879996928 + assert resource_allocation_config['skaled']['disk']['small'] == 554999808 + assert resource_allocation_config['skaled']['disk']['medium'] == 8879996928 + assert resource_allocation_config['skaled']['disk']['large'] == 71039975424 assert resource_allocation_config['ima']['cpu_shares'] == { 'large': 204, @@ -108,7 +108,7 @@ def test_generate_resource_allocation_config(): } assert isinstance(resource_allocation_config['ima']['mem'], dict) - assert resource_allocation_config['schain']['volume_limits'] == SCHAIN_VOLUME_PARTS + assert resource_allocation_config['skaled']['volume_limits'] == SCHAIN_VOLUME_PARTS def test_update_allocation_config(resource_alloc_config): @@ -194,7 +194,7 @@ def test_get_memory_alloc(params_by_env_type): def test_leveldb_limits(): with mock.patch('node_cli.core.resources.get_disk_size', return_value=NORMAL_DISK_SIZE): resource_allocation_config = compose_resource_allocation_config(DEFAULT_ENV_TYPE) - assert resource_allocation_config['schain']['leveldb_limits'] == { + assert resource_allocation_config['skaled']['leveldb_limits'] == { 'large': {'contract_storage': 12787195576, 'db_storage': 4262398525}, 'medium': {'contract_storage': 1598399446, 'db_storage': 532799815}, 'small': {'contract_storage': 99899965, 'db_storage': 33299988}, From 0c83e4976f704626c6685b35dc29e190886b4be5 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Mon, 8 Sep 2025 16:20:09 +0100 Subject: [PATCH 087/198] Fix fair-api service name --- node_cli/operations/fair.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node_cli/operations/fair.py b/node_cli/operations/fair.py index d6d4334b..4fc10b16 100644 --- a/node_cli/operations/fair.py +++ b/node_cli/operations/fair.py @@ -126,7 +126,7 @@ def init( ) compose_up(env=env, node_type=NodeType.FAIR, node_mode=node_mode) - wait_for_container(BASE_PASSIVE_FAIR_COMPOSE_SERVICES['fair-api']) + wait_for_container(BASE_PASSIVE_FAIR_COMPOSE_SERVICES['api']) time.sleep(REDIS_START_TIMEOUT) return True From 9903d20e9537e684dddcb46ab95872dc35c0aa04 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Wed, 10 Sep 2025 20:29:52 +0100 Subject: [PATCH 088/198] Add `turn on` and `turn off` commands for FAIR nodes --- node_cli/cli/fair_node.py | 31 ++++++++++++++++++++++++++++++- node_cli/fair/common.py | 34 +++++++++++++++++++++++++++++++++- 2 files changed, 63 insertions(+), 2 deletions(-) diff --git a/node_cli/cli/fair_node.py b/node_cli/cli/fair_node.py index 7022f51d..355d1e30 100644 --- a/node_cli/cli/fair_node.py +++ b/node_cli/cli/fair_node.py @@ -18,11 +18,13 @@ # along with this program. If not, see . import click - +from node_cli.cli.info import TYPE from node_cli.core.node import backup from node_cli.fair.active import change_ip as change_ip_fair from node_cli.fair.common import cleanup as cleanup_fair +from node_cli.fair.common import turn_off as turn_off_fair +from node_cli.fair.common import turn_on as turn_on_fair from node_cli.fair.active import exit as exit_fair from node_cli.fair.active import ( get_node_info, @@ -199,3 +201,30 @@ def exit_node() -> None: @streamed_cmd def set_domain_name(domain): set_domain_name_fair(domain) + + +@node.command('turn-off', help='Turn off the node') +@click.option( + '--yes', + is_flag=True, + callback=abort_if_false, + expose_value=False, + prompt='Are you sure you want to turn off the node?', +) +@streamed_cmd +def _turn_off(): + turn_off_fair(node_type=TYPE) + + +@node.command('turn-on', help='Turn on the node') +@click.option( + '--yes', + is_flag=True, + callback=abort_if_false, + expose_value=False, + prompt='Are you sure you want to turn on the node?', +) +@click.argument('env_file') +@streamed_cmd +def _turn_on(env_file): + turn_on_fair(env_file, node_type=TYPE) \ No newline at end of file diff --git a/node_cli/fair/common.py b/node_cli/fair/common.py index 9d24fa29..b1c3d8e0 100644 --- a/node_cli/fair/common.py +++ b/node_cli/fair/common.py @@ -20,7 +20,7 @@ import time import logging -from node_cli.configs import INIT_TIMEOUT, SKALE_DIR +from node_cli.configs import INIT_TIMEOUT, SKALE_DIR, TM_INIT_TIMEOUT from node_cli.configs.user import SKALE_DIR_ENV_FILEPATH from node_cli.core.docker_config import cleanup_docker_configuration from node_cli.core.node import compose_node_env, is_base_containers_alive @@ -32,6 +32,8 @@ repair_fair_op, update_fair_op, init_fair_op, + turn_off_op, + turn_on_op ) from node_cli.core.host import save_env_params from node_cli.utils.decorators import check_inited, check_not_inited, check_user @@ -134,3 +136,33 @@ def repair_chain(snapshot_from: str = 'any') -> None: SKALE_DIR_ENV_FILEPATH, save=False, node_type=NodeType.FAIR, node_mode=node_mode ) repair_fair_op(env=env, snapshot_from=snapshot_from) + + +@check_inited +@check_user +def turn_off(node_type: NodeType) -> None: + node_mode = upsert_node_mode() + env = compose_node_env( + SKALE_DIR_ENV_FILEPATH, save=False, node_type=node_type, node_mode=node_mode + ) + turn_off_op(node_type=node_type, node_mode=node_mode, env=env) + + +@check_inited +@check_user +def turn_on(sync_schains, env_file, node_type: NodeType) -> None: + node_mode = upsert_node_mode() + env = compose_node_env( + env_file, + inited_node=True, + sync_schains=sync_schains, + node_type=node_type, + node_mode=node_mode, + ) + turn_on_op(env=env, node_type=node_type, node_mode=node_mode) + logger.info('Waiting for containers initialization') + time.sleep(TM_INIT_TIMEOUT) + if not is_base_containers_alive(node_type=node_type, node_mode=node_mode): + print_node_cmd_error() + return + logger.info('Node turned on') \ No newline at end of file From 85cb965fb58b883d861f2ac52c8261875be28000 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Thu, 11 Sep 2025 13:50:09 +0100 Subject: [PATCH 089/198] Fix skaled container name in docker utils --- node_cli/utils/docker_utils.py | 4 ++-- tests/cli/health_test.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/node_cli/utils/docker_utils.py b/node_cli/utils/docker_utils.py index 26d16bd7..825886e6 100644 --- a/node_cli/utils/docker_utils.py +++ b/node_cli/utils/docker_utils.py @@ -120,7 +120,7 @@ def get_containers(container_name_filter=None, _all=True) -> list: def get_all_schain_containers(_all=True) -> list: - return docker_client().containers.list(all=_all, filters={'name': 'sk_schain_*'}) + return docker_client().containers.list(all=_all, filters={'name': 'sk_skaled_*'}) def get_all_ima_containers(_all=True) -> list: @@ -203,7 +203,7 @@ def start_container_by_name(container_name: str, dclient: Optional[DockerClient] def remove_schain_container_by_name( schain_name: str, dclient: Optional[DockerClient] = None ) -> None: - container_name = f'sk_schain_{schain_name}' + container_name = f'sk_skaled_{schain_name}' remove_container_by_name(container_name, timeout=SCHAIN_REMOVE_TIMEOUT, dclient=dclient) diff --git a/tests/cli/health_test.py b/tests/cli/health_test.py index 6f3fc022..8b3dc19b 100644 --- a/tests/cli/health_test.py +++ b/tests/cli/health_test.py @@ -9,7 +9,7 @@ 'payload': [ { 'image': 'skalenetwork/schain:1.46-develop.21', - 'name': 'sk_schain_shapely-alfecca-meridiana', + 'name': 'sk_skaled_shapely-alfecca-meridiana', 'state': { 'Status': 'running', 'Running': True, From c21ac67f2d21a17be16bde96a6f72766fe5c60eb Mon Sep 17 00:00:00 2001 From: Dmytro Date: Thu, 11 Sep 2025 15:46:55 +0100 Subject: [PATCH 090/198] Fix containers test --- tests/cli/health_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/cli/health_test.py b/tests/cli/health_test.py index 8b3dc19b..404d9a73 100644 --- a/tests/cli/health_test.py +++ b/tests/cli/health_test.py @@ -51,7 +51,7 @@ def test_containers(): assert result.exit_code == 0 assert ( result.output - == ' Name Status Started At Image \n----------------------------------------------------------------------------------------------------------\nsk_schain_shapely-alfecca-meridiana Running Jul 31 2020 11:56:35 skalenetwork/schain:1.46-develop.21\nsk_api Running Jul 31 2020 11:55:17 skale-admin:latest \n' # noqa + == ' Name Status Started At Image \n----------------------------------------------------------------------------------------------------------\nsk_skaled_shapely-alfecca-meridiana Running Jul 31 2020 11:56:35 skalenetwork/schain:1.46-develop.21\nsk_api Running Jul 31 2020 11:55:17 skale-admin:latest \n' # noqa ) From 0bfdb9e72eaafca361a86cf9fbb7cef75d43419f Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Thu, 11 Sep 2025 18:45:02 +0100 Subject: [PATCH 091/198] Remove `sync_schains` parameter --- node_cli/fair/common.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/node_cli/fair/common.py b/node_cli/fair/common.py index b1c3d8e0..7db0a975 100644 --- a/node_cli/fair/common.py +++ b/node_cli/fair/common.py @@ -150,12 +150,11 @@ def turn_off(node_type: NodeType) -> None: @check_inited @check_user -def turn_on(sync_schains, env_file, node_type: NodeType) -> None: +def turn_on(env_file, node_type: NodeType) -> None: node_mode = upsert_node_mode() env = compose_node_env( env_file, inited_node=True, - sync_schains=sync_schains, node_type=node_type, node_mode=node_mode, ) From 878419a1677933652044a91c939abd40891e0653 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Thu, 11 Sep 2025 18:46:31 +0100 Subject: [PATCH 092/198] Add `turn_on_node` and `turn_off_node` commands for passive FAIR nodes --- node_cli/cli/passive_fair_node.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/node_cli/cli/passive_fair_node.py b/node_cli/cli/passive_fair_node.py index 487ded44..4d389b51 100644 --- a/node_cli/cli/passive_fair_node.py +++ b/node_cli/cli/passive_fair_node.py @@ -22,6 +22,9 @@ from node_cli.fair.common import init as init_fair from node_cli.fair.common import update as update_fair from node_cli.fair.common import cleanup as cleanup_fair +from node_cli.fair.common import turn_off as turn_off_fair +from node_cli.fair.common import turn_on as turn_on_fair +from node_cli.cli.info import TYPE from node_cli.fair.passive import setup_fair_passive from node_cli.utils.helper import ( URL_OR_ANY_TYPE, @@ -119,3 +122,30 @@ def cleanup_node(): @click.option('--id', required=True, type=int, help=TEXTS['fair']['node']['setup']['id']) def _setup(id: int) -> None: setup_fair_passive(node_id=id) + + +@passive_node.command('turn-off', help='Turn off the node') +@click.option( + '--yes', + is_flag=True, + callback=abort_if_false, + expose_value=False, + prompt='Are you sure you want to turn off the node?', +) +@streamed_cmd +def turn_off_node() -> None: + turn_off_fair(node_type=TYPE) + + +@passive_node.command('turn-on', help='Turn on the node') +@click.option( + '--yes', + is_flag=True, + callback=abort_if_false, + expose_value=False, + prompt='Are you sure you want to turn on the node?', +) +@click.argument('env_filepath') +@streamed_cmd +def turn_on_node(env_filepath: str) -> None: + turn_on_fair(env_file=env_filepath, node_type=TYPE) From be228672ec9ef008ad451142d6c82cf9fe56c2f7 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Thu, 11 Sep 2025 18:47:39 +0100 Subject: [PATCH 093/198] Rename `turn on/off` functions --- node_cli/cli/fair_node.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/node_cli/cli/fair_node.py b/node_cli/cli/fair_node.py index 355d1e30..dbb5c559 100644 --- a/node_cli/cli/fair_node.py +++ b/node_cli/cli/fair_node.py @@ -212,7 +212,7 @@ def set_domain_name(domain): prompt='Are you sure you want to turn off the node?', ) @streamed_cmd -def _turn_off(): +def turn_off_node() -> None: turn_off_fair(node_type=TYPE) @@ -224,7 +224,7 @@ def _turn_off(): expose_value=False, prompt='Are you sure you want to turn on the node?', ) -@click.argument('env_file') +@click.argument('env_filepath') @streamed_cmd -def _turn_on(env_file): - turn_on_fair(env_file, node_type=TYPE) \ No newline at end of file +def turn_on_node(env_filepath: str) -> None: + turn_on_fair(env_file=env_filepath, node_type=TYPE) From 2baf27ef8a8370781bf10460ee85a3f2a4b5225d Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Thu, 11 Sep 2025 19:27:17 +0100 Subject: [PATCH 094/198] Update `turn_on` function to support FAIR nodes --- node_cli/operations/base.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index 2380d91c..2726413b 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -359,14 +359,23 @@ def turn_off(env: dict, node_type: NodeType, node_mode: NodeMode) -> None: def turn_on(env: dict, node_type: NodeType, node_mode: NodeMode) -> None: logger.info('Turning on the node...') - meta_manager = CliMetaManager() - meta_manager.update_meta( - VERSION, - env['NODE_VERSION'], - env['DOCKER_LVMPY_VERSION'], - distro.id(), - distro.version(), - ) + if node_type == NodeType.FAIR: + meta_manager = FairCliMetaManager() + meta_manager.update_meta( + VERSION, + env['NODE_VERSION'], + distro.id(), + distro.version(), + ) + else: + meta_manager = CliMetaManager() + meta_manager.update_meta( + VERSION, + env['NODE_VERSION'], + env['DOCKER_LVMPY_VERSION'], + distro.id(), + distro.version() + ) if env.get('SKIP_DOCKER_CONFIG') != 'True': configure_docker() From 9c3d290762363e6186f4074a2a004e015629fc1a Mon Sep 17 00:00:00 2001 From: Dmytro Date: Fri, 12 Sep 2025 13:52:46 +0100 Subject: [PATCH 095/198] Fix nginx container name --- node_cli/configs/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node_cli/configs/__init__.py b/node_cli/configs/__init__.py index 5a09dd83..ad4c2216 100644 --- a/node_cli/configs/__init__.py +++ b/node_cli/configs/__init__.py @@ -62,7 +62,7 @@ NGINX_TEMPLATE_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, 'nginx.conf.j2') NGINX_CONFIG_FILEPATH = os.path.join(NODE_DATA_PATH, 'nginx.conf') -NGINX_CONTAINER_NAME = 'skale_nginx' +NGINX_CONTAINER_NAME = 'sk_nginx' LOG_PATH = os.path.join(NODE_DATA_PATH, 'log') REMOVED_CONTAINERS_FOLDER_NAME = '.removed_containers' From d35f885e37c4f5bde12226c007a9fbae9b11751c Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Tue, 16 Sep 2025 15:40:13 +0100 Subject: [PATCH 096/198] Sort imports --- node_cli/cli/fair_node.py | 20 ++++++++------------ node_cli/cli/passive_fair_node.py | 6 +++--- node_cli/fair/common.py | 14 +++++++------- 3 files changed, 18 insertions(+), 22 deletions(-) diff --git a/node_cli/cli/fair_node.py b/node_cli/cli/fair_node.py index dbb5c559..2f7ccc4f 100644 --- a/node_cli/cli/fair_node.py +++ b/node_cli/cli/fair_node.py @@ -18,25 +18,21 @@ # along with this program. If not, see . import click + from node_cli.cli.info import TYPE from node_cli.core.node import backup - from node_cli.fair.active import change_ip as change_ip_fair -from node_cli.fair.common import cleanup as cleanup_fair -from node_cli.fair.common import turn_off as turn_off_fair -from node_cli.fair.common import turn_on as turn_on_fair from node_cli.fair.active import exit as exit_fair -from node_cli.fair.active import ( - get_node_info, - migrate_from_boot, - restore as restore_fair, -) -from node_cli.fair.common import init as init_fair +from node_cli.fair.active import get_node_info, migrate_from_boot from node_cli.fair.active import register as register_fair -from node_cli.fair.common import update as update_fair +from node_cli.fair.active import restore as restore_fair from node_cli.fair.active import set_domain_name as set_domain_name_fair +from node_cli.fair.common import cleanup as cleanup_fair +from node_cli.fair.common import init as init_fair from node_cli.fair.common import repair_chain - +from node_cli.fair.common import turn_off as turn_off_fair +from node_cli.fair.common import turn_on as turn_on_fair +from node_cli.fair.common import update as update_fair from node_cli.utils.helper import IP_TYPE, URL_OR_ANY_TYPE, abort_if_false, streamed_cmd from node_cli.utils.node_type import NodeMode from node_cli.utils.texts import safe_load_texts diff --git a/node_cli/cli/passive_fair_node.py b/node_cli/cli/passive_fair_node.py index 4d389b51..e48f0ce0 100644 --- a/node_cli/cli/passive_fair_node.py +++ b/node_cli/cli/passive_fair_node.py @@ -19,12 +19,12 @@ import click -from node_cli.fair.common import init as init_fair -from node_cli.fair.common import update as update_fair +from node_cli.cli.info import TYPE from node_cli.fair.common import cleanup as cleanup_fair +from node_cli.fair.common import init as init_fair from node_cli.fair.common import turn_off as turn_off_fair from node_cli.fair.common import turn_on as turn_on_fair -from node_cli.cli.info import TYPE +from node_cli.fair.common import update as update_fair from node_cli.fair.passive import setup_fair_passive from node_cli.utils.helper import ( URL_OR_ANY_TYPE, diff --git a/node_cli/fair/common.py b/node_cli/fair/common.py index 7db0a975..c8b4d9d5 100644 --- a/node_cli/fair/common.py +++ b/node_cli/fair/common.py @@ -17,31 +17,31 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import time import logging +import time from node_cli.configs import INIT_TIMEOUT, SKALE_DIR, TM_INIT_TIMEOUT from node_cli.configs.user import SKALE_DIR_ENV_FILEPATH from node_cli.core.docker_config import cleanup_docker_configuration +from node_cli.core.host import save_env_params from node_cli.core.node import compose_node_env, is_base_containers_alive from node_cli.core.node_options import upsert_node_mode from node_cli.fair.passive import setup_fair_passive from node_cli.operations import ( FairUpdateType, cleanup_fair_op, - repair_fair_op, - update_fair_op, init_fair_op, + repair_fair_op, turn_off_op, - turn_on_op + turn_on_op, + update_fair_op, ) -from node_cli.core.host import save_env_params from node_cli.utils.decorators import check_inited, check_not_inited, check_user +from node_cli.utils.exit_codes import CLIExitCodes +from node_cli.utils.helper import error_exit from node_cli.utils.node_type import NodeMode, NodeType from node_cli.utils.print_formatters import print_node_cmd_error from node_cli.utils.texts import safe_load_texts -from node_cli.utils.exit_codes import CLIExitCodes -from node_cli.utils.helper import error_exit logger = logging.getLogger(__name__) TEXTS = safe_load_texts() From 0b01cdea45cfce938db1ebb796d925c5dafe94b0 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Tue, 16 Sep 2025 16:03:55 +0100 Subject: [PATCH 097/198] Update README.md --- README.md | 58 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/README.md b/README.md index a50d8a3e..a9cc916a 100644 --- a/README.md +++ b/README.md @@ -828,6 +828,35 @@ Options: * `--yes` - Update without confirmation prompt. * `--force-skaled-start` - Force skaled container to start (hidden option). +#### Fair Node turn-off + +Turn off the Fair node containers. + +```shell +fair node turn-off [--yes] +``` + +Options: + +* `--yes` - Turn off without confirmation. + +#### Fair Node turn-on + +Turn on the Fair node containers. + +```shell +fair node turn-on [ENV_FILEPATH] [--yes] +``` + +Arguments: + +* `ENV_FILEPATH` - Path to the .env file. + +Options: + +* `--yes` - Turn on without additional confirmation. + + #### Fair Node Migrate Switch from boot phase to regular Fair node operation. @@ -1183,6 +1212,35 @@ Update software / configs for passive Fair node. fair passive-node update [--yes] ``` +#### Passive Fair Node turn-off + +Turn off the Fair passive node containers. + +```shell +fair passive-node turn-off [--yes] +``` + +Options: + +* `--yes` - Turn off without confirmation. + +#### Passive Fair Node turn-on + +Turn on the Fair passive node containers. + +```shell +fair passive-node turn-on [ENV_FILEPATH] [--yes] +``` + +Arguments: + +* `ENV_FILEPATH` - Path to the .env file. + +Options: + +* `--yes` - Turn on without additional confirmation. + + #### Passive Fair Node Cleanup Remove all passive Fair node data and containers. From da2d5b8acc93dc17ac52c46cb12604e62e3970cc Mon Sep 17 00:00:00 2001 From: Dmytro Date: Wed, 17 Sep 2025 19:17:37 +0100 Subject: [PATCH 098/198] Update staking functions --- README.md | 54 +++++++++++++++++++--------- node_cli/cli/staking.py | 52 +++++++++++++++++++-------- node_cli/configs/routes.py | 6 ++-- node_cli/fair/staking.py | 72 +++++++++++++++++++++++++++++++------- tests/routes_test.py | 6 ++-- 5 files changed, 144 insertions(+), 46 deletions(-) diff --git a/README.md b/README.md index a50d8a3e..9ccb80ca 100644 --- a/README.md +++ b/README.md @@ -1098,48 +1098,70 @@ Arguments: * `RECEIVER_ADDRESS` - Address to remove from the allowed receivers list. -#### Claim fees +Workflow (fees): request fees -> review exit requests -> claim request. -Claim a specific amount of fees or all fees to the node wallet. +#### Request fees + +Create a request to claim a specific amount of earned fees (FAIR). Use `--all` to request all. ```shell -fair staking claim-fees -fair staking claim-fees --all +fair staking request-fees +fair staking request-fees --all ``` -#### Set fee rate +#### Request send fees -Set the fee rate (uint16 value) used by the staking logic. +Create a request to send a specific amount (or all) of earned fees to an address. ```shell -fair staking set-fee-rate +fair staking request-send-fees +fair staking request-send-fees --all ``` Arguments: -* `FEE_RATE` - Fee rate value as integer (uint16). +* `TO_ADDRESS` - Destination address for the fee transfer. +* `AMOUNT` - Amount of fees to include in the request (FAIR). -#### Send fees +#### Claim request -Send a specific amount of fees to the default allowed receiver. +Claim a previously created request by its request ID once it is unlocked. ```shell -fair staking send-fees +fair staking claim-request ``` -Arguments: +#### Get exit requests -* `TO_ADDRESS` - Destination address for the fee transfer. -* `AMOUNT` - Amount of fees to send (FAIR). Use `--all` to send all. +List exit (fee withdrawal) requests for the current wallet. Use `--json` for raw JSON output. + +```shell +fair staking exit-requests +fair staking exit-requests --json +``` + +Default output (non-JSON) shows: `request_id`, `user`, `node_id`, `amount_wei`, `amount_fair`, `unlock_date (ISO)`. #### Get earned fee amount -Get the currently earned fee amount. +Get the currently earned (unrequested) fee amount. ```shell -fair staking get-earned-fee-amount +fair staking earned-fee-amount ``` +#### Set fee rate + +Set the fee rate (uint16 value) used by the staking logic. + +```shell +fair staking set-fee-rate +``` + +Arguments: + +* `FEE_RATE` - Fee rate value as integer (uint16). + ### Passive Fair Node commands > Prefix: `fair passive-node` (passive Fair build) diff --git a/node_cli/cli/staking.py b/node_cli/cli/staking.py index 8cde234c..8cb08d2d 100644 --- a/node_cli/cli/staking.py +++ b/node_cli/cli/staking.py @@ -23,9 +23,11 @@ add_allowed_receiver, remove_allowed_receiver, set_fee_rate, - claim_fees, - send_fees, + request_fees, + request_send_fees, + claim_request, get_earned_fee_amount, + get_exit_requests, ) from node_cli.utils.helper import abort_if_false @@ -79,39 +81,61 @@ def _set_fee_rate(fee_rate: int) -> None: set_fee_rate(fee_rate) -@staking.command('claim-fees', help='Claim fees amount (FAIR) or all with --all') +@staking.command('request-fees', help='Create a request to claim fees (FAIR) or all with --all') @click.argument('amount', type=float, required=False) -@click.option('--all', 'claim_all', is_flag=True, help='Claim all fees') +@click.option('--all', 'request_all', is_flag=True, help='Request all fees') @click.option( '--yes', is_flag=True, callback=abort_if_false, expose_value=False, - prompt='Are you sure you want to claim fees?', + prompt='Are you sure you want to request fees?', ) -def _claim_fees(amount: float | None, claim_all: bool) -> None: - if amount is None and not claim_all: +def _request_fees(amount: float | None, request_all: bool) -> None: + if amount is None and not request_all: raise click.UsageError('Provide or use --all') - claim_fees(None if claim_all else amount) + request_fees(None if request_all else amount) -@staking.command('send-fees', help='Send fees to address (or all with --all)') +@staking.command( + 'request-send-fees', + help='Create a request to send fees to address (or all with --all)', +) @click.argument('to') @click.argument('amount', type=float, required=False) -@click.option('--all', 'send_all', is_flag=True, help='Send all fees to address') +@click.option('--all', 'send_all', is_flag=True, help='Request to send all fees to address') @click.option( '--yes', is_flag=True, callback=abort_if_false, expose_value=False, - prompt='Are you sure you want to send fees?', + prompt='Are you sure you want to request to send fees?', ) -def _send_fees(to: str, amount: float | None, send_all: bool) -> None: +def _request_send_fees(to: str, amount: float | None, send_all: bool) -> None: if amount is None and not send_all: raise click.UsageError('Provide or use --all') - send_fees(to, None if send_all else amount) + request_send_fees(to, None if send_all else amount) -@staking.command('get-earned-fee-amount', help='Get earned fee amount') +@staking.command('claim-request', help='Claim previously created request by request ID') +@click.argument('request_id', type=int) +@click.option( + '--yes', + is_flag=True, + callback=abort_if_false, + expose_value=False, + prompt='Are you sure you want to claim this request?', +) +def _claim_request(request_id: int) -> None: + claim_request(request_id) + + +@staking.command('earned-fee-amount', help='Get earned fee amount') def _get_earned_fee_amount() -> None: get_earned_fee_amount() + + +@staking.command('exit-requests', help='Get exit requests for current wallet') +@click.option('--json', 'raw', is_flag=True, help='Output in JSON format') +def _get_exit_requests(raw: bool) -> None: + get_exit_requests(raw=raw) diff --git a/node_cli/configs/routes.py b/node_cli/configs/routes.py index 83bb2866..336fea3e 100644 --- a/node_cli/configs/routes.py +++ b/node_cli/configs/routes.py @@ -47,9 +47,11 @@ 'add-receiver', 'remove-receiver', 'set-fee-rate', - 'claim-fees', - 'send-fees', + 'request-fees', + 'request-send-fees', + 'claim-request', 'get-earned-fee-amount', + 'get-exit-requests', ], } } diff --git a/node_cli/fair/staking.py b/node_cli/fair/staking.py index 9e81e69f..4b7a4008 100644 --- a/node_cli/fair/staking.py +++ b/node_cli/fair/staking.py @@ -18,6 +18,8 @@ # along with this program. If not, see . from typing import Any +import json +from datetime import datetime, timezone from node_cli.utils.decorators import check_inited from node_cli.utils.exit_codes import CLIExitCodes @@ -50,12 +52,27 @@ def remove_allowed_receiver(receiver: str) -> None: @check_inited -def send_fees(to: str, amount: float | None) -> None: +def request_fees(amount: float | None) -> None: + json_data: dict[str, Any] = {} + if amount is not None: + json_data['amount'] = amount + status, payload = post_request(blueprint=BLUEPRINT_NAME, method='request-fees', json=json_data) + _handle_response( + status, + payload, + success='All fees requested' if amount is None else f'Fees requested: {amount}', + ) + + +@check_inited +def request_send_fees(to: str, amount: float | None) -> None: json_data: dict[str, Any] = {'to': to} if amount is not None: json_data['amount'] = amount - status, payload = post_request(blueprint=BLUEPRINT_NAME, method='send-fees', json=json_data) - _handle_response(status, payload, success=f'Fees sent to {to}') + status, payload = post_request( + blueprint=BLUEPRINT_NAME, method='request-send-fees', json=json_data + ) + _handle_response(status, payload, success=f'Fees request to send to {to} created') @check_inited @@ -67,16 +84,11 @@ def set_fee_rate(fee_rate: int) -> None: @check_inited -def claim_fees(amount: float | None) -> None: - json_data: dict[str, Any] = {} - if amount is not None: - json_data['amount'] = amount - status, payload = post_request(blueprint=BLUEPRINT_NAME, method='claim-fees', json=json_data) - _handle_response( - status, - payload, - success='All fees claimed' if amount is None else f'Fees claimed: {amount}', +def claim_request(request_id: int) -> None: + status, payload = post_request( + blueprint=BLUEPRINT_NAME, method='claim-request', json={'requestId': request_id} ) + _handle_response(status, payload, success=f'Request claimed: {request_id}') @check_inited @@ -88,3 +100,39 @@ def get_earned_fee_amount() -> None: print(f'Earned fee amount: {amount_wei} wei ({amount_ether} FAIR)') return error_exit(payload, exit_code=CLIExitCodes.BAD_API_RESPONSE) + + +@check_inited +def get_exit_requests(raw: bool = False) -> None: + status, payload = post_request(blueprint=BLUEPRINT_NAME, method='get-exit-requests') + if status == 'ok' and isinstance(payload, dict): + exit_requests = payload.get('exit_requests') + if not isinstance(exit_requests, list): + error_exit(payload, exit_code=CLIExitCodes.BAD_API_RESPONSE) + return + if raw: + print(json.dumps(exit_requests, indent=2)) + return + for req in exit_requests: + try: + request_id = req.get('request_id') + user = req.get('user') + node_id = req.get('node_id') + amount = req.get('amount') + unlock_date = req.get('unlock_date') + amount_fair = None + if isinstance(amount, int): + amount_fair = amount / 10**18 + unlock_iso = None + if isinstance(unlock_date, int): + unlock_iso = datetime.fromtimestamp(unlock_date, tz=timezone.utc).isoformat() + base = ( + f'request_id: {request_id} | user: {user} | node_id: {node_id} | ' + f'amount_wei: {amount} | amount_fair: {amount_fair} | ' + f'unlock_date: {unlock_date}' + ) + print(base + (f' ({unlock_iso})' if unlock_iso else '')) + except Exception: # noqa: BLE001 + print(req) + return + error_exit(payload, exit_code=CLIExitCodes.BAD_API_RESPONSE) diff --git a/tests/routes_test.py b/tests/routes_test.py index b64e9e0a..39490ab3 100644 --- a/tests/routes_test.py +++ b/tests/routes_test.py @@ -42,9 +42,11 @@ '/api/v1/fair-staking/add-receiver', '/api/v1/fair-staking/remove-receiver', '/api/v1/fair-staking/set-fee-rate', - '/api/v1/fair-staking/claim-fees', - '/api/v1/fair-staking/send-fees', + '/api/v1/fair-staking/request-fees', + '/api/v1/fair-staking/request-send-fees', + '/api/v1/fair-staking/claim-request', '/api/v1/fair-staking/get-earned-fee-amount', + '/api/v1/fair-staking/get-exit-requests', ] From 42d45335e182239e997b067bab5bdef7bfbe7820 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Thu, 18 Sep 2025 21:46:51 +0100 Subject: [PATCH 099/198] Fix repair cmd --- node_cli/cli/fair_node.py | 6 +++--- node_cli/fair/common.py | 2 +- text.yml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/node_cli/cli/fair_node.py b/node_cli/cli/fair_node.py index 2f7ccc4f..343be1de 100644 --- a/node_cli/cli/fair_node.py +++ b/node_cli/cli/fair_node.py @@ -138,7 +138,7 @@ def migrate_node(env_filepath: str) -> None: type=URL_OR_ANY_TYPE, default='any', hidden=True, - help=TEXTS['fair']['node']['repair']['snapshot_from'], + help=TEXTS['fair']['node']['repair']['snapshot'], ) @click.option( '--yes', @@ -148,8 +148,8 @@ def migrate_node(env_filepath: str) -> None: prompt=TEXTS['fair']['node']['repair']['warning'], ) @streamed_cmd -def repair(snapshot_from: str = 'any') -> None: - repair_chain(snapshot_from=snapshot_from) +def repair(snapshot: str = 'any') -> None: + repair_chain(snapshot_from=snapshot) @node.command('cleanup', help='Cleanup Fair node.') diff --git a/node_cli/fair/common.py b/node_cli/fair/common.py index c8b4d9d5..dc91e26e 100644 --- a/node_cli/fair/common.py +++ b/node_cli/fair/common.py @@ -164,4 +164,4 @@ def turn_on(env_file, node_type: NodeType) -> None: if not is_base_containers_alive(node_type=node_type, node_mode=node_mode): print_node_cmd_error() return - logger.info('Node turned on') \ No newline at end of file + logger.info('Node turned on') diff --git a/text.yml b/text.yml index c0cbe23a..6013f2d0 100644 --- a/text.yml +++ b/text.yml @@ -82,7 +82,7 @@ fair: repair: help: Repair Fair chain node warning: Are you sure you want to repair Fair chain node? In rare cases may cause data loss and require additional maintenance - snapshot_from: IP of the node to take snapshot from (put "any" to use any available node) + snapshot: IP of the node to take snapshot from (put "any" to use any available node) repair_requested: Repair mode is requested not_inited: Node should be initialized to proceed with operation From c38d88f95cc6843a8b3db464e3e191c34a28811c Mon Sep 17 00:00:00 2001 From: Dmytro Date: Thu, 25 Sep 2025 17:27:21 +0100 Subject: [PATCH 100/198] Fix snapshot option for passive nodes, add logs --- node_cli/fair/record/redis_record.py | 4 ++++ node_cli/operations/fair.py | 3 +++ node_cli/utils/helper.py | 4 ++-- tests/cli/passive_node_test.py | 31 ++++++++++++++++++++++++++++ 4 files changed, 40 insertions(+), 2 deletions(-) diff --git a/node_cli/fair/record/redis_record.py b/node_cli/fair/record/redis_record.py index b71aa9e4..f40aa898 100644 --- a/node_cli/fair/record/redis_record.py +++ b/node_cli/fair/record/redis_record.py @@ -18,6 +18,7 @@ # along with this program. If not, see . import abc +import logging from dataclasses import dataclass from datetime import datetime from typing import Any @@ -26,6 +27,8 @@ from node_cli.configs import REDIS_URI +logger = logging.getLogger(__name__) + cpool: redis.ConnectionPool = redis.ConnectionPool.from_url(REDIS_URI) rs: redis.Redis = redis.Redis(connection_pool=cpool) @@ -83,6 +86,7 @@ def _get_field(self, field_name: str): def _set_field(self, field_name: str, value) -> None: key = self._get_field_key(field_name) serialized_value = self._serialize_field(value, self._record_fields()[field_name].type) + logger.info('Setting field %s to value %s', field_name, serialized_value) rs.set(key, serialized_value) def _deserialize_field(self, value, field_type: type): diff --git a/node_cli/operations/fair.py b/node_cli/operations/fair.py index 4fc10b16..689b4a51 100644 --- a/node_cli/operations/fair.py +++ b/node_cli/operations/fair.py @@ -112,8 +112,10 @@ def init( upsert_node_mode(node_mode=node_mode) if node_mode == NodeMode.PASSIVE: + logger.info('Setting passive node options') set_passive_node_options(archive=archive, indexer=indexer) if snapshot: + logger.info('Waiting %s seconds for redis to start', REDIS_START_TIMEOUT) time.sleep(REDIS_START_TIMEOUT) trigger_skaled_snapshot_mode(env=env, snapshot_from=snapshot) @@ -293,6 +295,7 @@ def trigger_skaled_snapshot_mode(env: dict, snapshot_from: str = 'any') -> None: record = get_fair_chain_record(env) if not snapshot_from: snapshot_from = 'any' + logger.info('Triggering skaled snapshot mode, snapshot_from: %s', snapshot_from) record.set_snapshot_from(snapshot_from) diff --git a/node_cli/utils/helper.py b/node_cli/utils/helper.py index d299fbf6..c9c9bfb0 100644 --- a/node_cli/utils/helper.py +++ b/node_cli/utils/helper.py @@ -382,13 +382,13 @@ def convert(self, value, param, ctx): return value -class UrlOrAnyType(click.ParamType): +class UrlOrAnyType(UrlType): name = 'url' def convert(self, value, param, ctx): if value == 'any': return value - super().convert(value, param, ctx) + return super().convert(value, param, ctx) class IpType(click.ParamType): diff --git a/tests/cli/passive_node_test.py b/tests/cli/passive_node_test.py index 94cde559..e0342340 100644 --- a/tests/cli/passive_node_test.py +++ b/tests/cli/passive_node_test.py @@ -57,6 +57,37 @@ def test_init_passive(mocked_g_config, clean_node_options, passive_user_conf): assert result.exit_code == 0 +def test_init_passive_snapshot_any(mocked_g_config, clean_node_options, passive_user_conf): + pathlib.Path(NODE_DATA_PATH).mkdir(parents=True, exist_ok=True) + with ( + mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), + mock.patch('node_cli.operations.base.cleanup_volume_artifacts'), + mock.patch('node_cli.operations.base.download_skale_node'), + mock.patch('node_cli.operations.base.sync_skale_node'), + mock.patch('node_cli.operations.base.configure_docker'), + mock.patch('node_cli.operations.base.prepare_host'), + mock.patch('node_cli.operations.base.ensure_filestorage_mapping'), + mock.patch('node_cli.operations.base.link_env_file'), + mock.patch('node_cli.operations.base.generate_nginx_config'), + mock.patch('node_cli.operations.base.prepare_block_device'), + mock.patch('node_cli.operations.base.CliMetaManager.update_meta'), + mock.patch('node_cli.operations.base.update_resource_allocation'), + mock.patch('node_cli.operations.base.update_images'), + mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), + mock.patch('node_cli.operations.base.configure_nftables'), + mock.patch('node_cli.utils.decorators.is_node_inited', return_value=False), + mock.patch('node_cli.configs.user.validate_alias_or_address'), + mock.patch('node_cli.cli.node.TYPE', NodeType.SKALE), + mock.patch('node_cli.operations.base.compose_up') as compose_up_mock, + ): + result = run_command(_init_passive, [passive_user_conf.as_posix(), '--snapshot', 'any']) + assert result.exit_code == 0 + assert compose_up_mock.called + args, kwargs = compose_up_mock.call_args + assert 'snapshot' in kwargs + assert kwargs['snapshot'] == 'any' + + def test_init_passive_archive(mocked_g_config, clean_node_options, passive_user_conf): pathlib.Path(NODE_DATA_PATH).mkdir(parents=True, exist_ok=True) with ( From 273368bcead9c86dcc9da3d46d2997d80913d32e Mon Sep 17 00:00:00 2001 From: Dmytro Date: Thu, 25 Sep 2025 18:16:45 +0100 Subject: [PATCH 101/198] Add fair passive node tests --- tests/cli/fair_passive_node_test.py | 86 +++++++++++++++++++++++++++++ tests/cli/passive_node_test.py | 31 ----------- 2 files changed, 86 insertions(+), 31 deletions(-) create mode 100644 tests/cli/fair_passive_node_test.py diff --git a/tests/cli/fair_passive_node_test.py b/tests/cli/fair_passive_node_test.py new file mode 100644 index 00000000..34ececf9 --- /dev/null +++ b/tests/cli/fair_passive_node_test.py @@ -0,0 +1,86 @@ +import logging +import pathlib + +import mock + +from node_cli.cli.passive_fair_node import init_passive_node, update_node +from node_cli.configs import NODE_DATA_PATH, SKALE_DIR +from node_cli.utils.helper import init_default_logger +from tests.helper import run_command, subprocess_run_mock +from tests.resources_test import BIG_DISK_SIZE + +logger = logging.getLogger(__name__) +init_default_logger() + + +def test_init_fair_passive(mocked_g_config, tmp_path): + env_file = tmp_path / 'test-env' + env_file.write_text('') + pathlib.Path(SKALE_DIR).mkdir(parents=True, exist_ok=True) + with ( + mock.patch('subprocess.run', new=subprocess_run_mock), + mock.patch('node_cli.fair.common.init_fair_op', return_value=True), + mock.patch('node_cli.fair.common.compose_node_env', return_value={}), + mock.patch('node_cli.fair.common.save_env_params'), + mock.patch('node_cli.fair.passive.setup_fair_passive'), + mock.patch('node_cli.fair.common.time.sleep'), + mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), + mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), + mock.patch('node_cli.operations.base.configure_nftables'), + mock.patch('node_cli.utils.decorators.is_node_inited', return_value=False), + ): + result = run_command( + init_passive_node, + [ + env_file.as_posix(), + '--id', + '1', + ], + ) + assert result.exit_code == 0 + + +def test_init_fair_passive_snapshot_any(mocked_g_config, tmp_path): + env_file = tmp_path / 'test-env' + env_file.write_text('') + pathlib.Path(SKALE_DIR).mkdir(parents=True, exist_ok=True) + with ( + mock.patch('subprocess.run', new=subprocess_run_mock), + mock.patch('node_cli.fair.common.init_fair_op', return_value=True), + mock.patch('node_cli.fair.common.compose_node_env', return_value={}), + mock.patch('node_cli.fair.common.save_env_params'), + mock.patch('node_cli.fair.passive.setup_fair_passive'), + mock.patch('node_cli.fair.common.time.sleep'), + mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), + mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), + mock.patch('node_cli.operations.base.configure_nftables'), + mock.patch('node_cli.utils.decorators.is_node_inited', return_value=False), + ): + result = run_command( + init_passive_node, + [ + env_file.as_posix(), + '--id', + '2', + '--snapshot', + 'any', + ], + ) + assert result.exit_code == 0 + + +def test_update_fair_passive(mocked_g_config, tmp_path): + env_file = tmp_path / 'test-env' + env_file.write_text('') + pathlib.Path(NODE_DATA_PATH).mkdir(parents=True, exist_ok=True) + with ( + mock.patch('subprocess.run', new=subprocess_run_mock), + mock.patch('node_cli.fair.common.update_fair_op', return_value=True), + mock.patch('node_cli.fair.common.compose_node_env', return_value={}), + mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), + mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), + mock.patch('node_cli.operations.base.configure_nftables'), + mock.patch('node_cli.utils.decorators.is_node_inited', return_value=True), + ): + result = run_command(update_node, [env_file.as_posix(), '--yes']) + assert result.exit_code == 0 diff --git a/tests/cli/passive_node_test.py b/tests/cli/passive_node_test.py index e0342340..94cde559 100644 --- a/tests/cli/passive_node_test.py +++ b/tests/cli/passive_node_test.py @@ -57,37 +57,6 @@ def test_init_passive(mocked_g_config, clean_node_options, passive_user_conf): assert result.exit_code == 0 -def test_init_passive_snapshot_any(mocked_g_config, clean_node_options, passive_user_conf): - pathlib.Path(NODE_DATA_PATH).mkdir(parents=True, exist_ok=True) - with ( - mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), - mock.patch('node_cli.operations.base.cleanup_volume_artifacts'), - mock.patch('node_cli.operations.base.download_skale_node'), - mock.patch('node_cli.operations.base.sync_skale_node'), - mock.patch('node_cli.operations.base.configure_docker'), - mock.patch('node_cli.operations.base.prepare_host'), - mock.patch('node_cli.operations.base.ensure_filestorage_mapping'), - mock.patch('node_cli.operations.base.link_env_file'), - mock.patch('node_cli.operations.base.generate_nginx_config'), - mock.patch('node_cli.operations.base.prepare_block_device'), - mock.patch('node_cli.operations.base.CliMetaManager.update_meta'), - mock.patch('node_cli.operations.base.update_resource_allocation'), - mock.patch('node_cli.operations.base.update_images'), - mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), - mock.patch('node_cli.operations.base.configure_nftables'), - mock.patch('node_cli.utils.decorators.is_node_inited', return_value=False), - mock.patch('node_cli.configs.user.validate_alias_or_address'), - mock.patch('node_cli.cli.node.TYPE', NodeType.SKALE), - mock.patch('node_cli.operations.base.compose_up') as compose_up_mock, - ): - result = run_command(_init_passive, [passive_user_conf.as_posix(), '--snapshot', 'any']) - assert result.exit_code == 0 - assert compose_up_mock.called - args, kwargs = compose_up_mock.call_args - assert 'snapshot' in kwargs - assert kwargs['snapshot'] == 'any' - - def test_init_passive_archive(mocked_g_config, clean_node_options, passive_user_conf): pathlib.Path(NODE_DATA_PATH).mkdir(parents=True, exist_ok=True) with ( From ae3bfc6cf8515a727803778abe36e88429e49c05 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Mon, 27 Oct 2025 11:37:52 +0000 Subject: [PATCH 102/198] Fix init and update commands for SKALE nodes --- Dockerfile | 24 ++++++++++++------------ node_cli/core/node.py | 4 ++-- node_cli/operations/base.py | 21 ++++++++------------- 3 files changed, 22 insertions(+), 27 deletions(-) diff --git a/Dockerfile b/Dockerfile index c2fc6972..abc1fcd3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,18 +2,18 @@ FROM python:3.11-bookworm ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && apt install -y \ - git \ - build-essential \ - software-properties-common \ - zlib1g-dev \ - libssl-dev \ - libffi-dev \ - swig \ - iptables \ - nftables \ - python3-nftables \ - libxslt-dev \ - kmod + git \ + build-essential \ + software-properties-common \ + zlib1g-dev \ + libssl-dev \ + libffi-dev \ + swig \ + iptables \ + nftables \ + python3-nftables \ + libxslt-dev \ + kmod RUN mkdir /app diff --git a/node_cli/core/node.py b/node_cli/core/node.py index 3a704ddc..992c1e20 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -157,7 +157,7 @@ def init(env_filepath: str, node_type: NodeType) -> None: node_mode = NodeMode.ACTIVE env = compose_node_env(env_filepath=env_filepath, node_type=node_type, node_mode=node_mode) - init_op(env_filepath=env_filepath, env=env, node_type=node_type, node_mode=node_mode) + init_op(env_filepath=env_filepath, env=env, node_mode=node_mode) logger.info('Waiting for containers initialization') time.sleep(TM_INIT_TIMEOUT) if not is_base_containers_alive(node_type=node_type, node_mode=node_mode): @@ -314,7 +314,7 @@ def update( node_type=node_type, node_mode=node_mode, ) - update_ok = update_op(env_filepath, env, node_type=node_type, node_mode=node_mode) + update_ok = update_op(env_filepath, env, node_mode=node_mode) if update_ok: logger.info('Waiting for containers initialization') time.sleep(TM_INIT_TIMEOUT) diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index 2726413b..55c54914 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -114,8 +114,8 @@ def wrapper(env_filepath: str, env: Dict, node_mode: NodeMode, *args, **kwargs): @checked_host -def update(env_filepath: str, env: Dict, node_type: NodeType, node_mode: NodeMode) -> bool: - compose_rm(node_type=node_type, node_mode=node_mode, env=env) +def update(env_filepath: str, env: Dict, node_mode: NodeMode) -> bool: + compose_rm(node_type=NodeType.SKALE, node_mode=node_mode, env=env) remove_dynamic_containers() sync_skale_node() @@ -151,8 +151,8 @@ def update(env_filepath: str, env: Dict, node_type: NodeType, node_mode: NodeMod distro.id(), distro.version(), ) - update_images(env=env, node_type=node_type, node_mode=node_mode) - compose_up(env=env, node_type=node_type, node_mode=node_mode) + update_images(env=env, node_type=NodeType.SKALE, node_mode=node_mode) + compose_up(env=env, node_type=NodeType.SKALE, node_mode=node_mode) return True @@ -199,7 +199,7 @@ def update_fair_boot(env_filepath: str, env: Dict, node_mode: NodeMode = NodeMod @checked_host -def init(env_filepath: str, env: dict, node_type: NodeType, node_mode: NodeMode) -> None: +def init(env_filepath: str, env: dict, node_mode: NodeMode) -> None: sync_skale_node() ensure_btrfs_kernel_module_autoloaded() if env.get('SKIP_DOCKER_CONFIG') != 'True': @@ -229,9 +229,8 @@ def init(env_filepath: str, env: dict, node_type: NodeType, node_mode: NodeMode) distro.version(), ) update_resource_allocation(env_type=env['ENV_TYPE']) - update_images(env=env, node_type=node_type, node_mode=node_mode) - - compose_up(env=env, node_type=node_type, node_mode=node_mode) + update_images(env=env, node_type=NodeType.SKALE, node_mode=node_mode) + compose_up(env=env, node_type=NodeType.SKALE, node_mode=node_mode) @checked_host @@ -370,11 +369,7 @@ def turn_on(env: dict, node_type: NodeType, node_mode: NodeMode) -> None: else: meta_manager = CliMetaManager() meta_manager.update_meta( - VERSION, - env['NODE_VERSION'], - env['DOCKER_LVMPY_VERSION'], - distro.id(), - distro.version() + VERSION, env['NODE_VERSION'], env['DOCKER_LVMPY_VERSION'], distro.id(), distro.version() ) if env.get('SKIP_DOCKER_CONFIG') != 'True': configure_docker() From c756859520d02bccec49918c2269e2065c3ca7c2 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Tue, 28 Oct 2025 17:53:32 +0000 Subject: [PATCH 103/198] Add common cleanup function for SKALE active and passive nodes --- node_cli/operations/__init__.py | 2 +- node_cli/operations/base.py | 21 ++++++++++++++++----- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/node_cli/operations/__init__.py b/node_cli/operations/__init__.py index 5ae8103f..f2ac1a94 100644 --- a/node_cli/operations/__init__.py +++ b/node_cli/operations/__init__.py @@ -27,7 +27,7 @@ turn_off as turn_off_op, turn_on as turn_on_op, restore as restore_op, - cleanup_passive as cleanup_passive_op, + cleanup as cleanup_skale_op, configure_nftables, ) from node_cli.operations.fair import ( # noqa diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index 2726413b..a866c9ea 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -29,11 +29,12 @@ CONTAINER_CONFIG_PATH, CONTAINER_CONFIG_TMP_PATH, GLOBAL_SKALE_DIR, + NFTABLES_CHAIN_FOLDER_PATH, SKALE_DIR, ) from node_cli.core.checks import CheckType from node_cli.core.checks import run_checks as run_host_checks -from node_cli.core.docker_config import configure_docker +from node_cli.core.docker_config import cleanup_docker_configuration, configure_docker from node_cli.core.host import ( ensure_btrfs_kernel_module_autoloaded, link_env_file, @@ -69,7 +70,7 @@ docker_cleanup, remove_dynamic_containers, ) -from node_cli.utils.helper import rm_dir, str_to_bool +from node_cli.utils.helper import cleanup_dir_content, rm_dir, str_to_bool from node_cli.utils.meta import CliMetaManager, FairCliMetaManager from node_cli.utils.node_type import NodeType, NodeMode from node_cli.utils.print_formatters import print_failed_requirements_checks @@ -438,8 +439,18 @@ def restore(env, backup_path, node_type: NodeType, config_only=False): return True -def cleanup_passive(env, schain_name: str) -> None: - turn_off(env, node_type=NodeType.SKALE, node_mode=NodeMode.PASSIVE) - cleanup_no_lvm_datadir(chain_name=schain_name) +def cleanup_active(): + pass + + +def cleanup(node_mode: NodeMode, env: dict) -> None: + turn_off(env, node_type=NodeType.SKALE, node_mode=node_mode) + if node_mode == NodeMode.PASSIVE: + schain_name = env['SCHAIN_NAME'] + cleanup_no_lvm_datadir(chain_name=schain_name) + else: + cleanup_active() rm_dir(GLOBAL_SKALE_DIR) rm_dir(SKALE_DIR) + cleanup_dir_content(NFTABLES_CHAIN_FOLDER_PATH) + cleanup_docker_configuration() \ No newline at end of file From 26ba3ffd199de642a61f71ba57c7b2bbe5510d34 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Tue, 28 Oct 2025 17:55:55 +0000 Subject: [PATCH 104/198] Change `cleanup_passive` function by common one for all SKALE nodes --- node_cli/core/node.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/node_cli/core/node.py b/node_cli/core/node.py index 3a704ddc..143f6848 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -56,7 +56,7 @@ ) from node_cli.migrations.focal_to_jammy import migrate as migrate_2_6 from node_cli.operations import ( - cleanup_passive_op, + cleanup_skale_op, configure_nftables, init_op, init_passive_op, @@ -227,13 +227,13 @@ def update_passive(env_filepath: str, unsafe_ok: bool = False) -> None: @check_inited @check_user -def cleanup_passive() -> None: +def cleanup(node_mode: NodeMode) -> None: + node_mode = upsert_node_mode(node_mode=node_mode) env = compose_node_env( - SKALE_DIR_ENV_FILEPATH, save=False, node_type=NodeType.SKALE, node_mode=NodeMode.PASSIVE + SKALE_DIR_ENV_FILEPATH, save=False, node_type=NodeType.SKALE, node_mode=node_mode ) - schain_name = env['SCHAIN_NAME'] - cleanup_passive_op(env, schain_name) - logger.info('Passive node was cleaned up, all containers and data removed') + cleanup_skale_op(node_mode=node_mode, env=env) + logger.info('SKALE node was cleaned up, all containers and data removed') def compose_node_env( From f892699c768885c81a4e57b02cc09bd2f42cd2f8 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Tue, 28 Oct 2025 18:17:45 +0000 Subject: [PATCH 105/198] Add CLI cleanup commands for SKALE nodes --- node_cli/cli/node.py | 14 ++++++++++++++ node_cli/cli/passive_node.py | 11 ++++++----- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/node_cli/cli/node.py b/node_cli/cli/node.py index 20e2d701..8f55e93c 100644 --- a/node_cli/cli/node.py +++ b/node_cli/cli/node.py @@ -21,6 +21,7 @@ from node_cli.cli.info import TYPE from node_cli.core.node import ( + cleanup as cleanup_skale, configure_firewall_rules, get_node_signature, init, @@ -257,3 +258,16 @@ def version(raw: bool) -> None: print(meta_info) else: print_meta_info(meta_info) + + +@node.command('cleanup', help='Remove all SKALE node data and containers..') +@click.option( + '--yes', + is_flag=True, + callback=abort_if_false, + expose_value=False, + prompt='Are you sure you want to remove all SKALE node data and containers?', +) +@streamed_cmd +def cleanup_node(): + cleanup_skale(node_mode=NodeMode.ACTIVE) \ No newline at end of file diff --git a/node_cli/cli/passive_node.py b/node_cli/cli/passive_node.py index 6be74b28..adadfd48 100644 --- a/node_cli/cli/passive_node.py +++ b/node_cli/cli/passive_node.py @@ -21,8 +21,9 @@ import click -from node_cli.core.node import init_passive, update_passive, cleanup_passive +from node_cli.core.node import init_passive, update_passive, cleanup as cleanup_skale from node_cli.utils.helper import abort_if_false, error_exit, streamed_cmd, URL_TYPE +from node_cli.utils.node_type import NodeMode from node_cli.utils.texts import safe_load_texts @@ -72,14 +73,14 @@ def _update_passive(env_file, unsafe_ok): update_passive(env_file) -@passive_node.command('cleanup', help='Remove passive node data and containers') +@passive_node.command('cleanup', help='Remove all passive SKALE node data and containers.') @click.option( '--yes', is_flag=True, callback=abort_if_false, expose_value=False, - prompt='Are you sure you want to remove all node containers and data?', + prompt='Are you sure you want to remove all SKALE node data and containers?', ) @streamed_cmd -def _cleanup_passive() -> None: - cleanup_passive() +def cleanup_node(): + cleanup_skale(node_mode=NodeMode.PASSIVE) \ No newline at end of file From 84633ccb3849b5c6bf2407e3e32c83a88951eb81 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Tue, 28 Oct 2025 19:43:16 +0000 Subject: [PATCH 106/198] Add implementation for function `cleanup_active` --- node_cli/operations/base.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index a866c9ea..13abb0f3 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -70,9 +70,9 @@ docker_cleanup, remove_dynamic_containers, ) -from node_cli.utils.helper import cleanup_dir_content, rm_dir, str_to_bool +from node_cli.utils.helper import cleanup_dir_content, rm_dir, str_to_bool, run_cmd from node_cli.utils.meta import CliMetaManager, FairCliMetaManager -from node_cli.utils.node_type import NodeType, NodeMode +from node_cli.utils.node_type import NodeMode, NodeType from node_cli.utils.print_formatters import print_failed_requirements_checks logger = logging.getLogger(__name__) @@ -439,8 +439,22 @@ def restore(env, backup_path, node_type: NodeType, config_only=False): return True +def cleanup_passive(env, schain_name: str) -> None: + turn_off(env, node_type=NodeType.SKALE, node_mode=NodeMode.PASSIVE) + cleanup_no_lvm_datadir(chain_name=schain_name) + rm_dir(GLOBAL_SKALE_DIR) + rm_dir(SKALE_DIR) + + def cleanup_active(): - pass + logger.info('Starting cleanup for active node...') + logger.info('Unmounting /mnt/schains-shared-space...') + run_cmd(['sudo', 'umount', '/mnt/schains-shared-space'], check_code=False) + logger.info('Cleaning up /mnt directory content...') + cleanup_dir_content('/mnt/') + logger.info('Removing LVM volume group "schains"...') + run_cmd(['sudo', 'lvremove', '-f', 'schains'], check_code=False) + logger.info('Active node cleanup finished.') def cleanup(node_mode: NodeMode, env: dict) -> None: @@ -453,4 +467,4 @@ def cleanup(node_mode: NodeMode, env: dict) -> None: rm_dir(GLOBAL_SKALE_DIR) rm_dir(SKALE_DIR) cleanup_dir_content(NFTABLES_CHAIN_FOLDER_PATH) - cleanup_docker_configuration() \ No newline at end of file + cleanup_docker_configuration() From b3f11b6a3a1defe78e8abcf18698d7816a84f46b Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Tue, 28 Oct 2025 19:45:00 +0000 Subject: [PATCH 107/198] Add `prune` option for SKALE node clean up commands --- node_cli/cli/node.py | 5 +++-- node_cli/cli/passive_node.py | 5 +++-- node_cli/core/node.py | 4 ++-- node_cli/operations/base.py | 5 ++++- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/node_cli/cli/node.py b/node_cli/cli/node.py index 8f55e93c..6180fc0a 100644 --- a/node_cli/cli/node.py +++ b/node_cli/cli/node.py @@ -268,6 +268,7 @@ def version(raw: bool) -> None: expose_value=False, prompt='Are you sure you want to remove all SKALE node data and containers?', ) +@click.option('--prune', is_flag=True, help='Prune docker system.') @streamed_cmd -def cleanup_node(): - cleanup_skale(node_mode=NodeMode.ACTIVE) \ No newline at end of file +def cleanup_node(prune): + cleanup_skale(node_mode=NodeMode.ACTIVE, prune=prune) diff --git a/node_cli/cli/passive_node.py b/node_cli/cli/passive_node.py index adadfd48..ee466f06 100644 --- a/node_cli/cli/passive_node.py +++ b/node_cli/cli/passive_node.py @@ -81,6 +81,7 @@ def _update_passive(env_file, unsafe_ok): expose_value=False, prompt='Are you sure you want to remove all SKALE node data and containers?', ) +@click.option('--prune', is_flag=True, help='Prune docker system.') @streamed_cmd -def cleanup_node(): - cleanup_skale(node_mode=NodeMode.PASSIVE) \ No newline at end of file +def cleanup_node(prune): + cleanup_skale(node_mode=NodeMode.PASSIVE, prune=prune) \ No newline at end of file diff --git a/node_cli/core/node.py b/node_cli/core/node.py index 143f6848..46f30ebb 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -227,12 +227,12 @@ def update_passive(env_filepath: str, unsafe_ok: bool = False) -> None: @check_inited @check_user -def cleanup(node_mode: NodeMode) -> None: +def cleanup(node_mode: NodeMode, prune: bool = False) -> None: node_mode = upsert_node_mode(node_mode=node_mode) env = compose_node_env( SKALE_DIR_ENV_FILEPATH, save=False, node_type=NodeType.SKALE, node_mode=node_mode ) - cleanup_skale_op(node_mode=node_mode, env=env) + cleanup_skale_op(node_mode=node_mode, env=env, prune=prune) logger.info('SKALE node was cleaned up, all containers and data removed') diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index 13abb0f3..e6a9a53c 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -69,6 +69,7 @@ compose_up, docker_cleanup, remove_dynamic_containers, + system_prune, ) from node_cli.utils.helper import cleanup_dir_content, rm_dir, str_to_bool, run_cmd from node_cli.utils.meta import CliMetaManager, FairCliMetaManager @@ -457,8 +458,10 @@ def cleanup_active(): logger.info('Active node cleanup finished.') -def cleanup(node_mode: NodeMode, env: dict) -> None: +def cleanup(node_mode: NodeMode, env: dict, prune: bool = False) -> None: turn_off(env, node_type=NodeType.SKALE, node_mode=node_mode) + if prune: + system_prune() if node_mode == NodeMode.PASSIVE: schain_name = env['SCHAIN_NAME'] cleanup_no_lvm_datadir(chain_name=schain_name) From 0a1abc434a6ffbe467d69bba6cf845f82fbdb284 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Wed, 29 Oct 2025 13:57:54 +0000 Subject: [PATCH 108/198] Rename `cleanup_node` function to `_cleanup_node` --- node_cli/cli/node.py | 2 +- node_cli/cli/passive_node.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/node_cli/cli/node.py b/node_cli/cli/node.py index 6180fc0a..bdb4a3b7 100644 --- a/node_cli/cli/node.py +++ b/node_cli/cli/node.py @@ -270,5 +270,5 @@ def version(raw: bool) -> None: ) @click.option('--prune', is_flag=True, help='Prune docker system.') @streamed_cmd -def cleanup_node(prune): +def _cleanup_node(prune): cleanup_skale(node_mode=NodeMode.ACTIVE, prune=prune) diff --git a/node_cli/cli/passive_node.py b/node_cli/cli/passive_node.py index ee466f06..7c7ce7d7 100644 --- a/node_cli/cli/passive_node.py +++ b/node_cli/cli/passive_node.py @@ -83,5 +83,5 @@ def _update_passive(env_file, unsafe_ok): ) @click.option('--prune', is_flag=True, help='Prune docker system.') @streamed_cmd -def cleanup_node(prune): +def _cleanup_node(prune): cleanup_skale(node_mode=NodeMode.PASSIVE, prune=prune) \ No newline at end of file From 3e0d4469fd4b8eb1f82ff97f57c9d70892a3ef28 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Wed, 29 Oct 2025 15:30:53 +0000 Subject: [PATCH 109/198] Add `prune` option for FAIR node clean up commands --- node_cli/cli/fair_node.py | 9 +++++---- node_cli/cli/passive_fair_node.py | 9 +++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/node_cli/cli/fair_node.py b/node_cli/cli/fair_node.py index 343be1de..56ab9d75 100644 --- a/node_cli/cli/fair_node.py +++ b/node_cli/cli/fair_node.py @@ -152,17 +152,18 @@ def repair(snapshot: str = 'any') -> None: repair_chain(snapshot_from=snapshot) -@node.command('cleanup', help='Cleanup Fair node.') +@node.command('cleanup', help='Remove all FAIR node data and containers.') @click.option( '--yes', is_flag=True, callback=abort_if_false, expose_value=False, - prompt='Are you sure you want to cleanup Fair node?', + prompt='Are you sure you want to remove all FAIR node data and containers?', ) +@click.option('--prune', is_flag=True, help='Prune docker system.') @streamed_cmd -def cleanup_node(): - cleanup_fair(node_mode=NodeMode.ACTIVE) +def cleanup_node(prune): + cleanup_fair(node_mode=NodeMode.ACTIVE, prune=prune) @node.command('change-ip', help=TEXTS['fair']['node']['change-ip']['help']) diff --git a/node_cli/cli/passive_fair_node.py b/node_cli/cli/passive_fair_node.py index e48f0ce0..83eb1bd1 100644 --- a/node_cli/cli/passive_fair_node.py +++ b/node_cli/cli/passive_fair_node.py @@ -105,17 +105,18 @@ def update_node(env_filepath: str, pull_config_for_schain, force_skaled_start: b ) -@passive_node.command('cleanup', help='Cleanup Fair node.') +@passive_node.command('cleanup', help='Remove all FAIR node data and containers.') @click.option( '--yes', is_flag=True, callback=abort_if_false, expose_value=False, - prompt='Are you sure you want to cleanup Fair node?', + prompt='Are you sure you want to remove all FAIR node data and containers?', ) +@click.option('--prune', is_flag=True, help='Prune docker system.') @streamed_cmd -def cleanup_node(): - cleanup_fair(node_mode=NodeMode.PASSIVE) +def cleanup_node(prune): + cleanup_fair(node_mode=NodeMode.PASSIVE, prune=prune) @passive_node.command('setup', help=TEXTS['fair']['node']['setup']['help']) From 89cc3b060b8d1a8dea11eee5a8826b4aaec48e14 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Wed, 29 Oct 2025 15:36:53 +0000 Subject: [PATCH 110/198] Rename `_cleanup_node` function to `cleanup_node` --- node_cli/cli/node.py | 4 ++-- node_cli/cli/passive_node.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/node_cli/cli/node.py b/node_cli/cli/node.py index bdb4a3b7..55304356 100644 --- a/node_cli/cli/node.py +++ b/node_cli/cli/node.py @@ -260,7 +260,7 @@ def version(raw: bool) -> None: print_meta_info(meta_info) -@node.command('cleanup', help='Remove all SKALE node data and containers..') +@node.command('cleanup', help='Remove all SKALE node data and containers.') @click.option( '--yes', is_flag=True, @@ -270,5 +270,5 @@ def version(raw: bool) -> None: ) @click.option('--prune', is_flag=True, help='Prune docker system.') @streamed_cmd -def _cleanup_node(prune): +def cleanup_node(prune): cleanup_skale(node_mode=NodeMode.ACTIVE, prune=prune) diff --git a/node_cli/cli/passive_node.py b/node_cli/cli/passive_node.py index 7c7ce7d7..9c92c767 100644 --- a/node_cli/cli/passive_node.py +++ b/node_cli/cli/passive_node.py @@ -73,7 +73,7 @@ def _update_passive(env_file, unsafe_ok): update_passive(env_file) -@passive_node.command('cleanup', help='Remove all passive SKALE node data and containers.') +@passive_node.command('cleanup', help='Remove all SKALE node data and containers.') @click.option( '--yes', is_flag=True, @@ -83,5 +83,5 @@ def _update_passive(env_file, unsafe_ok): ) @click.option('--prune', is_flag=True, help='Prune docker system.') @streamed_cmd -def _cleanup_node(prune): +def cleanup_node(prune): cleanup_skale(node_mode=NodeMode.PASSIVE, prune=prune) \ No newline at end of file From fc2b09a05b90d486b79ab11e9e5f941d9ef3219c Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Wed, 29 Oct 2025 17:42:20 +0000 Subject: [PATCH 111/198] Update SKALE passive node clean up tests --- tests/cli/passive_node_test.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/cli/passive_node_test.py b/tests/cli/passive_node_test.py index 94cde559..790ad5f3 100644 --- a/tests/cli/passive_node_test.py +++ b/tests/cli/passive_node_test.py @@ -22,12 +22,12 @@ import mock -from node_cli.cli.passive_node import _cleanup_passive, _init_passive, _update_passive +from node_cli.cli.passive_node import cleanup_node, _init_passive, _update_passive from node_cli.configs import NODE_DATA_PATH, SKALE_DIR from node_cli.core.node_options import NodeOptions from node_cli.utils.helper import init_default_logger from node_cli.utils.meta import CliMeta -from node_cli.utils.node_type import NodeType +from node_cli.utils.node_type import NodeType, NodeMode from tests.conftest import set_env_var from tests.helper import run_command, subprocess_run_mock from tests.resources_test import BIG_DISK_SIZE @@ -127,12 +127,12 @@ def test_update_passive(passive_user_conf, mocked_g_config): assert result.exit_code == 0 -def test_cleanup_passive(mocked_g_config): +def test_cleanup_node(mocked_g_config): pathlib.Path(SKALE_DIR).mkdir(parents=True, exist_ok=True) with ( mock.patch('subprocess.run', new=subprocess_run_mock), - mock.patch('node_cli.core.node.cleanup_passive_op'), + mock.patch('node_cli.core.node.cleanup_skale_op') as cleanup_mock, mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), mock.patch('node_cli.operations.base.configure_nftables'), @@ -143,5 +143,6 @@ def test_cleanup_passive(mocked_g_config): return_value=CliMeta(version='2.6.0', config_stream='3.0.2'), ), ): - result = run_command(_cleanup_passive, ['--yes']) + result = run_command(cleanup_node, ['--yes']) assert result.exit_code == 0 + cleanup_mock.assert_called_once_with(node_mode=NodeMode.PASSIVE, prune=False, env={'SCHAIN_NAME': 'test'}) From 2780195679598e0b34d85220b4c9fea6d5f279a2 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Wed, 29 Oct 2025 17:53:54 +0000 Subject: [PATCH 112/198] Apply formatting according to Ruff rules --- tests/cli/passive_node_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/cli/passive_node_test.py b/tests/cli/passive_node_test.py index 790ad5f3..c9ea9527 100644 --- a/tests/cli/passive_node_test.py +++ b/tests/cli/passive_node_test.py @@ -145,4 +145,5 @@ def test_cleanup_node(mocked_g_config): ): result = run_command(cleanup_node, ['--yes']) assert result.exit_code == 0 - cleanup_mock.assert_called_once_with(node_mode=NodeMode.PASSIVE, prune=False, env={'SCHAIN_NAME': 'test'}) + cleanup_mock.assert_called_once_with( + node_mode=NodeMode.PASSIVE, prune=False, env={'SCHAIN_NAME': 'test'}) From f18fbbf00939523423b4c4bf0ee7367c610b2180 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Wed, 29 Oct 2025 18:03:34 +0000 Subject: [PATCH 113/198] Add SKALE active node clean up test --- tests/cli/node_test.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/cli/node_test.py b/tests/cli/node_test.py index bfd5263b..b31cddc0 100644 --- a/tests/cli/node_test.py +++ b/tests/cli/node_test.py @@ -30,6 +30,7 @@ _turn_off, _turn_on, backup_node, + cleanup_node, node_info, register_node, remove_node_from_maintenance, @@ -473,3 +474,23 @@ def test_node_version(meta_file_v2): result.output == "{'version': '0.1.1', 'config_stream': 'develop', 'docker_lvmpy_version': '1.1.2'}\n" ) + +def test_cleanup_node(mocked_g_config): + pathlib.Path(SKALE_DIR).mkdir(parents=True, exist_ok=True) + + with ( + mock.patch('subprocess.run', new=subprocess_run_mock), + mock.patch('node_cli.core.node.cleanup_skale_op') as cleanup_mock, + mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), + mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), + mock.patch('node_cli.operations.base.configure_nftables'), + mock.patch('node_cli.utils.decorators.is_node_inited', return_value=True), + mock.patch('node_cli.core.node.compose_node_env', return_value={}), + mock.patch( + 'node_cli.core.node.CliMetaManager.get_meta_info', + return_value=CliMeta(version='2.6.0', config_stream='3.0.2'), + ), + ): + result = run_command(cleanup_node, ['--yes']) + assert result.exit_code == 0 + cleanup_mock.assert_called_once_with(node_mode=NodeMode.ACTIVE, prune=False, env={}) \ No newline at end of file From 3f11ac2af292bc9629ba9b3d9fb25189c59e91a8 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Wed, 29 Oct 2025 19:12:23 +0000 Subject: [PATCH 114/198] Add `prune` option for FAIR node clean up functions --- node_cli/fair/common.py | 5 ++--- node_cli/operations/fair.py | 5 ++++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/node_cli/fair/common.py b/node_cli/fair/common.py index dc91e26e..23c88cd5 100644 --- a/node_cli/fair/common.py +++ b/node_cli/fair/common.py @@ -81,14 +81,13 @@ def init( @check_user -def cleanup(node_mode: NodeMode) -> None: +def cleanup(node_mode: NodeMode, prune: bool = False) -> None: node_mode = upsert_node_mode(node_mode=node_mode) env = compose_node_env( SKALE_DIR_ENV_FILEPATH, save=False, node_type=NodeType.FAIR, node_mode=node_mode ) - cleanup_fair_op(node_mode=node_mode, env=env) + cleanup_fair_op(node_mode=node_mode, env=env, prune=prune) logger.info('Fair node was cleaned up, all containers and data removed') - cleanup_docker_configuration() @check_inited diff --git a/node_cli/operations/fair.py b/node_cli/operations/fair.py index 689b4a51..ca7baef5 100644 --- a/node_cli/operations/fair.py +++ b/node_cli/operations/fair.py @@ -63,6 +63,7 @@ remove_dynamic_containers, start_container_by_name, stop_container_by_name, + system_prune, wait_for_container, ) from node_cli.utils.helper import cleanup_dir_content, rm_dir, str_to_bool @@ -282,8 +283,10 @@ def restore(node_mode: NodeMode, env, backup_path, config_only=False): return True -def cleanup(node_mode: NodeMode, env: dict) -> None: +def cleanup(node_mode: NodeMode, env: dict, prune: bool = False) -> None: turn_off(env, node_type=NodeType.FAIR, node_mode=node_mode) + if prune: + system_prune() cleanup_no_lvm_datadir() rm_dir(GLOBAL_SKALE_DIR) rm_dir(SKALE_DIR) From 64fc97f385275d61d7910e25fbe71954572b044a Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Wed, 29 Oct 2025 20:17:31 +0000 Subject: [PATCH 115/198] Delete unused import --- node_cli/fair/common.py | 1 - 1 file changed, 1 deletion(-) diff --git a/node_cli/fair/common.py b/node_cli/fair/common.py index 23c88cd5..47bfc50a 100644 --- a/node_cli/fair/common.py +++ b/node_cli/fair/common.py @@ -22,7 +22,6 @@ from node_cli.configs import INIT_TIMEOUT, SKALE_DIR, TM_INIT_TIMEOUT from node_cli.configs.user import SKALE_DIR_ENV_FILEPATH -from node_cli.core.docker_config import cleanup_docker_configuration from node_cli.core.host import save_env_params from node_cli.core.node import compose_node_env, is_base_containers_alive from node_cli.core.node_options import upsert_node_mode From 9e8f4896ee284ac3b1deda1c1a6fb6e08fbd25bd Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Wed, 29 Oct 2025 20:20:34 +0000 Subject: [PATCH 116/198] Add test for FAIR active node cleanup --- tests/cli/fair_cli_test.py | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/tests/cli/fair_cli_test.py b/tests/cli/fair_cli_test.py index 51fa0a9b..9d62d058 100644 --- a/tests/cli/fair_cli_test.py +++ b/tests/cli/fair_cli_test.py @@ -2,7 +2,6 @@ from unittest import mock from click.testing import CliRunner - from node_cli.cli.fair_boot import ( init_boot, register_boot, @@ -10,11 +9,16 @@ ) from node_cli.cli.fair_node import ( backup_node, + cleanup_node, migrate_node, exit_node, restore_node, ) - +from node_cli.configs import SKALE_DIR +from node_cli.utils.node_type import NodeMode +from node_cli.utils.meta import CliMeta +from tests.helper import run_command, subprocess_run_mock +from tests.resources_test import BIG_DISK_SIZE @mock.patch('node_cli.cli.fair_node.restore_fair') def test_fair_node_restore(mock_restore_core, valid_env_file, tmp_path): @@ -111,3 +115,25 @@ def test_fair_node_exit(mock_exit_core): assert result.exit_code == 0, f'Output: {result.output}\nException: {result.exception}' mock_exit_core.assert_called_once() + + +def test_cleanup_node(mocked_g_config): + pathlib.Path(SKALE_DIR).mkdir(parents=True, exist_ok=True) + + with ( + mock.patch('subprocess.run', new=subprocess_run_mock), + mock.patch('node_cli.fair.common.cleanup_fair_op') as cleanup_mock, + mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), + mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), + mock.patch('node_cli.operations.base.configure_nftables'), + mock.patch('node_cli.utils.decorators.is_node_inited', return_value=True), + mock.patch('node_cli.fair.common.compose_node_env', return_value={'SCHAIN_NAME': 'test'}), + mock.patch( + 'node_cli.core.node.CliMetaManager.get_meta_info', + return_value=CliMeta(version='2.6.0', config_stream='3.0.2'), + ), + ): + result = run_command(cleanup_node, ['--yes']) + assert result.exit_code == 0 + cleanup_mock.assert_called_once_with( + node_mode=NodeMode.ACTIVE, prune=False, env={'SCHAIN_NAME': 'test'}) From 59781ce2333374216bf5b165cd2833c76621a13a Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Wed, 29 Oct 2025 20:40:14 +0000 Subject: [PATCH 117/198] Add test for FAIR passive node cleanup --- tests/cli/fair_passive_node_test.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/tests/cli/fair_passive_node_test.py b/tests/cli/fair_passive_node_test.py index 34ececf9..d0e94634 100644 --- a/tests/cli/fair_passive_node_test.py +++ b/tests/cli/fair_passive_node_test.py @@ -3,9 +3,11 @@ import mock -from node_cli.cli.passive_fair_node import init_passive_node, update_node +from node_cli.cli.passive_fair_node import cleanup_node, init_passive_node, update_node from node_cli.configs import NODE_DATA_PATH, SKALE_DIR from node_cli.utils.helper import init_default_logger +from node_cli.utils.meta import CliMeta +from node_cli.utils.node_type import NodeMode from tests.helper import run_command, subprocess_run_mock from tests.resources_test import BIG_DISK_SIZE @@ -84,3 +86,25 @@ def test_update_fair_passive(mocked_g_config, tmp_path): ): result = run_command(update_node, [env_file.as_posix(), '--yes']) assert result.exit_code == 0 + + +def test_cleanup_node(mocked_g_config): + pathlib.Path(SKALE_DIR).mkdir(parents=True, exist_ok=True) + + with ( + mock.patch('subprocess.run', new=subprocess_run_mock), + mock.patch('node_cli.fair.common.cleanup_fair_op') as cleanup_mock, + mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), + mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), + mock.patch('node_cli.operations.base.configure_nftables'), + mock.patch('node_cli.utils.decorators.is_node_inited', return_value=True), + mock.patch('node_cli.fair.common.compose_node_env', return_value={'SCHAIN_NAME': 'test'}), + mock.patch( + 'node_cli.core.node.CliMetaManager.get_meta_info', + return_value=CliMeta(version='2.6.0', config_stream='3.0.2'), + ), + ): + result = run_command(cleanup_node, ['--yes']) + assert result.exit_code == 0 + cleanup_mock.assert_called_once_with( + node_mode=NodeMode.PASSIVE, prune=False, env={'SCHAIN_NAME': 'test'}) From 8840987ef70276dfb2561b0476f78e86a7947a40 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Thu, 30 Oct 2025 13:35:12 +0000 Subject: [PATCH 118/198] Update Fair node tests --- tests/fair/fair_node_test.py | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/tests/fair/fair_node_test.py b/tests/fair/fair_node_test.py index c3789ff1..4b63aec7 100644 --- a/tests/fair/fair_node_test.py +++ b/tests/fair/fair_node_test.py @@ -158,13 +158,11 @@ def test_migrate_from_boot( @mock.patch('node_cli.utils.decorators.is_user_valid', return_value=True) -@mock.patch('node_cli.fair.common.cleanup_docker_configuration') @mock.patch('node_cli.fair.common.cleanup_fair_op') @mock.patch('node_cli.fair.common.compose_node_env') def test_cleanup_success( mock_compose_env, mock_cleanup_fair_op, - mock_cleanup_docker_config, mock_is_user_valid, inited_node, resource_alloc, @@ -182,18 +180,16 @@ def test_cleanup_success( node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE, ) - mock_cleanup_fair_op.assert_called_once_with(node_mode=NodeMode.ACTIVE, env=mock_env) - mock_cleanup_docker_config.assert_called_once() + mock_cleanup_fair_op.assert_called_once_with( + node_mode=NodeMode.ACTIVE, env=mock_env, prune=False) @mock.patch('node_cli.utils.decorators.is_user_valid', return_value=True) -@mock.patch('node_cli.fair.common.cleanup_docker_configuration') @mock.patch('node_cli.fair.common.cleanup_fair_op') @mock.patch('node_cli.fair.common.compose_node_env') def test_cleanup_calls_operations_in_correct_order( mock_compose_env, mock_cleanup_fair_op, - mock_cleanup_docker_config, mock_is_user_valid, inited_node, resource_alloc, @@ -208,26 +204,22 @@ def test_cleanup_calls_operations_in_correct_order( manager = mock.Mock() manager.attach_mock(mock_compose_env, 'compose_env') manager.attach_mock(mock_cleanup_fair_op, 'cleanup_fair_op') - manager.attach_mock(mock_cleanup_docker_config, 'cleanup_docker_config') cleanup(node_mode=NodeMode.ACTIVE) expected_calls = [ mock.call.compose_env(mock.ANY, save=False, node_type=mock.ANY, node_mode=NodeMode.ACTIVE), - mock.call.cleanup_fair_op(node_mode=NodeMode.ACTIVE, env=mock_env), - mock.call.cleanup_docker_config(), + mock.call.cleanup_fair_op(node_mode=NodeMode.ACTIVE, env=mock_env, prune=False), ] manager.assert_has_calls(expected_calls, any_order=False) @mock.patch('node_cli.utils.decorators.is_user_valid', return_value=True) -@mock.patch('node_cli.fair.common.cleanup_docker_configuration') @mock.patch('node_cli.fair.common.cleanup_fair_op', side_effect=Exception('Cleanup failed')) @mock.patch('node_cli.fair.common.compose_node_env') def test_cleanup_continues_after_fair_op_error( mock_compose_env, mock_cleanup_fair_op, - mock_cleanup_docker_config, mock_is_user_valid, inited_node, resource_alloc, @@ -241,8 +233,8 @@ def test_cleanup_continues_after_fair_op_error( cleanup(node_mode=NodeMode.ACTIVE) mock_compose_env.assert_called_once() - mock_cleanup_fair_op.assert_called_once_with(node_mode=NodeMode.ACTIVE, env=mock_env) - mock_cleanup_docker_config.assert_not_called() + mock_cleanup_fair_op.assert_called_once_with( + node_mode=NodeMode.ACTIVE, env=mock_env, prune=False) @mock.patch('node_cli.utils.decorators.is_user_valid', return_value=False) @@ -269,7 +261,6 @@ def test_cleanup_fails_when_not_inited(ensure_meta_removed, active_node_option): @mock.patch('node_cli.utils.decorators.is_user_valid', return_value=True) -@mock.patch('node_cli.fair.common.cleanup_docker_configuration') @mock.patch('node_cli.fair.common.cleanup_fair_op') @mock.patch('node_cli.fair.common.compose_node_env') @mock.patch('node_cli.fair.common.logger') @@ -277,7 +268,6 @@ def test_cleanup_logs_success_message( mock_logger, mock_compose_env, mock_cleanup_fair_op, - mock_cleanup_docker_config, mock_is_user_valid, inited_node, resource_alloc, From b9346b7dc13de21764a3bdc3b3ab743866a00d07 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Thu, 30 Oct 2025 15:16:58 +0000 Subject: [PATCH 119/198] Fix FAIR node tests --- tests/cli/fair_cli_test.py | 2 +- tests/cli/fair_passive_node_test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/cli/fair_cli_test.py b/tests/cli/fair_cli_test.py index 9d62d058..3c6b9ef5 100644 --- a/tests/cli/fair_cli_test.py +++ b/tests/cli/fair_cli_test.py @@ -117,7 +117,7 @@ def test_fair_node_exit(mock_exit_core): mock_exit_core.assert_called_once() -def test_cleanup_node(mocked_g_config): +def test_cleanup_node(mocked_g_config, inited_node): pathlib.Path(SKALE_DIR).mkdir(parents=True, exist_ok=True) with ( diff --git a/tests/cli/fair_passive_node_test.py b/tests/cli/fair_passive_node_test.py index d0e94634..a8b7818a 100644 --- a/tests/cli/fair_passive_node_test.py +++ b/tests/cli/fair_passive_node_test.py @@ -71,7 +71,7 @@ def test_init_fair_passive_snapshot_any(mocked_g_config, tmp_path): assert result.exit_code == 0 -def test_update_fair_passive(mocked_g_config, tmp_path): +def test_update_fair_passive(mocked_g_config, tmp_path, clean_node_options): env_file = tmp_path / 'test-env' env_file.write_text('') pathlib.Path(NODE_DATA_PATH).mkdir(parents=True, exist_ok=True) From 17cdb8e89eb48db86405cf5b5b470d1cd2b97ecc Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Fri, 31 Oct 2025 19:25:25 +0000 Subject: [PATCH 120/198] Add test for SKALE node cleanup --- tests/core/core_node_test.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tests/core/core_node_test.py b/tests/core/core_node_test.py index 7188ffae..5eccc1d8 100644 --- a/tests/core/core_node_test.py +++ b/tests/core/core_node_test.py @@ -12,7 +12,9 @@ from node_cli.configs import NODE_DATA_PATH, SCHAINS_MNT_DIR_REGULAR, SCHAINS_MNT_DIR_SINGLE_CHAIN from node_cli.configs.resource_allocation import RESOURCE_ALLOCATION_FILEPATH +from node_cli.configs.user import SKALE_DIR_ENV_FILEPATH from node_cli.core.node import ( + cleanup, compose_node_env, get_expected_container_names, init, @@ -471,3 +473,32 @@ def test_is_update_safe_when_api_call_fails( mock_requests_get.side_effect = requests.exceptions.ConnectionError('Test connection error') assert is_update_safe(node_mode=node_mode) is False mock_requests_get.assert_called_once() + + +@mock.patch('node_cli.utils.decorators.is_user_valid', return_value=True) +@mock.patch('node_cli.utils.decorators.is_node_inited', return_value=True) +@mock.patch('node_cli.core.node.cleanup_skale_op') +@mock.patch('node_cli.core.node.compose_node_env') +def test_cleanup_success( + mock_compose_env, + mock_cleanup_skale_op, + mock_node_inited, + mock_is_user_valid, + inited_node, + resource_alloc, + meta_file_v3, + active_node_option, +): + mock_env = {'ENV_TYPE': 'devnet'} + mock_compose_env.return_value = mock_env + + cleanup(node_mode=NodeMode.ACTIVE) + + mock_compose_env.assert_called_once_with( + SKALE_DIR_ENV_FILEPATH, + save=False, + node_type=NodeType.SKALE, + node_mode=NodeMode.ACTIVE, + ) + mock_cleanup_skale_op.assert_called_once_with( + node_mode=NodeMode.ACTIVE, env=mock_env, prune=False) \ No newline at end of file From 894abc4f438dfc70f9a05e8b586a8cdff7c7d943 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Fri, 31 Oct 2025 19:39:32 +0000 Subject: [PATCH 121/198] Unmount `no_lvm_datadir` volume after cleanup --- node_cli/core/schains.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node_cli/core/schains.py b/node_cli/core/schains.py index fe3aeeb4..ac457371 100644 --- a/node_cli/core/schains.py +++ b/node_cli/core/schains.py @@ -299,6 +299,6 @@ def cleanup_no_lvm_datadir( if folder_name != 'shared-space': logger.info('Removing datadir content for %s', folder_path) cleanup_datadir_content(folder_path) - logger.info('Removing datadir content for %s', folder_path) if os.path.isdir(folder_path): shutil.rmtree(folder_path) + run_cmd(['umount', base_path]) From 438b1f4f744f933b1b4b4f97e1ebd23906a17e99 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Mon, 3 Nov 2025 17:17:27 +0000 Subject: [PATCH 122/198] Remove `@check_inited` decorator for SKALE node cleanup function --- node_cli/core/node.py | 1 - 1 file changed, 1 deletion(-) diff --git a/node_cli/core/node.py b/node_cli/core/node.py index 18bf473f..30036fcf 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -225,7 +225,6 @@ def update_passive(env_filepath: str, unsafe_ok: bool = False) -> None: logger.info('Node update finished') -@check_inited @check_user def cleanup(node_mode: NodeMode, prune: bool = False) -> None: node_mode = upsert_node_mode(node_mode=node_mode) From 78c46141dc764dad5962d4ffda59ca53743c2da0 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Mon, 3 Nov 2025 17:18:40 +0000 Subject: [PATCH 123/198] Update SKALE node cleanup test --- tests/core/core_node_test.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/core/core_node_test.py b/tests/core/core_node_test.py index 5eccc1d8..83117c29 100644 --- a/tests/core/core_node_test.py +++ b/tests/core/core_node_test.py @@ -476,13 +476,11 @@ def test_is_update_safe_when_api_call_fails( @mock.patch('node_cli.utils.decorators.is_user_valid', return_value=True) -@mock.patch('node_cli.utils.decorators.is_node_inited', return_value=True) @mock.patch('node_cli.core.node.cleanup_skale_op') @mock.patch('node_cli.core.node.compose_node_env') def test_cleanup_success( mock_compose_env, mock_cleanup_skale_op, - mock_node_inited, mock_is_user_valid, inited_node, resource_alloc, From 2fed20e73285a3da6c54378dfa17701562f198c1 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Mon, 3 Nov 2025 17:58:24 +0000 Subject: [PATCH 124/198] Fix `test_cleanup_passive_datadir` test --- tests/core/core_schains_test.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/core/core_schains_test.py b/tests/core/core_schains_test.py index ee868242..7016a262 100644 --- a/tests/core/core_schains_test.py +++ b/tests/core/core_schains_test.py @@ -80,6 +80,9 @@ def test_cleanup_passive_datadir(tmp_passive_datadir): hash_path = snapshot_folder.joinpath('snapshot_hash.txt') hash_path.touch() - with mock.patch('node_cli.core.schains.rm_btrfs_subvolume'): + with ( + mock.patch('node_cli.core.schains.rm_btrfs_subvolume'), + mock.patch('node_cli.core.schains.run_cmd'), + ): cleanup_no_lvm_datadir(schain_name, base_path=tmp_passive_datadir) assert not os.path.isdir(base_folder) From 26180971d8916cea9bbec30db89ed63ab0d71f68 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Tue, 4 Nov 2025 17:05:41 +0000 Subject: [PATCH 125/198] Skip user config validation during cleanup process --- node_cli/configs/user.py | 4 +++- node_cli/core/node.py | 9 ++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/node_cli/configs/user.py b/node_cli/configs/user.py index 3aa7b831..2625f200 100644 --- a/node_cli/configs/user.py +++ b/node_cli/configs/user.py @@ -143,6 +143,7 @@ def get_validated_user_config( node_mode: NodeMode, env_filepath: str = SKALE_DIR_ENV_FILEPATH, is_fair_boot: bool = False, + skip_usr_conf_validation: bool = False, ) -> BaseUserConfig: params = parse_env_file(env_filepath) user_config_class = get_user_config_class( @@ -160,7 +161,8 @@ def get_validated_user_config( params = to_lower_keys(params) user_config = user_config_class(**params) - validate_user_config(user_config) + if not skip_usr_conf_validation: + validate_user_config(user_config) return user_config diff --git a/node_cli/core/node.py b/node_cli/core/node.py index 30036fcf..ad2f2713 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -229,7 +229,11 @@ def update_passive(env_filepath: str, unsafe_ok: bool = False) -> None: def cleanup(node_mode: NodeMode, prune: bool = False) -> None: node_mode = upsert_node_mode(node_mode=node_mode) env = compose_node_env( - SKALE_DIR_ENV_FILEPATH, save=False, node_type=NodeType.SKALE, node_mode=node_mode + SKALE_DIR_ENV_FILEPATH, + save=False, + node_type=NodeType.SKALE, + node_mode=node_mode, + skip_usr_conf_validation=True, ) cleanup_skale_op(node_mode=node_mode, env=env, prune=prune) logger.info('SKALE node was cleaned up, all containers and data removed') @@ -244,6 +248,7 @@ def compose_node_env( pull_config_for_schain: Optional[str] = None, save: bool = True, is_fair_boot: bool = False, + skip_usr_conf_validation: bool = False, ) -> dict[str, str]: if env_filepath is not None: user_config = get_validated_user_config( @@ -251,6 +256,7 @@ def compose_node_env( node_mode=node_mode, env_filepath=env_filepath, is_fair_boot=is_fair_boot, + skip_usr_conf_validation=skip_usr_conf_validation, ) if save: save_env_params(env_filepath) @@ -259,6 +265,7 @@ def compose_node_env( node_type=node_type, env_filepath=INIT_ENV_FILEPATH, is_fair_boot=is_fair_boot, + skip_usr_conf_validation=skip_usr_conf_validation, ) if node_mode == NodeMode.PASSIVE or node_type == NodeType.FAIR: From c8b4760b483d41ae594fee82baec8625ccb4ed6e Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Tue, 4 Nov 2025 17:50:38 +0000 Subject: [PATCH 126/198] Update `test_cleanup_success` for SKALE node --- tests/core/core_node_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/core/core_node_test.py b/tests/core/core_node_test.py index 83117c29..e53a3351 100644 --- a/tests/core/core_node_test.py +++ b/tests/core/core_node_test.py @@ -497,6 +497,7 @@ def test_cleanup_success( save=False, node_type=NodeType.SKALE, node_mode=NodeMode.ACTIVE, + skip_usr_conf_validation=True, ) mock_cleanup_skale_op.assert_called_once_with( node_mode=NodeMode.ACTIVE, env=mock_env, prune=False) \ No newline at end of file From e5e7ee0349064cae0dcafd2d79088a24b54ea747 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Wed, 5 Nov 2025 19:04:09 +0000 Subject: [PATCH 127/198] Skip user config validation during cleanup of FAIR nodes --- node_cli/fair/common.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/node_cli/fair/common.py b/node_cli/fair/common.py index 47bfc50a..458d06f2 100644 --- a/node_cli/fair/common.py +++ b/node_cli/fair/common.py @@ -83,7 +83,11 @@ def init( def cleanup(node_mode: NodeMode, prune: bool = False) -> None: node_mode = upsert_node_mode(node_mode=node_mode) env = compose_node_env( - SKALE_DIR_ENV_FILEPATH, save=False, node_type=NodeType.FAIR, node_mode=node_mode + SKALE_DIR_ENV_FILEPATH, + save=False, + node_type=NodeType.FAIR, + node_mode=node_mode, + skip_usr_conf_validation=True, ) cleanup_fair_op(node_mode=node_mode, env=env, prune=prune) logger.info('Fair node was cleaned up, all containers and data removed') From ee186623b6d37f263fcf02e5a7dc064c945b838e Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Wed, 5 Nov 2025 19:26:03 +0000 Subject: [PATCH 128/198] Update tests --- tests/fair/fair_node_test.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/fair/fair_node_test.py b/tests/fair/fair_node_test.py index 4b63aec7..7fa2e912 100644 --- a/tests/fair/fair_node_test.py +++ b/tests/fair/fair_node_test.py @@ -179,6 +179,7 @@ def test_cleanup_success( save=False, node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE, + skip_usr_conf_validation=True, ) mock_cleanup_fair_op.assert_called_once_with( node_mode=NodeMode.ACTIVE, env=mock_env, prune=False) @@ -208,7 +209,12 @@ def test_cleanup_calls_operations_in_correct_order( cleanup(node_mode=NodeMode.ACTIVE) expected_calls = [ - mock.call.compose_env(mock.ANY, save=False, node_type=mock.ANY, node_mode=NodeMode.ACTIVE), + mock.call.compose_env( + mock.ANY, + save=False, + node_type=mock.ANY, + node_mode=NodeMode.ACTIVE, + skip_usr_conf_validation=True), mock.call.cleanup_fair_op(node_mode=NodeMode.ACTIVE, env=mock_env, prune=False), ] manager.assert_has_calls(expected_calls, any_order=False) From 2be08b8735bf96f8665d063d40c0b00ce00d28d3 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Tue, 18 Nov 2025 12:05:47 +0000 Subject: [PATCH 129/198] Rename `cleanup_active` function to `cleanup_lvm_datadir` and move to another module --- node_cli/core/schains.py | 12 ++++++++++++ node_cli/operations/base.py | 14 ++------------ 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/node_cli/core/schains.py b/node_cli/core/schains.py index ac457371..64c16031 100644 --- a/node_cli/core/schains.py +++ b/node_cli/core/schains.py @@ -38,6 +38,7 @@ from node_cli.utils.docker_utils import ensure_volume, is_volume_exists from node_cli.utils.exit_codes import CLIExitCodes from node_cli.utils.helper import ( + cleanup_dir_content, error_exit, get_request, is_btrfs_subvolume, @@ -302,3 +303,14 @@ def cleanup_no_lvm_datadir( if os.path.isdir(folder_path): shutil.rmtree(folder_path) run_cmd(['umount', base_path]) + + +def cleanup_lvm_datadir(): + logger.info('Starting cleanup for active node...') + logger.info('Unmounting /mnt/schains-shared-space...') + run_cmd(['sudo', 'umount', '/mnt/schains-shared-space'], check_code=False) + logger.info('Cleaning up /mnt directory content...') + cleanup_dir_content('/mnt/') + logger.info('Removing LVM volume group "schains"...') + run_cmd(['sudo', 'lvremove', '-f', 'schains'], check_code=False) + logger.info('Active node cleanup finished.') \ No newline at end of file diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index 26db1c2e..28be076d 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -49,6 +49,7 @@ ) from node_cli.core.resources import init_shared_space_volume, update_resource_allocation from node_cli.core.schains import ( + cleanup_lvm_datadir, cleanup_no_lvm_datadir, update_node_cli_schain_status, ) @@ -442,17 +443,6 @@ def cleanup_passive(env, schain_name: str) -> None: rm_dir(SKALE_DIR) -def cleanup_active(): - logger.info('Starting cleanup for active node...') - logger.info('Unmounting /mnt/schains-shared-space...') - run_cmd(['sudo', 'umount', '/mnt/schains-shared-space'], check_code=False) - logger.info('Cleaning up /mnt directory content...') - cleanup_dir_content('/mnt/') - logger.info('Removing LVM volume group "schains"...') - run_cmd(['sudo', 'lvremove', '-f', 'schains'], check_code=False) - logger.info('Active node cleanup finished.') - - def cleanup(node_mode: NodeMode, env: dict, prune: bool = False) -> None: turn_off(env, node_type=NodeType.SKALE, node_mode=node_mode) if prune: @@ -461,7 +451,7 @@ def cleanup(node_mode: NodeMode, env: dict, prune: bool = False) -> None: schain_name = env['SCHAIN_NAME'] cleanup_no_lvm_datadir(chain_name=schain_name) else: - cleanup_active() + cleanup_lvm_datadir() rm_dir(GLOBAL_SKALE_DIR) rm_dir(SKALE_DIR) cleanup_dir_content(NFTABLES_CHAIN_FOLDER_PATH) From 11300451ae14a2806f7c21737ff7425ee1ca6ebc Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Tue, 18 Nov 2025 12:10:46 +0000 Subject: [PATCH 130/198] Rename `skip_usr_conf_validation` parameter to `skip_user_conf_validation` --- node_cli/configs/user.py | 4 ++-- node_cli/core/node.py | 8 ++++---- node_cli/fair/common.py | 2 +- tests/core/core_node_test.py | 2 +- tests/fair/fair_node_test.py | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/node_cli/configs/user.py b/node_cli/configs/user.py index 2625f200..59ecda77 100644 --- a/node_cli/configs/user.py +++ b/node_cli/configs/user.py @@ -143,7 +143,7 @@ def get_validated_user_config( node_mode: NodeMode, env_filepath: str = SKALE_DIR_ENV_FILEPATH, is_fair_boot: bool = False, - skip_usr_conf_validation: bool = False, + skip_user_conf_validation: bool = False, ) -> BaseUserConfig: params = parse_env_file(env_filepath) user_config_class = get_user_config_class( @@ -161,7 +161,7 @@ def get_validated_user_config( params = to_lower_keys(params) user_config = user_config_class(**params) - if not skip_usr_conf_validation: + if not skip_user_conf_validation: validate_user_config(user_config) return user_config diff --git a/node_cli/core/node.py b/node_cli/core/node.py index ad2f2713..725deee3 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -233,7 +233,7 @@ def cleanup(node_mode: NodeMode, prune: bool = False) -> None: save=False, node_type=NodeType.SKALE, node_mode=node_mode, - skip_usr_conf_validation=True, + skip_user_conf_validation=True, ) cleanup_skale_op(node_mode=node_mode, env=env, prune=prune) logger.info('SKALE node was cleaned up, all containers and data removed') @@ -248,7 +248,7 @@ def compose_node_env( pull_config_for_schain: Optional[str] = None, save: bool = True, is_fair_boot: bool = False, - skip_usr_conf_validation: bool = False, + skip_user_conf_validation: bool = False, ) -> dict[str, str]: if env_filepath is not None: user_config = get_validated_user_config( @@ -256,7 +256,7 @@ def compose_node_env( node_mode=node_mode, env_filepath=env_filepath, is_fair_boot=is_fair_boot, - skip_usr_conf_validation=skip_usr_conf_validation, + skip_user_conf_validation=skip_user_conf_validation, ) if save: save_env_params(env_filepath) @@ -265,7 +265,7 @@ def compose_node_env( node_type=node_type, env_filepath=INIT_ENV_FILEPATH, is_fair_boot=is_fair_boot, - skip_usr_conf_validation=skip_usr_conf_validation, + skip_user_conf_validation=skip_user_conf_validation, ) if node_mode == NodeMode.PASSIVE or node_type == NodeType.FAIR: diff --git a/node_cli/fair/common.py b/node_cli/fair/common.py index 458d06f2..968891d4 100644 --- a/node_cli/fair/common.py +++ b/node_cli/fair/common.py @@ -87,7 +87,7 @@ def cleanup(node_mode: NodeMode, prune: bool = False) -> None: save=False, node_type=NodeType.FAIR, node_mode=node_mode, - skip_usr_conf_validation=True, + skip_user_conf_validation=True, ) cleanup_fair_op(node_mode=node_mode, env=env, prune=prune) logger.info('Fair node was cleaned up, all containers and data removed') diff --git a/tests/core/core_node_test.py b/tests/core/core_node_test.py index e53a3351..953e7ca5 100644 --- a/tests/core/core_node_test.py +++ b/tests/core/core_node_test.py @@ -497,7 +497,7 @@ def test_cleanup_success( save=False, node_type=NodeType.SKALE, node_mode=NodeMode.ACTIVE, - skip_usr_conf_validation=True, + skip_user_conf_validation=True, ) mock_cleanup_skale_op.assert_called_once_with( node_mode=NodeMode.ACTIVE, env=mock_env, prune=False) \ No newline at end of file diff --git a/tests/fair/fair_node_test.py b/tests/fair/fair_node_test.py index 7fa2e912..a0408caa 100644 --- a/tests/fair/fair_node_test.py +++ b/tests/fair/fair_node_test.py @@ -179,7 +179,7 @@ def test_cleanup_success( save=False, node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE, - skip_usr_conf_validation=True, + skip_user_conf_validation=True, ) mock_cleanup_fair_op.assert_called_once_with( node_mode=NodeMode.ACTIVE, env=mock_env, prune=False) @@ -214,7 +214,7 @@ def test_cleanup_calls_operations_in_correct_order( save=False, node_type=mock.ANY, node_mode=NodeMode.ACTIVE, - skip_usr_conf_validation=True), + skip_user_conf_validation=True), mock.call.cleanup_fair_op(node_mode=NodeMode.ACTIVE, env=mock_env, prune=False), ] manager.assert_has_calls(expected_calls, any_order=False) From bec919eb04c2e5ca7c0598c8712bea00b1b952a1 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Tue, 18 Nov 2025 12:15:42 +0000 Subject: [PATCH 131/198] Delete unused import --- node_cli/operations/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index 28be076d..cbd0cbc7 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -72,7 +72,7 @@ remove_dynamic_containers, system_prune, ) -from node_cli.utils.helper import cleanup_dir_content, rm_dir, str_to_bool, run_cmd +from node_cli.utils.helper import cleanup_dir_content, rm_dir, str_to_bool from node_cli.utils.meta import CliMetaManager, FairCliMetaManager from node_cli.utils.node_type import NodeMode, NodeType from node_cli.utils.print_formatters import print_failed_requirements_checks From 371d71271c114df2a175b71b1238024e837b1fd8 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Wed, 19 Nov 2025 20:23:36 +0000 Subject: [PATCH 132/198] Add pyproject.toml --- pyproject.toml | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..3aa139d4 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,64 @@ +[build-system] +requires = ["setuptools>=75", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "node-cli" +version = "3.2.0" +description = "Node CLI tools" +readme = "README.md" +requires-python = ">=3.11" +license = { file = "LICENSE" } + +authors = [ + { name = "SKALE Labs", email = "support@skalelabs.com" } +] + +dependencies = [ + "click==8.1.7", + "distro==1.9.0", + "docker==6.0.1", + "texttable==1.6.7", + "python-dateutil==2.8.2", + "Jinja2==3.1.4", + "psutil==5.9.4", + "python-dotenv==0.21.0", + "terminaltables==3.1.10", + "requests==2.28.1", + "GitPython==3.1.41", + "packaging==23.0", + "python-debian==0.1.49", + "PyYAML==6.0.3", + "pyOpenSSL==24.2.1", + "MarkupSafe==3.0.2", + "Flask==2.3.3", + "itsdangerous==2.1.2", + "cryptography==42.0.4", + "filelock==3.0.12", + "sh==1.14.2", + "python-crontab==2.6.0", + "requests-mock==1.12.1", + "redis==6.2.0", +] + +[project.optional-dependencies] +dev = [ + "ruff==0.14.0", + "bumpversion==0.6.0", + "pytest==8.4.2", + "pytest-cov==7.0.0", + "twine==4.0.2", + "mock==4.0.3", + "freezegun==1.5.5", + "PyInstaller>=6.14.0", +] + +[tool.setuptools] +package-dir = { "" = "." } + +[tool.setuptools.packages.find] +where = ["."] + +[tool.ruff] +line-length = 100 +target-version = "py311" From 0eeb193f0de9831b68a35c1cbf62b72b21d5af7e Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Thu, 20 Nov 2025 13:31:44 +0000 Subject: [PATCH 133/198] Add classifiers and keywords --- pyproject.toml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3aa139d4..fa2c13cb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,10 +9,17 @@ description = "Node CLI tools" readme = "README.md" requires-python = ">=3.11" license = { file = "LICENSE" } - +keywords = ["skale", "cli"] authors = [ { name = "SKALE Labs", email = "support@skalelabs.com" } ] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "License :: OSI Approved :: GNU Affero General Public License v3", + "Natural Language :: English", + "Programming Language :: Python :: 3.11", +] dependencies = [ "click==8.1.7", From 798d2739d9a541b3223b3daa4ba2f8cd2f927fd2 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Thu, 20 Nov 2025 13:32:55 +0000 Subject: [PATCH 134/198] Add homepage url --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index fa2c13cb..b33b8efe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,9 @@ dependencies = [ "redis==6.2.0", ] +[project.urls] +Homepage = "https://github.com/skalenetwork/node-cli" + [project.optional-dependencies] dev = [ "ruff==0.14.0", From 1f458f5dcffb88f0bda362282fef9a39d8410fd4 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Thu, 20 Nov 2025 13:34:20 +0000 Subject: [PATCH 135/198] Exclude the tests directory from the final package --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index b33b8efe..088b9059 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,6 +68,7 @@ package-dir = { "" = "." } [tool.setuptools.packages.find] where = ["."] +exclude = ["tests"] [tool.ruff] line-length = 100 From e6d4b126f1db5c6cb2d4d2d53820649c180be811 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Thu, 20 Nov 2025 13:40:27 +0000 Subject: [PATCH 136/198] Set `include_package_data` to `True` --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 088b9059..8486cf81 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,6 +65,7 @@ dev = [ [tool.setuptools] package-dir = { "" = "." } +include-package-data = true [tool.setuptools.packages.find] where = ["."] From 87ecf0d75b49f517151f681cb744a8b9a676ecac Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Thu, 20 Nov 2025 15:20:51 +0000 Subject: [PATCH 137/198] Bump Python version to 3.13 --- pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8486cf81..2864eb25 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ name = "node-cli" version = "3.2.0" description = "Node CLI tools" readme = "README.md" -requires-python = ">=3.11" +requires-python = ">=3.13" license = { file = "LICENSE" } keywords = ["skale", "cli"] authors = [ @@ -18,7 +18,7 @@ classifiers = [ "Intended Audience :: Developers", "License :: OSI Approved :: GNU Affero General Public License v3", "Natural Language :: English", - "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.13", ] dependencies = [ @@ -73,4 +73,4 @@ exclude = ["tests"] [tool.ruff] line-length = 100 -target-version = "py311" +target-version = "py313" From 74c5719aa8352b80aadd8a6476ee0cb0caa57f78 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Thu, 20 Nov 2025 18:55:59 +0000 Subject: [PATCH 138/198] Update dependencies --- pyproject.toml | 56 +++++++++++++++++++++++++------------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2864eb25..bd45c729 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,30 +22,31 @@ classifiers = [ ] dependencies = [ - "click==8.1.7", + "click==8.3.1", "distro==1.9.0", - "docker==6.0.1", - "texttable==1.6.7", - "python-dateutil==2.8.2", - "Jinja2==3.1.4", - "psutil==5.9.4", - "python-dotenv==0.21.0", + "docker==7.1.0", + "texttable==1.7.0", + "python-dateutil==2.9.0", + "Jinja2==3.1.6", + "psutil==7.1.3", + "python-dotenv==1.0.1", "terminaltables==3.1.10", - "requests==2.28.1", - "GitPython==3.1.41", - "packaging==23.0", - "python-debian==0.1.49", - "PyYAML==6.0.3", - "pyOpenSSL==24.2.1", - "MarkupSafe==3.0.2", - "Flask==2.3.3", - "itsdangerous==2.1.2", - "cryptography==42.0.4", - "filelock==3.0.12", - "sh==1.14.2", - "python-crontab==2.6.0", + "requests==2.32.3", + "GitPython==3.1.43", + "packaging==24.1", + "python-debian==1.0.1", + "PyYAML==6.0.1", + "pyOpenSSL==25.3.0", + "MarkupSafe==2.1.5", + "Flask==3.0.3", + "itsdangerous==2.2.0", + "cryptography==45.0.7", + "filelock==3.15.4", + "sh==2.0.6", + "python-crontab==3.0.0", "requests-mock==1.12.1", - "redis==6.2.0", + "redis==5.0.7", + "PyInstaller==6.16.0", ] [project.urls] @@ -53,14 +54,13 @@ Homepage = "https://github.com/skalenetwork/node-cli" [project.optional-dependencies] dev = [ - "ruff==0.14.0", + "ruff==0.5.5", "bumpversion==0.6.0", - "pytest==8.4.2", - "pytest-cov==7.0.0", - "twine==4.0.2", - "mock==4.0.3", - "freezegun==1.5.5", - "PyInstaller>=6.14.0", + "pytest==8.2.2", + "pytest-cov==5.0.0", + "twine==5.1.1", + "mock==5.1.0", + "freezegun==1.5.1", ] [tool.setuptools] From 357d12d568a658d0a17685e03709e046c6d3ecd8 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Thu, 20 Nov 2025 18:56:54 +0000 Subject: [PATCH 139/198] Delete outdated setup.py --- setup.py | 87 -------------------------------------------------------- 1 file changed, 87 deletions(-) delete mode 100644 setup.py diff --git a/setup.py b/setup.py deleted file mode 100644 index 5ea3e6ce..00000000 --- a/setup.py +++ /dev/null @@ -1,87 +0,0 @@ -import os -import re -from setuptools import find_packages, setup - - -def read(*parts): - path = os.path.join(os.path.dirname(__file__), *parts) - f = open(path, 'r') - return f.read() - - -def find_version(*file_paths): - version_file = read(*file_paths) - version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", version_file, re.M) - if version_match: - return version_match.group(1) - raise RuntimeError("Couldn't parse version from file.") - - -extras_require = { - 'linter': [ - 'isort>=4.2.15,<5.10.2', - 'ruff==0.9.9', - ], - 'dev': [ - 'bumpversion==0.6.0', - 'pytest==8.3.2', - 'pytest-cov==5.0.0', - 'twine==4.0.2', - 'mock==4.0.3', - 'freezegun==1.2.2', - ], -} - -extras_require['dev'] = extras_require['linter'] + extras_require['dev'] - - -setup( - name='node-cli', - # *IMPORTANT*: Don't manually change the version here. - # Use the 'bumpversion' utility instead. - version=find_version('node_cli', 'cli', '__init__.py'), - include_package_data=True, - description='SKALE client tools', - long_description_markdown_filename='README.md', - author='SKALE Labs', - author_email='support@skalelabs.com', - url='https://github.com/skalenetwork/node-cli', - install_requires=[ - 'click==8.1.7', - 'PyInstaller==5.12.0', - 'distro==1.9.0', - 'docker==6.0.1', - 'texttable==1.6.7', - 'python-dateutil==2.8.2', - 'Jinja2==3.1.4', - 'psutil==5.9.4', - 'python-dotenv==0.21.0', - 'terminaltables==3.1.10', - 'requests==2.28.1', - 'GitPython==3.1.41', - 'packaging==23.0', - 'python-debian==0.1.49', - 'PyYAML==6.0', - 'pyOpenSSL==24.2.1', - 'MarkupSafe==3.0.2', - 'Flask==2.3.3', - 'itsdangerous==2.1.2', - 'cryptography==42.0.4', - 'filelock==3.0.12', - 'sh==1.14.2', - 'python-crontab==2.6.0', - 'requests-mock==1.12.1', - 'redis==6.2.0', - ], - python_requires='>=3.8,<4', - extras_require=extras_require, - keywords=['skale', 'cli'], - packages=find_packages(exclude=['tests']), - classifiers=[ - 'Development Status :: 5 - Production/Stable', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: GNU Affero General Public License v3', - 'Natural Language :: English', - 'Programming Language :: Python :: 3.11', - ], -) From 0c1e0d853b6595822a109980bfe0e577ee52a84d Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Thu, 20 Nov 2025 19:16:23 +0000 Subject: [PATCH 140/198] Use `python:3.13-bookworm` --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index abc1fcd3..a7c14258 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11-bookworm +FROM python:3.13-bookworm ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && apt install -y \ @@ -10,8 +10,8 @@ RUN apt-get update && apt install -y \ libffi-dev \ swig \ iptables \ - nftables \ - python3-nftables \ + nftables \ + python3-nftables \ libxslt-dev \ kmod From 1768b73cf2d2fdf5e568c3a5a873ed20d45d65b5 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Thu, 20 Nov 2025 19:17:59 +0000 Subject: [PATCH 141/198] Bump Python and actions version --- .github/workflows/test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 42173eeb..5a2500aa 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -6,9 +6,9 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: - python-version: [3.11] + python-version: ['3.13'] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: submodules: true @@ -16,7 +16,7 @@ jobs: run: git submodule update --init - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} From a80db48360f58d69847cd68c4f186a960ea403bd Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Thu, 20 Nov 2025 19:19:14 +0000 Subject: [PATCH 142/198] Delete unused ruff.toml --- ruff.toml | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 ruff.toml diff --git a/ruff.toml b/ruff.toml deleted file mode 100644 index 978b38de..00000000 --- a/ruff.toml +++ /dev/null @@ -1,8 +0,0 @@ -line-length = 100 - -[format] -quote-style = "single" - -[lint] -# Add the `line-too-long` rule to the enforced rule set. -extend-select = ["E501"] From 239bcf1eb2dfffecbd592cabf6bd44a11fc1c6b6 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Thu, 20 Nov 2025 19:39:27 +0000 Subject: [PATCH 143/198] Handle `OSError` exception --- node_cli/utils/global_config.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/node_cli/utils/global_config.py b/node_cli/utils/global_config.py index 4347c6b0..974c573a 100644 --- a/node_cli/utils/global_config.py +++ b/node_cli/utils/global_config.py @@ -52,6 +52,10 @@ def generate_g_config_file(g_skale_dir: str, g_skale_conf_filepath: str) -> dict json.dump(g_config, outfile, indent=4) except PermissionError as e: logger.exception(e) - print('No permissions to write into /etc directory') + print(f'No permissions to write into {g_skale_dir} directory') + sys.exit(7) + except OSError as e: + logger.exception(e) + print(f'Error writing to {g_skale_conf_filepath}: {e}') sys.exit(7) return g_config From 37781b98d5db441330b9082ae5c475b4e42edd75 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Fri, 21 Nov 2025 14:07:53 +0000 Subject: [PATCH 144/198] Use 'Group' instead of deprecated 'MultiCommand' --- node_cli/main.py | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/node_cli/main.py b/node_cli/main.py index a18b4d67..479099ca 100644 --- a/node_cli/main.py +++ b/node_cli/main.py @@ -54,9 +54,20 @@ logger = logging.getLogger(__name__) -@click.group() -def cli(): - pass +@click.group(invoke_without_command=True) +@click.pass_context +def cli(ctx): + if ctx.invoked_subcommand is None: + print(ctx.get_help()) + ctx.exit(0) + + start_time = time.time() + init_logs_dir() + init_default_logger() + args = sys.argv + # todo: hide secret variables (passwords, private keys) + logger.debug(f'cmd: {" ".join(str(x) for x in args)}, v.{__version__}') + ctx.call_on_close(lambda: logger.debug('Execution time: %d seconds', time.time() - start_time)) @cli.command('version', help='Show SKALE node CLI version') @@ -84,10 +95,9 @@ def info(): ) -def get_sources_list() -> List[click.MultiCommand]: +def get_command_groups() -> List[click.Group]: if TYPE == NodeType.FAIR: return [ - cli, logs_cli, fair_boot_cli, fair_node_cli, @@ -99,7 +109,6 @@ def get_sources_list() -> List[click.MultiCommand]: ] else: return [ - cli, health_cli, schains_cli, logs_cli, @@ -122,19 +131,12 @@ def handle_exception(exc_type, exc_value, exc_traceback): sys.excepthook = handle_exception if __name__ == '__main__': - start_time = time.time() - init_logs_dir() - init_default_logger() - args = sys.argv - # todo: hide secret variables (passwords, private keys) - logger.debug(f'cmd: {" ".join(str(x) for x in args)}, v.{__version__}') - sources = get_sources_list() - cmd_collection = click.CommandCollection(sources=sources) + for group in get_command_groups(): + for cmd_name, cmd_obj in group.commands.items(): + cli.add_command(cmd_obj, cmd_name) try: - cmd_collection() + cli() except Exception as err: traceback.print_exc() - logger.debug('Execution time: %d seconds', time.time() - start_time) error_exit(err) - logger.debug('Execution time: %d seconds', time.time() - start_time) From 2c77239178b5398413dd34c86af994ffaad9bcb2 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Fri, 21 Nov 2025 14:24:19 +0000 Subject: [PATCH 145/198] Disable warnings from static analyzer --- node_cli/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/node_cli/main.py b/node_cli/main.py index 479099ca..9016f79d 100644 --- a/node_cli/main.py +++ b/node_cli/main.py @@ -97,7 +97,7 @@ def info(): def get_command_groups() -> List[click.Group]: if TYPE == NodeType.FAIR: - return [ + return [ # type: ignore logs_cli, fair_boot_cli, fair_node_cli, @@ -108,7 +108,7 @@ def get_command_groups() -> List[click.Group]: ssl_cli, ] else: - return [ + return [ # type: ignore health_cli, schains_cli, logs_cli, From 6c5b526a42b4fe8a26e33cbe9446930113fa8f54 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Fri, 21 Nov 2025 17:20:02 +0000 Subject: [PATCH 146/198] Replace `str_to_bool` with a new implementation without deleted `distutils` dependency --- node_cli/utils/helper.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/node_cli/utils/helper.py b/node_cli/utils/helper.py index c9c9bfb0..dd1a7151 100644 --- a/node_cli/utils/helper.py +++ b/node_cli/utils/helper.py @@ -17,8 +17,6 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import distutils -import distutils.util import ipaddress import json import logging @@ -145,8 +143,14 @@ def get_username(): return os.environ.get('USERNAME') or os.environ.get('USER') -def str_to_bool(val): - return bool(distutils.util.strtobool(val)) +def str_to_bool(val: str) -> bool: + val = val.lower() + if val in ('y', 'yes', 't', 'true', 'on', '1'): + return True + elif val in ('n', 'no', 'f', 'false', 'off', '0'): + return False + else: + raise ValueError(f'Invalid truth value {val!r}') def error_exit(error_payload: Any, exit_code: CLIExitCodes = CLIExitCodes.FAILURE) -> NoReturn: From f4c68cc04a5222ca04cfb2ae6e21eb8580594d10 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Fri, 21 Nov 2025 18:15:44 +0000 Subject: [PATCH 147/198] Update all dependencies to their latest versions --- pyproject.toml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index bd45c729..323b052a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,23 +29,23 @@ dependencies = [ "python-dateutil==2.9.0", "Jinja2==3.1.6", "psutil==7.1.3", - "python-dotenv==1.0.1", + "python-dotenv==1.2.1", "terminaltables==3.1.10", - "requests==2.32.3", - "GitPython==3.1.43", - "packaging==24.1", + "requests==2.32.5", + "GitPython==3.1.45", + "packaging==25.0", "python-debian==1.0.1", - "PyYAML==6.0.1", + "PyYAML==6.0.3", "pyOpenSSL==25.3.0", - "MarkupSafe==2.1.5", - "Flask==3.0.3", + "MarkupSafe==3.0.3", + "Flask==3.1.2", "itsdangerous==2.2.0", - "cryptography==45.0.7", - "filelock==3.15.4", - "sh==2.0.6", - "python-crontab==3.0.0", + "cryptography==46.0.3", + "filelock==3.20.0", + "sh==2.2.2", + "python-crontab==3.3.0", "requests-mock==1.12.1", - "redis==5.0.7", + "redis==7.1.0", "PyInstaller==6.16.0", ] From 108257d3058b52446427113cbf09d4dc42bc7757 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Fri, 21 Nov 2025 18:17:23 +0000 Subject: [PATCH 148/198] Bump `setuptools` version to 80.9.0 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index a7c14258..1b9162ab 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,5 +25,5 @@ ENV PATH=/app/buildvenv/bin:$PATH ENV PYTHONPATH="{PYTHONPATH}:/usr/lib/python3/dist-packages" RUN pip install --upgrade pip && \ - pip install wheel setuptools==63.2.0 && \ + pip install wheel setuptools==80.9.0 && \ pip install -e '.[dev]' From e3dbf31f820479ed044bcec1cb4a52c7a36efef1 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Fri, 21 Nov 2025 18:36:08 +0000 Subject: [PATCH 149/198] Update `dev` dependencies to their latest versions --- pyproject.toml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 323b052a..47914c7e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,13 +54,13 @@ Homepage = "https://github.com/skalenetwork/node-cli" [project.optional-dependencies] dev = [ - "ruff==0.5.5", + "ruff==0.14.6", "bumpversion==0.6.0", - "pytest==8.2.2", - "pytest-cov==5.0.0", - "twine==5.1.1", - "mock==5.1.0", - "freezegun==1.5.1", + "pytest==9.0.1", + "pytest-cov==7.0.0", + "twine==6.2.0", + "mock==5.2.0", + "freezegun==1.5.5", ] [tool.setuptools] From 258d9646f99a2779951bbf4f3926889949ce08be Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Fri, 21 Nov 2025 19:15:20 +0000 Subject: [PATCH 150/198] Use builder, slim image and uv --- Dockerfile | 40 +++++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1b9162ab..91deca29 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,29 +1,31 @@ -FROM python:3.13-bookworm +FROM python:3.13-slim-bookworm AS builder -ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt install -y \ +COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv + +WORKDIR /app + +COPY pyproject.toml ./ + +RUN uv pip install --system --no-cache ".[dev]" + +FROM python:3.13-slim-bookworm + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ git \ - build-essential \ - software-properties-common \ - zlib1g-dev \ - libssl-dev \ - libffi-dev \ - swig \ iptables \ nftables \ python3-nftables \ - libxslt-dev \ - kmod - + kmod \ + wget && \ + rm -rf /var/lib/apt/lists/* -RUN mkdir /app WORKDIR /app -COPY . . +COPY --from=builder /usr/local/lib/python3.13/site-packages /usr/local/lib/python3.13/site-packages +COPY --from=builder /usr/local/bin /usr/local/bin -ENV PATH=/app/buildvenv/bin:$PATH -ENV PYTHONPATH="{PYTHONPATH}:/usr/lib/python3/dist-packages" +COPY . . -RUN pip install --upgrade pip && \ - pip install wheel setuptools==80.9.0 && \ - pip install -e '.[dev]' +ENV PYTHONPATH="/app:/usr/lib/python3/dist-packages" +ENV COLUMNS=80 From 52e8a60d1f3bd6e30f3bd54cdf801f41c06f79f8 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Fri, 21 Nov 2025 19:41:24 +0000 Subject: [PATCH 151/198] Install `binutils` --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 91deca29..7fc37545 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,7 +17,8 @@ RUN apt-get update && \ nftables \ python3-nftables \ kmod \ - wget && \ + wget \ + binutils && \ rm -rf /var/lib/apt/lists/* WORKDIR /app From 3ff9c2faa839a37a74bca5e279fc9e730dbc62b9 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Mon, 24 Nov 2025 12:57:27 +0000 Subject: [PATCH 152/198] Bump `python-dateutil` version to 2.9.0.post0 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 47914c7e..970112dd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,7 @@ dependencies = [ "distro==1.9.0", "docker==7.1.0", "texttable==1.7.0", - "python-dateutil==2.9.0", + "python-dateutil==2.9.0.post0", "Jinja2==3.1.6", "psutil==7.1.3", "python-dotenv==1.2.1", From bff05d8c20ca88180a4fd12eb8d37aba76e46468 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Mon, 24 Nov 2025 12:58:15 +0000 Subject: [PATCH 153/198] Set `quote-style` to "single" --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 970112dd..608092b3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -74,3 +74,6 @@ exclude = ["tests"] [tool.ruff] line-length = 100 target-version = "py313" + +[tool.ruff.format] +quote-style = "single" \ No newline at end of file From e791fb8fa87b7d8a81ae52861cbdc106aaa4715e Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Mon, 24 Nov 2025 13:09:06 +0000 Subject: [PATCH 154/198] Use `uv` for installing dependencies --- .github/workflows/test.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5a2500aa..8f348515 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,6 +20,15 @@ jobs: with: python-version: ${{ matrix.python-version }} + - name: Install uv + run: curl -LsSf https://astral.sh/uv/install.sh | sh + + - name: Cache uv + uses: actions/cache@v4 + with: + path: ~/.cache/uv + key: ${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('pyproject.toml') }} + - name: Install ubuntu dependencies run: | sudo apt-get update @@ -27,8 +36,7 @@ jobs: - name: Install python dependencies run: | - python -m pip install --upgrade pip - pip install -e ".[dev]" + uv pip install -e ".[dev]" - name: Generate info run: bash ./scripts/generate_info.sh 1.0.0 my-branch skale From 4bd90cc26ce747f7dbb0ed5b47f8bb6e2997d7f8 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Mon, 24 Nov 2025 13:37:53 +0000 Subject: [PATCH 155/198] Create `venv` --- .github/workflows/test.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8f348515..b985132b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -36,6 +36,7 @@ jobs: - name: Install python dependencies run: | + uv venv uv pip install -e ".[dev]" - name: Generate info @@ -43,7 +44,7 @@ jobs: - name: Check with ruff run: | - ruff check + uv run ruff check - name: Build binary - skale run: | @@ -76,8 +77,8 @@ jobs: - name: Run tests run: | export PYTHONPATH=${PYTHONPATH}:/usr/lib/python3/dist-packages/ - bash ./scripts/run_tests.sh + uv run bash ./scripts/run_tests.sh - name: Run nftables tests run: | - scripts/run_nftables_test.sh + uv run scripts/run_nftables_test.sh From 623e821852e6b6b8f70ed31bd146b17ed8ce39db Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Mon, 24 Nov 2025 13:45:35 +0000 Subject: [PATCH 156/198] Run `prepare test build` in `venv` --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b985132b..e5c1dd8f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -68,7 +68,7 @@ jobs: - name: Run prepare test build run: | - bash scripts/build.sh test test skale + uv run bash scripts/build.sh test test skale - name: Run redis run: | From c72148b1561f7a87fecdaa0aa0a3e3abf12e47be Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Mon, 24 Nov 2025 16:25:38 +0000 Subject: [PATCH 157/198] Bump Python version to 3.13 --- .github/workflows/publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 660ec07d..95797b24 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -79,10 +79,10 @@ jobs: with: submodules: true - - name: Set up Python 3.11 + - name: Set up Python 3.13 uses: actions/setup-python@v5 with: - python-version: 3.11 + python-version: 3.13 - name: Install ubuntu dependencies if: matrix.os == 'ubuntu-22.04' From 7f0f2f0bad639d7e573459d7fab4c9b4f9075cf1 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Mon, 24 Nov 2025 16:26:19 +0000 Subject: [PATCH 158/198] Use importlib.metadata to get package version --- node_cli/cli/__init__.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/node_cli/cli/__init__.py b/node_cli/cli/__init__.py index 46419e54..c081741c 100644 --- a/node_cli/cli/__init__.py +++ b/node_cli/cli/__init__.py @@ -1,4 +1,9 @@ -__version__ = '3.2.0' +from importlib.metadata import version, PackageNotFoundError + +try: + __version__ = version("node-cli") +except PackageNotFoundError: + __version__ = "0.0.0-dev" if __name__ == '__main__': print(__version__) From 6c94e9a9fb2a3093cdcb7e5a916e1e39368525f9 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Mon, 24 Nov 2025 18:57:02 +0000 Subject: [PATCH 159/198] Get version from 'pyproject.toml' --- scripts/set_versions_ga.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/set_versions_ga.sh b/scripts/set_versions_ga.sh index ddcc7aaa..7333357f 100644 --- a/scripts/set_versions_ga.sh +++ b/scripts/set_versions_ga.sh @@ -7,7 +7,7 @@ echo PROJECT_DIR: $GITHUB_WORKSPACE export BRANCH=${GITHUB_REF##*/} echo "Branch $BRANCH" -export VERSION=$(python setup.py --version) +export VERSION=$(python -c "import tomllib; print(tomllib.load(open('pyproject.toml', 'rb'))['project']['version'])") export VERSION=$(bash ./helper-scripts/calculate_version.sh) echo "VERSION=$VERSION" >> $GITHUB_ENV From 0fdbbdd0dfc137dbb738735fd705d46ea5be1a5e Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Mon, 24 Nov 2025 19:08:46 +0000 Subject: [PATCH 160/198] Modernize and simplify the release workflow --- .github/workflows/publish.yml | 147 ++++++++++------------------------ 1 file changed, 42 insertions(+), 105 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 95797b24..c57d3bc1 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,140 +1,77 @@ -name: Build and publish +name: Build and Publish + on: pull_request: types: [closed] branches: - - master - develop - beta - stable - 'v*.*.*' - - 'fair' - - 'fair-*' jobs: - create_release: + build_and_release: if: github.event.pull_request.merged - name: Create release + name: Build and Create Release runs-on: ubuntu-22.04 - outputs: - upload_url: ${{ steps.create_release.outputs.upload_url }} - version: ${{ steps.export_outputs.outputs.version }} - branch: ${{ steps.export_outputs.outputs.branch }} + permissions: + contents: write steps: - name: Checkout code uses: actions/checkout@v4 with: submodules: true - - name: Checkout submodules - run: git submodule update --init --recursive - - - name: Install ubuntu dependencies - run: | - sudo apt-get update - sudo apt-get install python-setuptools - - - name: Set Versions - run: | - bash ./scripts/set_versions_ga.sh - - - name: Set release - run: | - if [[ "$BRANCH" == "stable" ]]; then - export PRERELEASE=false - else - export PRERELEASE=true - fi - echo "PRERELEASE=$PRERELEASE" >> $GITHUB_ENV - - - name: Create Release - id: create_release - uses: actions/create-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - tag_name: ${{ env.VERSION }} - release_name: ${{ env.VERSION }} - draft: false - prerelease: ${{ env.PRERELEASE }} - - - name: Export outputs - id: export_outputs - run: | - echo "::set-output name=version::$VERSION" - echo "::set-output name=branch::$BRANCH" - - build_and_publish: - if: github.event.pull_request.merged - needs: create_release - name: Build and publish ${{ matrix.build_type }} for ${{ matrix.os }} - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-22.04] - build_type: [skale, fair] - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - submodules: true - - name: Set up Python 3.13 - uses: actions/setup-python@v5 + uses: actions/setup-python@v5 with: python-version: 3.13 - - name: Install ubuntu dependencies - if: matrix.os == 'ubuntu-22.04' + - name: Calculate Version + id: versioning run: | - sudo apt-get update - - - name: Ensure submodules are updated - run: git submodule update --init --recursive - - - name: Define Asset Name - id: asset_details + bash ./scripts/set_versions_ga.sh + + - name: Determine Prerelease status + id: release_info run: | - ASSET_BASE_NAME="skale-${{ needs.create_release.outputs.version }}-Linux-x86_64" - if [[ "${{ matrix.build_type }}" == "skale" ]]; then - echo "FINAL_ASSET_NAME=${ASSET_BASE_NAME}" >> $GITHUB_OUTPUT + if [[ "${{ env.BRANCH }}" == "stable" ]]; then + echo "prerelease=false" >> $GITHUB_OUTPUT else - echo "FINAL_ASSET_NAME=${ASSET_BASE_NAME}-${{ matrix.build_type }}" >> $GITHUB_OUTPUT + echo "prerelease=true" >> $GITHUB_OUTPUT fi - - name: Build ${{ matrix.build_type }} release binary + - name: Build binaries + id: build run: | mkdir -p ${{ github.workspace }}/dist docker build . -t node-cli-builder + docker run --rm -v ${{ github.workspace }}/dist:/app/dist node-cli-builder \ - bash scripts/build.sh ${{ needs.create_release.outputs.version }} ${{ needs.create_release.outputs.branch }} ${{ matrix.build_type }} - echo "Contents of dist directory:" - ls -altr ${{ github.workspace }}/dist/ - docker rm -f $(docker ps -aq) || true - - - name: Save sha512sum for ${{ steps.asset_details.outputs.FINAL_ASSET_NAME }} + bash scripts/build.sh ${{ env.VERSION }} ${{ env.BRANCH }} skale + + docker run --rm -v ${{ github.workspace }}/dist:/app/dist node-cli-builder \ + bash scripts/build.sh ${{ env.VERSION }} ${{ env.BRANCH }} fair + + echo "dist_path=${{ github.workspace }}/dist" >> $GITHUB_OUTPUT + + - name: Generate checksums run: | - cd ${{ github.workspace }}/dist - sha512sum ${{ steps.asset_details.outputs.FINAL_ASSET_NAME }} > ${{ steps.asset_details.outputs.FINAL_ASSET_NAME }}.sha512sum - echo "Checksum file created: ${{ steps.asset_details.outputs.FINAL_ASSET_NAME }}.sha512sum" - cat ${{ steps.asset_details.outputs.FINAL_ASSET_NAME }}.sha512sum - - - name: Upload release binary (${{ matrix.build_type }}) - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + cd ${{ steps.build.outputs.dist_path }} + for file in skale-*; do + sha512sum "$file" > "$file.sha512" + done + echo "Checksums generated:" + ls -l *.sha512 + + - name: Create GitHub Release + uses: softprops/action-gh-release@v2 with: - upload_url: ${{ needs.create_release.outputs.upload_url }} - asset_path: ${{ github.workspace }}/dist/${{ steps.asset_details.outputs.FINAL_ASSET_NAME }} - asset_name: ${{ steps.asset_details.outputs.FINAL_ASSET_NAME }} - asset_content_type: application/octet-stream - - - name: Upload release checksum (${{ matrix.build_type }}) - uses: actions/upload-release-asset@v1 + tag_name: ${{ env.VERSION }} + name: Release ${{ env.VERSION }} + draft: false + prerelease: ${{ steps.release_info.outputs.prerelease }} + files: | + ${{ steps.build.outputs.dist_path }}/skale-* env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ needs.create_release.outputs.upload_url }} - asset_path: ${{ github.workspace }}/dist/${{ steps.asset_details.outputs.FINAL_ASSET_NAME }}.sha512sum - asset_name: ${{ steps.asset_details.outputs.FINAL_ASSET_NAME }}.sha512 - asset_content_type: text/plain \ No newline at end of file From 1fc9118e50d8971695d29d16ca17d17b2900f009 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Mon, 24 Nov 2025 19:14:54 +0000 Subject: [PATCH 161/198] Add new branch for new release workflow testing --- .github/workflows/publish.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index c57d3bc1..bc6ef0ce 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -5,6 +5,7 @@ on: types: [closed] branches: - develop + - migrate-to-pyproject - beta - stable - 'v*.*.*' From cac1d9b54dfbdba14c0e7fac07c9a91db2707d8d Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Mon, 24 Nov 2025 19:42:07 +0000 Subject: [PATCH 162/198] Generate `generate_release_notes` --- .github/workflows/publish.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index bc6ef0ce..37aeff40 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -72,6 +72,7 @@ jobs: name: Release ${{ env.VERSION }} draft: false prerelease: ${{ steps.release_info.outputs.prerelease }} + generate_release_notes: true files: | ${{ steps.build.outputs.dist_path }}/skale-* env: From a7d74799002d455330b69e148ff77ab7b3b925a6 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Tue, 25 Nov 2025 12:21:48 +0000 Subject: [PATCH 163/198] Remove test branch and improve task names --- .github/workflows/publish.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 37aeff40..dd86ba23 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,11 +1,10 @@ -name: Build and Publish +name: Build and publish on: pull_request: types: [closed] branches: - develop - - migrate-to-pyproject - beta - stable - 'v*.*.*' @@ -13,7 +12,7 @@ on: jobs: build_and_release: if: github.event.pull_request.merged - name: Build and Create Release + name: Build and create release runs-on: ubuntu-22.04 permissions: contents: write @@ -28,12 +27,12 @@ jobs: with: python-version: 3.13 - - name: Calculate Version + - name: Calculate version id: versioning run: | bash ./scripts/set_versions_ga.sh - - name: Determine Prerelease status + - name: Determine prerelease status id: release_info run: | if [[ "${{ env.BRANCH }}" == "stable" ]]; then @@ -65,7 +64,7 @@ jobs: echo "Checksums generated:" ls -l *.sha512 - - name: Create GitHub Release + - name: Create GitHub release uses: softprops/action-gh-release@v2 with: tag_name: ${{ env.VERSION }} From 46baadec937dc2070744f828f70c8a3d6f93d33b Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Tue, 25 Nov 2025 13:11:41 +0000 Subject: [PATCH 164/198] Build `node-cli-builder` only once --- .github/workflows/test.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e5c1dd8f..5f589d47 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -46,10 +46,12 @@ jobs: run: | uv run ruff check + - name: Build docker image + run: docker build . -t node-cli-builder + - name: Build binary - skale run: | mkdir -p ./dist - docker build . -t node-cli-builder docker run -v /home/ubuntu/dist:/app/dist node-cli-builder bash scripts/build.sh test test skale docker rm -f $(docker ps -aq) @@ -59,7 +61,6 @@ jobs: - name: Build binary - fair run: | mkdir -p ./dist - docker build . -t node-cli-builder docker run -v /home/ubuntu/dist:/app/dist node-cli-builder bash scripts/build.sh test test fair docker rm -f $(docker ps -aq) From 94122eabd8cd8a4605d5e3d13c715f8799bded96 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Tue, 25 Nov 2025 13:52:12 +0000 Subject: [PATCH 165/198] Make PyInstaller spec paths absolute --- main.spec | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/main.spec b/main.spec index e3844bc1..8b7da407 100644 --- a/main.spec +++ b/main.spec @@ -1,16 +1,15 @@ # -*- mode: python -*- -import importlib.util - +import os block_cipher = None a = Analysis( ['node_cli/main.py'], - pathex=['.'], + pathex=[SPECPATH], datas=[ - ("./text.yml", "data"), - ("./datafiles/skaled-ssl-test", "data/datafiles") + (os.path.join(SPECPATH, "text.yml"), "data"), + (os.path.join(SPECPATH, "datafiles/skaled-ssl-test"), "data/datafiles") ], hiddenimports=[], hookspath=[], From e0b77cd90078c6dd2aaca52e2ac7bf5e93098061 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Tue, 25 Nov 2025 13:54:03 +0000 Subject: [PATCH 166/198] Remove `PackageNotFoundError` handling --- node_cli/cli/__init__.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/node_cli/cli/__init__.py b/node_cli/cli/__init__.py index c081741c..9a2285f7 100644 --- a/node_cli/cli/__init__.py +++ b/node_cli/cli/__init__.py @@ -1,9 +1,6 @@ -from importlib.metadata import version, PackageNotFoundError +from importlib.metadata import version -try: - __version__ = version("node-cli") -except PackageNotFoundError: - __version__ = "0.0.0-dev" +__version__ = version("node-cli") if __name__ == '__main__': print(__version__) From 3a0de05dd342318bd7c797a065d2bb504db67546 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Tue, 16 Dec 2025 20:07:51 +0000 Subject: [PATCH 167/198] Update __init__.py --- node_cli/cli/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node_cli/cli/__init__.py b/node_cli/cli/__init__.py index e1e6add7..1fe20f2e 100644 --- a/node_cli/cli/__init__.py +++ b/node_cli/cli/__init__.py @@ -1,4 +1,4 @@ -__version__ = '2.6.2' +__version__ = '2.6.3' if __name__ == '__main__': print(__version__) From 89a28c4af1bc8bd1281b078acfd5b2082f35fce6 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Fri, 26 Dec 2025 20:08:27 +0000 Subject: [PATCH 168/198] Fix skale passive node options --- helper-scripts | 2 +- node_cli/operations/base.py | 2 +- node_cli/utils/docker_utils.py | 5 +---- node_cli/utils/meta.py | 2 +- 4 files changed, 4 insertions(+), 7 deletions(-) diff --git a/helper-scripts b/helper-scripts index 808c768f..c1f67269 160000 --- a/helper-scripts +++ b/helper-scripts @@ -1 +1 @@ -Subproject commit 808c768feebfa99d9148e076b5b6b24b1b340734 +Subproject commit c1f67269955126ac400c2445d2aa81f1a387d964 diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index cbd0cbc7..94139959 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -343,7 +343,7 @@ def update_passive(env_filepath: str, env: Dict) -> bool: meta_manager.update_meta( VERSION, env['NODE_VERSION'], - env['DOCKER_LVMPY_VERSION'], + None, distro.id(), distro.version(), ) diff --git a/node_cli/utils/docker_utils.py b/node_cli/utils/docker_utils.py index 825886e6..9aa95f4f 100644 --- a/node_cli/utils/docker_utils.py +++ b/node_cli/utils/docker_utils.py @@ -77,10 +77,7 @@ 'boot-api': 'sk_boot_api', } -BASE_PASSIVE_COMPOSE_SERVICES = { - 'admin': 'sk_admin', - 'nginx': 'sk_nginx', -} +BASE_PASSIVE_COMPOSE_SERVICES = {'admin': 'sk_admin', 'nginx': 'sk_nginx', **REDIS_SERVICE_DICT} BASE_PASSIVE_FAIR_COMPOSE_SERVICES = { 'admin': 'sk_admin', diff --git a/node_cli/utils/meta.py b/node_cli/utils/meta.py index bb8ad8af..651efc1e 100644 --- a/node_cli/utils/meta.py +++ b/node_cli/utils/meta.py @@ -26,7 +26,7 @@ def asdict(self) -> dict: @dataclass class CliMeta(CliMetaBase): - docker_lvmpy_version: str = DEFAULT_DOCKER_LVMPY_VERSION + docker_lvmpy_version: str | None = DEFAULT_DOCKER_LVMPY_VERSION def asdict(self) -> dict: return { From 1f953bb0c8b37db54341343c1a6512035a68f08c Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Mon, 29 Dec 2025 17:49:19 +0000 Subject: [PATCH 169/198] Move the configuration from `pytest.ini` to `pyproject.toml` --- pyproject.toml | 11 ++++++++++- pytest.ini | 6 ------ 2 files changed, 10 insertions(+), 7 deletions(-) delete mode 100644 pytest.ini diff --git a/pyproject.toml b/pyproject.toml index 608092b3..cf813bfa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -76,4 +76,13 @@ line-length = 100 target-version = "py313" [tool.ruff.format] -quote-style = "single" \ No newline at end of file +quote-style = "single" + +[tool.pytest.ini_options] +log_cli = false +log_cli_level = "INFO" +log_cli_format = "%(asctime)s [%(levelname)8s] %(message)s (%(filename)s:%(lineno)s)" +log_cli_date_format = "%Y-%m-%d %H:%M:%S" +filterwarnings = [ + "ignore::DeprecationWarning", +] diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index 5785bf3f..00000000 --- a/pytest.ini +++ /dev/null @@ -1,6 +0,0 @@ -[pytest] -log_cli = 0 -log_cli_level = INFO -log_cli_format = %(asctime)s [%(levelname)8s] %(message)s (%(filename)s:%(lineno)s) -log_cli_date_format=%Y-%m-%d %H:%M:%S -filterwarnings = ignore::DeprecationWarning From 4923b5478a1d8b1ae2104337d4c6fa3c40f9cb75 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Tue, 30 Dec 2025 20:33:22 +0000 Subject: [PATCH 170/198] Remove PASSIVE_COMPOSE_PATH --- node_cli/configs/__init__.py | 1 - node_cli/utils/docker_utils.py | 5 +---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/node_cli/configs/__init__.py b/node_cli/configs/__init__.py index ad4c2216..35d776d8 100644 --- a/node_cli/configs/__init__.py +++ b/node_cli/configs/__init__.py @@ -55,7 +55,6 @@ SGX_CERTIFICATES_DIR_NAME = 'sgx_certs' COMPOSE_PATH = os.path.join(CONTAINER_CONFIG_PATH, 'docker-compose.yml') -PASSIVE_COMPOSE_PATH = os.path.join(CONTAINER_CONFIG_PATH, 'docker-compose-passive.yml') FAIR_COMPOSE_PATH = os.path.join(CONTAINER_CONFIG_PATH, 'docker-compose-fair.yml') STATIC_PARAMS_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, 'static_params.yaml') FAIR_STATIC_PARAMS_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, 'fair_static_params.yaml') diff --git a/node_cli/utils/docker_utils.py b/node_cli/utils/docker_utils.py index 9aa95f4f..2bc7b306 100644 --- a/node_cli/utils/docker_utils.py +++ b/node_cli/utils/docker_utils.py @@ -35,7 +35,6 @@ NGINX_CONTAINER_NAME, REMOVED_CONTAINERS_FOLDER_PATH, SGX_CERTIFICATES_DIR_NAME, - PASSIVE_COMPOSE_PATH, ) from node_cli.core.node_options import active_fair, active_skale, passive_fair, passive_skale from node_cli.utils.helper import run_cmd, str_to_bool @@ -292,9 +291,7 @@ def compose_build(env: dict, node_type: NodeType, node_mode: NodeMode): def get_compose_path(node_type: NodeType, node_mode: NodeMode) -> str: - if passive_skale(node_type, node_mode): - return PASSIVE_COMPOSE_PATH - elif active_fair(node_type, node_mode) or passive_fair(node_type, node_mode): + if node_type == NodeType.FAIR: return FAIR_COMPOSE_PATH return COMPOSE_PATH From 45de1fb4c530063753b9de2d5e766d131a38fda2 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Mon, 5 Jan 2026 20:17:31 +0000 Subject: [PATCH 171/198] Add api and watchdog containers to `BASE_PASSIVE_COMPOSE_SERVICES` --- node_cli/utils/docker_utils.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/node_cli/utils/docker_utils.py b/node_cli/utils/docker_utils.py index 2bc7b306..098a61e7 100644 --- a/node_cli/utils/docker_utils.py +++ b/node_cli/utils/docker_utils.py @@ -76,7 +76,13 @@ 'boot-api': 'sk_boot_api', } -BASE_PASSIVE_COMPOSE_SERVICES = {'admin': 'sk_admin', 'nginx': 'sk_nginx', **REDIS_SERVICE_DICT} +BASE_PASSIVE_COMPOSE_SERVICES = { + 'admin': 'sk_admin', + 'nginx': 'sk_nginx', + 'api': 'sk_api', + 'watchdog': 'sk_watchdog', + **REDIS_SERVICE_DICT, +} BASE_PASSIVE_FAIR_COMPOSE_SERVICES = { 'admin': 'sk_admin', From 7b7d589c769e77ca1d5ee68ebd643582073f664c Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Wed, 7 Jan 2026 17:47:15 +0000 Subject: [PATCH 172/198] Set Redis data directory ownership --- node_cli/core/host.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/node_cli/core/host.py b/node_cli/core/host.py index 8c684455..da4be640 100644 --- a/node_cli/core/host.py +++ b/node_cli/core/host.py @@ -19,7 +19,7 @@ import logging import os -from shutil import copyfile +from shutil import copyfile, chown from urllib.parse import urlparse from node_cli.core.resources import update_resource_allocation @@ -94,6 +94,7 @@ def prepare_host(env_filepath: str, env_type: str, allocation: bool = False) -> try: logger.info('Preparing host started') make_dirs() + chown(REDIS_DATA_PATH, user=999, group=1000) save_env_params(env_filepath) if allocation: From fae6a38a0b9183322aa2e0082a5e45594a399fd8 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Fri, 9 Jan 2026 19:50:22 +0000 Subject: [PATCH 173/198] Run `run_host_checks` during passive SKALE node init --- node_cli/operations/base.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index 94139959..0429ae37 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -291,6 +291,16 @@ def init_passive( env_filepath, env_type=env['ENV_TYPE'], ) + failed_checks = run_host_checks( + env['BLOCK_DEVICE'], + TYPE, + NodeMode.PASSIVE, + env['ENV_TYPE'], + CONTAINER_CONFIG_PATH, + check_type=CheckType.PREINSTALL + ) + if failed_checks: + print_failed_requirements_checks(failed_checks) set_passive_node_options(archive=archive, indexer=indexer) From 0bc7f3a982ac678a3b0718cef8b8f01d1f2fc51f Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Fri, 9 Jan 2026 19:54:49 +0000 Subject: [PATCH 174/198] Run `run_host_checks` during passive SKALE node update --- node_cli/operations/base.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index 0429ae37..05132995 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -349,6 +349,17 @@ def update_passive(env_filepath: str, env: Dict) -> bool: prepare_host(env_filepath, env['ENV_TYPE'], allocation=True) + failed_checks = run_host_checks( + env['BLOCK_DEVICE'], + TYPE, + NodeMode.PASSIVE, + env['ENV_TYPE'], + CONTAINER_CONFIG_PATH, + check_type=CheckType.PREINSTALL + ) + if failed_checks: + print_failed_requirements_checks(failed_checks) + meta_manager = CliMetaManager() meta_manager.update_meta( VERSION, From 67b68617c710eed098d58c2e97c6361ed982aaf7 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Tue, 13 Jan 2026 12:35:18 +0000 Subject: [PATCH 175/198] Move `init_fair_boot` and `update_fair_boot` functions to `fair.py` --- node_cli/operations/__init__.py | 4 +- node_cli/operations/base.py | 75 --------------------------------- node_cli/operations/fair.py | 35 ++++++++++++++- 3 files changed, 36 insertions(+), 78 deletions(-) diff --git a/node_cli/operations/__init__.py b/node_cli/operations/__init__.py index f2ac1a94..7a72b442 100644 --- a/node_cli/operations/__init__.py +++ b/node_cli/operations/__init__.py @@ -21,8 +21,6 @@ update as update_op, init as init_op, init_passive as init_passive_op, - init_fair_boot as init_fair_boot_op, - update_fair_boot as update_fair_boot_op, update_passive as update_passive_op, turn_off as turn_off_op, turn_on as turn_on_op, @@ -31,7 +29,9 @@ configure_nftables, ) from node_cli.operations.fair import ( # noqa + init_fair_boot as init_fair_boot_op, init as init_fair_op, + update_fair_boot as update_fair_boot_op, update as update_fair_op, FairUpdateType, restore as restore_fair_op, diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index 05132995..1a64d585 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -159,48 +159,6 @@ def update(env_filepath: str, env: Dict, node_mode: NodeMode) -> bool: return True -@checked_host -def update_fair_boot(env_filepath: str, env: Dict, node_mode: NodeMode = NodeMode.ACTIVE) -> bool: - compose_rm(node_type=NodeType.FAIR, node_mode=node_mode, env=env) - remove_dynamic_containers() - cleanup_volume_artifacts(env['BLOCK_DEVICE']) - - sync_skale_node() - ensure_btrfs_kernel_module_autoloaded() - - if env.get('SKIP_DOCKER_CONFIG') != 'True': - configure_docker() - - enable_monitoring = str_to_bool(env.get('MONITORING_CONTAINERS', 'False')) - configure_nftables(enable_monitoring=enable_monitoring) - - generate_nginx_config() - prepare_block_device(env['BLOCK_DEVICE'], force=env['ENFORCE_BTRFS'] == 'True') - - prepare_host(env_filepath, env['ENV_TYPE']) - - meta_manager = FairCliMetaManager() - current_stream = meta_manager.get_meta_info().config_stream - skip_cleanup = env.get('SKIP_DOCKER_CLEANUP') == 'True' - if not skip_cleanup and current_stream != env['NODE_VERSION']: - logger.info( - 'Stream version was changed from %s to %s', - current_stream, - env['NODE_VERSION'], - ) - docker_cleanup() - - meta_manager.update_meta( - VERSION, - env['NODE_VERSION'], - distro.id(), - distro.version(), - ) - update_images(env=env, node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE) - compose_up(env=env, node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE, is_fair_boot=True) - return True - - @checked_host def init(env_filepath: str, env: dict, node_mode: NodeMode) -> None: sync_skale_node() @@ -236,39 +194,6 @@ def init(env_filepath: str, env: dict, node_mode: NodeMode) -> None: compose_up(env=env, node_type=NodeType.SKALE, node_mode=node_mode) -@checked_host -def init_fair_boot(env_filepath: str, env: dict, node_mode: NodeMode = NodeMode.ACTIVE) -> None: - sync_skale_node() - cleanup_volume_artifacts(env['BLOCK_DEVICE']) - - ensure_btrfs_kernel_module_autoloaded() - if env.get('SKIP_DOCKER_CONFIG') != 'True': - configure_docker() - - enable_monitoring = str_to_bool(env.get('MONITORING_CONTAINERS', 'False')) - configure_nftables(enable_monitoring=enable_monitoring) - - prepare_host(env_filepath, env_type=env['ENV_TYPE']) - link_env_file() - mark_active_node() - - configure_filebeat() - configure_flask() - generate_nginx_config() - prepare_block_device(env['BLOCK_DEVICE'], force=env['ENFORCE_BTRFS'] == 'True') - - meta_manager = FairCliMetaManager() - meta_manager.update_meta( - VERSION, - env['NODE_VERSION'], - distro.id(), - distro.version(), - ) - update_images(env=env, node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE) - - compose_up(env=env, node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE, is_fair_boot=True) - - def init_passive( env_filepath: str, env: dict, diff --git a/node_cli/operations/fair.py b/node_cli/operations/fair.py index ca7baef5..f1e62305 100644 --- a/node_cli/operations/fair.py +++ b/node_cli/operations/fair.py @@ -38,7 +38,7 @@ from node_cli.core.nginx import generate_nginx_config from node_cli.core.schains import cleanup_no_lvm_datadir from node_cli.core.static_config import get_fair_chain_name -from node_cli.core.node_options import set_passive_node_options, upsert_node_mode +from node_cli.core.node_options import mark_active_node, set_passive_node_options, upsert_node_mode from node_cli.fair.record.chain_record import ( get_fair_chain_record, migrate_chain_record, @@ -80,6 +80,39 @@ class FairUpdateType(Enum): FROM_BOOT = 'from_boot' +@checked_host +def init_fair_boot(env_filepath: str, env: dict) -> None: + sync_skale_node() + cleanup_volume_artifacts(env['BLOCK_DEVICE']) + + ensure_btrfs_kernel_module_autoloaded() + if env.get('SKIP_DOCKER_CONFIG') != 'True': + configure_docker() + + enable_monitoring = str_to_bool(env.get('MONITORING_CONTAINERS', 'False')) + configure_nftables(enable_monitoring=enable_monitoring) + + prepare_host(env_filepath, env_type=env['ENV_TYPE']) + link_env_file() + mark_active_node() + + configure_filebeat() + configure_flask() + generate_nginx_config() + prepare_block_device(env['BLOCK_DEVICE'], force=env['ENFORCE_BTRFS'] == 'True') + + meta_manager = FairCliMetaManager() + meta_manager.update_meta( + VERSION, + env['NODE_VERSION'], + distro.id(), + distro.version(), + ) + update_images(env=env, node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE) + + compose_up(env=env, node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE, is_fair_boot=True) + + @checked_host def init( env_filepath: str, From 3b0e92dd31c0dbe1355fbf08b3242be0f88950db Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Tue, 13 Jan 2026 12:36:01 +0000 Subject: [PATCH 176/198] Delete unused constant `BASE_CONTAINERS_AMOUNT` --- node_cli/core/node.py | 1 - 1 file changed, 1 deletion(-) diff --git a/node_cli/core/node.py b/node_cli/core/node.py index 725deee3..43df3c3a 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -94,7 +94,6 @@ logger = logging.getLogger(__name__) TEXTS = safe_load_texts() -BASE_CONTAINERS_AMOUNT = 5 BLUEPRINT_NAME = 'node' From 05d1251e88e722e6bece8371ca7c719dbfb8e324 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Tue, 13 Jan 2026 13:03:19 +0000 Subject: [PATCH 177/198] Delete unused parameter `unsafe_ok` --- node_cli/cli/passive_node.py | 2 +- node_cli/core/node.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/node_cli/cli/passive_node.py b/node_cli/cli/passive_node.py index 9c92c767..c1f406da 100644 --- a/node_cli/cli/passive_node.py +++ b/node_cli/cli/passive_node.py @@ -69,7 +69,7 @@ def _init_passive( @click.option('--unsafe', 'unsafe_ok', help='Allow unsafe update', hidden=True, is_flag=True) @click.argument('env_file') @streamed_cmd -def _update_passive(env_file, unsafe_ok): +def _update_passive(env_file): update_passive(env_file) diff --git a/node_cli/core/node.py b/node_cli/core/node.py index 43df3c3a..c1264346 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -206,7 +206,7 @@ def init_passive( @check_inited @check_user -def update_passive(env_filepath: str, unsafe_ok: bool = False) -> None: +def update_passive(env_filepath: str) -> None: logger.info('Node update started') prev_version = CliMetaManager().get_meta_info().version if (__version__ == 'test' or __version__.startswith('2.6')) and prev_version == '2.5.0': From e63a42d9aa86366727e219f48e805a926dd6dff0 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Tue, 13 Jan 2026 13:59:25 +0000 Subject: [PATCH 178/198] Remove redundant parentheses --- node_cli/core/resources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node_cli/core/resources.py b/node_cli/core/resources.py index 68686738..f75a5c1c 100644 --- a/node_cli/core/resources.py +++ b/node_cli/core/resources.py @@ -151,7 +151,7 @@ def get_cpu_alloc(common_config: Dict) -> ResourceAlloc: cpu_proportions = common_config['schain']['cpu'] schain_max_cpu_shares = int(cpu_proportions['skaled'] * MAX_CPU_SHARES) ima_max_cpu_shares = int(cpu_proportions['ima'] * MAX_CPU_SHARES) - return (ResourceAlloc(schain_max_cpu_shares), ResourceAlloc(ima_max_cpu_shares)) + return ResourceAlloc(schain_max_cpu_shares), ResourceAlloc(ima_max_cpu_shares) def verify_disk_size( From 5ba2a8cd111d66289a3754e919ccb6c11fa5d315 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Tue, 13 Jan 2026 15:24:22 +0000 Subject: [PATCH 179/198] Fix `tail` parameter type --- node_cli/utils/docker_utils.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/node_cli/utils/docker_utils.py b/node_cli/utils/docker_utils.py index 098a61e7..96e1c10a 100644 --- a/node_cli/utils/docker_utils.py +++ b/node_cli/utils/docker_utils.py @@ -211,12 +211,11 @@ def remove_schain_container_by_name( def backup_container_logs( container: Container, - head: int = DOCKER_DEFAULT_HEAD_LINES, - tail: int = DOCKER_DEFAULT_TAIL_LINES, + tail: int | str = DOCKER_DEFAULT_TAIL_LINES, ) -> None: logger.info(f'Going to backup container logs: {container.name}') logs_backup_filepath = get_logs_backup_filepath(container) - save_container_logs(container, logs_backup_filepath, tail) + save_container_logs(container, logs_backup_filepath, tail=tail) logger.info(f'Old container logs saved to {logs_backup_filepath}, tail: {tail}') @@ -224,7 +223,7 @@ def save_container_logs( container: Container, log_filepath: str, head: int = DOCKER_DEFAULT_HEAD_LINES, - tail: int = DOCKER_DEFAULT_TAIL_LINES, + tail: int | str = DOCKER_DEFAULT_TAIL_LINES, ) -> None: separator = b'=' * 80 + b'\n' tail_lines = container.logs(tail=tail) From 5debc47cff5ce916394a20be420a66c5b26c8232 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Tue, 13 Jan 2026 15:25:05 +0000 Subject: [PATCH 180/198] Add `node_mode` parameter --- node_cli/core/node.py | 1 + 1 file changed, 1 insertion(+) diff --git a/node_cli/core/node.py b/node_cli/core/node.py index c1264346..2128d43f 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -262,6 +262,7 @@ def compose_node_env( else: user_config = get_validated_user_config( node_type=node_type, + node_mode=node_mode, env_filepath=INIT_ENV_FILEPATH, is_fair_boot=is_fair_boot, skip_user_conf_validation=skip_user_conf_validation, From afc79eea94288fce7226f83ca66b51f3fb8c8cf8 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Tue, 13 Jan 2026 15:25:35 +0000 Subject: [PATCH 181/198] Clean code --- node_cli/core/checks.py | 1 - node_cli/core/resources.py | 5 +---- node_cli/fair/staking.py | 1 - 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/node_cli/core/checks.py b/node_cli/core/checks.py index 9c312476..a4c439c9 100644 --- a/node_cli/core/checks.py +++ b/node_cli/core/checks.py @@ -385,7 +385,6 @@ def docker_compose(self) -> CheckResult: ) output = v_cmd_result.stdout.decode('utf-8').rstrip() if v_cmd_result.returncode != 0: - info = f'Checking docker compose version failed with: {output}' return self._failed(name=name, info=output) actual_version = output.split(',')[0].split()[-1].strip() diff --git a/node_cli/core/resources.py b/node_cli/core/resources.py index f75a5c1c..430eee86 100644 --- a/node_cli/core/resources.py +++ b/node_cli/core/resources.py @@ -154,10 +154,7 @@ def get_cpu_alloc(common_config: Dict) -> ResourceAlloc: return ResourceAlloc(schain_max_cpu_shares), ResourceAlloc(ima_max_cpu_shares) -def verify_disk_size( - disk_device: str, - env_configs: dict, -) -> Dict: +def verify_disk_size(disk_device: str, env_configs: dict): disk_size = get_disk_size(disk_device) env_disk_size = env_configs['server']['disk'] check_disk_size(disk_size, env_disk_size) diff --git a/node_cli/fair/staking.py b/node_cli/fair/staking.py index 4b7a4008..95b93adf 100644 --- a/node_cli/fair/staking.py +++ b/node_cli/fair/staking.py @@ -109,7 +109,6 @@ def get_exit_requests(raw: bool = False) -> None: exit_requests = payload.get('exit_requests') if not isinstance(exit_requests, list): error_exit(payload, exit_code=CLIExitCodes.BAD_API_RESPONSE) - return if raw: print(json.dumps(exit_requests, indent=2)) return From 88f6959a6612b95dbcac97e173c2a0656b250d3e Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Tue, 13 Jan 2026 15:47:06 +0000 Subject: [PATCH 182/198] Delete duplicated function `ensure_filestorage_mapping` --- node_cli/operations/docker_lvmpy.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/node_cli/operations/docker_lvmpy.py b/node_cli/operations/docker_lvmpy.py index fb70e480..c21328f8 100644 --- a/node_cli/operations/docker_lvmpy.py +++ b/node_cli/operations/docker_lvmpy.py @@ -35,6 +35,7 @@ SCHAINS_MNT_DIR_REGULAR, VOLUME_GROUP, ) +from node_cli.operations.volume import ensure_filestorage_mapping from lvmpy.src.install import setup as setup_lvmpy logger = logging.getLogger(__name__) @@ -49,11 +50,6 @@ def update_docker_lvmpy_env(env): return env -def ensure_filestorage_mapping(mapping_dir=FILESTORAGE_MAPPING): - if not os.path.isdir(FILESTORAGE_MAPPING): - os.makedirs(FILESTORAGE_MAPPING) - - def sync_docker_lvmpy_repo(env): if os.path.isdir(DOCKER_LVMPY_PATH): shutil.rmtree(DOCKER_LVMPY_PATH) From 795f1a363037a0701a17710e79e024c47f3a7841 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Tue, 13 Jan 2026 16:38:56 +0000 Subject: [PATCH 183/198] Clean code --- node_cli/core/nftables.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/node_cli/core/nftables.py b/node_cli/core/nftables.py index 042ff2e1..e0059fff 100644 --- a/node_cli/core/nftables.py +++ b/node_cli/core/nftables.py @@ -529,7 +529,6 @@ def add_loopback_rule(self, chain) -> None: def get_base_ruleset(self) -> str: self.nft.set_json_output(False) - output = '' try: cmd = f'list chain {self.family} {self.table} {self.chain}' rc, output, error = self.nft.cmd(cmd) @@ -539,7 +538,6 @@ def get_base_ruleset(self) -> str: finally: self.nft.set_json_output(True) - return output def setup_firewall(self, enable_monitoring: bool = False) -> None: """Setup firewall rules.""" From 6ba32375bad95775dc537e01b85db323e85601ac Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Tue, 13 Jan 2026 19:15:34 +0000 Subject: [PATCH 184/198] Fix `get_containers` and `create_logs_dump` functions --- node_cli/core/logs.py | 2 +- node_cli/utils/docker_utils.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/node_cli/core/logs.py b/node_cli/core/logs.py index 67472e96..1563ddfc 100644 --- a/node_cli/core/logs.py +++ b/node_cli/core/logs.py @@ -42,7 +42,7 @@ def create_logs_dump(path, filter_container=None): if filter_container: containers = get_containers(filter_container) else: - containers = get_containers('skale') + containers = get_containers('sk_*') for container in containers: log_filepath = os.path.join(containers_logs_path, f'{container.name}.log') diff --git a/node_cli/utils/docker_utils.py b/node_cli/utils/docker_utils.py index 96e1c10a..e30092a8 100644 --- a/node_cli/utils/docker_utils.py +++ b/node_cli/utils/docker_utils.py @@ -118,8 +118,7 @@ def get_sanitized_container_name(container_info: dict) -> str: def get_containers(container_name_filter=None, _all=True) -> list: - return docker_client().containers.list(all=_all) - + return docker_client().containers.list(all=_all, filters={'name': container_name_filter}) def get_all_schain_containers(_all=True) -> list: return docker_client().containers.list(all=_all, filters={'name': 'sk_skaled_*'}) From 475a2117ae2d47ea65ed49dd943ecff2d5d3d5a2 Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Tue, 13 Jan 2026 19:48:23 +0000 Subject: [PATCH 185/198] Delete unused optional parameter `unsafe_ok` --- node_cli/cli/passive_node.py | 1 - 1 file changed, 1 deletion(-) diff --git a/node_cli/cli/passive_node.py b/node_cli/cli/passive_node.py index c1f406da..f86ba99b 100644 --- a/node_cli/cli/passive_node.py +++ b/node_cli/cli/passive_node.py @@ -66,7 +66,6 @@ def _init_passive( expose_value=False, prompt='Are you sure you want to update SKALE node software?', ) -@click.option('--unsafe', 'unsafe_ok', help='Allow unsafe update', hidden=True, is_flag=True) @click.argument('env_file') @streamed_cmd def _update_passive(env_file): From b8dca6562a119d889c6351a235f8720605f9cc7e Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Tue, 13 Jan 2026 19:50:29 +0000 Subject: [PATCH 186/198] Fix `get_containers` function --- node_cli/utils/docker_utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/node_cli/utils/docker_utils.py b/node_cli/utils/docker_utils.py index e30092a8..695a1253 100644 --- a/node_cli/utils/docker_utils.py +++ b/node_cli/utils/docker_utils.py @@ -118,7 +118,11 @@ def get_sanitized_container_name(container_info: dict) -> str: def get_containers(container_name_filter=None, _all=True) -> list: - return docker_client().containers.list(all=_all, filters={'name': container_name_filter}) + filters = {} + if container_name_filter: + filters['name'] = container_name_filter + return docker_client().containers.list(all=_all, filters=filters) + def get_all_schain_containers(_all=True) -> list: return docker_client().containers.list(all=_all, filters={'name': 'sk_skaled_*'}) From d00ec041f5da8dfb7f0337044b7a09c670b3533d Mon Sep 17 00:00:00 2001 From: Alex Sheverdin Date: Sun, 25 Jan 2026 21:48:34 +0000 Subject: [PATCH 187/198] Add `bite` variable --- node_cli/configs/user.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/node_cli/configs/user.py b/node_cli/configs/user.py index 59ecda77..a4f52c2c 100644 --- a/node_cli/configs/user.py +++ b/node_cli/configs/user.py @@ -127,6 +127,7 @@ class SkaleUserConfig(BaseUserConfig): disable_dry_run: str = '' default_gas_limit: str = '' default_gas_price_wei: str = '' + bite: str = '' @dataclass @@ -136,6 +137,7 @@ class PassiveSkaleUserConfig(BaseUserConfig): schain_name: str = '' ima_contracts: str = '' enforce_btrfs: str = '' + bite: str = '' def get_validated_user_config( From c190221f392c52342297b0d20c6145c51e9700f7 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Fri, 13 Feb 2026 18:39:00 +0000 Subject: [PATCH 188/198] update settings structure - wip --- .github/copilot-instructions.md | 9 ++-- .gitignore | 1 + node_cli/cli/node.py | 18 ++++--- node_cli/configs/__init__.py | 11 ++-- node_cli/configs/{user.py => _user.py} | 6 +-- node_cli/core/host.py | 27 +--------- node_cli/core/node.py | 72 +++++--------------------- node_cli/operations/base.py | 10 ++-- node_cli/operations/common.py | 13 ----- node_cli/operations/fair.py | 9 ++-- node_cli/utils/docker_utils.py | 3 -- node_cli/utils/helper.py | 11 ++-- node_cli/utils/node_type.py | 6 +-- node_cli/utils/print_formatters.py | 3 -- node_cli/utils/settings.py | 47 +++++++++++++++++ pyproject.toml | 29 ++++++----- scripts/export_env.sh | 8 +++ scripts/run_tests.sh | 10 +--- tests/cli/node_test.py | 4 +- tests/conftest.py | 2 +- tests/core/core_node_test.py | 11 +--- tests/utils/settings_test.py | 20 +++++++ 22 files changed, 155 insertions(+), 175 deletions(-) rename node_cli/configs/{user.py => _user.py} (98%) create mode 100644 node_cli/utils/settings.py create mode 100644 scripts/export_env.sh create mode 100644 tests/utils/settings_test.py diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index a5c8ebdf..5df55bfe 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -11,7 +11,10 @@ - no redundant code - move repeated logic into helper functions - use type hints to specify the expected types of function arguments and return values -- check `ruff.toml` for formatting rules -- always lint changes using `ruff check` +- check `pyproject.toml` for formatting rules +- always lint changes using `uv run ruff check` - tests should be placed in `tests/` directory, follow the existing structure and code style -- to run a test always use `bash scripts/run_tests.sh tests/path_to_test.py -k [TEST_NAME]` command \ No newline at end of file +- always use `uv` to run all commands in the repo (e.g., `uv run ruff`, `uv run pytest`, etc.) +- for running tests, export environment variables in the terminal before running the tests: `. ./scripts/export_env.sh` + +- additional external context is located in context directory \ No newline at end of file diff --git a/.gitignore b/.gitignore index 87a0d8f8..65232943 100644 --- a/.gitignore +++ b/.gitignore @@ -125,3 +125,4 @@ tests/.skale/node_data/node_options.json tests/.skale/config/nginx.conf.j2 .zed +uv.lock \ No newline at end of file diff --git a/node_cli/cli/node.py b/node_cli/cli/node.py index 55304356..f13825d9 100644 --- a/node_cli/cli/node.py +++ b/node_cli/cli/node.py @@ -17,8 +17,11 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +from typing import get_args + import click +from skale.core.types import EnvType from node_cli.cli.info import TYPE from node_cli.core.node import ( cleanup as cleanup_skale, @@ -38,7 +41,6 @@ run_checks, ) from node_cli.configs import DEFAULT_NODE_BASE_PORT -from node_cli.configs.user import ALLOWED_ENV_TYPES from node_cli.core.node_options import upsert_node_mode from node_cli.utils.decorators import check_inited from node_cli.utils.helper import abort_if_false, streamed_cmd, IP_TYPE @@ -85,10 +87,10 @@ def register_node(name, ip, port, domain): @node.command('init', help='Initialize SKALE node') -@click.argument('env_file') +@click.argument('config_file') @streamed_cmd -def init_node(env_file): - init(env_filepath=env_file, node_type=TYPE) +def init_node(config_file): + init(config_file=config_file, node_type=TYPE) @node.command('update', help='Update node from .env file') @@ -101,12 +103,12 @@ def init_node(env_file): ) @click.option('--pull-config', 'pull_config_for_schain', hidden=True, type=str) @click.option('--unsafe', 'unsafe_ok', help='Allow unsafe update', hidden=True, is_flag=True) -@click.argument('env_file') +@click.argument('config_file') @streamed_cmd -def update_node(env_file, pull_config_for_schain, unsafe_ok): +def update_node(config_file, pull_config_for_schain, unsafe_ok): update( node_mode=NodeMode.ACTIVE, - env_filepath=env_file, + env_filepath=config_file, pull_config_for_schain=pull_config_for_schain, node_type=TYPE, unsafe_ok=unsafe_ok, @@ -227,7 +229,7 @@ def _set_domain_name(domain): @click.option( '--network', '-n', - type=click.Choice(ALLOWED_ENV_TYPES), + type=click.Choice(get_args(EnvType)), default='mainnet', help='Network to check', ) diff --git a/node_cli/configs/__init__.py b/node_cli/configs/__init__.py index 35d776d8..f9fc0aa5 100644 --- a/node_cli/configs/__init__.py +++ b/node_cli/configs/__init__.py @@ -19,6 +19,7 @@ import os import sys +from pathlib import Path from node_cli.utils.global_config import read_g_config @@ -43,6 +44,11 @@ NODE_DATA_PATH = os.path.join(SKALE_DIR, 'node_data') SCHAIN_NODE_DATA_PATH = os.path.join(NODE_DATA_PATH, 'schains') NODE_CLI_STATUS_FILENAME = 'node_cli.status' + +SETTINGS_DIR = Path(NODE_DATA_PATH) / 'settings' +NODE_SETTINGS_PATH = SETTINGS_DIR / 'node.toml' +INTERNAL_SETTINGS_PATH = SETTINGS_DIR / 'internal.toml' + NODE_CONFIG_PATH = os.path.join(NODE_DATA_PATH, 'node_config.json') CONTAINER_CONFIG_PATH = os.path.join(SKALE_DIR, 'config') CONTAINER_CONFIG_TMP_PATH = os.path.join(SKALE_TMP_DIR, 'config') @@ -52,8 +58,6 @@ INIT_ENV_FILEPATH = os.path.join(SKALE_DIR, '.env') SKALE_RUN_DIR = '/var/run/skale' -SGX_CERTIFICATES_DIR_NAME = 'sgx_certs' - COMPOSE_PATH = os.path.join(CONTAINER_CONFIG_PATH, 'docker-compose.yml') FAIR_COMPOSE_PATH = os.path.join(CONTAINER_CONFIG_PATH, 'docker-compose-fair.yml') STATIC_PARAMS_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, 'static_params.yaml') @@ -95,9 +99,6 @@ IPTABLES_RULES_STATE_FILEPATH = os.path.join(IPTABLES_DIR, 'rules.v4') DEFAULT_SSH_PORT = 22 -FLASK_SECRET_KEY_FILENAME = 'flask_db_key.txt' -FLASK_SECRET_KEY_FILE = os.path.join(NODE_DATA_PATH, FLASK_SECRET_KEY_FILENAME) - DOCKER_CONFIG_FILEPATH = '/etc/docker/daemon.json' HIDE_STREAM_LOG = os.getenv('HIDE_STREAM_LOG') diff --git a/node_cli/configs/user.py b/node_cli/configs/_user.py similarity index 98% rename from node_cli/configs/user.py rename to node_cli/configs/_user.py index a4f52c2c..6ab7b6f5 100644 --- a/node_cli/configs/user.py +++ b/node_cli/configs/_user.py @@ -52,6 +52,7 @@ class ValidationResult(NamedTuple): class BaseUserConfig(ABC): node_version: str env_type: str + endpoint: str filebeat_host: str block_device: str @@ -89,7 +90,6 @@ def validate_params(cls, params: Dict) -> ValidationResult: @dataclass class FairUserConfig(BaseUserConfig): fair_contracts: str - boot_endpoint: str sgx_server_url: str enforce_btrfs: str = '' telegraf: str = '' @@ -99,13 +99,11 @@ class FairUserConfig(BaseUserConfig): @dataclass class PassiveFairUserConfig(BaseUserConfig): fair_contracts: str - boot_endpoint: str enforce_btrfs: str = '' @dataclass class FairBootUserConfig(BaseUserConfig): - endpoint: str manager_contracts: str ima_contracts: str sgx_server_url: str @@ -114,7 +112,6 @@ class FairBootUserConfig(BaseUserConfig): @dataclass class SkaleUserConfig(BaseUserConfig): - endpoint: str manager_contracts: str ima_contracts: str docker_lvmpy_version: str @@ -132,7 +129,6 @@ class SkaleUserConfig(BaseUserConfig): @dataclass class PassiveSkaleUserConfig(BaseUserConfig): - endpoint: str manager_contracts: str schain_name: str = '' ima_contracts: str = '' diff --git a/node_cli/core/host.py b/node_cli/core/host.py index da4be640..920b651e 100644 --- a/node_cli/core/host.py +++ b/node_cli/core/host.py @@ -39,6 +39,7 @@ SGX_CERTS_PATH, REPORTS_PATH, REDIS_DATA_PATH, + SETTINGS_DIR, SCHAINS_DATA_PATH, LOG_PATH, REMOVED_CONTAINERS_FOLDER_PATH, @@ -50,7 +51,6 @@ NGINX_CONFIG_FILEPATH, ) from node_cli.configs.cli_logger import LOG_DATA_PATH -from node_cli.configs.user import SKALE_DIR_ENV_FILEPATH, CONFIGS_ENV_FILEPATH from node_cli.core.nftables import NFTablesManager from node_cli.utils.helper import safe_mkdir @@ -73,20 +73,6 @@ def fix_url(url): return False -def get_flask_secret_key() -> str: - secret_key_filepath = os.path.join(NODE_DATA_PATH, 'flask_db_key.txt') - - if not os.path.exists(secret_key_filepath): - error_exit(f'Flask secret key file not found at {secret_key_filepath}') - - try: - with open(secret_key_filepath, 'r') as key_file: - secret_key = key_file.read().strip() - return secret_key - except (IOError, OSError) as e: - error_exit(f'Failed to read Flask secret key: {e}') - - def prepare_host(env_filepath: str, env_type: str, allocation: bool = False) -> None: if not env_filepath or not env_type: error_exit('Missing required parameters for host initialization') @@ -121,6 +107,7 @@ def make_dirs(): LOG_PATH, REPORTS_PATH, REDIS_DATA_PATH, + SETTINGS_DIR, SKALE_RUN_DIR, SKALE_STATE_DIR, SKALE_TMP_DIR, @@ -128,16 +115,6 @@ def make_dirs(): safe_mkdir(dir_path) -def save_env_params(env_filepath: str) -> None: - copyfile(env_filepath, SKALE_DIR_ENV_FILEPATH) - - -def link_env_file(): - if not (os.path.islink(CONFIGS_ENV_FILEPATH) or os.path.isfile(CONFIGS_ENV_FILEPATH)): - logger.info('Creating symlink %s → %s', SKALE_DIR_ENV_FILEPATH, CONFIGS_ENV_FILEPATH) - os.symlink(SKALE_DIR_ENV_FILEPATH, CONFIGS_ENV_FILEPATH) - - def init_logs_dir(): safe_mkdir(LOG_DATA_PATH) safe_mkdir(REMOVED_CONTAINERS_FOLDER_PATH) diff --git a/node_cli/core/node.py b/node_cli/core/node.py index 2128d43f..191d2077 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -43,9 +43,8 @@ TM_INIT_TIMEOUT, ) from node_cli.configs.cli_logger import LOG_DATA_PATH as CLI_LOG_DATA_PATH -from node_cli.configs.user import SKALE_DIR_ENV_FILEPATH, get_validated_user_config from node_cli.core.checks import run_checks as run_host_checks -from node_cli.core.host import get_flask_secret_key, is_node_inited, save_env_params +from node_cli.core.host import is_node_inited from node_cli.core.resources import update_resource_allocation from node_cli.core.node_options import ( active_fair, @@ -152,11 +151,11 @@ def register_node(name, p2p_ip, public_ip, port, domain_name): @check_not_inited -def init(env_filepath: str, node_type: NodeType) -> None: +def init(config_file: str, node_type: NodeType) -> None: node_mode = NodeMode.ACTIVE - env = compose_node_env(env_filepath=env_filepath, node_type=node_type, node_mode=node_mode) + env = compose_node_env(node_type=node_type, node_mode=node_mode) - init_op(env_filepath=env_filepath, env=env, node_mode=node_mode) + init_op(env_filepath=config_file, env=env, node_mode=node_mode) logger.info('Waiting for containers initialization') time.sleep(TM_INIT_TIMEOUT) if not is_base_containers_alive(node_type=node_type, node_mode=node_mode): @@ -169,7 +168,7 @@ def init(env_filepath: str, node_type: NodeType) -> None: @check_not_inited def restore(backup_path, env_filepath, node_type: NodeType, no_snapshot=False, config_only=False): node_mode = NodeMode.ACTIVE - env = compose_node_env(env_filepath=env_filepath, node_type=node_type, node_mode=node_mode) + env = compose_node_env(node_type=node_type, node_mode=node_mode) if env is None: return save_env_params(env_filepath) @@ -211,7 +210,7 @@ def update_passive(env_filepath: str) -> None: prev_version = CliMetaManager().get_meta_info().version if (__version__ == 'test' or __version__.startswith('2.6')) and prev_version == '2.5.0': migrate_2_6() - env = compose_node_env(env_filepath, node_type=NodeType.SKALE, node_mode=NodeMode.PASSIVE) + env = compose_node_env(node_type=NodeType.SKALE, node_mode=NodeMode.PASSIVE) update_ok = update_passive_op(env_filepath, env) if update_ok: logger.info('Waiting for containers initialization') @@ -227,69 +226,22 @@ def update_passive(env_filepath: str) -> None: @check_user def cleanup(node_mode: NodeMode, prune: bool = False) -> None: node_mode = upsert_node_mode(node_mode=node_mode) - env = compose_node_env( - SKALE_DIR_ENV_FILEPATH, - save=False, - node_type=NodeType.SKALE, - node_mode=node_mode, - skip_user_conf_validation=True, - ) + env = compose_node_env(NodeType.SKALE, node_mode) cleanup_skale_op(node_mode=node_mode, env=env, prune=prune) logger.info('SKALE node was cleaned up, all containers and data removed') -def compose_node_env( - env_filepath: str, - node_type: NodeType, - node_mode: NodeMode, - inited_node: bool = False, - sync_schains: Optional[bool] = None, - pull_config_for_schain: Optional[str] = None, - save: bool = True, - is_fair_boot: bool = False, - skip_user_conf_validation: bool = False, -) -> dict[str, str]: - if env_filepath is not None: - user_config = get_validated_user_config( - node_type=node_type, - node_mode=node_mode, - env_filepath=env_filepath, - is_fair_boot=is_fair_boot, - skip_user_conf_validation=skip_user_conf_validation, - ) - if save: - save_env_params(env_filepath) - else: - user_config = get_validated_user_config( - node_type=node_type, - node_mode=node_mode, - env_filepath=INIT_ENV_FILEPATH, - is_fair_boot=is_fair_boot, - skip_user_conf_validation=skip_user_conf_validation, - ) - +def compose_node_env(node_type: NodeType, node_mode: NodeMode) -> dict[str, str]: if node_mode == NodeMode.PASSIVE or node_type == NodeType.FAIR: mnt_dir = SCHAINS_MNT_DIR_SINGLE_CHAIN else: mnt_dir = SCHAINS_MNT_DIR_REGULAR - env = { 'SKALE_DIR': SKALE_DIR, 'SCHAINS_MNT_DIR': mnt_dir, 'FILESTORAGE_MAPPING': FILESTORAGE_MAPPING, 'SKALE_LIB_PATH': SKALE_STATE_DIR, - **user_config.to_env(), } - - if inited_node and not node_mode == NodeMode.PASSIVE: - env['FLASK_SECRET_KEY'] = get_flask_secret_key() - - if sync_schains and not node_mode == NodeMode.PASSIVE: - env['BACKUP_RUN'] = 'True' - - if pull_config_for_schain: - env['PULL_CONFIG_FOR_SCHAIN'] = pull_config_for_schain - return {k: v for k, v in env.items() if v != ''} @@ -313,10 +265,10 @@ def update( migrate_2_6() logger.info('Node update started') env = compose_node_env( - env_filepath, - inited_node=True, - sync_schains=False, - pull_config_for_schain=pull_config_for_schain, + # env_filepath, + # inited_node=True, + # sync_schains=False, + # pull_config_for_schain=pull_config_for_schain, node_type=node_type, node_mode=node_mode, ) diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index 1a64d585..713a6775 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -53,7 +53,7 @@ cleanup_no_lvm_datadir, update_node_cli_schain_status, ) -from node_cli.operations.common import configure_filebeat, configure_flask, unpack_backup_archive +from node_cli.operations.common import configure_filebeat, unpack_backup_archive from node_cli.operations.config_repo import ( download_skale_node, sync_skale_node, @@ -76,6 +76,7 @@ from node_cli.utils.meta import CliMetaManager, FairCliMetaManager from node_cli.utils.node_type import NodeMode, NodeType from node_cli.utils.print_formatters import print_failed_requirements_checks +from node_cli.utils.settings import save_settings logger = logging.getLogger(__name__) @@ -134,6 +135,7 @@ def update(env_filepath: str, env: Dict, node_mode: NodeMode) -> bool: generate_nginx_config() prepare_host(env_filepath, env['ENV_TYPE'], allocation=True) + save_settings(node_type=NodeType.SKALE, node_mode=node_mode) init_shared_space_volume(env['ENV_TYPE']) meta_manager = CliMetaManager() @@ -170,12 +172,12 @@ def init(env_filepath: str, env: dict, node_mode: NodeMode) -> None: configure_nftables(enable_monitoring=enable_monitoring) prepare_host(env_filepath, env_type=env['ENV_TYPE']) + save_settings(node_type=NodeType.SKALE, node_mode=node_mode) link_env_file() mark_active_node() configure_filebeat() - configure_flask() generate_nginx_config() lvmpy_install(env) @@ -222,7 +224,7 @@ def init_passive( NodeMode.PASSIVE, env['ENV_TYPE'], CONTAINER_CONFIG_PATH, - check_type=CheckType.PREINSTALL + check_type=CheckType.PREINSTALL, ) if failed_checks: print_failed_requirements_checks(failed_checks) @@ -280,7 +282,7 @@ def update_passive(env_filepath: str, env: Dict) -> bool: NodeMode.PASSIVE, env['ENV_TYPE'], CONTAINER_CONFIG_PATH, - check_type=CheckType.PREINSTALL + check_type=CheckType.PREINSTALL, ) if failed_checks: print_failed_requirements_checks(failed_checks) diff --git a/node_cli/operations/common.py b/node_cli/operations/common.py index 7c876fa8..c595a3b8 100644 --- a/node_cli/operations/common.py +++ b/node_cli/operations/common.py @@ -19,7 +19,6 @@ import logging import os -import secrets import shutil import stat import tarfile @@ -27,7 +26,6 @@ from node_cli.configs import ( FILEBEAT_CONFIG_PATH, - FLASK_SECRET_KEY_FILE, G_CONF_HOME, SRC_FILEBEAT_CONFIG_PATH, ) @@ -43,17 +41,6 @@ def configure_filebeat(): logger.info('Filebeat configured') -def configure_flask(): - if os.path.isfile(FLASK_SECRET_KEY_FILE): - logger.info('Flask secret key already exists') - else: - logger.info('Generating Flask secret key...') - flask_secret_key = secrets.token_urlsafe(16) - with open(FLASK_SECRET_KEY_FILE, 'w') as f: - f.write(flask_secret_key) - logger.info('Flask secret key generated and saved') - - def unpack_backup_archive(backup_path: str) -> None: logger.info('Unpacking backup archive...') with tarfile.open(backup_path) as tar: diff --git a/node_cli/operations/fair.py b/node_cli/operations/fair.py index f1e62305..096d1623 100644 --- a/node_cli/operations/fair.py +++ b/node_cli/operations/fair.py @@ -46,7 +46,7 @@ ) from node_cli.migrations.fair.from_boot import migrate_nftables_from_boot from node_cli.operations.base import checked_host, turn_off -from node_cli.operations.common import configure_filebeat, configure_flask, unpack_backup_archive +from node_cli.operations.common import configure_filebeat, unpack_backup_archive from node_cli.operations.config_repo import ( sync_skale_node, update_images, @@ -70,6 +70,7 @@ from node_cli.utils.meta import FairCliMetaManager from node_cli.utils.print_formatters import print_failed_requirements_checks from node_cli.utils.node_type import NodeMode, NodeType +from node_cli.utils.settings import save_settings logger = logging.getLogger(__name__) @@ -93,11 +94,11 @@ def init_fair_boot(env_filepath: str, env: dict) -> None: configure_nftables(enable_monitoring=enable_monitoring) prepare_host(env_filepath, env_type=env['ENV_TYPE']) + save_settings(node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE) link_env_file() mark_active_node() configure_filebeat() - configure_flask() generate_nginx_config() prepare_block_device(env['BLOCK_DEVICE'], force=env['ENFORCE_BTRFS'] == 'True') @@ -131,10 +132,10 @@ def init( configure_nftables() configure_filebeat() - configure_flask() generate_nginx_config() prepare_host(env_filepath, env_type=env['ENV_TYPE']) + save_settings(node_type=NodeType.FAIR, node_mode=node_mode) link_env_file() prepare_block_device(env['BLOCK_DEVICE'], force=env['ENFORCE_BTRFS'] == 'True') @@ -186,6 +187,7 @@ def update_fair_boot(env_filepath: str, env: dict, node_mode: NodeMode = NodeMod prepare_block_device(env['BLOCK_DEVICE'], force=env['ENFORCE_BTRFS'] == 'True') prepare_host(env_filepath, env['ENV_TYPE']) + save_settings(node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE) meta_manager = FairCliMetaManager() current_stream = meta_manager.get_meta_info().config_stream @@ -231,6 +233,7 @@ def update( generate_nginx_config() prepare_host(env_filepath, env['ENV_TYPE'], allocation=True) + save_settings(node_type=NodeType.FAIR, node_mode=node_mode) meta_manager = FairCliMetaManager() current_stream = meta_manager.get_meta_info().config_stream skip_cleanup = env.get('SKIP_DOCKER_CLEANUP') == 'True' diff --git a/node_cli/utils/docker_utils.py b/node_cli/utils/docker_utils.py index 695a1253..37e3fab9 100644 --- a/node_cli/utils/docker_utils.py +++ b/node_cli/utils/docker_utils.py @@ -338,9 +338,6 @@ def compose_up( run_cmd(cmd=get_up_compose_cmd(node_type=node_type, node_mode=node_mode), env=env) return - if 'SGX_CERTIFICATES_DIR_NAME' not in env: - env['SGX_CERTIFICATES_DIR_NAME'] = SGX_CERTIFICATES_DIR_NAME - if active_fair(node_type, node_mode): logger.info('Running fair base set of containers') if is_fair_boot: diff --git a/node_cli/utils/helper.py b/node_cli/utils/helper.py index dd1a7151..9d96a3ca 100644 --- a/node_cli/utils/helper.py +++ b/node_cli/utils/helper.py @@ -30,6 +30,7 @@ import urllib.parse import urllib.request import uuid +from pathlib import Path from functools import wraps from logging import Formatter, StreamHandler from typing import Any, NoReturn, Optional @@ -85,13 +86,13 @@ def write_json(path: str, content: dict) -> None: json.dump(content, outfile, indent=4) -def save_json(path: str, content: dict) -> None: +def save_json(path: str | Path, content: dict) -> None: tmp_path = get_tmp_path(path) write_json(tmp_path, content) shutil.move(tmp_path, path) -def init_file(path, content=None): +def init_file(path: str | Path, content=None): if not os.path.exists(path): write_json(path, content) @@ -339,7 +340,7 @@ def cleanup_dir_content(folder: str) -> None: shutil.rmtree(file_path) -def safe_mkdir(path: str, print_res: bool = False) -> None: +def safe_mkdir(path: str | Path, print_res: bool = False) -> None: if os.path.exists(path): logger.debug(f'Directory {path} already exists') return @@ -411,8 +412,8 @@ def convert(self, value, param, ctx): IP_TYPE = IpType() -def get_tmp_path(path: str) -> str: - base, ext = os.path.splitext(path) +def get_tmp_path(path: str | Path) -> str: + base, ext = os.path.splitext(str(path)) salt = uuid.uuid4().hex[:5] return base + salt + '.tmp' + ext diff --git a/node_cli/utils/node_type.py b/node_cli/utils/node_type.py index bf3f6d4f..754a1d69 100644 --- a/node_cli/utils/node_type.py +++ b/node_cli/utils/node_type.py @@ -20,9 +20,9 @@ from enum import Enum -class NodeType(Enum): - SKALE = 0 - FAIR = 1 +class NodeType(str, Enum): + SKALE = 'skale' + FAIR = 'fair' class NodeMode(str, Enum): diff --git a/node_cli/utils/print_formatters.py b/node_cli/utils/print_formatters.py index 7edec113..1da07d51 100644 --- a/node_cli/utils/print_formatters.py +++ b/node_cli/utils/print_formatters.py @@ -339,9 +339,6 @@ def format_timestamp(value): return str(value) -1 - - def print_chain_record(record): print( inspect.cleandoc(f""" diff --git a/node_cli/utils/settings.py b/node_cli/utils/settings.py new file mode 100644 index 00000000..c0b3aa76 --- /dev/null +++ b/node_cli/utils/settings.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- +# +# This file is part of node-cli +# +# Copyright (C) 2026 SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from skale.core.settings import ( + SETTINGS_MAP, + write_node_settings_file, + write_internal_settings_file, + InternalSettings, + SkaleSettings, + SkalePassiveSettings, + FairSettings, + FairBaseSettings, +) + +from node_cli.configs import NODE_SETTINGS_PATH, INTERNAL_SETTINGS_PATH + +from node_cli.utils.node_type import NodeMode, NodeType + +InternalSettings.model_config['toml_file'] = INTERNAL_SETTINGS_PATH +SkaleSettings.model_config['toml_file'] = NODE_SETTINGS_PATH +SkalePassiveSettings.model_config['toml_file'] = NODE_SETTINGS_PATH +FairSettings.model_config['toml_file'] = NODE_SETTINGS_PATH +FairBaseSettings.model_config['toml_file'] = NODE_SETTINGS_PATH + + +def save_settings(node_type: NodeType, node_mode: NodeMode) -> None: + write_internal_settings_file(path=INTERNAL_SETTINGS_PATH, data={}) # todof: fix + settings_type = SETTINGS_MAP[(node_type.value, node_mode.value)] + write_node_settings_file( + path=NODE_SETTINGS_PATH, settings_type=settings_type, data={} + ) # todof: fix diff --git a/pyproject.toml b/pyproject.toml index cf813bfa..3f2e25d1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,15 +10,13 @@ readme = "README.md" requires-python = ">=3.13" license = { file = "LICENSE" } keywords = ["skale", "cli"] -authors = [ - { name = "SKALE Labs", email = "support@skalelabs.com" } -] +authors = [{ name = "SKALE Labs", email = "support@skalelabs.com" }] classifiers = [ - "Development Status :: 5 - Production/Stable", - "Intended Audience :: Developers", - "License :: OSI Approved :: GNU Affero General Public License v3", - "Natural Language :: English", - "Programming Language :: Python :: 3.13", + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "License :: OSI Approved :: GNU Affero General Public License v3", + "Natural Language :: English", + "Programming Language :: Python :: 3.13", ] dependencies = [ @@ -40,13 +38,14 @@ dependencies = [ "MarkupSafe==3.0.3", "Flask==3.1.2", "itsdangerous==2.2.0", - "cryptography==46.0.3", + "cryptography==46.0.5", "filelock==3.20.0", "sh==2.2.2", "python-crontab==3.3.0", "requests-mock==1.12.1", - "redis==7.1.0", - "PyInstaller==6.16.0", + "redis==7.1.1", + "PyInstaller==6.18.0", + "skale.py==7.12dev2", ] [project.urls] @@ -78,11 +77,13 @@ target-version = "py313" [tool.ruff.format] quote-style = "single" +[tool.uv] +prerelease = "allow" + + [tool.pytest.ini_options] log_cli = false log_cli_level = "INFO" log_cli_format = "%(asctime)s [%(levelname)8s] %(message)s (%(filename)s:%(lineno)s)" log_cli_date_format = "%Y-%m-%d %H:%M:%S" -filterwarnings = [ - "ignore::DeprecationWarning", -] +filterwarnings = ["ignore::DeprecationWarning"] diff --git a/scripts/export_env.sh b/scripts/export_env.sh new file mode 100644 index 00000000..af30b4ab --- /dev/null +++ b/scripts/export_env.sh @@ -0,0 +1,8 @@ +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +PROJECT_DIR=$(dirname $DIR) + +export LVMPY_LOG_DIR="$PROJECT_DIR/tests/" +export HIDE_STREAM_LOG=true +export TEST_HOME_DIR="$PROJECT_DIR/tests/" +export GLOBAL_SKALE_DIR="$PROJECT_DIR/tests/etc/skale" +export DOTENV_FILEPATH='tests/test-env' \ No newline at end of file diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh index efc72c6c..23592396 100755 --- a/scripts/run_tests.sh +++ b/scripts/run_tests.sh @@ -1,11 +1,3 @@ #!/usr/bin/env bash -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -PROJECT_DIR=$(dirname $DIR) - -LVMPY_LOG_DIR="$PROJECT_DIR/tests/" \ - HIDE_STREAM_LOG=true \ - TEST_HOME_DIR="$PROJECT_DIR/tests/" \ - GLOBAL_SKALE_DIR="$PROJECT_DIR/tests/etc/skale" \ - DOTENV_FILEPATH='tests/test-env' \ - py.test --cov=$PROJECT_DIR/ --ignore=tests/core/nftables_test.py --ignore=tests/core/migration_test.py tests/ $@ +py.test --cov=$PROJECT_DIR/ --ignore=tests/core/nftables_test.py --ignore=tests/core/migration_test.py tests/ $@ diff --git a/tests/cli/node_test.py b/tests/cli/node_test.py index b31cddc0..bb5aa6d0 100644 --- a/tests/cli/node_test.py +++ b/tests/cli/node_test.py @@ -422,7 +422,6 @@ def test_turn_on_maintenance_off(mocked_g_config, regular_user_conf, active_node resp_mock = response_mock(requests.codes.ok, {'status': 'ok', 'payload': None}) with ( mock.patch('subprocess.run', new=subprocess_run_mock), - mock.patch('node_cli.core.node.get_flask_secret_key'), mock.patch('node_cli.core.node.turn_on_op'), mock.patch('node_cli.core.node.is_base_containers_alive'), mock.patch('node_cli.utils.decorators.is_node_inited', return_value=True), @@ -475,6 +474,7 @@ def test_node_version(meta_file_v2): == "{'version': '0.1.1', 'config_stream': 'develop', 'docker_lvmpy_version': '1.1.2'}\n" ) + def test_cleanup_node(mocked_g_config): pathlib.Path(SKALE_DIR).mkdir(parents=True, exist_ok=True) @@ -493,4 +493,4 @@ def test_cleanup_node(mocked_g_config): ): result = run_command(cleanup_node, ['--yes']) assert result.exit_code == 0 - cleanup_mock.assert_called_once_with(node_mode=NodeMode.ACTIVE, prune=False, env={}) \ No newline at end of file + cleanup_mock.assert_called_once_with(node_mode=NodeMode.ACTIVE, prune=False, env={}) diff --git a/tests/conftest.py b/tests/conftest.py index 5434e697..cc5a5fa3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -380,7 +380,7 @@ def fair_user_conf(tmp_path): test_env_path = pathlib.Path(tmp_path / 'test-env') try: test_env = """ - BOOT_ENDPOINT=http://localhost:8545 + ENDPOINT=http://localhost:8545 NODE_VERSION='main' FILEBEAT_HOST=127.0.0.1:3010 SGX_SERVER_URL=http://127.0.0.1 diff --git a/tests/core/core_node_test.py b/tests/core/core_node_test.py index 953e7ca5..0fe113b2 100644 --- a/tests/core/core_node_test.py +++ b/tests/core/core_node_test.py @@ -243,7 +243,6 @@ def test_compose_node_env( inited_node, sync_schains, expected_mnt_dir, - expect_flask_key, expect_backup_run, ): user_config_path = request.getfixturevalue(test_user_conf) @@ -251,7 +250,6 @@ def test_compose_node_env( with ( mock.patch('node_cli.configs.user.validate_alias_or_address'), mock.patch('node_cli.core.node.save_env_params'), - mock.patch('node_cli.core.node.get_flask_secret_key', return_value='mock_secret'), ): result_env = compose_node_env( env_filepath=user_config_path.as_posix(), @@ -264,11 +262,6 @@ def test_compose_node_env( ) assert result_env['SCHAINS_MNT_DIR'] == expected_mnt_dir - assert ( - 'FLASK_SECRET_KEY' in result_env and result_env['FLASK_SECRET_KEY'] is not None - ) == expect_flask_key - if expect_flask_key: - assert result_env['FLASK_SECRET_KEY'] == 'mock_secret' should_have_backup = sync_schains and node_mode != NodeMode.PASSIVE assert ('BACKUP_RUN' in result_env and result_env['BACKUP_RUN'] == 'True') == should_have_backup @@ -358,7 +351,6 @@ def test_update_node(regular_user_conf, mocked_g_config, resource_file, inited_n with ( mock.patch('subprocess.run', new=subprocess_run_mock), mock.patch('node_cli.core.node.update_op'), - mock.patch('node_cli.core.node.get_flask_secret_key'), mock.patch('node_cli.core.node.save_env_params'), mock.patch('node_cli.operations.base.configure_nftables'), mock.patch('node_cli.core.host.prepare_host'), @@ -500,4 +492,5 @@ def test_cleanup_success( skip_user_conf_validation=True, ) mock_cleanup_skale_op.assert_called_once_with( - node_mode=NodeMode.ACTIVE, env=mock_env, prune=False) \ No newline at end of file + node_mode=NodeMode.ACTIVE, env=mock_env, prune=False + ) diff --git a/tests/utils/settings_test.py b/tests/utils/settings_test.py new file mode 100644 index 00000000..1d9deeb1 --- /dev/null +++ b/tests/utils/settings_test.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# +# This file is part of node-cli +# +# Copyright (C) 2026 SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from node_cli.utils.node_type import NodeMode, NodeType From 6e9c17318e3080358d281dbfe7efe7f421cd472b Mon Sep 17 00:00:00 2001 From: Dmytro Date: Mon, 16 Feb 2026 11:09:42 +0000 Subject: [PATCH 189/198] new settings structure --- node_cli/cli/fair_boot.py | 2 +- node_cli/cli/fair_node.py | 30 +-- node_cli/cli/node.py | 4 +- node_cli/cli/passive_fair_node.py | 18 +- node_cli/cli/schains.py | 16 +- node_cli/configs/__init__.py | 4 - node_cli/configs/_user.py | 212 ----------------- node_cli/core/checks.py | 4 +- node_cli/core/host.py | 10 +- node_cli/core/node.py | 101 ++++---- node_cli/core/resources.py | 18 +- node_cli/core/schains.py | 14 +- node_cli/core/static_config.py | 8 +- node_cli/fair/active.py | 35 ++- node_cli/fair/boot.py | 30 +-- node_cli/fair/common.py | 72 ++---- node_cli/fair/record/chain_record.py | 19 +- node_cli/operations/base.py | 254 ++++++++++++--------- node_cli/operations/config_repo.py | 11 +- node_cli/operations/docker_lvmpy.py | 6 +- node_cli/operations/fair.py | 205 ++++++++++------- node_cli/utils/docker_utils.py | 10 +- node_cli/utils/helper.py | 10 - node_cli/utils/settings.py | 54 ++++- tests/cli/fair_cli_test.py | 6 +- tests/cli/node_test.py | 4 +- tests/configs/configs_env_validate_test.py | 72 ------ tests/conftest.py | 6 + tests/core/core_node_test.py | 88 ++----- tests/fair/fair_node_test.py | 13 +- tests/fixtures/__init__.py | 0 tests/fixtures/settings.py | 131 +++++++++++ tests/utils/settings_test.py | 1 - 33 files changed, 683 insertions(+), 785 deletions(-) delete mode 100644 node_cli/configs/_user.py create mode 100644 tests/fixtures/__init__.py create mode 100644 tests/fixtures/settings.py diff --git a/node_cli/cli/fair_boot.py b/node_cli/cli/fair_boot.py index f5dce9b1..d5ff6039 100644 --- a/node_cli/cli/fair_boot.py +++ b/node_cli/cli/fair_boot.py @@ -88,6 +88,6 @@ def signature_boot(validator_id): @streamed_cmd def update_node(env_file, pull_config_for_schain): update( - env_filepath=env_file, + config_file=env_file, pull_config_for_schain=pull_config_for_schain, ) diff --git a/node_cli/cli/fair_node.py b/node_cli/cli/fair_node.py index 56ab9d75..efdf3149 100644 --- a/node_cli/cli/fair_node.py +++ b/node_cli/cli/fair_node.py @@ -57,10 +57,10 @@ def fair_node_info(format): @node.command('init', help='Initialize regular Fair node') -@click.argument('env_filepath') +@click.argument('config_file') @streamed_cmd -def init_node(env_filepath: str): - init_fair(node_mode=NodeMode.ACTIVE, env_filepath=env_filepath) +def init_node(config_file: str): + init_fair(node_mode=NodeMode.ACTIVE, config_file=config_file) @node.command('register', help=TEXTS['fair']['node']['register']['help']) @@ -70,7 +70,7 @@ def register(ip: str) -> None: @node.command('update', help='Update Fair node') -@click.argument('env_filepath') +@click.argument('config_file') @click.option( '--yes', is_flag=True, @@ -88,10 +88,10 @@ def register(ip: str) -> None: is_flag=True, ) @streamed_cmd -def update_node(env_filepath: str, pull_config_for_schain, force_skaled_start: bool): +def update_node(config_file: str, pull_config_for_schain, force_skaled_start: bool): update_fair( node_mode=NodeMode.ACTIVE, - env_filepath=env_filepath, + config_file=config_file, pull_config_for_schain=pull_config_for_schain, force_skaled_start=force_skaled_start, ) @@ -106,7 +106,7 @@ def backup_node(backup_folder_path): @node.command('restore', help='Restore Fair node from a backup file.') @click.argument('backup_path') -@click.argument('env_file') +@click.argument('config_file') @click.option( '--config-only', help='Only restore configuration files in .skale and artifacts', @@ -114,12 +114,12 @@ def backup_node(backup_folder_path): hidden=True, ) @streamed_cmd -def restore_node(backup_path, env_file, config_only): - restore_fair(backup_path, env_file, config_only) +def restore_node(backup_path, config_file, config_only): + restore_fair(backup_path, config_file, config_only) @node.command('migrate', help='Switch from boot to regular Fair node.') -@click.argument('env_filepath') +@click.argument('config_file') @click.option( '--yes', is_flag=True, @@ -128,8 +128,8 @@ def restore_node(backup_path, env_file, config_only): prompt='Are you sure you want to migrate to regular Fair node? The action cannot be undone', ) @streamed_cmd -def migrate_node(env_filepath: str) -> None: - migrate_from_boot(env_filepath=env_filepath) +def migrate_node(config_file: str) -> None: + migrate_from_boot(config_file=config_file) @node.command('repair', help='Toggle fair chain repair mode') @@ -221,7 +221,7 @@ def turn_off_node() -> None: expose_value=False, prompt='Are you sure you want to turn on the node?', ) -@click.argument('env_filepath') +@click.argument('config_file') @streamed_cmd -def turn_on_node(env_filepath: str) -> None: - turn_on_fair(env_file=env_filepath, node_type=TYPE) +def turn_on_node(config_file: str) -> None: + turn_on_fair(env_file=config_file, node_type=TYPE) diff --git a/node_cli/cli/node.py b/node_cli/cli/node.py index f13825d9..cb798a1c 100644 --- a/node_cli/cli/node.py +++ b/node_cli/cli/node.py @@ -108,7 +108,7 @@ def init_node(config_file): def update_node(config_file, pull_config_for_schain, unsafe_ok): update( node_mode=NodeMode.ACTIVE, - env_filepath=config_file, + config_file=config_file, pull_config_for_schain=pull_config_for_schain, node_type=TYPE, unsafe_ok=unsafe_ok, @@ -145,7 +145,7 @@ def backup_node(backup_folder_path): def restore_node(backup_path, env_file, no_snapshot, config_only): restore( backup_path=backup_path, - env_filepath=env_file, + config_file=env_file, no_snapshot=no_snapshot, config_only=config_only, node_type=TYPE, diff --git a/node_cli/cli/passive_fair_node.py b/node_cli/cli/passive_fair_node.py index 83eb1bd1..7bac3581 100644 --- a/node_cli/cli/passive_fair_node.py +++ b/node_cli/cli/passive_fair_node.py @@ -49,7 +49,7 @@ def passive_node(): @passive_node.command('init', help='Initialize a passive Fair node') -@click.argument('env_filepath') +@click.argument('config_file') @click.option('--id', required=True, type=int, help=TEXTS['fair']['node']['setup']['id']) @click.option('--indexer', help=TEXTS['passive_node']['init']['indexer'], is_flag=True) @click.option('--archive', help=TEXTS['passive_node']['init']['archive'], is_flag=True) @@ -61,7 +61,7 @@ def passive_node(): ) @streamed_cmd def init_passive_node( - env_filepath: str, id: int, indexer: bool, archive: bool, snapshot: str | None + config_file: str, id: int, indexer: bool, archive: bool, snapshot: str | None ): if indexer and archive: error_exit('Cannot use both --indexer and --archive options') @@ -69,7 +69,7 @@ def init_passive_node( error_exit('Cannot use any for indexer/archive node') init_fair( node_mode=NodeMode.PASSIVE, - env_filepath=env_filepath, + config_file=config_file, node_id=id, indexer=indexer, archive=archive, @@ -78,7 +78,7 @@ def init_passive_node( @passive_node.command('update', help='Update Fair node') -@click.argument('env_filepath') +@click.argument('config_file') @click.option( '--yes', is_flag=True, @@ -96,10 +96,10 @@ def init_passive_node( is_flag=True, ) @streamed_cmd -def update_node(env_filepath: str, pull_config_for_schain, force_skaled_start: bool): +def update_node(config_file: str, pull_config_for_schain, force_skaled_start: bool): update_fair( node_mode=NodeMode.PASSIVE, - env_filepath=env_filepath, + config_file=config_file, pull_config_for_schain=pull_config_for_schain, force_skaled_start=force_skaled_start, ) @@ -146,7 +146,7 @@ def turn_off_node() -> None: expose_value=False, prompt='Are you sure you want to turn on the node?', ) -@click.argument('env_filepath') +@click.argument('config_file') @streamed_cmd -def turn_on_node(env_filepath: str) -> None: - turn_on_fair(env_file=env_filepath, node_type=TYPE) +def turn_on_node(config_file: str) -> None: + turn_on_fair(env_file=config_file, node_type=TYPE) diff --git a/node_cli/cli/schains.py b/node_cli/cli/schains.py index 803ea754..40f1ab0c 100644 --- a/node_cli/cli/schains.py +++ b/node_cli/cli/schains.py @@ -21,6 +21,8 @@ import click +from skale.core.settings import get_settings + from node_cli.utils.helper import abort_if_false, URL_TYPE from node_cli.core.schains import ( describe, @@ -104,8 +106,12 @@ def info_(schain_name: str, json_format: bool) -> None: @click.argument('schain_name') @click.argument('snapshot_path') @click.option('--schain-type', default='medium') -@click.option('--env-type', default=None) -def restore( - schain_name: str, snapshot_path: str, schain_type: str, env_type: Optional[str] -) -> None: - restore_schain_from_snapshot(schain_name, snapshot_path, node_type=TYPE) +def restore(schain_name: str, snapshot_path: str, schain_type: str) -> None: + settings = get_settings() + restore_schain_from_snapshot( + schain_name, + snapshot_path, + node_type=TYPE, + env_type=settings.env_type, + schain_type=schain_type, + ) diff --git a/node_cli/configs/__init__.py b/node_cli/configs/__init__.py index f9fc0aa5..7ed6afee 100644 --- a/node_cli/configs/__init__.py +++ b/node_cli/configs/__init__.py @@ -55,7 +55,6 @@ CONTRACTS_PATH = os.path.join(SKALE_DIR, 'contracts_info') REPORTS_PATH = os.path.join(SKALE_DIR, 'reports') BACKUP_CONTRACTS_PATH = os.path.join(SKALE_DIR, '.old_contracts_info') -INIT_ENV_FILEPATH = os.path.join(SKALE_DIR, '.env') SKALE_RUN_DIR = '/var/run/skale' COMPOSE_PATH = os.path.join(CONTAINER_CONFIG_PATH, 'docker-compose.yml') @@ -77,9 +76,6 @@ SGX_CERTS_PATH = os.path.join(NODE_DATA_PATH, 'sgx_certs') SCHAINS_DATA_PATH = os.path.join(NODE_DATA_PATH, 'schains') -CURRENT_FILE_LOCATION = os.path.dirname(os.path.realpath(__file__)) -DOTENV_FILEPATH = os.path.join(os.path.dirname(CURRENT_FILE_LOCATION), '.env') - SRC_FILEBEAT_CONFIG_PATH = os.path.join(CONTAINER_CONFIG_PATH, 'filebeat.yml') FILEBEAT_CONFIG_PATH = os.path.join(NODE_DATA_PATH, 'filebeat.yml') diff --git a/node_cli/configs/_user.py b/node_cli/configs/_user.py deleted file mode 100644 index 6ab7b6f5..00000000 --- a/node_cli/configs/_user.py +++ /dev/null @@ -1,212 +0,0 @@ -# -*- coding: utf-8 -*- -# -# This file is part of node-cli -# -# Copyright (C) 2019-Present SKALE Labs -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -import inspect -import os -from abc import ABC -from dataclasses import dataclass -from typing import Dict, NamedTuple - -from dotenv.main import DotEnv - -from node_cli.configs import CONTAINER_CONFIG_PATH, SKALE_DIR -from node_cli.configs.alias_address_validation import ContractType, validate_alias_or_address -from node_cli.utils.helper import error_exit -from node_cli.utils.node_type import NodeMode, NodeType -from node_cli.core.node_options import ( - active_fair, - active_skale, - passive_skale, - passive_fair, -) - -SKALE_DIR_ENV_FILEPATH = os.path.join(SKALE_DIR, '.env') -CONFIGS_ENV_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, '.env') - -ALLOWED_ENV_TYPES = ['mainnet', 'testnet', 'qanet', 'devnet'] - - -class ValidationResult(NamedTuple): - result: bool - missing: set - extra: set - - -@dataclass(kw_only=True) -class BaseUserConfig(ABC): - node_version: str - env_type: str - endpoint: str - filebeat_host: str - block_device: str - - container_configs_dir: str = '' - skip_docker_config: str = '' - skip_docker_cleanup: str = '' - - def to_env(self) -> Dict[str, str]: - result = {} - for field_name, field_value in self.__dict__.items(): - upper_key = field_name.upper() - result[upper_key] = str(field_value) if field_value is not None else '' - return result - - @classmethod - def validate_params(cls, params: Dict) -> ValidationResult: - parameters = inspect.signature(cls.__init__).parameters - missing = [] - keys = params.keys() - expected_keys = { - name.upper() - for name, value in parameters.items() - if name != 'self' and value.default == inspect._empty - } - optional_keys = { - name.upper() - for name, value in parameters.items() - if name != 'self' and value.default != inspect._empty - } - missing = expected_keys - keys - extra = keys - expected_keys - optional_keys - return ValidationResult(missing == set() and extra == set(), missing, extra) - - -@dataclass -class FairUserConfig(BaseUserConfig): - fair_contracts: str - sgx_server_url: str - enforce_btrfs: str = '' - telegraf: str = '' - influx_url: str = '' - - -@dataclass -class PassiveFairUserConfig(BaseUserConfig): - fair_contracts: str - enforce_btrfs: str = '' - - -@dataclass -class FairBootUserConfig(BaseUserConfig): - manager_contracts: str - ima_contracts: str - sgx_server_url: str - enforce_btrfs: str = '' - - -@dataclass -class SkaleUserConfig(BaseUserConfig): - manager_contracts: str - ima_contracts: str - docker_lvmpy_version: str - sgx_server_url: str - monitoring_containers: str = '' - telegraf: str = '' - influx_url: str = '' - tg_api_key: str = '' - tg_chat_id: str = '' - disable_dry_run: str = '' - default_gas_limit: str = '' - default_gas_price_wei: str = '' - bite: str = '' - - -@dataclass -class PassiveSkaleUserConfig(BaseUserConfig): - manager_contracts: str - schain_name: str = '' - ima_contracts: str = '' - enforce_btrfs: str = '' - bite: str = '' - - -def get_validated_user_config( - node_type: NodeType, - node_mode: NodeMode, - env_filepath: str = SKALE_DIR_ENV_FILEPATH, - is_fair_boot: bool = False, - skip_user_conf_validation: bool = False, -) -> BaseUserConfig: - params = parse_env_file(env_filepath) - user_config_class = get_user_config_class( - node_type=node_type, - node_mode=node_mode, - is_fair_boot=is_fair_boot, - ) - _, missing_params, extra_params = user_config_class.validate_params(params) - - if len(missing_params) > 0: - error_exit(f'Missing required parameters: {missing_params}') - - if len(extra_params) > 0: - error_exit(f'Extra parameters: {extra_params}') - - params = to_lower_keys(params) - user_config = user_config_class(**params) - if not skip_user_conf_validation: - validate_user_config(user_config) - - return user_config - - -def validate_user_config(user_config: BaseUserConfig) -> None: - validate_env_type(env_type=user_config.env_type) - - if not isinstance(user_config, FairUserConfig) and not isinstance( - user_config, PassiveFairUserConfig - ): - validate_alias_or_address( - user_config.manager_contracts, ContractType.MANAGER, user_config.endpoint - ) - - if isinstance(user_config, (SkaleUserConfig, FairBootUserConfig)): - validate_alias_or_address(user_config.ima_contracts, ContractType.IMA, user_config.endpoint) - - -def to_lower_keys(params: Dict[str, str]) -> Dict[str, str]: - return {key.lower(): value for key, value in params.items()} - - -def parse_env_file(env_filepath: str) -> Dict: - if not os.path.isfile(env_filepath): - error_exit(f'Failed to load environment from {env_filepath}') - return DotEnv(env_filepath).dict() - - -def get_user_config_class( - node_type: NodeType, - node_mode: NodeMode, - is_fair_boot: bool, -) -> type[BaseUserConfig]: - if node_type == NodeType.FAIR and is_fair_boot: - user_config_class = FairBootUserConfig - elif passive_fair(node_type, node_mode): - user_config_class = PassiveFairUserConfig - elif active_fair(node_type, node_mode): - user_config_class = FairUserConfig - elif passive_skale(node_type, node_mode): - user_config_class = PassiveSkaleUserConfig - elif active_skale(node_type, node_mode): - user_config_class = SkaleUserConfig - return user_config_class - - -def validate_env_type(env_type: str) -> None: - if env_type not in ALLOWED_ENV_TYPES: - error_exit(f'Allowed ENV_TYPE values are {ALLOWED_ENV_TYPES}. Actual: "{env_type}"') diff --git a/node_cli/core/checks.py b/node_cli/core/checks.py index a4c439c9..e7677847 100644 --- a/node_cli/core/checks.py +++ b/node_cli/core/checks.py @@ -48,6 +48,8 @@ from debian import debian_support from packaging.version import parse as version_parse +from skale.core.types import EnvType + from node_cli.configs import ( CHECK_REPORT_PATH, CONTAINER_CONFIG_PATH, @@ -471,7 +473,7 @@ def run_checks( disk: str, node_type: NodeType, node_mode: NodeMode, - env_type: str = 'mainnet', + env_type: EnvType = 'mainnet', config_path: str = CONTAINER_CONFIG_PATH, check_type: CheckType = CheckType.ALL, ) -> ResultList: diff --git a/node_cli/core/host.py b/node_cli/core/host.py index 920b651e..3a4b036a 100644 --- a/node_cli/core/host.py +++ b/node_cli/core/host.py @@ -19,9 +19,11 @@ import logging import os -from shutil import copyfile, chown +from shutil import chown from urllib.parse import urlparse +from skale.core.types import EnvType + from node_cli.core.resources import update_resource_allocation from node_cli.utils.helper import error_exit @@ -73,15 +75,11 @@ def fix_url(url): return False -def prepare_host(env_filepath: str, env_type: str, allocation: bool = False) -> None: - if not env_filepath or not env_type: - error_exit('Missing required parameters for host initialization') - +def prepare_host(env_type: EnvType, allocation: bool = False) -> None: try: logger.info('Preparing host started') make_dirs() chown(REDIS_DATA_PATH, user=999, group=1000) - save_env_params(env_filepath) if allocation: update_resource_allocation(env_type) diff --git a/node_cli/core/node.py b/node_cli/core/node.py index 191d2077..8bef033c 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -33,7 +33,6 @@ BACKUP_ARCHIVE_NAME, CONTAINER_CONFIG_PATH, FILESTORAGE_MAPPING, - INIT_ENV_FILEPATH, LOG_PATH, RESTORE_SLEEP_TIMEOUT, SCHAINS_MNT_DIR_REGULAR, @@ -88,6 +87,8 @@ print_node_cmd_error, print_node_info, ) +from node_cli.utils.settings import validate_and_save_node_settings +from skale.core.settings import get_settings from node_cli.utils.texts import safe_load_texts logger = logging.getLogger(__name__) @@ -153,49 +154,62 @@ def register_node(name, p2p_ip, public_ip, port, domain_name): @check_not_inited def init(config_file: str, node_type: NodeType) -> None: node_mode = NodeMode.ACTIVE - env = compose_node_env(node_type=node_type, node_mode=node_mode) + settings = validate_and_save_node_settings(config_file, node_type, node_mode) + compose_env = compose_node_env(node_type=node_type, node_mode=node_mode) - init_op(env_filepath=config_file, env=env, node_mode=node_mode) + init_op(settings=settings, compose_env=compose_env, node_mode=node_mode) logger.info('Waiting for containers initialization') time.sleep(TM_INIT_TIMEOUT) if not is_base_containers_alive(node_type=node_type, node_mode=node_mode): error_exit('Containers are not running', exit_code=CLIExitCodes.OPERATION_EXECUTION_ERROR) logger.info('Generating resource allocation file ...') - update_resource_allocation(env['ENV_TYPE']) + update_resource_allocation(settings.env_type) logger.info('Init procedure finished') @check_not_inited -def restore(backup_path, env_filepath, node_type: NodeType, no_snapshot=False, config_only=False): +def restore( + backup_path: str, + config_file: str, + node_type: NodeType, + no_snapshot: bool = False, + config_only: bool = False, +): node_mode = NodeMode.ACTIVE - env = compose_node_env(node_type=node_type, node_mode=node_mode) - if env is None: - return - save_env_params(env_filepath) - env['SKALE_DIR'] = SKALE_DIR - - if not no_snapshot: - logger.info('Adding BACKUP_RUN to env ...') - env['BACKUP_RUN'] = 'True' # should be str + settings = validate_and_save_node_settings(config_file, node_type, node_mode) + compose_env = compose_node_env(node_type=node_type, node_mode=node_mode) - restored_ok = restore_op(env, backup_path, node_type=node_type, config_only=config_only) + restored_ok = restore_op( + settings=settings, + compose_env=compose_env, + backup_path=backup_path, + node_type=node_type, + config_only=config_only, + backup_run=not no_snapshot, + ) if not restored_ok: error_exit('Restore operation failed', exit_code=CLIExitCodes.OPERATION_EXECUTION_ERROR) time.sleep(RESTORE_SLEEP_TIMEOUT) logger.info('Generating resource allocation file ...') - update_resource_allocation(env['ENV_TYPE']) + update_resource_allocation(settings.env_type) print('Node is restored from backup') @check_not_inited def init_passive( - env_filepath: str, indexer: bool, archive: bool, snapshot: bool, snapshot_from: Optional[str] + config_file: str, indexer: bool, archive: bool, snapshot: bool, snapshot_from: Optional[str] ) -> None: node_mode = NodeMode.PASSIVE - env = compose_node_env(env_filepath, node_type=NodeType.SKALE, node_mode=node_mode) - if env is None: - return - init_passive_op(env_filepath, env, indexer, archive, snapshot, snapshot_from) + settings = validate_and_save_node_settings(config_file, NodeType.SKALE, node_mode) + compose_env = compose_node_env(node_type=NodeType.SKALE, node_mode=node_mode) + init_passive_op( + settings=settings, + compose_env=compose_env, + indexer=indexer, + archive=archive, + snapshot=snapshot, + snapshot_from=snapshot_from, + ) logger.info('Waiting for containers initialization') time.sleep(TM_INIT_TIMEOUT) if not is_base_containers_alive(node_type=NodeType.SKALE, node_mode=node_mode): @@ -205,13 +219,14 @@ def init_passive( @check_inited @check_user -def update_passive(env_filepath: str) -> None: +def update_passive(config_file: str) -> None: logger.info('Node update started') prev_version = CliMetaManager().get_meta_info().version if (__version__ == 'test' or __version__.startswith('2.6')) and prev_version == '2.5.0': migrate_2_6() - env = compose_node_env(node_type=NodeType.SKALE, node_mode=NodeMode.PASSIVE) - update_ok = update_passive_op(env_filepath, env) + settings = validate_and_save_node_settings(config_file, NodeType.SKALE, NodeMode.PASSIVE) + compose_env = compose_node_env(node_type=NodeType.SKALE, node_mode=NodeMode.PASSIVE) + update_ok = update_passive_op(settings=settings, compose_env=compose_env) if update_ok: logger.info('Waiting for containers initialization') time.sleep(TM_INIT_TIMEOUT) @@ -248,7 +263,7 @@ def compose_node_env(node_type: NodeType, node_mode: NodeMode) -> dict[str, str] @check_inited @check_user def update( - env_filepath: str, + config_file: str, pull_config_for_schain: Optional[str], node_type: NodeType, node_mode: NodeMode, @@ -264,15 +279,9 @@ def update( if (__version__ == 'test' or __version__.startswith('2.6')) and prev_version == '2.5.0': migrate_2_6() logger.info('Node update started') - env = compose_node_env( - # env_filepath, - # inited_node=True, - # sync_schains=False, - # pull_config_for_schain=pull_config_for_schain, - node_type=node_type, - node_mode=node_mode, - ) - update_ok = update_op(env_filepath, env, node_mode=node_mode) + settings = validate_and_save_node_settings(config_file, node_type, node_mode) + compose_env = compose_node_env(node_type=node_type, node_mode=node_mode) + update_ok = update_op(settings=settings, compose_env=compose_env, node_mode=node_mode) if update_ok: logger.info('Waiting for containers initialization') time.sleep(TM_INIT_TIMEOUT) @@ -385,24 +394,24 @@ def turn_off(node_type: NodeType, maintenance_on: bool = False, unsafe_ok: bool error_exit(error_msg, exit_code=CLIExitCodes.UNSAFE_UPDATE) if maintenance_on: set_maintenance_mode_on() - env = compose_node_env( - SKALE_DIR_ENV_FILEPATH, save=False, node_type=node_type, node_mode=node_mode - ) - turn_off_op(node_type=node_type, node_mode=node_mode, env=env) + compose_env = compose_node_env(node_type=node_type, node_mode=node_mode) + turn_off_op(compose_env=compose_env, node_type=node_type, node_mode=node_mode) @check_inited @check_user -def turn_on(maintenance_off, sync_schains, env_file, node_type: NodeType) -> None: +def turn_on(maintenance_off: bool, sync_schains: bool, env_file: str, node_type: NodeType) -> None: node_mode = upsert_node_mode() - env = compose_node_env( - env_file, - inited_node=True, - sync_schains=sync_schains, + settings = validate_and_save_node_settings(env_file, node_type, node_mode) + compose_env = compose_node_env(node_type=node_type, node_mode=node_mode) + backup_run = sync_schains and node_mode != NodeMode.PASSIVE + turn_on_op( + settings=settings, + compose_env=compose_env, node_type=node_type, node_mode=node_mode, + backup_run=backup_run, ) - turn_on_op(env=env, node_type=node_type, node_mode=node_mode) logger.info('Waiting for containers initialization') time.sleep(TM_INIT_TIMEOUT) if not is_base_containers_alive(node_type=node_type, node_mode=node_mode): @@ -497,8 +506,8 @@ def run_checks( return if disk is None: - env_config = get_validated_user_config(node_type=node_type, node_mode=node_mode) - disk = env_config.block_device + settings = get_settings() + disk = settings.block_device failed_checks = run_host_checks(disk, node_type, node_mode, network, container_config_path) if not failed_checks: print('Requirements checking successfully finished!') diff --git a/node_cli/core/resources.py b/node_cli/core/resources.py index 430eee86..9a02d0b9 100644 --- a/node_cli/core/resources.py +++ b/node_cli/core/resources.py @@ -24,7 +24,9 @@ import psutil -from node_cli.configs.user import get_validated_user_config +from skale.core.types import EnvType + +from node_cli.utils.settings import validate_and_save_node_settings from node_cli.utils.docker_utils import ensure_volume from node_cli.utils.schain_types import SchainTypes from node_cli.utils.helper import write_json, read_json, run_cmd, safe_load_yml @@ -73,7 +75,7 @@ def get_resource_allocation_info(): return None -def compose_resource_allocation_config(env_type: str, params_by_env_type: Dict = None) -> Dict: +def compose_resource_allocation_config(env_type: EnvType, params_by_env_type: Dict = None) -> Dict: params_by_env_type = params_by_env_type or safe_load_yml(STATIC_PARAMS_FILEPATH) common_config = params_by_env_type['common'] schain_cpu_alloc, ima_cpu_alloc = get_cpu_alloc(common_config) @@ -93,22 +95,20 @@ def compose_resource_allocation_config(env_type: str, params_by_env_type: Dict = def generate_resource_allocation_config( - env_file, + env_file: str, node_type: NodeType, node_mode: NodeMode, - force=False, + force: bool = False, ) -> None: if not force and os.path.isfile(RESOURCE_ALLOCATION_FILEPATH): msg = 'Resource allocation file already exists' logger.debug(msg) print(msg) return - user_config = get_validated_user_config( - node_type=node_type, node_mode=node_mode, env_filepath=env_file - ) + settings = validate_and_save_node_settings(env_file, node_type, node_mode) logger.info('Generating resource allocation file ...') try: - update_resource_allocation(user_config.env_type) + update_resource_allocation(settings.env_type) except Exception as e: logger.exception(e) print("Can't generate resource allocation file, check out CLI logs") @@ -116,7 +116,7 @@ def generate_resource_allocation_config( print(f'Resource allocation file generated: {RESOURCE_ALLOCATION_FILEPATH}') -def update_resource_allocation(env_type: str) -> None: +def update_resource_allocation(env_type: EnvType) -> None: resource_allocation_config = compose_resource_allocation_config(env_type) write_json(RESOURCE_ALLOCATION_FILEPATH, resource_allocation_config) diff --git a/node_cli/core/schains.py b/node_cli/core/schains.py index 64c16031..174430e7 100644 --- a/node_cli/core/schains.py +++ b/node_cli/core/schains.py @@ -26,6 +26,8 @@ from pathlib import Path from typing import Dict, Optional +from skale.core.types import EnvType + from lvmpy.src.core import mount, volume_mountpoint from node_cli.configs import ( ALLOCATION_FILEPATH, @@ -34,7 +36,6 @@ SCHAIN_NODE_DATA_PATH, SCHAINS_MNT_DIR_SINGLE_CHAIN, ) -from node_cli.configs.user import get_validated_user_config from node_cli.utils.docker_utils import ensure_volume, is_volume_exists from node_cli.utils.exit_codes import CLIExitCodes from node_cli.utils.helper import ( @@ -214,12 +215,9 @@ def restore_schain_from_snapshot( schain: str, snapshot_path: str, node_type: NodeType, - env_type: Optional[str] = None, + env_type: EnvType, schain_type: str = 'medium', ) -> None: - if env_type is None: - user_config = get_validated_user_config(node_type=node_type) - env_type = user_config.env_type ensure_schain_volume(schain, schain_type, env_type) block_number = get_block_number_from_path(snapshot_path) if block_number == -1: @@ -242,12 +240,12 @@ def get_schains_by_artifacts() -> str: return '\n'.join(os.listdir(SCHAIN_NODE_DATA_PATH)) -def get_schain_volume_size(schain_type: str, env_type: str) -> int: +def get_schain_volume_size(schain_type: str, env_type: EnvType) -> int: alloc = safe_load_yml(ALLOCATION_FILEPATH) return alloc[env_type]['disk'][schain_type] -def ensure_schain_volume(schain: str, schain_type: str, env_type: str) -> None: +def ensure_schain_volume(schain: str, schain_type: str, env_type: EnvType) -> None: if not is_volume_exists(schain): size = get_schain_volume_size(schain_type, env_type) ensure_volume(schain, size) @@ -313,4 +311,4 @@ def cleanup_lvm_datadir(): cleanup_dir_content('/mnt/') logger.info('Removing LVM volume group "schains"...') run_cmd(['sudo', 'lvremove', '-f', 'schains'], check_code=False) - logger.info('Active node cleanup finished.') \ No newline at end of file + logger.info('Active node cleanup finished.') diff --git a/node_cli/core/static_config.py b/node_cli/core/static_config.py index a93a7a60..c2b90641 100644 --- a/node_cli/core/static_config.py +++ b/node_cli/core/static_config.py @@ -28,10 +28,12 @@ ) from node_cli.utils.node_type import NodeType +from skale.core.types import EnvType + def get_static_params( node_type: NodeType, - env_type: str = 'mainnet', + env_type: EnvType = 'mainnet', config_path: str = CONTAINER_CONFIG_PATH, ) -> dict: if node_type == NodeType.FAIR: @@ -46,7 +48,7 @@ def get_static_params( return ydata['envs'][env_type] -def get_fair_chain_name(env: dict) -> str: +def get_fair_chain_name(env_type: EnvType) -> str: node_type = NodeType.FAIR - params = get_static_params(node_type, env['ENV_TYPE']) + params = get_static_params(node_type, env_type) return params['info']['chain_name'] diff --git a/node_cli/fair/active.py b/node_cli/fair/active.py index 963686fe..2ec42bde 100644 --- a/node_cli/fair/active.py +++ b/node_cli/fair/active.py @@ -22,8 +22,8 @@ import time from typing import cast -from node_cli.configs import DEFAULT_SKALED_BASE_PORT, RESTORE_SLEEP_TIMEOUT, SKALE_DIR -from node_cli.core.host import is_node_inited, save_env_params +from node_cli.configs import DEFAULT_SKALED_BASE_PORT, RESTORE_SLEEP_TIMEOUT +from node_cli.core.host import is_node_inited from node_cli.core.node import compose_node_env, is_base_containers_alive from node_cli.operations import ( FairUpdateType, @@ -35,6 +35,7 @@ from node_cli.utils.helper import error_exit, get_request, post_request from node_cli.utils.node_type import NodeMode, NodeType from node_cli.utils.print_formatters import print_node_cmd_error, print_node_info_fair +from node_cli.utils.settings import validate_and_save_node_settings from node_cli.utils.texts import safe_load_texts logger = logging.getLogger(__name__) @@ -63,19 +64,14 @@ def get_node_info(format): @check_inited @check_user def migrate_from_boot( - env_filepath: str, + config_file: str, ) -> None: logger.info('Migrating from boot to fair node...') - env = compose_node_env( - env_filepath, - inited_node=True, - sync_schains=False, - node_type=NodeType.FAIR, - node_mode=NodeMode.ACTIVE, - ) + settings = validate_and_save_node_settings(config_file, NodeType.FAIR, NodeMode.ACTIVE) + compose_env = compose_node_env(node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE) migrate_ok = update_fair_op( - env_filepath, - env, + settings=settings, + compose_env=compose_env, node_mode=NodeMode.ACTIVE, update_type=FairUpdateType.FROM_BOOT, force_skaled_start=False, @@ -145,16 +141,17 @@ def exit() -> None: @check_not_inited -def restore(backup_path, env_filepath, config_only=False): +def restore(backup_path: str, config_file: str, config_only: bool = False): node_mode = NodeMode.ACTIVE - env = compose_node_env(env_filepath, node_type=NodeType.FAIR, node_mode=node_mode) - if env is None: - return - save_env_params(env_filepath) - env['SKALE_DIR'] = SKALE_DIR + settings = validate_and_save_node_settings(config_file, NodeType.FAIR, node_mode) + compose_env = compose_node_env(node_type=NodeType.FAIR, node_mode=node_mode) restored_ok = restore_fair_op( - node_mode=node_mode, env=env, backup_path=backup_path, config_only=config_only + node_mode=node_mode, + settings=settings, + compose_env=compose_env, + backup_path=backup_path, + config_only=config_only, ) if not restored_ok: error_exit('Restore operation failed', exit_code=CLIExitCodes.OPERATION_EXECUTION_ERROR) diff --git a/node_cli/fair/boot.py b/node_cli/fair/boot.py index a6415f75..d9da98fa 100644 --- a/node_cli/fair/boot.py +++ b/node_cli/fair/boot.py @@ -30,22 +30,19 @@ from node_cli.utils.helper import error_exit from node_cli.utils.node_type import NodeMode, NodeType from node_cli.utils.print_formatters import print_node_cmd_error +from node_cli.utils.settings import validate_and_save_node_settings logger = logging.getLogger(__name__) @check_not_inited -def init(env_filepath: str) -> None: +def init(config_file: str) -> None: node_mode = NodeMode.ACTIVE node_type = NodeType.FAIR - env = compose_node_env( - env_filepath, - node_type=node_type, - node_mode=node_mode, - is_fair_boot=True, - ) + settings = validate_and_save_node_settings(config_file, node_type, node_mode) + compose_env = compose_node_env(node_type=node_type, node_mode=node_mode) - init_fair_boot_op(env_filepath, env, node_mode) + init_fair_boot_op(settings=settings, compose_env=compose_env, node_mode=node_mode) logger.info('Waiting for fair containers initialization') time.sleep(TM_INIT_TIMEOUT) if not is_base_containers_alive(node_type=node_type, node_mode=node_mode, is_fair_boot=True): @@ -55,19 +52,16 @@ def init(env_filepath: str) -> None: @check_inited @check_user -def update(env_filepath: str, pull_config_for_schain: str) -> None: +def update(config_file: str, pull_config_for_schain: str) -> None: logger.info('Fair boot node update started') node_mode = upsert_node_mode(node_mode=NodeMode.ACTIVE) - env = compose_node_env( - env_filepath, - inited_node=True, - sync_schains=False, - pull_config_for_schain=pull_config_for_schain, - node_type=NodeType.FAIR, - node_mode=node_mode, - is_fair_boot=True, + settings = validate_and_save_node_settings(config_file, NodeType.FAIR, node_mode) + compose_env = compose_node_env(node_type=NodeType.FAIR, node_mode=node_mode) + migrate_ok = update_fair_boot_op( + settings=settings, + compose_env=compose_env, + node_mode=NodeMode.ACTIVE, ) - migrate_ok = update_fair_boot_op(env_filepath, env, node_mode=NodeMode.ACTIVE) if migrate_ok: logger.info('Waiting for containers initialization') time.sleep(TM_INIT_TIMEOUT) diff --git a/node_cli/fair/common.py b/node_cli/fair/common.py index 968891d4..3b506c74 100644 --- a/node_cli/fair/common.py +++ b/node_cli/fair/common.py @@ -20,9 +20,7 @@ import logging import time -from node_cli.configs import INIT_TIMEOUT, SKALE_DIR, TM_INIT_TIMEOUT -from node_cli.configs.user import SKALE_DIR_ENV_FILEPATH -from node_cli.core.host import save_env_params +from node_cli.configs import INIT_TIMEOUT, TM_INIT_TIMEOUT from node_cli.core.node import compose_node_env, is_base_containers_alive from node_cli.core.node_options import upsert_node_mode from node_cli.fair.passive import setup_fair_passive @@ -40,7 +38,9 @@ from node_cli.utils.helper import error_exit from node_cli.utils.node_type import NodeMode, NodeType from node_cli.utils.print_formatters import print_node_cmd_error +from node_cli.utils.settings import validate_and_save_node_settings from node_cli.utils.texts import safe_load_texts +from skale.core.settings import get_settings logger = logging.getLogger(__name__) TEXTS = safe_load_texts() @@ -49,21 +49,18 @@ @check_not_inited def init( node_mode: NodeMode, - env_filepath: str, + config_file: str, node_id: int | None = None, indexer: bool = False, archive: bool = False, snapshot: str | None = None, ) -> None: - env = compose_node_env(env_filepath, node_type=NodeType.FAIR, node_mode=node_mode) - if env is None: - return - save_env_params(env_filepath) - env['SKALE_DIR'] = SKALE_DIR + settings = validate_and_save_node_settings(config_file, NodeType.FAIR, node_mode) + compose_env = compose_node_env(node_type=NodeType.FAIR, node_mode=node_mode) init_ok = init_fair_op( - env_filepath, - env, + settings=settings, + compose_env=compose_env, node_mode=node_mode, indexer=indexer, archive=archive, @@ -82,14 +79,8 @@ def init( @check_user def cleanup(node_mode: NodeMode, prune: bool = False) -> None: node_mode = upsert_node_mode(node_mode=node_mode) - env = compose_node_env( - SKALE_DIR_ENV_FILEPATH, - save=False, - node_type=NodeType.FAIR, - node_mode=node_mode, - skip_user_conf_validation=True, - ) - cleanup_fair_op(node_mode=node_mode, env=env, prune=prune) + compose_env = compose_node_env(node_type=NodeType.FAIR, node_mode=node_mode) + cleanup_fair_op(node_mode=node_mode, compose_env=compose_env, prune=prune) logger.info('Fair node was cleaned up, all containers and data removed') @@ -97,29 +88,23 @@ def cleanup(node_mode: NodeMode, prune: bool = False) -> None: @check_user def update( node_mode: NodeMode, - env_filepath: str, + config_file: str, pull_config_for_schain: str | None = None, force_skaled_start: bool = False, ) -> None: logger.info( 'Updating fair node: %s, pull_config_for_schain: %s, force_skaled_start: %s', - env_filepath, + config_file, pull_config_for_schain, force_skaled_start, ) node_mode = upsert_node_mode(node_mode=node_mode) - env = compose_node_env( - env_filepath, - inited_node=True, - sync_schains=False, - node_type=NodeType.FAIR, - node_mode=node_mode, - pull_config_for_schain=pull_config_for_schain, - ) + settings = validate_and_save_node_settings(config_file, NodeType.FAIR, node_mode) + compose_env = compose_node_env(node_type=NodeType.FAIR, node_mode=node_mode) update_ok = update_fair_op( - env_filepath, - env, + settings=settings, + compose_env=compose_env, node_mode=node_mode, update_type=FairUpdateType.REGULAR, force_skaled_start=force_skaled_start, @@ -133,34 +118,25 @@ def update( def repair_chain(snapshot_from: str = 'any') -> None: - node_mode = upsert_node_mode() - env = compose_node_env( - SKALE_DIR_ENV_FILEPATH, save=False, node_type=NodeType.FAIR, node_mode=node_mode - ) - repair_fair_op(env=env, snapshot_from=snapshot_from) + settings = get_settings() + repair_fair_op(env_type=settings.env_type, snapshot_from=snapshot_from) @check_inited @check_user def turn_off(node_type: NodeType) -> None: node_mode = upsert_node_mode() - env = compose_node_env( - SKALE_DIR_ENV_FILEPATH, save=False, node_type=node_type, node_mode=node_mode - ) - turn_off_op(node_type=node_type, node_mode=node_mode, env=env) + compose_env = compose_node_env(node_type=node_type, node_mode=node_mode) + turn_off_op(compose_env=compose_env, node_type=node_type, node_mode=node_mode) @check_inited @check_user -def turn_on(env_file, node_type: NodeType) -> None: +def turn_on(env_file: str, node_type: NodeType) -> None: node_mode = upsert_node_mode() - env = compose_node_env( - env_file, - inited_node=True, - node_type=node_type, - node_mode=node_mode, - ) - turn_on_op(env=env, node_type=node_type, node_mode=node_mode) + settings = validate_and_save_node_settings(env_file, node_type, node_mode) + compose_env = compose_node_env(node_type=node_type, node_mode=node_mode) + turn_on_op(settings=settings, compose_env=compose_env, node_type=node_type, node_mode=node_mode) logger.info('Waiting for containers initialization') time.sleep(TM_INIT_TIMEOUT) if not is_base_containers_alive(node_type=node_type, node_mode=node_mode): diff --git a/node_cli/fair/record/chain_record.py b/node_cli/fair/record/chain_record.py index d4ac740c..16ff8a72 100644 --- a/node_cli/fair/record/chain_record.py +++ b/node_cli/fair/record/chain_record.py @@ -22,6 +22,8 @@ from typing import cast from datetime import datetime +from skale.core.types import EnvType + from node_cli.core.static_config import get_fair_chain_name from node_cli.fair.record.redis_record import FlatRedisRecord, FieldInfo @@ -78,18 +80,17 @@ def set_force_skaled_start(self, value: bool) -> None: self._set_field('force_skaled_start', value) -def get_fair_chain_record(env: dict) -> ChainRecord: - return ChainRecord(get_fair_chain_name(env)) +def get_fair_chain_record(env_type: EnvType) -> ChainRecord: + return ChainRecord(get_fair_chain_name(env_type)) -def migrate_chain_record(env: dict) -> None: - version = env['NODE_VERSION'] - logger.info('Migrating fair chain record, setting config version to %s', version) - record = get_fair_chain_record(env) - record.set_config_version(version) +def migrate_chain_record(env_type: EnvType, node_version: str) -> None: + logger.info('Migrating fair chain record, setting config version to %s', node_version) + record = get_fair_chain_record(env_type) + record.set_config_version(node_version) -def update_chain_record(env: dict, force_skaled_start: bool) -> None: - record = get_fair_chain_record(env) +def update_chain_record(env_type: EnvType, force_skaled_start: bool) -> None: + record = get_fair_chain_record(env_type) record.set_force_skaled_start(force_skaled_start) logger.info('Updated fair chain record with force_skaled_start=%s', force_skaled_start) diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index 713a6775..c164b11a 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -20,10 +20,12 @@ import functools import logging import time -from typing import Dict, Optional +from typing import Optional import distro +from skale.core.settings import BaseNodeSettings, SkalePassiveSettings, SkaleSettings, get_settings + from node_cli.cli.info import TYPE, VERSION from node_cli.configs import ( CONTAINER_CONFIG_PATH, @@ -37,7 +39,6 @@ from node_cli.core.docker_config import cleanup_docker_configuration, configure_docker from node_cli.core.host import ( ensure_btrfs_kernel_module_autoloaded, - link_env_file, prepare_host, ) from node_cli.core.nftables import configure_nftables @@ -72,24 +73,26 @@ remove_dynamic_containers, system_prune, ) -from node_cli.utils.helper import cleanup_dir_content, rm_dir, str_to_bool +from node_cli.utils.helper import cleanup_dir_content, rm_dir from node_cli.utils.meta import CliMetaManager, FairCliMetaManager from node_cli.utils.node_type import NodeMode, NodeType from node_cli.utils.print_formatters import print_failed_requirements_checks -from node_cli.utils.settings import save_settings +from node_cli.utils.settings import save_internal_settings logger = logging.getLogger(__name__) def checked_host(func): @functools.wraps(func) - def wrapper(env_filepath: str, env: Dict, node_mode: NodeMode, *args, **kwargs): - download_skale_node(env.get('NODE_VERSION'), env.get('CONTAINER_CONFIGS_DIR')) + def wrapper( + settings: BaseNodeSettings, compose_env: dict, node_mode: NodeMode, *args, **kwargs + ): + download_skale_node(settings.node_version, settings.container_configs_dir or None) failed_checks = run_host_checks( - env['BLOCK_DEVICE'], + settings.block_device, TYPE, node_mode, - env['ENV_TYPE'], + settings.env_type, CONTAINER_CONFIG_TMP_PATH, check_type=CheckType.PREINSTALL, ) @@ -97,15 +100,15 @@ def wrapper(env_filepath: str, env: Dict, node_mode: NodeMode, *args, **kwargs): print_failed_requirements_checks(failed_checks) return False - result = func(env_filepath, env, node_mode, *args, **kwargs) + result = func(settings, compose_env, node_mode, *args, **kwargs) if not result: return result failed_checks = run_host_checks( - env['BLOCK_DEVICE'], + settings.block_device, TYPE, node_mode, - env['ENV_TYPE'], + settings.env_type, CONTAINER_CONFIG_PATH, check_type=CheckType.POSTINSTALL, ) @@ -118,111 +121,116 @@ def wrapper(env_filepath: str, env: Dict, node_mode: NodeMode, *args, **kwargs): @checked_host -def update(env_filepath: str, env: Dict, node_mode: NodeMode) -> bool: - compose_rm(node_type=NodeType.SKALE, node_mode=node_mode, env=env) +def update(settings: BaseNodeSettings, compose_env: dict, node_mode: NodeMode) -> bool: + compose_rm(node_type=NodeType.SKALE, node_mode=node_mode, env=compose_env) remove_dynamic_containers() sync_skale_node() ensure_btrfs_kernel_module_autoloaded() - if env.get('SKIP_DOCKER_CONFIG') != 'True': + if not settings.skip_docker_config: configure_docker() - enable_monitoring = str_to_bool(env.get('MONITORING_CONTAINERS', 'False')) - configure_nftables(enable_monitoring=enable_monitoring) + configure_nftables(enable_monitoring=settings.monitoring_containers) - lvmpy_install(env) + lvmpy_install(settings.block_device) generate_nginx_config() - prepare_host(env_filepath, env['ENV_TYPE'], allocation=True) - save_settings(node_type=NodeType.SKALE, node_mode=node_mode) - init_shared_space_volume(env['ENV_TYPE']) + prepare_host(settings.env_type, allocation=True) + save_internal_settings(node_type=NodeType.SKALE, node_mode=node_mode) + init_shared_space_volume(settings.env_type) meta_manager = CliMetaManager() current_stream = meta_manager.get_meta_info().config_stream - skip_cleanup = env.get('SKIP_DOCKER_CLEANUP') == 'True' - if not skip_cleanup and current_stream != env['NODE_VERSION']: + if not settings.skip_docker_cleanup and current_stream != settings.node_version: logger.info( 'Stream version was changed from %s to %s', current_stream, - env['NODE_VERSION'], + settings.node_version, ) docker_cleanup() + skale_settings = get_settings(SkaleSettings) meta_manager.update_meta( VERSION, - env['NODE_VERSION'], - env['DOCKER_LVMPY_VERSION'], + settings.node_version, + skale_settings.docker_lvmpy_version, distro.id(), distro.version(), ) - update_images(env=env, node_type=NodeType.SKALE, node_mode=node_mode) - compose_up(env=env, node_type=NodeType.SKALE, node_mode=node_mode) + update_images( + compose_env=compose_env, + container_configs_dir=settings.container_configs_dir, + node_type=NodeType.SKALE, + node_mode=node_mode, + ) + compose_up(env=compose_env, settings=settings, node_type=NodeType.SKALE, node_mode=node_mode) return True @checked_host -def init(env_filepath: str, env: dict, node_mode: NodeMode) -> None: +def init(settings: BaseNodeSettings, compose_env: dict, node_mode: NodeMode) -> None: sync_skale_node() ensure_btrfs_kernel_module_autoloaded() - if env.get('SKIP_DOCKER_CONFIG') != 'True': + if not settings.skip_docker_config: configure_docker() - enable_monitoring = str_to_bool(env.get('MONITORING_CONTAINERS', 'False')) - configure_nftables(enable_monitoring=enable_monitoring) + configure_nftables(enable_monitoring=settings.monitoring_containers) - prepare_host(env_filepath, env_type=env['ENV_TYPE']) - save_settings(node_type=NodeType.SKALE, node_mode=node_mode) - link_env_file() + prepare_host(env_type=settings.env_type) + save_internal_settings(node_type=NodeType.SKALE, node_mode=node_mode) mark_active_node() configure_filebeat() generate_nginx_config() - lvmpy_install(env) - init_shared_space_volume(env['ENV_TYPE']) + lvmpy_install(settings.block_device) + init_shared_space_volume(settings.env_type) + skale_settings = get_settings(SkaleSettings) meta_manager = CliMetaManager() meta_manager.update_meta( VERSION, - env['NODE_VERSION'], - env['DOCKER_LVMPY_VERSION'], + settings.node_version, + skale_settings.docker_lvmpy_version, distro.id(), distro.version(), ) - update_resource_allocation(env_type=env['ENV_TYPE']) - update_images(env=env, node_type=NodeType.SKALE, node_mode=node_mode) - compose_up(env=env, node_type=NodeType.SKALE, node_mode=node_mode) + update_resource_allocation(env_type=settings.env_type) + update_images( + compose_env=compose_env, + container_configs_dir=settings.container_configs_dir, + node_type=NodeType.SKALE, + node_mode=node_mode, + ) + compose_up(env=compose_env, settings=settings, node_type=NodeType.SKALE, node_mode=node_mode) def init_passive( - env_filepath: str, - env: dict, + settings: BaseNodeSettings, + compose_env: dict, indexer: bool, archive: bool, snapshot: bool, snapshot_from: Optional[str], ) -> None: - cleanup_volume_artifacts(env['BLOCK_DEVICE']) - download_skale_node(env.get('NODE_VERSION'), env.get('CONTAINER_CONFIGS_DIR')) + cleanup_volume_artifacts(settings.block_device) + download_skale_node(settings.node_version, settings.container_configs_dir or None) sync_skale_node() - if env.get('SKIP_DOCKER_CONFIG') != 'True': + if not settings.skip_docker_config: configure_docker() - enable_monitoring = str_to_bool(env.get('MONITORING_CONTAINERS', 'False')) - configure_nftables(enable_monitoring=enable_monitoring) + configure_nftables(enable_monitoring=settings.monitoring_containers) - prepare_host( - env_filepath, - env_type=env['ENV_TYPE'], - ) + prepare_host(env_type=settings.env_type) + save_internal_settings(node_type=NodeType.SKALE, node_mode=NodeMode.PASSIVE) failed_checks = run_host_checks( - env['BLOCK_DEVICE'], + settings.block_device, TYPE, NodeMode.PASSIVE, - env['ENV_TYPE'], + settings.env_type, CONTAINER_CONFIG_PATH, check_type=CheckType.PREINSTALL, ) @@ -232,55 +240,64 @@ def init_passive( set_passive_node_options(archive=archive, indexer=indexer) ensure_filestorage_mapping() - link_env_file() generate_nginx_config() - prepare_block_device(env['BLOCK_DEVICE'], force=env['ENFORCE_BTRFS'] == 'True') + passive_settings = get_settings(SkalePassiveSettings) + prepare_block_device(settings.block_device, force=passive_settings.enforce_btrfs) meta_manager = CliMetaManager() meta_manager.update_meta( VERSION, - env['NODE_VERSION'], + settings.node_version, None, distro.id(), distro.version(), ) - update_resource_allocation(env_type=env['ENV_TYPE']) + update_resource_allocation(env_type=settings.env_type) - schain_name = env['SCHAIN_NAME'] - if snapshot or snapshot_from: + if passive_settings.schain_name and (snapshot or snapshot_from): ts = int(time.time()) - update_node_cli_schain_status(schain_name, repair_ts=ts, snapshot_from=snapshot_from) + update_node_cli_schain_status( + passive_settings.schain_name, repair_ts=ts, snapshot_from=snapshot_from + ) - update_images(env=env, node_type=NodeType.SKALE, node_mode=NodeMode.PASSIVE) - compose_up(env=env, node_type=NodeType.SKALE, node_mode=NodeMode.PASSIVE) + update_images( + compose_env=compose_env, + container_configs_dir=settings.container_configs_dir, + node_type=NodeType.SKALE, + node_mode=NodeMode.PASSIVE, + ) + compose_up( + env=compose_env, settings=settings, node_type=NodeType.SKALE, node_mode=NodeMode.PASSIVE + ) -def update_passive(env_filepath: str, env: Dict) -> bool: - compose_rm(env=env, node_type=NodeType.SKALE, node_mode=NodeMode.PASSIVE) +def update_passive(settings: BaseNodeSettings, compose_env: dict) -> bool: + compose_rm(env=compose_env, node_type=NodeType.SKALE, node_mode=NodeMode.PASSIVE) remove_dynamic_containers() - cleanup_volume_artifacts(env['BLOCK_DEVICE']) - download_skale_node(env['NODE_VERSION'], env.get('CONTAINER_CONFIGS_DIR')) + cleanup_volume_artifacts(settings.block_device) + download_skale_node(settings.node_version, settings.container_configs_dir or None) sync_skale_node() - if env.get('SKIP_DOCKER_CONFIG') != 'True': + if not settings.skip_docker_config: configure_docker() - enable_monitoring = str_to_bool(env.get('MONITORING_CONTAINERS', 'False')) - configure_nftables(enable_monitoring=enable_monitoring) + configure_nftables(enable_monitoring=settings.monitoring_containers) ensure_filestorage_mapping() - prepare_block_device(env['BLOCK_DEVICE'], force=env['ENFORCE_BTRFS'] == 'True') + passive_settings = get_settings(SkalePassiveSettings) + prepare_block_device(settings.block_device, force=passive_settings.enforce_btrfs) generate_nginx_config() - prepare_host(env_filepath, env['ENV_TYPE'], allocation=True) + prepare_host(settings.env_type, allocation=True) + save_internal_settings(node_type=NodeType.SKALE, node_mode=NodeMode.PASSIVE) failed_checks = run_host_checks( - env['BLOCK_DEVICE'], + settings.block_device, TYPE, NodeMode.PASSIVE, - env['ENV_TYPE'], + settings.env_type, CONTAINER_CONFIG_PATH, check_type=CheckType.PREINSTALL, ) @@ -290,56 +307,82 @@ def update_passive(env_filepath: str, env: Dict) -> bool: meta_manager = CliMetaManager() meta_manager.update_meta( VERSION, - env['NODE_VERSION'], + settings.node_version, None, distro.id(), distro.version(), ) - update_images(env=env, node_type=NodeType.SKALE, node_mode=NodeMode.PASSIVE) - compose_up(env=env, node_type=NodeType.SKALE, node_mode=NodeMode.PASSIVE) + update_images( + compose_env=compose_env, + container_configs_dir=settings.container_configs_dir, + node_type=NodeType.SKALE, + node_mode=NodeMode.PASSIVE, + ) + compose_up( + env=compose_env, settings=settings, node_type=NodeType.SKALE, node_mode=NodeMode.PASSIVE + ) return True -def turn_off(env: dict, node_type: NodeType, node_mode: NodeMode) -> None: +def turn_off(compose_env: dict, node_type: NodeType, node_mode: NodeMode) -> None: logger.info('Turning off the node...') - compose_rm(env=env, node_type=node_type, node_mode=node_mode) + compose_rm(env=compose_env, node_type=node_type, node_mode=node_mode) remove_dynamic_containers() logger.info('Node was successfully turned off') -def turn_on(env: dict, node_type: NodeType, node_mode: NodeMode) -> None: +def turn_on( + settings: BaseNodeSettings, + compose_env: dict, + node_type: NodeType, + node_mode: NodeMode, + backup_run: bool = False, +) -> None: logger.info('Turning on the node...') if node_type == NodeType.FAIR: meta_manager = FairCliMetaManager() meta_manager.update_meta( VERSION, - env['NODE_VERSION'], + settings.node_version, distro.id(), distro.version(), ) else: + skale_settings = get_settings((SkaleSettings, SkalePassiveSettings)) + docker_lvmpy_version = ( + skale_settings.docker_lvmpy_version + if isinstance(skale_settings, SkaleSettings) + else None + ) meta_manager = CliMetaManager() meta_manager.update_meta( - VERSION, env['NODE_VERSION'], env['DOCKER_LVMPY_VERSION'], distro.id(), distro.version() + VERSION, settings.node_version, docker_lvmpy_version, distro.id(), distro.version() ) - if env.get('SKIP_DOCKER_CONFIG') != 'True': + if not settings.skip_docker_config: configure_docker() - enable_monitoring = str_to_bool(env.get('MONITORING_CONTAINERS', 'False')) - configure_nftables(enable_monitoring=enable_monitoring) + configure_nftables(enable_monitoring=settings.monitoring_containers) + save_internal_settings(node_type=node_type, node_mode=node_mode, backup_run=backup_run) logger.info('Launching containers on the node...') - compose_up(env=env, node_type=node_type, node_mode=node_mode) + compose_up(env=compose_env, settings=settings, node_type=node_type, node_mode=node_mode) -def restore(env, backup_path, node_type: NodeType, config_only=False): +def restore( + settings: BaseNodeSettings, + compose_env: dict, + backup_path: str, + node_type: NodeType, + config_only: bool = False, + backup_run: bool = False, +) -> bool: node_mode = upsert_node_mode(node_mode=NodeMode.ACTIVE) unpack_backup_archive(backup_path) failed_checks = run_host_checks( - env['BLOCK_DEVICE'], + settings.block_device, TYPE, node_mode, - env['ENV_TYPE'], + settings.env_type, CONTAINER_CONFIG_PATH, check_type=CheckType.PREINSTALL, ) @@ -349,32 +392,32 @@ def restore(env, backup_path, node_type: NodeType, config_only=False): ensure_btrfs_kernel_module_autoloaded() - if env.get('SKIP_DOCKER_CONFIG') != 'True': + if not settings.skip_docker_config: configure_docker() - enable_monitoring = str_to_bool(env.get('MONITORING_CONTAINERS', 'False')) - configure_nftables(enable_monitoring=enable_monitoring) + configure_nftables(enable_monitoring=settings.monitoring_containers) - link_env_file() - lvmpy_install(env) - init_shared_space_volume(env['ENV_TYPE']) + lvmpy_install(settings.block_device) + init_shared_space_volume(settings.env_type) + skale_settings = get_settings(SkaleSettings) meta_manager = CliMetaManager() meta_manager.update_meta( VERSION, - env['NODE_VERSION'], - env['DOCKER_LVMPY_VERSION'], + settings.node_version, + skale_settings.docker_lvmpy_version, distro.id(), distro.version(), ) + save_internal_settings(node_type=node_type, node_mode=node_mode, backup_run=backup_run) if not config_only: - compose_up(env=env, node_type=node_type, node_mode=node_mode) + compose_up(env=compose_env, settings=settings, node_type=node_type, node_mode=node_mode) failed_checks = run_host_checks( - env['BLOCK_DEVICE'], + settings.block_device, TYPE, node_mode, - env['ENV_TYPE'], + settings.env_type, CONTAINER_CONFIG_PATH, check_type=CheckType.POSTINSTALL, ) @@ -384,19 +427,20 @@ def restore(env, backup_path, node_type: NodeType, config_only=False): return True -def cleanup_passive(env, schain_name: str) -> None: - turn_off(env, node_type=NodeType.SKALE, node_mode=NodeMode.PASSIVE) +def cleanup_passive(compose_env: dict, schain_name: str) -> None: + turn_off(compose_env, node_type=NodeType.SKALE, node_mode=NodeMode.PASSIVE) cleanup_no_lvm_datadir(chain_name=schain_name) rm_dir(GLOBAL_SKALE_DIR) rm_dir(SKALE_DIR) -def cleanup(node_mode: NodeMode, env: dict, prune: bool = False) -> None: - turn_off(env, node_type=NodeType.SKALE, node_mode=node_mode) +def cleanup( + node_mode: NodeMode, compose_env: dict, schain_name: Optional[str] = None, prune: bool = False +) -> None: + turn_off(compose_env, node_type=NodeType.SKALE, node_mode=node_mode) if prune: system_prune() if node_mode == NodeMode.PASSIVE: - schain_name = env['SCHAIN_NAME'] cleanup_no_lvm_datadir(chain_name=schain_name) else: cleanup_lvm_datadir() diff --git a/node_cli/operations/config_repo.py b/node_cli/operations/config_repo.py index b8456db6..19cf284a 100644 --- a/node_cli/operations/config_repo.py +++ b/node_cli/operations/config_repo.py @@ -33,12 +33,13 @@ logger = logging.getLogger(__name__) -def update_images(env: dict, node_type: NodeType, node_mode: NodeMode) -> None: - local = env.get('CONTAINER_CONFIGS_DIR') != '' - if local: - compose_build(env=env, node_type=node_type, node_mode=node_mode) +def update_images( + compose_env: dict, container_configs_dir: str, node_type: NodeType, node_mode: NodeMode +) -> None: + if container_configs_dir: + compose_build(env=compose_env, node_type=node_type, node_mode=node_mode) else: - compose_pull(env=env, node_type=node_type, node_mode=node_mode) + compose_pull(env=compose_env, node_type=node_type, node_mode=node_mode) def download_skale_node(stream: Optional[str] = None, src: Optional[str] = None) -> None: diff --git a/node_cli/operations/docker_lvmpy.py b/node_cli/operations/docker_lvmpy.py index c21328f8..eb307d55 100644 --- a/node_cli/operations/docker_lvmpy.py +++ b/node_cli/operations/docker_lvmpy.py @@ -56,12 +56,10 @@ def sync_docker_lvmpy_repo(env): sync_repo(DOCKER_LVMPY_REPO_URL, DOCKER_LVMPY_PATH, env['DOCKER_LVMPY_VERSION']) -def lvmpy_install(env): +def lvmpy_install(block_device: str) -> None: ensure_filestorage_mapping() logging.info('Configuring and starting lvmpy') - setup_lvmpy( - block_device=env['BLOCK_DEVICE'], volume_group=VOLUME_GROUP, exec_start=LVMPY_RUN_CMD - ) + setup_lvmpy(block_device=block_device, volume_group=VOLUME_GROUP, exec_start=LVMPY_RUN_CMD) init_healing_cron() logger.info('docker-lvmpy is configured and started') diff --git a/node_cli/operations/fair.py b/node_cli/operations/fair.py index 096d1623..015abfe0 100644 --- a/node_cli/operations/fair.py +++ b/node_cli/operations/fair.py @@ -23,6 +23,9 @@ import distro +from skale.core.settings import BaseNodeSettings, FairBaseSettings, FairSettings, get_settings +from skale.core.types import EnvType + from node_cli.cli.info import TYPE, VERSION from node_cli.configs import ( CONTAINER_CONFIG_PATH, @@ -33,7 +36,7 @@ from node_cli.core.checks import CheckType from node_cli.core.checks import run_checks as run_host_checks from node_cli.core.docker_config import cleanup_docker_configuration, configure_docker -from node_cli.core.host import ensure_btrfs_kernel_module_autoloaded, link_env_file, prepare_host +from node_cli.core.host import ensure_btrfs_kernel_module_autoloaded, prepare_host from node_cli.core.nftables import configure_nftables from node_cli.core.nginx import generate_nginx_config from node_cli.core.schains import cleanup_no_lvm_datadir @@ -66,11 +69,11 @@ system_prune, wait_for_container, ) -from node_cli.utils.helper import cleanup_dir_content, rm_dir, str_to_bool +from node_cli.utils.helper import cleanup_dir_content, rm_dir from node_cli.utils.meta import FairCliMetaManager from node_cli.utils.print_formatters import print_failed_requirements_checks from node_cli.utils.node_type import NodeMode, NodeType -from node_cli.utils.settings import save_settings +from node_cli.utils.settings import save_internal_settings logger = logging.getLogger(__name__) @@ -82,42 +85,56 @@ class FairUpdateType(Enum): @checked_host -def init_fair_boot(env_filepath: str, env: dict) -> None: +def init_fair_boot( + settings: BaseNodeSettings, + compose_env: dict, + node_mode: NodeMode, +) -> None: sync_skale_node() - cleanup_volume_artifacts(env['BLOCK_DEVICE']) + cleanup_volume_artifacts(settings.block_device) ensure_btrfs_kernel_module_autoloaded() - if env.get('SKIP_DOCKER_CONFIG') != 'True': + if not settings.skip_docker_config: configure_docker() - enable_monitoring = str_to_bool(env.get('MONITORING_CONTAINERS', 'False')) - configure_nftables(enable_monitoring=enable_monitoring) + configure_nftables(enable_monitoring=settings.monitoring_containers) - prepare_host(env_filepath, env_type=env['ENV_TYPE']) - save_settings(node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE) - link_env_file() + prepare_host(env_type=settings.env_type) + save_internal_settings(node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE) mark_active_node() configure_filebeat() generate_nginx_config() - prepare_block_device(env['BLOCK_DEVICE'], force=env['ENFORCE_BTRFS'] == 'True') + fair_settings = get_settings((FairSettings, FairBaseSettings)) + prepare_block_device(settings.block_device, force=fair_settings.enforce_btrfs) meta_manager = FairCliMetaManager() meta_manager.update_meta( VERSION, - env['NODE_VERSION'], + settings.node_version, distro.id(), distro.version(), ) - update_images(env=env, node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE) + update_images( + compose_env=compose_env, + container_configs_dir=settings.container_configs_dir, + node_type=NodeType.FAIR, + node_mode=NodeMode.ACTIVE, + ) - compose_up(env=env, node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE, is_fair_boot=True) + compose_up( + env=compose_env, + settings=settings, + node_type=NodeType.FAIR, + node_mode=NodeMode.ACTIVE, + is_fair_boot=True, + ) @checked_host def init( - env_filepath: str, - env: dict, + settings: BaseNodeSettings, + compose_env: dict, node_mode: NodeMode, indexer: bool, archive: bool, @@ -125,24 +142,33 @@ def init( ) -> bool: sync_skale_node() ensure_btrfs_kernel_module_autoloaded() - cleanup_volume_artifacts(env['BLOCK_DEVICE']) + cleanup_volume_artifacts(settings.block_device) - if env.get('SKIP_DOCKER_CONFIG') != 'True': + if not settings.skip_docker_config: configure_docker() configure_nftables() configure_filebeat() generate_nginx_config() - prepare_host(env_filepath, env_type=env['ENV_TYPE']) - save_settings(node_type=NodeType.FAIR, node_mode=node_mode) - link_env_file() + prepare_host(env_type=settings.env_type) + save_internal_settings(node_type=NodeType.FAIR, node_mode=node_mode) - prepare_block_device(env['BLOCK_DEVICE'], force=env['ENFORCE_BTRFS'] == 'True') + fair_settings = get_settings((FairSettings, FairBaseSettings)) + prepare_block_device(settings.block_device, force=fair_settings.enforce_btrfs) - update_images(env=env, node_type=NodeType.FAIR, node_mode=node_mode) + update_images( + compose_env=compose_env, + container_configs_dir=settings.container_configs_dir, + node_type=NodeType.FAIR, + node_mode=node_mode, + ) compose_up( - env=env, node_type=NodeType.FAIR, node_mode=node_mode, services=list(REDIS_SERVICE_DICT) + env=compose_env, + settings=settings, + node_type=NodeType.FAIR, + node_mode=node_mode, + services=list(REDIS_SERVICE_DICT), ) upsert_node_mode(node_mode=node_mode) @@ -152,131 +178,159 @@ def init( if snapshot: logger.info('Waiting %s seconds for redis to start', REDIS_START_TIMEOUT) time.sleep(REDIS_START_TIMEOUT) - trigger_skaled_snapshot_mode(env=env, snapshot_from=snapshot) + trigger_skaled_snapshot_mode(env_type=settings.env_type, snapshot_from=snapshot) meta_manager = FairCliMetaManager() meta_manager.update_meta( VERSION, - env['NODE_VERSION'], + settings.node_version, distro.id(), distro.version(), ) - compose_up(env=env, node_type=NodeType.FAIR, node_mode=node_mode) + compose_up(env=compose_env, settings=settings, node_type=NodeType.FAIR, node_mode=node_mode) wait_for_container(BASE_PASSIVE_FAIR_COMPOSE_SERVICES['api']) time.sleep(REDIS_START_TIMEOUT) return True @checked_host -def update_fair_boot(env_filepath: str, env: dict, node_mode: NodeMode = NodeMode.ACTIVE) -> bool: - compose_rm(node_type=NodeType.FAIR, node_mode=node_mode, env=env) +def update_fair_boot( + settings: BaseNodeSettings, + compose_env: dict, + node_mode: NodeMode = NodeMode.ACTIVE, +) -> bool: + compose_rm(node_type=NodeType.FAIR, node_mode=node_mode, env=compose_env) remove_dynamic_containers() - cleanup_volume_artifacts(env['BLOCK_DEVICE']) + cleanup_volume_artifacts(settings.block_device) sync_skale_node() ensure_btrfs_kernel_module_autoloaded() - if env.get('SKIP_DOCKER_CONFIG') != 'True': + if not settings.skip_docker_config: configure_docker() - enable_monitoring = str_to_bool(env.get('MONITORING_CONTAINERS', 'False')) - configure_nftables(enable_monitoring=enable_monitoring) + configure_nftables(enable_monitoring=settings.monitoring_containers) generate_nginx_config() - prepare_block_device(env['BLOCK_DEVICE'], force=env['ENFORCE_BTRFS'] == 'True') + fair_settings = get_settings((FairSettings, FairBaseSettings)) + prepare_block_device(settings.block_device, force=fair_settings.enforce_btrfs) - prepare_host(env_filepath, env['ENV_TYPE']) - save_settings(node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE) + prepare_host(settings.env_type) + save_internal_settings(node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE) meta_manager = FairCliMetaManager() current_stream = meta_manager.get_meta_info().config_stream - skip_cleanup = env.get('SKIP_DOCKER_CLEANUP') == 'True' - if not skip_cleanup and current_stream != env['NODE_VERSION']: + if not settings.skip_docker_cleanup and current_stream != settings.node_version: logger.info( 'Stream version was changed from %s to %s', current_stream, - env['NODE_VERSION'], + settings.node_version, ) docker_cleanup() meta_manager.update_meta( VERSION, - env['NODE_VERSION'], + settings.node_version, distro.id(), distro.version(), ) - update_images(env=env, node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE) - compose_up(env=env, node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE, is_fair_boot=True) + update_images( + compose_env=compose_env, + container_configs_dir=settings.container_configs_dir, + node_type=NodeType.FAIR, + node_mode=NodeMode.ACTIVE, + ) + compose_up( + env=compose_env, + settings=settings, + node_type=NodeType.FAIR, + node_mode=NodeMode.ACTIVE, + is_fair_boot=True, + ) return True @checked_host def update( - env_filepath: str, - env: dict, + settings: BaseNodeSettings, + compose_env: dict, node_mode: NodeMode, update_type: FairUpdateType, force_skaled_start: bool, ) -> bool: - compose_rm(node_type=NodeType.FAIR, node_mode=node_mode, env=env) + compose_rm(node_type=NodeType.FAIR, node_mode=node_mode, env=compose_env) if update_type not in (FairUpdateType.INFRA_ONLY, FairUpdateType.FROM_BOOT): remove_dynamic_containers() sync_skale_node() ensure_btrfs_kernel_module_autoloaded() - if env.get('SKIP_DOCKER_CONFIG') != 'True': + if not settings.skip_docker_config: configure_docker() configure_nftables() generate_nginx_config() - prepare_host(env_filepath, env['ENV_TYPE'], allocation=True) - save_settings(node_type=NodeType.FAIR, node_mode=node_mode) + prepare_host(settings.env_type, allocation=True) + save_internal_settings(node_type=NodeType.FAIR, node_mode=node_mode) meta_manager = FairCliMetaManager() current_stream = meta_manager.get_meta_info().config_stream - skip_cleanup = env.get('SKIP_DOCKER_CLEANUP') == 'True' - if not skip_cleanup and current_stream != env['NODE_VERSION']: + if not settings.skip_docker_cleanup and current_stream != settings.node_version: logger.info( 'Stream version was changed from %s to %s', current_stream, - env['NODE_VERSION'], + settings.node_version, ) docker_cleanup() meta_manager.update_meta( VERSION, - env['NODE_VERSION'], + settings.node_version, distro.id(), distro.version(), ) - fair_chain_name = get_fair_chain_name(env) + fair_chain_name = get_fair_chain_name(settings.env_type) if update_type == FairUpdateType.FROM_BOOT: migrate_nftables_from_boot(chain_name=fair_chain_name) - update_images(env=env, node_type=NodeType.FAIR, node_mode=node_mode) + update_images( + compose_env=compose_env, + container_configs_dir=settings.container_configs_dir, + node_type=NodeType.FAIR, + node_mode=node_mode, + ) compose_up( - env=env, node_type=NodeType.FAIR, node_mode=node_mode, services=list(REDIS_SERVICE_DICT) + env=compose_env, + settings=settings, + node_type=NodeType.FAIR, + node_mode=node_mode, + services=list(REDIS_SERVICE_DICT), ) wait_for_container(REDIS_SERVICE_DICT['redis']) time.sleep(REDIS_START_TIMEOUT) if update_type == FairUpdateType.FROM_BOOT: - migrate_chain_record(env) - update_chain_record(env, force_skaled_start=force_skaled_start) - compose_up(env=env, node_type=NodeType.FAIR, node_mode=node_mode) + migrate_chain_record(settings.env_type, settings.node_version) + update_chain_record(settings.env_type, force_skaled_start=force_skaled_start) + compose_up(env=compose_env, settings=settings, node_type=NodeType.FAIR, node_mode=node_mode) return True -def restore(node_mode: NodeMode, env, backup_path, config_only=False): +def restore( + node_mode: NodeMode, + settings: BaseNodeSettings, + compose_env: dict, + backup_path: str, + config_only: bool = False, +) -> bool: unpack_backup_archive(backup_path) failed_checks = run_host_checks( - env['BLOCK_DEVICE'], + settings.block_device, TYPE, node_mode, - env['ENV_TYPE'], + settings.env_type, CONTAINER_CONFIG_PATH, check_type=CheckType.PREINSTALL, ) @@ -286,30 +340,27 @@ def restore(node_mode: NodeMode, env, backup_path, config_only=False): ensure_btrfs_kernel_module_autoloaded() - if env.get('SKIP_DOCKER_CONFIG') != 'True': + if not settings.skip_docker_config: configure_docker() - enable_monitoring = str_to_bool(env.get('MONITORING_CONTAINERS', 'False')) - configure_nftables(enable_monitoring=enable_monitoring) - - link_env_file() + configure_nftables(enable_monitoring=settings.monitoring_containers) meta_manager = FairCliMetaManager() meta_manager.update_meta( VERSION, - env['NODE_VERSION'], + settings.node_version, distro.id(), distro.version(), ) if not config_only: - compose_up(env=env, node_type=NodeType.FAIR, node_mode=node_mode) + compose_up(env=compose_env, settings=settings, node_type=NodeType.FAIR, node_mode=node_mode) failed_checks = run_host_checks( - env['BLOCK_DEVICE'], + settings.block_device, TYPE, node_mode, - env['ENV_TYPE'], + settings.env_type, CONTAINER_CONFIG_PATH, check_type=CheckType.POSTINSTALL, ) @@ -319,8 +370,8 @@ def restore(node_mode: NodeMode, env, backup_path, config_only=False): return True -def cleanup(node_mode: NodeMode, env: dict, prune: bool = False) -> None: - turn_off(env, node_type=NodeType.FAIR, node_mode=node_mode) +def cleanup(node_mode: NodeMode, compose_env: dict, prune: bool = False) -> None: + turn_off(compose_env, node_type=NodeType.FAIR, node_mode=node_mode) if prune: system_prune() cleanup_no_lvm_datadir() @@ -330,15 +381,15 @@ def cleanup(node_mode: NodeMode, env: dict, prune: bool = False) -> None: cleanup_docker_configuration() -def trigger_skaled_snapshot_mode(env: dict, snapshot_from: str = 'any') -> None: - record = get_fair_chain_record(env) +def trigger_skaled_snapshot_mode(env_type: EnvType, snapshot_from: str = 'any') -> None: + record = get_fair_chain_record(env_type) if not snapshot_from: snapshot_from = 'any' logger.info('Triggering skaled snapshot mode, snapshot_from: %s', snapshot_from) record.set_snapshot_from(snapshot_from) -def repair(env: dict, snapshot_from: str = 'any') -> None: +def repair(env_type: EnvType, snapshot_from: str = 'any') -> None: logger.info('Starting fair node repair') container_name = 'sk_admin' if is_admin_running(): @@ -349,7 +400,7 @@ def repair(env: dict, snapshot_from: str = 'any') -> None: logger.info('Cleaning up datadir') cleanup_no_lvm_datadir() logger.info('Requesting fair node repair') - trigger_skaled_snapshot_mode(env=env, snapshot_from=snapshot_from) + trigger_skaled_snapshot_mode(env_type=env_type, snapshot_from=snapshot_from) logger.info('Starting admin') start_container_by_name(container_name=container_name) logger.info('Fair node repair completed successfully') diff --git a/node_cli/utils/docker_utils.py b/node_cli/utils/docker_utils.py index 37e3fab9..8f0e1bb1 100644 --- a/node_cli/utils/docker_utils.py +++ b/node_cli/utils/docker_utils.py @@ -29,15 +29,16 @@ from docker.errors import NotFound from docker.models.containers import Container +from skale.core.settings import BaseNodeSettings + from node_cli.configs import ( COMPOSE_PATH, FAIR_COMPOSE_PATH, NGINX_CONTAINER_NAME, REMOVED_CONTAINERS_FOLDER_PATH, - SGX_CERTIFICATES_DIR_NAME, ) from node_cli.core.node_options import active_fair, active_skale, passive_fair, passive_skale -from node_cli.utils.helper import run_cmd, str_to_bool +from node_cli.utils.helper import run_cmd from node_cli.utils.node_type import NodeMode, NodeType logger = logging.getLogger(__name__) @@ -327,6 +328,7 @@ def get_up_compose_cmd( def compose_up( env, + settings: BaseNodeSettings, node_type: NodeType, node_mode: NodeMode, is_fair_boot: bool = False, @@ -365,7 +367,7 @@ def compose_up( logger.debug('Launching skale node containers with env %s', env) run_cmd(cmd=get_up_compose_cmd(node_type=node_type, node_mode=node_mode), env=env) - if 'TG_API_KEY' in env and 'TG_CHAT_ID' in env: + if settings.tg_api_key and settings.tg_chat_id: logger.info('Running containers for Telegram notifications') run_cmd( cmd=get_up_compose_cmd( @@ -376,7 +378,7 @@ def compose_up( env=env, ) - if str_to_bool(env.get('MONITORING_CONTAINERS', 'False')): + if settings.monitoring_containers: logger.info('Running monitoring containers') run_cmd( cmd=get_up_compose_cmd( diff --git a/node_cli/utils/helper.py b/node_cli/utils/helper.py index 9d96a3ca..d2c598fe 100644 --- a/node_cli/utils/helper.py +++ b/node_cli/utils/helper.py @@ -144,16 +144,6 @@ def get_username(): return os.environ.get('USERNAME') or os.environ.get('USER') -def str_to_bool(val: str) -> bool: - val = val.lower() - if val in ('y', 'yes', 't', 'true', 'on', '1'): - return True - elif val in ('n', 'no', 'f', 'false', 'off', '0'): - return False - else: - raise ValueError(f'Invalid truth value {val!r}') - - def error_exit(error_payload: Any, exit_code: CLIExitCodes = CLIExitCodes.FAILURE) -> NoReturn: """Print error message and exit the program with specified exit code. diff --git a/node_cli/utils/settings.py b/node_cli/utils/settings.py index c0b3aa76..e10e68ce 100644 --- a/node_cli/utils/settings.py +++ b/node_cli/utils/settings.py @@ -17,19 +17,23 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import tomllib + +from dotenv.main import DotEnv + from skale.core.settings import ( SETTINGS_MAP, - write_node_settings_file, - write_internal_settings_file, + BaseNodeSettings, + FairBaseSettings, + FairSettings, InternalSettings, - SkaleSettings, SkalePassiveSettings, - FairSettings, - FairBaseSettings, + SkaleSettings, + write_internal_settings_file, + write_node_settings_file, ) -from node_cli.configs import NODE_SETTINGS_PATH, INTERNAL_SETTINGS_PATH - +from node_cli.configs import INTERNAL_SETTINGS_PATH, NODE_SETTINGS_PATH, SKALE_DIR from node_cli.utils.node_type import NodeMode, NodeType InternalSettings.model_config['toml_file'] = INTERNAL_SETTINGS_PATH @@ -39,9 +43,35 @@ FairBaseSettings.model_config['toml_file'] = NODE_SETTINGS_PATH -def save_settings(node_type: NodeType, node_mode: NodeMode) -> None: - write_internal_settings_file(path=INTERNAL_SETTINGS_PATH, data={}) # todof: fix +def load_config_file(filepath: str) -> dict: + if filepath.endswith('.toml'): + with open(filepath, 'rb') as f: + return tomllib.load(f) + return {k.lower(): v for k, v in DotEnv(filepath).dict().items()} + + +def validate_and_save_node_settings( + config_filepath: str, + node_type: NodeType, + node_mode: NodeMode, +) -> BaseNodeSettings: + data = load_config_file(config_filepath) settings_type = SETTINGS_MAP[(node_type.value, node_mode.value)] - write_node_settings_file( - path=NODE_SETTINGS_PATH, settings_type=settings_type, data={} - ) # todof: fix + write_node_settings_file(path=NODE_SETTINGS_PATH, settings_type=settings_type, data=data) + return settings_type() + + +def save_internal_settings( + node_type: NodeType, + node_mode: NodeMode, + backup_run: bool = False, + pull_config_for_schain: str | None = None, +) -> None: + data = { + 'node_type': node_type.value, + 'node_mode': node_mode.value, + 'skale_dir_host': str(SKALE_DIR), + 'backup_run': backup_run, + 'pull_config_for_schain': pull_config_for_schain, + } + write_internal_settings_file(path=INTERNAL_SETTINGS_PATH, data=data) diff --git a/tests/cli/fair_cli_test.py b/tests/cli/fair_cli_test.py index 3c6b9ef5..2f93d1a7 100644 --- a/tests/cli/fair_cli_test.py +++ b/tests/cli/fair_cli_test.py @@ -20,6 +20,7 @@ from tests.helper import run_command, subprocess_run_mock from tests.resources_test import BIG_DISK_SIZE + @mock.patch('node_cli.cli.fair_node.restore_fair') def test_fair_node_restore(mock_restore_core, valid_env_file, tmp_path): runner = CliRunner() @@ -105,7 +106,7 @@ def test_fair_node_migrate(mock_migrate_core, valid_env_file): result = runner.invoke(migrate_node, ['--yes', valid_env_file]) assert result.exit_code == 0, f'Output: {result.output}\nException: {result.exception}' - mock_migrate_core.assert_called_once_with(env_filepath=valid_env_file) + mock_migrate_core.assert_called_once_with(config_file=valid_env_file) @mock.patch('node_cli.cli.fair_node.exit_fair') @@ -136,4 +137,5 @@ def test_cleanup_node(mocked_g_config, inited_node): result = run_command(cleanup_node, ['--yes']) assert result.exit_code == 0 cleanup_mock.assert_called_once_with( - node_mode=NodeMode.ACTIVE, prune=False, env={'SCHAIN_NAME': 'test'}) + node_mode=NodeMode.ACTIVE, prune=False, env={'SCHAIN_NAME': 'test'} + ) diff --git a/tests/cli/node_test.py b/tests/cli/node_test.py index bb5aa6d0..63e96259 100644 --- a/tests/cli/node_test.py +++ b/tests/cli/node_test.py @@ -353,12 +353,12 @@ def test_restore(request, node_type, node_mode, test_user_conf, mocked_g_config, result = run_command(restore_node, [backup_path, user_conf_path]) assert result.exit_code == 0 assert 'Node is restored from backup\n' in result.output # noqa - assert mock_restore_op.call_args[0][0].get('BACKUP_RUN') == 'True' + assert mock_restore_op.call_args.kwargs.get('backup_run') is True result = run_command(restore_node, [backup_path, user_conf_path, '--no-snapshot']) assert result.exit_code == 0 assert 'Node is restored from backup\n' in result.output # noqa - assert mock_restore_op.call_args[0][0].get('BACKUP_RUN') is None + assert mock_restore_op.call_args.kwargs.get('backup_run') is False def test_maintenance_on(): diff --git a/tests/configs/configs_env_validate_test.py b/tests/configs/configs_env_validate_test.py index 2db057ff..52622f73 100644 --- a/tests/configs/configs_env_validate_test.py +++ b/tests/configs/configs_env_validate_test.py @@ -1,4 +1,3 @@ -import os from typing import Optional import pytest @@ -12,18 +11,6 @@ validate_contract_address, validate_contract_alias, ) -from node_cli.configs.user import ( - ALLOWED_ENV_TYPES, - FairBootUserConfig, - FairUserConfig, - PassiveFairUserConfig, - SkaleUserConfig, - PassiveSkaleUserConfig, - get_user_config_class, - get_validated_user_config, - validate_env_type, -) -from node_cli.utils.node_type import NodeType, NodeMode ENDPOINT = 'http://localhost:8545' @@ -37,44 +24,6 @@ def json(self): return self._json_data -@pytest.mark.parametrize( - 'node_type, node_mode, is_fair_boot, expected_type', - [ - (NodeType.SKALE, NodeMode.ACTIVE, False, SkaleUserConfig), - (NodeType.SKALE, NodeMode.PASSIVE, False, PassiveSkaleUserConfig), - (NodeType.FAIR, NodeMode.ACTIVE, True, FairBootUserConfig), - (NodeType.FAIR, NodeMode.ACTIVE, False, FairUserConfig), - (NodeType.FAIR, NodeMode.PASSIVE, False, PassiveFairUserConfig), - ], - ids=['skale_active', 'skale_passive', 'fair_boot', 'fair_active', 'fair_passive'], -) -def test_build_env_params_keys(node_type, node_mode, is_fair_boot, expected_type): - env_type = get_user_config_class( - node_type=node_type, node_mode=node_mode, is_fair_boot=is_fair_boot - ) - assert env_type == expected_type - - -@pytest.mark.parametrize( - 'env_types, should_fail', - [ - (ALLOWED_ENV_TYPES, False), - (['invalid'], True), - ], - ids=[ - 'correct_env', - 'invalid_env', - ], -) -def test_env_types(env_types, should_fail): - for env_type in env_types: - if should_fail: - with pytest.raises(SystemExit): - validate_env_type(env_type=env_type) - else: - validate_env_type(env_type=env_type) - - def test_get_chain_id_success(monkeypatch): fake_response = FakeResponse(200, {'result': '0x1'}) @@ -166,24 +115,3 @@ def test_validate_env_alias_or_address_with_alias(requests_mock): alias_url = 'https://raw.githubusercontent.com/skalenetwork/skale-contracts/refs/heads/deployments/mainnet/mainnet-ima/test-alias.json' requests_mock.get(alias_url, status_code=200) validate_alias_or_address('test-alias', ContractType.IMA, ENDPOINT) - - -def test_get_validated_env_config_missing_file(): - with pytest.raises(SystemExit): - get_validated_user_config( - env_filepath='nonexistent.env', node_type=NodeType.SKALE, node_mode=NodeMode.ACTIVE - ) - - -def test_get_validated_env_config_unreadable_file(tmp_path): - env_file = tmp_path / 'unreadable.env' - env_file.touch() - original_mode = env_file.stat().st_mode - try: - os.chmod(env_file, 0o000) - with pytest.raises(PermissionError): - get_validated_user_config( - env_filepath=str(env_file), node_type=NodeType.SKALE, node_mode=NodeMode.ACTIVE - ) - finally: - os.chmod(env_file, original_mode) diff --git a/tests/conftest.py b/tests/conftest.py index cc5a5fa3..e9d4b6cc 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -48,6 +48,12 @@ from node_cli.utils.docker_utils import docker_client from node_cli.utils.global_config import generate_g_config_file from node_cli.utils.node_type import NodeMode +from tests.fixtures.settings import ( # noqa: F401 + fair_active_settings, + fair_passive_settings, + skale_active_settings, + skale_passive_settings, +) from tests.helper import TEST_META_V1, TEST_META_V2, TEST_META_V3, TEST_SCHAINS_MNT_DIR_SINGLE_CHAIN diff --git a/tests/core/core_node_test.py b/tests/core/core_node_test.py index 0fe113b2..ead55528 100644 --- a/tests/core/core_node_test.py +++ b/tests/core/core_node_test.py @@ -10,9 +10,14 @@ import pytest import requests -from node_cli.configs import NODE_DATA_PATH, SCHAINS_MNT_DIR_REGULAR, SCHAINS_MNT_DIR_SINGLE_CHAIN +from node_cli.configs import ( + NODE_DATA_PATH, + SCHAINS_MNT_DIR_REGULAR, + SCHAINS_MNT_DIR_SINGLE_CHAIN, + SKALE_DIR, +) from node_cli.configs.resource_allocation import RESOURCE_ALLOCATION_FILEPATH -from node_cli.configs.user import SKALE_DIR_ENV_FILEPATH + from node_cli.core.node import ( cleanup, compose_node_env, @@ -33,6 +38,8 @@ ALPINE_IMAGE_NAME = 'alpine:3.12' CMD = 'sleep 60' +SKALE_DIR_ENV_FILEPATH = os.path.join(SKALE_DIR, '.env') + WRONG_CONTAINERS = [ 'WRONG_CONTAINER_1', 'skale_WRONG_CONTAINER_4', @@ -165,105 +172,42 @@ def test_is_base_containers_alive_empty(node_type, node_mode, is_boot): @pytest.mark.parametrize( - ( - 'node_type, node_mode, test_user_conf, is_boot, inited_node, sync_schains,' - 'expected_mnt_dir, expect_flask_key, expect_backup_run' - ), + 'node_type, node_mode, expected_mnt_dir', [ ( NodeType.SKALE, NodeMode.ACTIVE, - 'regular_user_conf', - False, - True, - False, SCHAINS_MNT_DIR_REGULAR, - True, - False, - ), - ( - NodeType.SKALE, - NodeMode.ACTIVE, - 'regular_user_conf', - False, - True, - True, - SCHAINS_MNT_DIR_REGULAR, - True, - True, ), ( NodeType.SKALE, NodeMode.PASSIVE, - 'passive_user_conf', - False, - False, - False, SCHAINS_MNT_DIR_SINGLE_CHAIN, - False, - False, ), ( NodeType.FAIR, NodeMode.ACTIVE, - 'fair_boot_user_conf', - True, - True, - False, SCHAINS_MNT_DIR_SINGLE_CHAIN, - True, - False, - ), - ( - NodeType.FAIR, - NodeMode.ACTIVE, - 'fair_user_conf', - False, - True, - False, - SCHAINS_MNT_DIR_SINGLE_CHAIN, - True, - False, ), ], ids=[ 'regular', - 'regular_passive_flag', 'passive', - 'fair_boot', - 'fair_regular', + 'fair', ], ) def test_compose_node_env( - request, node_type, node_mode, - test_user_conf, - is_boot, - inited_node, - sync_schains, expected_mnt_dir, - expect_backup_run, ): - user_config_path = request.getfixturevalue(test_user_conf) - - with ( - mock.patch('node_cli.configs.user.validate_alias_or_address'), - mock.patch('node_cli.core.node.save_env_params'), - ): - result_env = compose_node_env( - env_filepath=user_config_path.as_posix(), - inited_node=inited_node, - sync_schains=sync_schains, - node_type=node_type, - node_mode=node_mode, - is_fair_boot=is_boot, - save=True, - ) + result_env = compose_node_env( + node_type=node_type, + node_mode=node_mode, + ) assert result_env['SCHAINS_MNT_DIR'] == expected_mnt_dir - should_have_backup = sync_schains and node_mode != NodeMode.PASSIVE - assert ('BACKUP_RUN' in result_env and result_env['BACKUP_RUN'] == 'True') == should_have_backup + assert 'BACKUP_RUN' not in result_env @pytest.fixture diff --git a/tests/fair/fair_node_test.py b/tests/fair/fair_node_test.py index a0408caa..62fcf271 100644 --- a/tests/fair/fair_node_test.py +++ b/tests/fair/fair_node_test.py @@ -1,9 +1,9 @@ +import os from unittest import mock import pytest from node_cli.configs import SKALE_DIR -from node_cli.configs.user import SKALE_DIR_ENV_FILEPATH from node_cli.fair.boot import init as init_boot from node_cli.fair.boot import update from node_cli.fair.common import cleanup @@ -11,6 +11,8 @@ from node_cli.operations.fair import FairUpdateType from node_cli.utils.node_type import NodeMode, NodeType +SKALE_DIR_ENV_FILEPATH = os.path.join(SKALE_DIR, '.env') + @mock.patch('node_cli.fair.active.time.sleep') @mock.patch('node_cli.fair.active.restore_fair_op') @@ -182,7 +184,8 @@ def test_cleanup_success( skip_user_conf_validation=True, ) mock_cleanup_fair_op.assert_called_once_with( - node_mode=NodeMode.ACTIVE, env=mock_env, prune=False) + node_mode=NodeMode.ACTIVE, env=mock_env, prune=False + ) @mock.patch('node_cli.utils.decorators.is_user_valid', return_value=True) @@ -214,7 +217,8 @@ def test_cleanup_calls_operations_in_correct_order( save=False, node_type=mock.ANY, node_mode=NodeMode.ACTIVE, - skip_user_conf_validation=True), + skip_user_conf_validation=True, + ), mock.call.cleanup_fair_op(node_mode=NodeMode.ACTIVE, env=mock_env, prune=False), ] manager.assert_has_calls(expected_calls, any_order=False) @@ -240,7 +244,8 @@ def test_cleanup_continues_after_fair_op_error( mock_compose_env.assert_called_once() mock_cleanup_fair_op.assert_called_once_with( - node_mode=NodeMode.ACTIVE, env=mock_env, prune=False) + node_mode=NodeMode.ACTIVE, env=mock_env, prune=False + ) @mock.patch('node_cli.utils.decorators.is_user_valid', return_value=False) diff --git a/tests/fixtures/__init__.py b/tests/fixtures/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/fixtures/settings.py b/tests/fixtures/settings.py new file mode 100644 index 00000000..11e53ede --- /dev/null +++ b/tests/fixtures/settings.py @@ -0,0 +1,131 @@ +# -*- coding: utf-8 -*- +# +# This file is part of node-cli +# +# Copyright (C) 2026-Present SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import pytest +import tomli_w + +from skale.core.settings import get_internal_settings + +from node_cli.configs import INTERNAL_SETTINGS_PATH, NODE_SETTINGS_PATH + +SKALE_DIR_HOST = './skale-data/' + +INTERNAL_SKALE_ACTIVE = { + 'node_type': 'skale', + 'node_mode': 'active', + 'skale_dir_host': SKALE_DIR_HOST, +} + +INTERNAL_SKALE_PASSIVE = { + 'node_type': 'skale', + 'node_mode': 'passive', + 'skale_dir_host': SKALE_DIR_HOST, +} + +INTERNAL_FAIR_ACTIVE = { + 'node_type': 'fair', + 'node_mode': 'active', + 'skale_dir_host': SKALE_DIR_HOST, +} + +INTERNAL_FAIR_PASSIVE = { + 'node_type': 'fair', + 'node_mode': 'passive', + 'skale_dir_host': SKALE_DIR_HOST, +} + +_BASE_NODE = { + 'env_type': 'devnet', + 'endpoint': 'http://127.0.0.1:8545', + 'container_stop_timeout': 1, + 'tg_api_key': '123', + 'tg_chat_id': '-1231232', + 'node_version': '0.0.0', + 'block_device': '/dev/sda', +} + +NODE_SKALE_ACTIVE = { + **_BASE_NODE, + 'sgx_url': 'https://localhost:1026', + 'docker_lvmpy_version': '0.0.0', + 'manager_contracts': 'test-manager', + 'ima_contracts': 'test-ima', +} + +NODE_SKALE_PASSIVE = { + **_BASE_NODE, + 'manager_contracts': 'test-manager', + 'ima_contracts': 'test-ima', + 'schain_name': 'test-schain', + 'enforce_btrfs': False, +} + +NODE_FAIR_ACTIVE = { + **_BASE_NODE, + 'sgx_url': 'https://localhost:1026', + 'fair_contracts': 'test-fair', + 'enforce_btrfs': False, +} + +NODE_FAIR_PASSIVE = { + **_BASE_NODE, + 'fair_contracts': 'test-fair', + 'enforce_btrfs': False, +} + + +def _write_settings(internal: dict, node: dict) -> None: + INTERNAL_SETTINGS_PATH.parent.mkdir(parents=True, exist_ok=True) + INTERNAL_SETTINGS_PATH.write_bytes(tomli_w.dumps(internal).encode()) + NODE_SETTINGS_PATH.write_bytes(tomli_w.dumps(node).encode()) + get_internal_settings.cache_clear() + + +def _cleanup_settings() -> None: + INTERNAL_SETTINGS_PATH.unlink(missing_ok=True) + NODE_SETTINGS_PATH.unlink(missing_ok=True) + get_internal_settings.cache_clear() + + +@pytest.fixture +def skale_active_settings(): + _write_settings(INTERNAL_SKALE_ACTIVE, NODE_SKALE_ACTIVE) + yield + _cleanup_settings() + + +@pytest.fixture +def skale_passive_settings(): + _write_settings(INTERNAL_SKALE_PASSIVE, NODE_SKALE_PASSIVE) + yield + _cleanup_settings() + + +@pytest.fixture +def fair_active_settings(): + _write_settings(INTERNAL_FAIR_ACTIVE, NODE_FAIR_ACTIVE) + yield + _cleanup_settings() + + +@pytest.fixture +def fair_passive_settings(): + _write_settings(INTERNAL_FAIR_PASSIVE, NODE_FAIR_PASSIVE) + yield + _cleanup_settings() diff --git a/tests/utils/settings_test.py b/tests/utils/settings_test.py index 1d9deeb1..e7b3f4be 100644 --- a/tests/utils/settings_test.py +++ b/tests/utils/settings_test.py @@ -17,4 +17,3 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -from node_cli.utils.node_type import NodeMode, NodeType From 070111caad2797fb794ad95aa1515154a09bb350 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Mon, 16 Feb 2026 13:15:06 +0000 Subject: [PATCH 190/198] Update node-cli to use skale core package --- node_cli/cli/node.py | 2 +- node_cli/cli/schains.py | 2 +- node_cli/core/checks.py | 2 +- node_cli/core/host.py | 2 +- node_cli/core/node.py | 4 +- node_cli/core/resources.py | 2 +- node_cli/core/schains.py | 2 +- node_cli/core/static_config.py | 2 +- node_cli/fair/common.py | 2 +- node_cli/fair/record/chain_record.py | 2 +- node_cli/operations/base.py | 2 +- node_cli/operations/fair.py | 4 +- node_cli/utils/docker_utils.py | 2 +- node_cli/utils/settings.py | 2 +- pyproject.toml | 2 +- tests/cli/fair_cli_test.py | 2 +- tests/cli/node_test.py | 6 +-- tests/cli/passive_node_test.py | 6 +-- tests/core/core_node_test.py | 18 ++------ tests/fair/fair_node_test.py | 63 ++++++++-------------------- tests/fixtures/settings.py | 2 +- 21 files changed, 43 insertions(+), 88 deletions(-) diff --git a/node_cli/cli/node.py b/node_cli/cli/node.py index cb798a1c..b34fd155 100644 --- a/node_cli/cli/node.py +++ b/node_cli/cli/node.py @@ -21,7 +21,7 @@ import click -from skale.core.types import EnvType +from skale_core.types import EnvType from node_cli.cli.info import TYPE from node_cli.core.node import ( cleanup as cleanup_skale, diff --git a/node_cli/cli/schains.py b/node_cli/cli/schains.py index 40f1ab0c..6bb1fce2 100644 --- a/node_cli/cli/schains.py +++ b/node_cli/cli/schains.py @@ -21,7 +21,7 @@ import click -from skale.core.settings import get_settings +from skale_core.settings import get_settings from node_cli.utils.helper import abort_if_false, URL_TYPE from node_cli.core.schains import ( diff --git a/node_cli/core/checks.py b/node_cli/core/checks.py index e7677847..0424577f 100644 --- a/node_cli/core/checks.py +++ b/node_cli/core/checks.py @@ -48,7 +48,7 @@ from debian import debian_support from packaging.version import parse as version_parse -from skale.core.types import EnvType +from skale_core.types import EnvType from node_cli.configs import ( CHECK_REPORT_PATH, diff --git a/node_cli/core/host.py b/node_cli/core/host.py index 3a4b036a..a040b8c4 100644 --- a/node_cli/core/host.py +++ b/node_cli/core/host.py @@ -22,7 +22,7 @@ from shutil import chown from urllib.parse import urlparse -from skale.core.types import EnvType +from skale_core.types import EnvType from node_cli.core.resources import update_resource_allocation from node_cli.utils.helper import error_exit diff --git a/node_cli/core/node.py b/node_cli/core/node.py index 8bef033c..0398010f 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -88,7 +88,7 @@ print_node_info, ) from node_cli.utils.settings import validate_and_save_node_settings -from skale.core.settings import get_settings +from skale_core.settings import get_settings from node_cli.utils.texts import safe_load_texts logger = logging.getLogger(__name__) @@ -242,7 +242,7 @@ def update_passive(config_file: str) -> None: def cleanup(node_mode: NodeMode, prune: bool = False) -> None: node_mode = upsert_node_mode(node_mode=node_mode) env = compose_node_env(NodeType.SKALE, node_mode) - cleanup_skale_op(node_mode=node_mode, env=env, prune=prune) + cleanup_skale_op(node_mode=node_mode, compose_env=env, prune=prune) logger.info('SKALE node was cleaned up, all containers and data removed') diff --git a/node_cli/core/resources.py b/node_cli/core/resources.py index 9a02d0b9..34f4cac5 100644 --- a/node_cli/core/resources.py +++ b/node_cli/core/resources.py @@ -24,7 +24,7 @@ import psutil -from skale.core.types import EnvType +from skale_core.types import EnvType from node_cli.utils.settings import validate_and_save_node_settings from node_cli.utils.docker_utils import ensure_volume diff --git a/node_cli/core/schains.py b/node_cli/core/schains.py index 174430e7..8aac42f1 100644 --- a/node_cli/core/schains.py +++ b/node_cli/core/schains.py @@ -26,7 +26,7 @@ from pathlib import Path from typing import Dict, Optional -from skale.core.types import EnvType +from skale_core.types import EnvType from lvmpy.src.core import mount, volume_mountpoint from node_cli.configs import ( diff --git a/node_cli/core/static_config.py b/node_cli/core/static_config.py index c2b90641..06e0e7f4 100644 --- a/node_cli/core/static_config.py +++ b/node_cli/core/static_config.py @@ -28,7 +28,7 @@ ) from node_cli.utils.node_type import NodeType -from skale.core.types import EnvType +from skale_core.types import EnvType def get_static_params( diff --git a/node_cli/fair/common.py b/node_cli/fair/common.py index 3b506c74..37fdba67 100644 --- a/node_cli/fair/common.py +++ b/node_cli/fair/common.py @@ -40,7 +40,7 @@ from node_cli.utils.print_formatters import print_node_cmd_error from node_cli.utils.settings import validate_and_save_node_settings from node_cli.utils.texts import safe_load_texts -from skale.core.settings import get_settings +from skale_core.settings import get_settings logger = logging.getLogger(__name__) TEXTS = safe_load_texts() diff --git a/node_cli/fair/record/chain_record.py b/node_cli/fair/record/chain_record.py index 16ff8a72..b6d48886 100644 --- a/node_cli/fair/record/chain_record.py +++ b/node_cli/fair/record/chain_record.py @@ -22,7 +22,7 @@ from typing import cast from datetime import datetime -from skale.core.types import EnvType +from skale_core.types import EnvType from node_cli.core.static_config import get_fair_chain_name from node_cli.fair.record.redis_record import FlatRedisRecord, FieldInfo diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index c164b11a..803fe299 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -24,7 +24,7 @@ import distro -from skale.core.settings import BaseNodeSettings, SkalePassiveSettings, SkaleSettings, get_settings +from skale_core.settings import BaseNodeSettings, SkalePassiveSettings, SkaleSettings, get_settings from node_cli.cli.info import TYPE, VERSION from node_cli.configs import ( diff --git a/node_cli/operations/fair.py b/node_cli/operations/fair.py index 015abfe0..3a48fdbd 100644 --- a/node_cli/operations/fair.py +++ b/node_cli/operations/fair.py @@ -23,8 +23,8 @@ import distro -from skale.core.settings import BaseNodeSettings, FairBaseSettings, FairSettings, get_settings -from skale.core.types import EnvType +from skale_core.settings import BaseNodeSettings, FairBaseSettings, FairSettings, get_settings +from skale_core.types import EnvType from node_cli.cli.info import TYPE, VERSION from node_cli.configs import ( diff --git a/node_cli/utils/docker_utils.py b/node_cli/utils/docker_utils.py index 8f0e1bb1..98a4946c 100644 --- a/node_cli/utils/docker_utils.py +++ b/node_cli/utils/docker_utils.py @@ -29,7 +29,7 @@ from docker.errors import NotFound from docker.models.containers import Container -from skale.core.settings import BaseNodeSettings +from skale_core.settings import BaseNodeSettings from node_cli.configs import ( COMPOSE_PATH, diff --git a/node_cli/utils/settings.py b/node_cli/utils/settings.py index e10e68ce..8e7f957c 100644 --- a/node_cli/utils/settings.py +++ b/node_cli/utils/settings.py @@ -21,7 +21,7 @@ from dotenv.main import DotEnv -from skale.core.settings import ( +from skale_core.settings import ( SETTINGS_MAP, BaseNodeSettings, FairBaseSettings, diff --git a/pyproject.toml b/pyproject.toml index 3f2e25d1..4ab3258d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,7 @@ dependencies = [ "requests-mock==1.12.1", "redis==7.1.1", "PyInstaller==6.18.0", - "skale.py==7.12dev2", + "skale.py-core==7.13.dev1", ] [project.urls] diff --git a/tests/cli/fair_cli_test.py b/tests/cli/fair_cli_test.py index 2f93d1a7..14e2aee9 100644 --- a/tests/cli/fair_cli_test.py +++ b/tests/cli/fair_cli_test.py @@ -137,5 +137,5 @@ def test_cleanup_node(mocked_g_config, inited_node): result = run_command(cleanup_node, ['--yes']) assert result.exit_code == 0 cleanup_mock.assert_called_once_with( - node_mode=NodeMode.ACTIVE, prune=False, env={'SCHAIN_NAME': 'test'} + node_mode=NodeMode.ACTIVE, prune=False, compose_env={'SCHAIN_NAME': 'test'} ) diff --git a/tests/cli/node_test.py b/tests/cli/node_test.py index 63e96259..04317f98 100644 --- a/tests/cli/node_test.py +++ b/tests/cli/node_test.py @@ -347,7 +347,6 @@ def test_restore(request, node_type, node_mode, test_user_conf, mocked_g_config, return_value=CliMeta(version='2.4.0', config_stream='3.0.2'), ), patch('node_cli.operations.base.configure_nftables'), - patch('node_cli.configs.user.validate_alias_or_address'), ): user_conf_path = request.getfixturevalue(test_user_conf).as_posix() result = run_command(restore_node, [backup_path, user_conf_path]) @@ -389,10 +388,8 @@ def test_turn_off_maintenance_on(mocked_g_config, regular_user_conf, active_node resp_mock = response_mock(requests.codes.ok, {'status': 'ok', 'payload': None}) with ( mock.patch('subprocess.run', new=subprocess_run_mock), - mock.patch('node_cli.core.node.SKALE_DIR_ENV_FILEPATH', regular_user_conf.as_posix()), mock.patch('node_cli.core.node.turn_off_op'), mock.patch('node_cli.utils.decorators.is_node_inited', return_value=True), - mock.patch('node_cli.configs.user.validate_alias_or_address'), mock.patch('node_cli.cli.node.TYPE', NodeType.SKALE), ): result = run_command_mock( @@ -425,7 +422,6 @@ def test_turn_on_maintenance_off(mocked_g_config, regular_user_conf, active_node mock.patch('node_cli.core.node.turn_on_op'), mock.patch('node_cli.core.node.is_base_containers_alive'), mock.patch('node_cli.utils.decorators.is_node_inited', return_value=True), - mock.patch('node_cli.configs.user.validate_alias_or_address'), mock.patch('node_cli.cli.node.TYPE', NodeType.SKALE), ): result = run_command_mock( @@ -493,4 +489,4 @@ def test_cleanup_node(mocked_g_config): ): result = run_command(cleanup_node, ['--yes']) assert result.exit_code == 0 - cleanup_mock.assert_called_once_with(node_mode=NodeMode.ACTIVE, prune=False, env={}) + cleanup_mock.assert_called_once_with(node_mode=NodeMode.ACTIVE, prune=False, compose_env={}) diff --git a/tests/cli/passive_node_test.py b/tests/cli/passive_node_test.py index c9ea9527..504c5876 100644 --- a/tests/cli/passive_node_test.py +++ b/tests/cli/passive_node_test.py @@ -45,7 +45,6 @@ def test_init_passive(mocked_g_config, clean_node_options, passive_user_conf): mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), mock.patch('node_cli.operations.base.configure_nftables'), mock.patch('node_cli.utils.decorators.is_node_inited', return_value=False), - mock.patch('node_cli.configs.user.validate_alias_or_address'), ): result = run_command(_init_passive, [passive_user_conf.as_posix()]) @@ -77,7 +76,6 @@ def test_init_passive_archive(mocked_g_config, clean_node_options, passive_user_ mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), mock.patch('node_cli.operations.base.configure_nftables'), mock.patch('node_cli.utils.decorators.is_node_inited', return_value=False), - mock.patch('node_cli.configs.user.validate_alias_or_address'), mock.patch('node_cli.cli.node.TYPE', NodeType.SKALE), ): result = run_command(_init_passive, [passive_user_conf.as_posix(), '--archive']) @@ -121,7 +119,6 @@ def test_update_passive(passive_user_conf, mocked_g_config): 'node_cli.core.node.CliMetaManager.get_meta_info', return_value=CliMeta(version='2.6.0', config_stream='3.0.2'), ), - mock.patch('node_cli.configs.user.validate_alias_or_address'), ): result = run_command(_update_passive, [passive_user_conf.as_posix(), '--yes']) assert result.exit_code == 0 @@ -146,4 +143,5 @@ def test_cleanup_node(mocked_g_config): result = run_command(cleanup_node, ['--yes']) assert result.exit_code == 0 cleanup_mock.assert_called_once_with( - node_mode=NodeMode.PASSIVE, prune=False, env={'SCHAIN_NAME': 'test'}) + node_mode=NodeMode.PASSIVE, prune=False, compose_env={'SCHAIN_NAME': 'test'} + ) diff --git a/tests/core/core_node_test.py b/tests/core/core_node_test.py index ead55528..b9d66960 100644 --- a/tests/core/core_node_test.py +++ b/tests/core/core_node_test.py @@ -14,7 +14,6 @@ NODE_DATA_PATH, SCHAINS_MNT_DIR_REGULAR, SCHAINS_MNT_DIR_SINGLE_CHAIN, - SKALE_DIR, ) from node_cli.configs.resource_allocation import RESOURCE_ALLOCATION_FILEPATH @@ -38,8 +37,6 @@ ALPINE_IMAGE_NAME = 'alpine:3.12' CMD = 'sleep 60' -SKALE_DIR_ENV_FILEPATH = os.path.join(SKALE_DIR, '.env') - WRONG_CONTAINERS = [ 'WRONG_CONTAINER_1', 'skale_WRONG_CONTAINER_4', @@ -283,9 +280,8 @@ def test_init_node(regular_user_conf, no_resource_file): # todo: write new init mock.patch('node_cli.core.node.init_op'), mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), mock.patch('node_cli.utils.helper.post_request', resp_mock), - mock.patch('node_cli.configs.user.validate_alias_or_address'), ): - init(env_filepath=regular_user_conf.as_posix(), node_type=NodeType.SKALE) + init(config_file=regular_user_conf.as_posix(), node_type=NodeType.SKALE) assert os.path.isfile(RESOURCE_ALLOCATION_FILEPATH) @@ -295,7 +291,6 @@ def test_update_node(regular_user_conf, mocked_g_config, resource_file, inited_n with ( mock.patch('subprocess.run', new=subprocess_run_mock), mock.patch('node_cli.core.node.update_op'), - mock.patch('node_cli.core.node.save_env_params'), mock.patch('node_cli.operations.base.configure_nftables'), mock.patch('node_cli.core.host.prepare_host'), mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), @@ -306,7 +301,6 @@ def test_update_node(regular_user_conf, mocked_g_config, resource_file, inited_n 'node_cli.core.node.CliMetaManager.get_meta_info', return_value=CliMeta(version='2.6.0', config_stream='3.0.2'), ), - mock.patch('node_cli.configs.user.validate_alias_or_address'), ): with mock.patch( 'node_cli.utils.helper.requests.get', return_value=safe_update_api_response() @@ -428,13 +422,7 @@ def test_cleanup_success( cleanup(node_mode=NodeMode.ACTIVE) - mock_compose_env.assert_called_once_with( - SKALE_DIR_ENV_FILEPATH, - save=False, - node_type=NodeType.SKALE, - node_mode=NodeMode.ACTIVE, - skip_user_conf_validation=True, - ) + mock_compose_env.assert_called_once_with(NodeType.SKALE, NodeMode.ACTIVE) mock_cleanup_skale_op.assert_called_once_with( - node_mode=NodeMode.ACTIVE, env=mock_env, prune=False + node_mode=NodeMode.ACTIVE, compose_env=mock_env, prune=False ) diff --git a/tests/fair/fair_node_test.py b/tests/fair/fair_node_test.py index 62fcf271..d1908ef6 100644 --- a/tests/fair/fair_node_test.py +++ b/tests/fair/fair_node_test.py @@ -1,9 +1,7 @@ -import os from unittest import mock import pytest -from node_cli.configs import SKALE_DIR from node_cli.fair.boot import init as init_boot from node_cli.fair.boot import update from node_cli.fair.common import cleanup @@ -11,16 +9,12 @@ from node_cli.operations.fair import FairUpdateType from node_cli.utils.node_type import NodeMode, NodeType -SKALE_DIR_ENV_FILEPATH = os.path.join(SKALE_DIR, '.env') - @mock.patch('node_cli.fair.active.time.sleep') @mock.patch('node_cli.fair.active.restore_fair_op') -@mock.patch('node_cli.fair.active.save_env_params') @mock.patch('node_cli.fair.active.compose_node_env') def test_restore_fair( mock_compose_env, - mock_save_env, mock_restore_op, mock_sleep, valid_env_file, @@ -35,13 +29,12 @@ def test_restore_fair( restore(backup_path, valid_env_file) mock_compose_env.assert_called_once_with( - valid_env_file, node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE + node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE ) - mock_save_env.assert_called_once_with(valid_env_file) - expected_env = {**mock_env, 'SKALE_DIR': SKALE_DIR} mock_restore_op.assert_called_once_with( node_mode=NodeMode.ACTIVE, - env=expected_env, + settings=mock.ANY, + compose_env=mock_env, backup_path=backup_path, config_only=False, ) @@ -66,15 +59,12 @@ def test_init_fair_boot( init_boot(valid_env_file) mock_compose_env.assert_called_once_with( - valid_env_file, - node_type=NodeType.FAIR, - node_mode=NodeMode.ACTIVE, - is_fair_boot=True, + node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE ) mock_init_op.assert_called_once_with( - valid_env_file, - mock_env, - NodeMode.ACTIVE, + settings=mock.ANY, + compose_env=mock_env, + node_mode=NodeMode.ACTIVE, ) mock_sleep.assert_called_once() mock_is_alive.assert_called_once_with( @@ -106,17 +96,11 @@ def test_update_fair_boot( update(valid_env_file, pull_config_for_schain) mock_compose_env.assert_called_once_with( - valid_env_file, - inited_node=True, - sync_schains=False, - pull_config_for_schain=pull_config_for_schain, - node_type=NodeType.FAIR, - node_mode=NodeMode.ACTIVE, - is_fair_boot=True, + node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE ) mock_update_op.assert_called_once_with( - valid_env_file, - mock_env, + settings=mock.ANY, + compose_env=mock_env, node_mode=NodeMode.ACTIVE, ) mock_sleep.assert_called_once() @@ -144,15 +128,11 @@ def test_migrate_from_boot( migrate_from_boot(valid_env_file) mock_compose_env.assert_called_once_with( - valid_env_file, - inited_node=True, - sync_schains=False, - node_type=NodeType.FAIR, - node_mode=NodeMode.ACTIVE, + node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE ) mock_migrate_op.assert_called_once_with( - valid_env_file, - mock_env, + settings=mock.ANY, + compose_env=mock_env, node_mode=NodeMode.ACTIVE, update_type=FairUpdateType.FROM_BOOT, force_skaled_start=False, @@ -177,14 +157,10 @@ def test_cleanup_success( cleanup(node_mode=NodeMode.ACTIVE) mock_compose_env.assert_called_once_with( - SKALE_DIR_ENV_FILEPATH, - save=False, - node_type=NodeType.FAIR, - node_mode=NodeMode.ACTIVE, - skip_user_conf_validation=True, + node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE ) mock_cleanup_fair_op.assert_called_once_with( - node_mode=NodeMode.ACTIVE, env=mock_env, prune=False + node_mode=NodeMode.ACTIVE, compose_env=mock_env, prune=False ) @@ -213,13 +189,10 @@ def test_cleanup_calls_operations_in_correct_order( expected_calls = [ mock.call.compose_env( - mock.ANY, - save=False, - node_type=mock.ANY, + node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE, - skip_user_conf_validation=True, ), - mock.call.cleanup_fair_op(node_mode=NodeMode.ACTIVE, env=mock_env, prune=False), + mock.call.cleanup_fair_op(node_mode=NodeMode.ACTIVE, compose_env=mock_env, prune=False), ] manager.assert_has_calls(expected_calls, any_order=False) @@ -244,7 +217,7 @@ def test_cleanup_continues_after_fair_op_error( mock_compose_env.assert_called_once() mock_cleanup_fair_op.assert_called_once_with( - node_mode=NodeMode.ACTIVE, env=mock_env, prune=False + node_mode=NodeMode.ACTIVE, compose_env=mock_env, prune=False ) diff --git a/tests/fixtures/settings.py b/tests/fixtures/settings.py index 11e53ede..55b080b2 100644 --- a/tests/fixtures/settings.py +++ b/tests/fixtures/settings.py @@ -20,7 +20,7 @@ import pytest import tomli_w -from skale.core.settings import get_internal_settings +from skale_core.settings import get_internal_settings from node_cli.configs import INTERNAL_SETTINGS_PATH, NODE_SETTINGS_PATH From 19d277e5d5d0371543669d049b7e04195ebc8246 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Mon, 16 Feb 2026 15:58:40 +0000 Subject: [PATCH 191/198] remove unused modules, fix test pipeline --- node_cli/configs/alias_address_validation.py | 118 ------------------- node_cli/utils/helper.py | 4 - scripts/run_tests.sh | 5 + tests/configs/configs_env_validate_test.py | 117 ------------------ tests/conftest.py | 17 ++- 5 files changed, 12 insertions(+), 249 deletions(-) delete mode 100644 node_cli/configs/alias_address_validation.py delete mode 100644 tests/configs/configs_env_validate_test.py diff --git a/node_cli/configs/alias_address_validation.py b/node_cli/configs/alias_address_validation.py deleted file mode 100644 index ffdbb2ee..00000000 --- a/node_cli/configs/alias_address_validation.py +++ /dev/null @@ -1,118 +0,0 @@ -# -*- coding: utf-8 -*- -# -# This file is part of node-cli -# -# Copyright (C) 2025-Present SKALE Labs -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -from enum import Enum -from typing import Dict, Optional - -import requests - -from node_cli.utils.helper import error_exit, is_contract_address - - -METADATA_URL: str = ( - 'https://raw.githubusercontent.com/skalenetwork/skale-contracts/' - 'refs/heads/deployments/metadata.json' -) - - -class ContractType(Enum): - """Contract types supported by the system using skale-contracts library.""" - - IMA = 'mainnet-ima' - MANAGER = 'skale-manager' - - -def validate_alias_or_address( - alias_or_address: str, contract_type: ContractType, endpoint: str -) -> None: - if is_contract_address(alias_or_address): - validate_contract_address(alias_or_address, endpoint) - else: - validate_contract_alias(alias_or_address, contract_type, endpoint) - - -def validate_contract_address(contract_address: str, endpoint: str) -> None: - try: - response = requests.post( - endpoint, - json={ - 'jsonrpc': '2.0', - 'method': 'eth_getCode', - 'params': [contract_address, 'latest'], - 'id': 1, - }, - ) - if response.status_code != 200: - error_exit(f'Failed to verify contract at address {contract_address}') - result = response.json().get('result') - if not result or result in ['0x', '0x0']: - error_exit(f'No contract code found at address {contract_address}') - except requests.RequestException as e: - error_exit(f'Failed to validate contract address: {str(e)}') - - -def get_deployment_url(alias: str, contract_type: ContractType, network_path: str) -> str: - return ( - f'https://raw.githubusercontent.com/skalenetwork/skale-contracts/' - f'refs/heads/deployments/{network_path}/{contract_type.value}/{alias}.json' - ) - - -def validate_contract_alias(alias: str, contract_type: ContractType, endpoint: str) -> None: - try: - chain_id = get_chain_id(endpoint) - metadata = get_network_metadata() - networks = metadata.get('networks', []) - network_path: Optional[str] = None - for net in networks: - if net.get('chainId') == chain_id: - network_path = net.get('path') - break - if not network_path: - error_exit(f'Network with chain ID {chain_id} not found in metadata') - if not isinstance(network_path, str): - error_exit(f'Invalid network path type: {network_path}') - deployment_url = get_deployment_url(alias, contract_type, network_path) - if requests.get(deployment_url).status_code != 200: - error_exit(f"Contract alias '{alias}' not found for {contract_type.value}") - except requests.RequestException as e: - error_exit(f"Failed to validate contract alias '{alias}': {str(e)}") - - -def get_chain_id(endpoint: str) -> int: - try: - response = requests.post( - endpoint, - json={'jsonrpc': '2.0', 'method': 'eth_chainId', 'params': [], 'id': 1}, - ) - if response.status_code != 200: - error_exit('Failed to get chain ID from endpoint') - return int(response.json()['result'], 16) - except requests.RequestException as e: - error_exit(f'Failed to get chain ID: {str(e)}') - - -def get_network_metadata() -> Dict: - try: - response = requests.get(METADATA_URL) - if response.status_code != 200: - error_exit('Failed to fetch networks metadata') - return response.json() - except requests.RequestException as e: - error_exit(f'Failed to fetch networks metadata: {str(e)}') diff --git a/node_cli/utils/helper.py b/node_cli/utils/helper.py index d2c598fe..8a642aee 100644 --- a/node_cli/utils/helper.py +++ b/node_cli/utils/helper.py @@ -416,10 +416,6 @@ def get_ssh_port(ssh_service_name='ssh'): return DEFAULT_SSH_PORT -def is_contract_address(value: str) -> bool: - return bool(re.fullmatch(r'0x[a-fA-F0-9]{40}', value)) - - def is_btrfs_subvolume(path: str) -> bool: """Check if the given path is a Btrfs subvolume.""" try: diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh index 23592396..1d3a3fe6 100755 --- a/scripts/run_tests.sh +++ b/scripts/run_tests.sh @@ -1,3 +1,8 @@ #!/usr/bin/env bash +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +PROJECT_DIR=$(dirname $DIR) + +. "$DIR/export_env.sh" + py.test --cov=$PROJECT_DIR/ --ignore=tests/core/nftables_test.py --ignore=tests/core/migration_test.py tests/ $@ diff --git a/tests/configs/configs_env_validate_test.py b/tests/configs/configs_env_validate_test.py deleted file mode 100644 index 52622f73..00000000 --- a/tests/configs/configs_env_validate_test.py +++ /dev/null @@ -1,117 +0,0 @@ -from typing import Optional - -import pytest -import requests - -from node_cli.configs.alias_address_validation import ( - ContractType, - get_chain_id, - get_network_metadata, - validate_alias_or_address, - validate_contract_address, - validate_contract_alias, -) - -ENDPOINT = 'http://localhost:8545' - - -class FakeResponse: - def __init__(self, status_code: int, json_data: Optional[dict] = None): - self.status_code = status_code - self._json_data = json_data or {} - - def json(self): - return self._json_data - - -def test_get_chain_id_success(monkeypatch): - fake_response = FakeResponse(200, {'result': '0x1'}) - - def fake_post(url, json): - return fake_response - - monkeypatch.setattr(requests, 'post', fake_post) - assert get_chain_id(ENDPOINT) == 1 - - -def test_get_chain_id_failure(monkeypatch): - fake_response = FakeResponse(404) - - def fake_post(url, json): - return fake_response - - monkeypatch.setattr(requests, 'post', fake_post) - with pytest.raises(SystemExit): - get_chain_id(ENDPOINT) - - -@pytest.mark.parametrize( - 'metadata,status_code,should_raise', - [ - ({'networks': [{'chainId': 1, 'path': 'mainnet'}]}, 200, False), - (None, 404, True), - ], -) -def test_get_network_metadata(requests_mock, metadata, status_code, should_raise): - metadata_url = 'https://raw.githubusercontent.com/skalenetwork/skale-contracts/refs/heads/deployments/metadata.json' - requests_mock.get(metadata_url, json=metadata, status_code=status_code) - - if should_raise: - with pytest.raises(SystemExit): - get_network_metadata() - else: - assert get_network_metadata() == metadata - - -@pytest.mark.parametrize( - 'code,should_raise', - [ - ('0x123', False), - ('0x', True), - ], -) -def test_validate_contract_address(requests_mock, code, should_raise): - requests_mock.post(ENDPOINT, json={'result': code}) - addr = '0x' + 'a' * 40 - if should_raise: - with pytest.raises(SystemExit): - validate_contract_address(addr, ENDPOINT) - else: - validate_contract_address(addr, ENDPOINT) - - -@pytest.mark.parametrize( - 'networks,should_raise', - [ - ([{'chainId': 1, 'path': 'mainnet'}], False), - ([], True), - ], -) -def test_validate_contract_alias(requests_mock, networks, should_raise): - requests_mock.post(ENDPOINT, json={'result': '0x1'}) - metadata_url = 'https://raw.githubusercontent.com/skalenetwork/skale-contracts/refs/heads/deployments/metadata.json' - requests_mock.get(metadata_url, json={'networks': networks}, status_code=200) - - if not should_raise: - alias_url = 'https://raw.githubusercontent.com/skalenetwork/skale-contracts/refs/heads/deployments/mainnet/skale-manager/test-alias.json' - requests_mock.get(alias_url, status_code=200) - validate_contract_alias('test-alias', ContractType.MANAGER, ENDPOINT) - else: - with pytest.raises(SystemExit): - validate_contract_alias('test-alias', ContractType.MANAGER, ENDPOINT) - - -def test_validate_env_alias_or_address_with_address(requests_mock): - addr = '0x' + 'b' * 40 - requests_mock.post(ENDPOINT, json={'result': '0x1'}) - validate_alias_or_address(addr, ContractType.IMA, ENDPOINT) - - -def test_validate_env_alias_or_address_with_alias(requests_mock): - requests_mock.post(ENDPOINT, json={'result': '0x1'}) - metadata_url = 'https://raw.githubusercontent.com/skalenetwork/skale-contracts/refs/heads/deployments/metadata.json' - metadata = {'networks': [{'chainId': 1, 'path': 'mainnet'}]} - requests_mock.get(metadata_url, json=metadata, status_code=200) - alias_url = 'https://raw.githubusercontent.com/skalenetwork/skale-contracts/refs/heads/deployments/mainnet/mainnet-ima/test-alias.json' - requests_mock.get(alias_url, status_code=200) - validate_alias_or_address('test-alias', ContractType.IMA, ENDPOINT) diff --git a/tests/conftest.py b/tests/conftest.py index e9d4b6cc..7dc5cbda 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -296,13 +296,9 @@ def tmp_passive_datadir(): def valid_env_params(): return { 'ENDPOINT': 'http://localhost:8545', - 'IMA_ENDPOINT': 'http://127.0.01', - 'DB_USER': 'user', - 'DB_PASSWORD': 'pass', - 'DB_PORT': '3307', 'NODE_VERSION': 'master', 'FILEBEAT_HOST': '127.0.0.1:3010', - 'SGX_SERVER_URL': 'http://127.0.0.1', + 'SGX_URL': 'http://127.0.0.1', 'BLOCK_DEVICE': '/dev/sss', 'DOCKER_LVMPY_VERSION': 'master', 'ENV_TYPE': 'devnet', @@ -310,6 +306,7 @@ def valid_env_params(): 'ENFORCE_BTRFS': 'False', 'MANAGER_CONTRACTS': 'test-manager', 'IMA_CONTRACTS': 'test-ima', + 'FAIR_CONTRACTS': 'test-fair', } @@ -367,7 +364,7 @@ def regular_user_conf(tmp_path): ENDPOINT=http://localhost:8545 NODE_VERSION='main' FILEBEAT_HOST=127.0.0.1:3010 - SGX_SERVER_URL=http://127.0.0.1 + SGX_URL=http://127.0.0.1 BLOCK_DEVICE=/dev/sss DOCKER_LVMPY_VERSION='master' ENV_TYPE='devnet' @@ -389,7 +386,7 @@ def fair_user_conf(tmp_path): ENDPOINT=http://localhost:8545 NODE_VERSION='main' FILEBEAT_HOST=127.0.0.1:3010 - SGX_SERVER_URL=http://127.0.0.1 + SGX_URL=http://127.0.0.1 BLOCK_DEVICE=/dev/sss ENV_TYPE='devnet' ENFORCE_BTRFS=False @@ -410,11 +407,10 @@ def fair_boot_user_conf(tmp_path): ENDPOINT=http://localhost:8545 NODE_VERSION='main' FILEBEAT_HOST=127.0.0.1:3010 - SGX_SERVER_URL=http://127.0.0.1 + SGX_URL=http://127.0.0.1 BLOCK_DEVICE=/dev/sss ENV_TYPE='devnet' - MANAGER_CONTRACTS='test-manager' - IMA_CONTRACTS='test-ima' + FAIR_CONTRACTS='test-fair' """ with open(test_env_path, 'w') as env_file: env_file.write(test_env) @@ -436,6 +432,7 @@ def passive_user_conf(tmp_path): SCHAIN_NAME='test-schain' ENFORCE_BTRFS=False MANAGER_CONTRACTS='test-manager' + IMA_CONTRACTS='test-ima' """ with open(test_env_path, 'w') as env_file: env_file.write(test_env) From 24ffcb0863e8833ad20cbfe352a87e0d1494d412 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Mon, 16 Feb 2026 16:57:50 +0000 Subject: [PATCH 192/198] fix passive node tests --- tests/.skale/node_data/settings/node.toml | 17 +++++++++++++ tests/cli/fair_passive_node_test.py | 27 +++++++++----------- tests/cli/passive_node_test.py | 5 +++- tests/conftest.py | 31 ++++++++++++++++++++--- 4 files changed, 60 insertions(+), 20 deletions(-) create mode 100644 tests/.skale/node_data/settings/node.toml diff --git a/tests/.skale/node_data/settings/node.toml b/tests/.skale/node_data/settings/node.toml new file mode 100644 index 00000000..cbc85451 --- /dev/null +++ b/tests/.skale/node_data/settings/node.toml @@ -0,0 +1,17 @@ +env_type = "devnet" +endpoint = "http://localhost:8545/" +bite = false +container_stop_timeout = 300 +max_skaled_restart_count = 5 +disable_colors = false +node_version = "main" +block_device = "/dev/sss" +filebeat_host = "127.0.0.1:3010" +container_configs_dir = "" +skip_docker_config = false +skip_docker_cleanup = false +monitoring_containers = false +manager_contracts = "test-manager" +ima_contracts = "test-ima" +schain_name = "test-schain" +enforce_btrfs = false diff --git a/tests/cli/fair_passive_node_test.py b/tests/cli/fair_passive_node_test.py index a8b7818a..00c61ca0 100644 --- a/tests/cli/fair_passive_node_test.py +++ b/tests/cli/fair_passive_node_test.py @@ -15,15 +15,12 @@ init_default_logger() -def test_init_fair_passive(mocked_g_config, tmp_path): - env_file = tmp_path / 'test-env' - env_file.write_text('') +def test_init_fair_passive(mocked_g_config, fair_passive_settings, fair_passive_user_conf): pathlib.Path(SKALE_DIR).mkdir(parents=True, exist_ok=True) with ( mock.patch('subprocess.run', new=subprocess_run_mock), mock.patch('node_cli.fair.common.init_fair_op', return_value=True), mock.patch('node_cli.fair.common.compose_node_env', return_value={}), - mock.patch('node_cli.fair.common.save_env_params'), mock.patch('node_cli.fair.passive.setup_fair_passive'), mock.patch('node_cli.fair.common.time.sleep'), mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), @@ -34,7 +31,7 @@ def test_init_fair_passive(mocked_g_config, tmp_path): result = run_command( init_passive_node, [ - env_file.as_posix(), + fair_passive_user_conf.as_posix(), '--id', '1', ], @@ -42,15 +39,14 @@ def test_init_fair_passive(mocked_g_config, tmp_path): assert result.exit_code == 0 -def test_init_fair_passive_snapshot_any(mocked_g_config, tmp_path): - env_file = tmp_path / 'test-env' - env_file.write_text('') +def test_init_fair_passive_snapshot_any( + mocked_g_config, fair_passive_settings, fair_passive_user_conf +): pathlib.Path(SKALE_DIR).mkdir(parents=True, exist_ok=True) with ( mock.patch('subprocess.run', new=subprocess_run_mock), mock.patch('node_cli.fair.common.init_fair_op', return_value=True), mock.patch('node_cli.fair.common.compose_node_env', return_value={}), - mock.patch('node_cli.fair.common.save_env_params'), mock.patch('node_cli.fair.passive.setup_fair_passive'), mock.patch('node_cli.fair.common.time.sleep'), mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), @@ -61,7 +57,7 @@ def test_init_fair_passive_snapshot_any(mocked_g_config, tmp_path): result = run_command( init_passive_node, [ - env_file.as_posix(), + fair_passive_user_conf.as_posix(), '--id', '2', '--snapshot', @@ -71,9 +67,9 @@ def test_init_fair_passive_snapshot_any(mocked_g_config, tmp_path): assert result.exit_code == 0 -def test_update_fair_passive(mocked_g_config, tmp_path, clean_node_options): - env_file = tmp_path / 'test-env' - env_file.write_text('') +def test_update_fair_passive( + mocked_g_config, fair_passive_settings, fair_passive_user_conf, clean_node_options +): pathlib.Path(NODE_DATA_PATH).mkdir(parents=True, exist_ok=True) with ( mock.patch('subprocess.run', new=subprocess_run_mock), @@ -84,7 +80,7 @@ def test_update_fair_passive(mocked_g_config, tmp_path, clean_node_options): mock.patch('node_cli.operations.base.configure_nftables'), mock.patch('node_cli.utils.decorators.is_node_inited', return_value=True), ): - result = run_command(update_node, [env_file.as_posix(), '--yes']) + result = run_command(update_node, [fair_passive_user_conf.as_posix(), '--yes']) assert result.exit_code == 0 @@ -107,4 +103,5 @@ def test_cleanup_node(mocked_g_config): result = run_command(cleanup_node, ['--yes']) assert result.exit_code == 0 cleanup_mock.assert_called_once_with( - node_mode=NodeMode.PASSIVE, prune=False, env={'SCHAIN_NAME': 'test'}) + node_mode=NodeMode.PASSIVE, compose_env={'SCHAIN_NAME': 'test'}, prune=False + ) diff --git a/tests/cli/passive_node_test.py b/tests/cli/passive_node_test.py index 504c5876..5bb15227 100644 --- a/tests/cli/passive_node_test.py +++ b/tests/cli/passive_node_test.py @@ -65,9 +65,12 @@ def test_init_passive_archive(mocked_g_config, clean_node_options, passive_user_ mock.patch('node_cli.operations.base.sync_skale_node'), mock.patch('node_cli.operations.base.configure_docker'), mock.patch('node_cli.operations.base.prepare_host'), + mock.patch('node_cli.operations.base.save_internal_settings'), + mock.patch('node_cli.operations.base.run_host_checks', return_value=[]), + mock.patch('node_cli.operations.base.set_passive_node_options'), mock.patch('node_cli.operations.base.ensure_filestorage_mapping'), - mock.patch('node_cli.operations.base.link_env_file'), mock.patch('node_cli.operations.base.generate_nginx_config'), + mock.patch('node_cli.operations.base.get_settings'), mock.patch('node_cli.operations.base.prepare_block_device'), mock.patch('node_cli.operations.base.CliMetaManager.update_meta'), mock.patch('node_cli.operations.base.update_resource_allocation'), diff --git a/tests/conftest.py b/tests/conftest.py index 7dc5cbda..dd65b2cb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -49,6 +49,7 @@ from node_cli.utils.global_config import generate_g_config_file from node_cli.utils.node_type import NodeMode from tests.fixtures.settings import ( # noqa: F401 + _cleanup_settings, fair_active_settings, fair_passive_settings, skale_active_settings, @@ -300,12 +301,8 @@ def valid_env_params(): 'FILEBEAT_HOST': '127.0.0.1:3010', 'SGX_URL': 'http://127.0.0.1', 'BLOCK_DEVICE': '/dev/sss', - 'DOCKER_LVMPY_VERSION': 'master', 'ENV_TYPE': 'devnet', - 'SCHAIN_NAME': 'test', 'ENFORCE_BTRFS': 'False', - 'MANAGER_CONTRACTS': 'test-manager', - 'IMA_CONTRACTS': 'test-ima', 'FAIR_CONTRACTS': 'test-fair', } @@ -322,6 +319,7 @@ def valid_env_file(valid_env_params): finally: if file_name: os.unlink(file_name) + _cleanup_settings() @pytest.fixture @@ -376,6 +374,7 @@ def regular_user_conf(tmp_path): yield test_env_path finally: test_env_path.unlink() + _cleanup_settings() @pytest.fixture @@ -397,6 +396,7 @@ def fair_user_conf(tmp_path): yield test_env_path finally: test_env_path.unlink() + _cleanup_settings() @pytest.fixture @@ -417,6 +417,28 @@ def fair_boot_user_conf(tmp_path): yield test_env_path finally: test_env_path.unlink() + _cleanup_settings() + + +@pytest.fixture +def fair_passive_user_conf(tmp_path): + test_env_path = pathlib.Path(tmp_path / 'test-env') + try: + test_env = """ + ENDPOINT=http://localhost:8545 + NODE_VERSION='main' + FILEBEAT_HOST=127.0.0.1:3010 + BLOCK_DEVICE=/dev/sss + ENV_TYPE='devnet' + ENFORCE_BTRFS=False + FAIR_CONTRACTS='test-fair' + """ + with open(test_env_path, 'w') as env_file: + env_file.write(test_env) + yield test_env_path + finally: + test_env_path.unlink() + _cleanup_settings() @pytest.fixture @@ -439,6 +461,7 @@ def passive_user_conf(tmp_path): yield test_env_path finally: test_env_path.unlink() + _cleanup_settings() @pytest.fixture From 34a83f50366f1c8783574ea2d2e33958eb58cab4 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Mon, 16 Feb 2026 17:25:58 +0000 Subject: [PATCH 193/198] fix passive node tests --- node_cli/core/node.py | 2 ++ node_cli/fair/common.py | 1 + tests/.skale/node_data/settings/node.toml | 17 ----------------- tests/cli/passive_node_test.py | 1 - 4 files changed, 3 insertions(+), 18 deletions(-) delete mode 100644 tests/.skale/node_data/settings/node.toml diff --git a/node_cli/core/node.py b/node_cli/core/node.py index 0398010f..7c9e2409 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -247,6 +247,7 @@ def cleanup(node_mode: NodeMode, prune: bool = False) -> None: def compose_node_env(node_type: NodeType, node_mode: NodeMode) -> dict[str, str]: + st = get_settings() if node_mode == NodeMode.PASSIVE or node_type == NodeType.FAIR: mnt_dir = SCHAINS_MNT_DIR_SINGLE_CHAIN else: @@ -256,6 +257,7 @@ def compose_node_env(node_type: NodeType, node_mode: NodeMode) -> dict[str, str] 'SCHAINS_MNT_DIR': mnt_dir, 'FILESTORAGE_MAPPING': FILESTORAGE_MAPPING, 'SKALE_LIB_PATH': SKALE_STATE_DIR, + 'FILEBEAT_HOST': st.filebeat_host, } return {k: v for k, v in env.items() if v != ''} diff --git a/node_cli/fair/common.py b/node_cli/fair/common.py index 37fdba67..98f10a7f 100644 --- a/node_cli/fair/common.py +++ b/node_cli/fair/common.py @@ -76,6 +76,7 @@ def init( print('Fair node is initialized') +@check_inited @check_user def cleanup(node_mode: NodeMode, prune: bool = False) -> None: node_mode = upsert_node_mode(node_mode=node_mode) diff --git a/tests/.skale/node_data/settings/node.toml b/tests/.skale/node_data/settings/node.toml deleted file mode 100644 index cbc85451..00000000 --- a/tests/.skale/node_data/settings/node.toml +++ /dev/null @@ -1,17 +0,0 @@ -env_type = "devnet" -endpoint = "http://localhost:8545/" -bite = false -container_stop_timeout = 300 -max_skaled_restart_count = 5 -disable_colors = false -node_version = "main" -block_device = "/dev/sss" -filebeat_host = "127.0.0.1:3010" -container_configs_dir = "" -skip_docker_config = false -skip_docker_cleanup = false -monitoring_containers = false -manager_contracts = "test-manager" -ima_contracts = "test-ima" -schain_name = "test-schain" -enforce_btrfs = false diff --git a/tests/cli/passive_node_test.py b/tests/cli/passive_node_test.py index 5bb15227..e8419220 100644 --- a/tests/cli/passive_node_test.py +++ b/tests/cli/passive_node_test.py @@ -67,7 +67,6 @@ def test_init_passive_archive(mocked_g_config, clean_node_options, passive_user_ mock.patch('node_cli.operations.base.prepare_host'), mock.patch('node_cli.operations.base.save_internal_settings'), mock.patch('node_cli.operations.base.run_host_checks', return_value=[]), - mock.patch('node_cli.operations.base.set_passive_node_options'), mock.patch('node_cli.operations.base.ensure_filestorage_mapping'), mock.patch('node_cli.operations.base.generate_nginx_config'), mock.patch('node_cli.operations.base.get_settings'), From 7afb2318253bfb6b90ecbf6c13ef0ecd0c4f37ce Mon Sep 17 00:00:00 2001 From: Dmytro Date: Mon, 16 Feb 2026 18:18:24 +0000 Subject: [PATCH 194/198] update fixtures --- tests/conftest.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index dd65b2cb..f42034bd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -49,7 +49,16 @@ from node_cli.utils.global_config import generate_g_config_file from node_cli.utils.node_type import NodeMode from tests.fixtures.settings import ( # noqa: F401 + INTERNAL_FAIR_ACTIVE, + INTERNAL_FAIR_PASSIVE, + INTERNAL_SKALE_ACTIVE, + INTERNAL_SKALE_PASSIVE, + NODE_FAIR_ACTIVE, + NODE_FAIR_PASSIVE, + NODE_SKALE_ACTIVE, + NODE_SKALE_PASSIVE, _cleanup_settings, + _write_settings, fair_active_settings, fair_passive_settings, skale_active_settings, @@ -357,6 +366,7 @@ def set_env_var(name, value): @pytest.fixture def regular_user_conf(tmp_path): test_env_path = pathlib.Path(tmp_path / 'test-env') + _write_settings(INTERNAL_SKALE_ACTIVE, NODE_SKALE_ACTIVE) try: test_env = """ ENDPOINT=http://localhost:8545 @@ -380,6 +390,7 @@ def regular_user_conf(tmp_path): @pytest.fixture def fair_user_conf(tmp_path): test_env_path = pathlib.Path(tmp_path / 'test-env') + _write_settings(INTERNAL_FAIR_ACTIVE, NODE_FAIR_ACTIVE) try: test_env = """ ENDPOINT=http://localhost:8545 @@ -402,6 +413,7 @@ def fair_user_conf(tmp_path): @pytest.fixture def fair_boot_user_conf(tmp_path): test_env_path = pathlib.Path(tmp_path / 'test-env') + _write_settings(INTERNAL_FAIR_ACTIVE, NODE_FAIR_ACTIVE) try: test_env = """ ENDPOINT=http://localhost:8545 @@ -423,6 +435,7 @@ def fair_boot_user_conf(tmp_path): @pytest.fixture def fair_passive_user_conf(tmp_path): test_env_path = pathlib.Path(tmp_path / 'test-env') + _write_settings(INTERNAL_FAIR_PASSIVE, NODE_FAIR_PASSIVE) try: test_env = """ ENDPOINT=http://localhost:8545 @@ -444,6 +457,7 @@ def fair_passive_user_conf(tmp_path): @pytest.fixture def passive_user_conf(tmp_path): test_env_path = pathlib.Path(tmp_path / 'test-env') + _write_settings(INTERNAL_SKALE_PASSIVE, NODE_SKALE_PASSIVE) try: test_env = """ ENDPOINT=http://localhost:8545 From df5bdded17b2d896abeafbd28a5041b3b97aba0a Mon Sep 17 00:00:00 2001 From: Dmytro Date: Mon, 16 Feb 2026 19:18:13 +0000 Subject: [PATCH 195/198] fix fair node tests --- node_cli/fair/common.py | 1 - tests/cli/node_test.py | 4 ++-- tests/conftest.py | 22 ++++++++++++++++++++++ tests/fair/fair_node_test.py | 30 ++++++++++-------------------- 4 files changed, 34 insertions(+), 23 deletions(-) diff --git a/node_cli/fair/common.py b/node_cli/fair/common.py index 98f10a7f..37fdba67 100644 --- a/node_cli/fair/common.py +++ b/node_cli/fair/common.py @@ -76,7 +76,6 @@ def init( print('Fair node is initialized') -@check_inited @check_user def cleanup(node_mode: NodeMode, prune: bool = False) -> None: node_mode = upsert_node_mode(node_mode=node_mode) diff --git a/tests/cli/node_test.py b/tests/cli/node_test.py index 04317f98..a0c064f0 100644 --- a/tests/cli/node_test.py +++ b/tests/cli/node_test.py @@ -384,7 +384,7 @@ def test_maintenance_off(mocked_g_config): ) -def test_turn_off_maintenance_on(mocked_g_config, regular_user_conf, active_node_option): +def test_turn_off_maintenance_on(mocked_g_config, regular_user_conf, active_node_option, skale_active_settings): resp_mock = response_mock(requests.codes.ok, {'status': 'ok', 'payload': None}) with ( mock.patch('subprocess.run', new=subprocess_run_mock), @@ -415,7 +415,7 @@ def test_turn_off_maintenance_on(mocked_g_config, regular_user_conf, active_node assert result.exit_code == CLIExitCodes.UNSAFE_UPDATE -def test_turn_on_maintenance_off(mocked_g_config, regular_user_conf, active_node_option): +def test_turn_on_maintenance_off(mocked_g_config, regular_user_conf, active_node_option, skale_active_settings): resp_mock = response_mock(requests.codes.ok, {'status': 'ok', 'payload': None}) with ( mock.patch('subprocess.run', new=subprocess_run_mock), diff --git a/tests/conftest.py b/tests/conftest.py index f42034bd..3d6cc858 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -66,6 +66,28 @@ ) from tests.helper import TEST_META_V1, TEST_META_V2, TEST_META_V3, TEST_SCHAINS_MNT_DIR_SINGLE_CHAIN +TIMEOUT_PATCHES = [ + 'node_cli.configs.TM_INIT_TIMEOUT', + 'node_cli.configs.RESTORE_SLEEP_TIMEOUT', + 'node_cli.configs.INIT_TIMEOUT', + 'node_cli.core.node.TM_INIT_TIMEOUT', + 'node_cli.core.node.RESTORE_SLEEP_TIMEOUT', + 'node_cli.fair.common.TM_INIT_TIMEOUT', + 'node_cli.fair.common.INIT_TIMEOUT', + 'node_cli.fair.boot.TM_INIT_TIMEOUT', + 'node_cli.fair.active.RESTORE_SLEEP_TIMEOUT', +] + + +@pytest.fixture(autouse=True, scope='session') +def _fast_timeouts(): + patchers = [mock.patch(target, 1) for target in TIMEOUT_PATCHES] + for p in patchers: + p.start() + yield + for p in patchers: + p.stop() + @pytest.fixture() def tmp_dir_path(): diff --git a/tests/fair/fair_node_test.py b/tests/fair/fair_node_test.py index d1908ef6..28ade830 100644 --- a/tests/fair/fair_node_test.py +++ b/tests/fair/fair_node_test.py @@ -28,9 +28,7 @@ def test_restore_fair( restore(backup_path, valid_env_file) - mock_compose_env.assert_called_once_with( - node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE - ) + mock_compose_env.assert_called_once_with(node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE) mock_restore_op.assert_called_once_with( node_mode=NodeMode.ACTIVE, settings=mock.ANY, @@ -58,9 +56,7 @@ def test_init_fair_boot( init_boot(valid_env_file) - mock_compose_env.assert_called_once_with( - node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE - ) + mock_compose_env.assert_called_once_with(node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE) mock_init_op.assert_called_once_with( settings=mock.ANY, compose_env=mock_env, @@ -95,9 +91,7 @@ def test_update_fair_boot( update(valid_env_file, pull_config_for_schain) - mock_compose_env.assert_called_once_with( - node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE - ) + mock_compose_env.assert_called_once_with(node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE) mock_update_op.assert_called_once_with( settings=mock.ANY, compose_env=mock_env, @@ -127,9 +121,7 @@ def test_migrate_from_boot( migrate_from_boot(valid_env_file) - mock_compose_env.assert_called_once_with( - node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE - ) + mock_compose_env.assert_called_once_with(node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE) mock_migrate_op.assert_called_once_with( settings=mock.ANY, compose_env=mock_env, @@ -146,19 +138,17 @@ def test_cleanup_success( mock_compose_env, mock_cleanup_fair_op, mock_is_user_valid, - inited_node, resource_alloc, meta_file_v3, active_node_option, + inited_node, ): mock_env = {'ENV_TYPE': 'devnet'} mock_compose_env.return_value = mock_env cleanup(node_mode=NodeMode.ACTIVE) - mock_compose_env.assert_called_once_with( - node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE - ) + mock_compose_env.assert_called_once_with(node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE) mock_cleanup_fair_op.assert_called_once_with( node_mode=NodeMode.ACTIVE, compose_env=mock_env, prune=False ) @@ -171,10 +161,10 @@ def test_cleanup_calls_operations_in_correct_order( mock_compose_env, mock_cleanup_fair_op, mock_is_user_valid, - inited_node, resource_alloc, meta_file_v3, active_node_option, + inited_node, ): from node_cli.fair.common import cleanup @@ -204,10 +194,10 @@ def test_cleanup_continues_after_fair_op_error( mock_compose_env, mock_cleanup_fair_op, mock_is_user_valid, - inited_node, resource_alloc, meta_file_v3, active_node_option, + inited_node, ): mock_env = {'ENV_TYPE': 'devnet'} mock_compose_env.return_value = mock_env @@ -224,9 +214,9 @@ def test_cleanup_continues_after_fair_op_error( @mock.patch('node_cli.utils.decorators.is_user_valid', return_value=False) def test_cleanup_fails_when_user_invalid( mock_is_user_valid, - inited_node, resource_alloc, meta_file_v3, + inited_node, ): """Test that cleanup fails when user validation fails""" import pytest @@ -253,10 +243,10 @@ def test_cleanup_logs_success_message( mock_compose_env, mock_cleanup_fair_op, mock_is_user_valid, - inited_node, resource_alloc, meta_file_v3, active_node_option, + inited_node, ): mock_env = {'ENV_TYPE': 'devnet'} mock_compose_env.return_value = mock_env From 6447de3be388d859fd4b172bc1f72d5b0dcb2684 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Mon, 16 Feb 2026 19:59:55 +0000 Subject: [PATCH 196/198] fix node tests --- node_cli/fair/common.py | 1 + tests/.skale/config/docker-compose-fair.yml | 5 +++++ tests/core/core_node_test.py | 6 +----- tests/fair/fair_node_test.py | 7 ++++--- 4 files changed, 11 insertions(+), 8 deletions(-) create mode 100644 tests/.skale/config/docker-compose-fair.yml diff --git a/node_cli/fair/common.py b/node_cli/fair/common.py index 37fdba67..98f10a7f 100644 --- a/node_cli/fair/common.py +++ b/node_cli/fair/common.py @@ -76,6 +76,7 @@ def init( print('Fair node is initialized') +@check_inited @check_user def cleanup(node_mode: NodeMode, prune: bool = False) -> None: node_mode = upsert_node_mode(node_mode=node_mode) diff --git a/tests/.skale/config/docker-compose-fair.yml b/tests/.skale/config/docker-compose-fair.yml new file mode 100644 index 00000000..c09f2c08 --- /dev/null +++ b/tests/.skale/config/docker-compose-fair.yml @@ -0,0 +1,5 @@ +services: + test: + container_name: test + image: alpine:latest + network_mode: host diff --git a/tests/core/core_node_test.py b/tests/core/core_node_test.py index b9d66960..b28719d4 100644 --- a/tests/core/core_node_test.py +++ b/tests/core/core_node_test.py @@ -193,11 +193,7 @@ def test_is_base_containers_alive_empty(node_type, node_mode, is_boot): 'fair', ], ) -def test_compose_node_env( - node_type, - node_mode, - expected_mnt_dir, -): +def test_compose_node_env(node_type, node_mode, expected_mnt_dir, regular_user_conf): result_env = compose_node_env( node_type=node_type, node_mode=node_mode, diff --git a/tests/fair/fair_node_test.py b/tests/fair/fair_node_test.py index 28ade830..6554cd3f 100644 --- a/tests/fair/fair_node_test.py +++ b/tests/fair/fair_node_test.py @@ -227,11 +227,12 @@ def test_cleanup_fails_when_user_invalid( cleanup(node_mode=NodeMode.ACTIVE) -def test_cleanup_fails_when_not_inited(ensure_meta_removed, active_node_option): +def test_cleanup_fails_when_not_inited(ensure_meta_removed, active_node_option, fair_user_conf): import pytest - with pytest.raises(SystemExit): - cleanup(node_mode=NodeMode.ACTIVE) + with mock.patch('node_cli.operations.cleanup_fair_op', return_value=None): + with pytest.raises(SystemExit): + cleanup(node_mode=NodeMode.ACTIVE) @mock.patch('node_cli.utils.decorators.is_user_valid', return_value=True) From a31d653d99285702aff26643a999dd82c9dd7a5d Mon Sep 17 00:00:00 2001 From: Dmytro Date: Mon, 16 Feb 2026 20:30:13 +0000 Subject: [PATCH 197/198] fix nftables test --- tests/conftest.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 3d6cc858..8b610d96 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -70,12 +70,6 @@ 'node_cli.configs.TM_INIT_TIMEOUT', 'node_cli.configs.RESTORE_SLEEP_TIMEOUT', 'node_cli.configs.INIT_TIMEOUT', - 'node_cli.core.node.TM_INIT_TIMEOUT', - 'node_cli.core.node.RESTORE_SLEEP_TIMEOUT', - 'node_cli.fair.common.TM_INIT_TIMEOUT', - 'node_cli.fair.common.INIT_TIMEOUT', - 'node_cli.fair.boot.TM_INIT_TIMEOUT', - 'node_cli.fair.active.RESTORE_SLEEP_TIMEOUT', ] From 1b0066e8fe62b1c14c941751229c81122b112631 Mon Sep 17 00:00:00 2001 From: Dmytro Date: Thu, 19 Feb 2026 20:08:30 +0000 Subject: [PATCH 198/198] fix internal settings generation --- node_cli/core/node.py | 4 +++- node_cli/operations/base.py | 1 - 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/node_cli/core/node.py b/node_cli/core/node.py index 7c9e2409..1ab450c0 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -87,7 +87,7 @@ print_node_cmd_error, print_node_info, ) -from node_cli.utils.settings import validate_and_save_node_settings +from node_cli.utils.settings import save_internal_settings, validate_and_save_node_settings from skale_core.settings import get_settings from node_cli.utils.texts import safe_load_texts @@ -154,6 +154,7 @@ def register_node(name, p2p_ip, public_ip, port, domain_name): @check_not_inited def init(config_file: str, node_type: NodeType) -> None: node_mode = NodeMode.ACTIVE + save_internal_settings(node_type=node_type, node_mode=node_mode) settings = validate_and_save_node_settings(config_file, node_type, node_mode) compose_env = compose_node_env(node_type=node_type, node_mode=node_mode) @@ -281,6 +282,7 @@ def update( if (__version__ == 'test' or __version__.startswith('2.6')) and prev_version == '2.5.0': migrate_2_6() logger.info('Node update started') + save_internal_settings(node_type=node_type, node_mode=node_mode) settings = validate_and_save_node_settings(config_file, node_type, node_mode) compose_env = compose_node_env(node_type=node_type, node_mode=node_mode) update_ok = update_op(settings=settings, compose_env=compose_env, node_mode=node_mode) diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index 803fe299..f8f13664 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -178,7 +178,6 @@ def init(settings: BaseNodeSettings, compose_env: dict, node_mode: NodeMode) -> configure_nftables(enable_monitoring=settings.monitoring_containers) prepare_host(env_type=settings.env_type) - save_internal_settings(node_type=NodeType.SKALE, node_mode=node_mode) mark_active_node()