diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 00000000..5df55bfe --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,20 @@ +- always keep the changes minimal and purposeful +- focus on fixing the exact problem or implementing the exact feature +- keep the code simple, do not write defensive code +- do not describe your changes in details after you made changes, focus on writing code +- do not generate any documentation, the code should be self-explanatory +- do not generate any in-line comments +- for the new files, always add a license header, same format as in the existing files +- no commented out code +- no console logs in production code +- no unused imports +- no redundant code - move repeated logic into helper functions +- use type hints to specify the expected types of function arguments and return values + +- check `pyproject.toml` for formatting rules +- always lint changes using `uv run ruff check` +- tests should be placed in `tests/` directory, follow the existing structure and code style +- always use `uv` to run all commands in the repo (e.g., `uv run ruff`, `uv run pytest`, etc.) +- for running tests, export environment variables in the terminal before running the tests: `. ./scripts/export_env.sh` + +- additional external context is located in context directory \ No newline at end of file diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 57852a13..dd86ba23 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,140 +1,78 @@ name: Build and publish + on: pull_request: types: [closed] branches: - - master - develop - beta - stable - 'v*.*.*' - - 'fair' - - 'fair-*' jobs: - create_release: + build_and_release: if: github.event.pull_request.merged - name: Create release + name: Build and create release runs-on: ubuntu-22.04 - outputs: - upload_url: ${{ steps.create_release.outputs.upload_url }} - version: ${{ steps.export_outputs.outputs.version }} - branch: ${{ steps.export_outputs.outputs.branch }} + permissions: + contents: write steps: - name: Checkout code uses: actions/checkout@v4 with: submodules: true - - name: Checkout submodules - run: git submodule update --init --recursive - - - name: Install ubuntu dependencies - run: | - sudo apt-get update - sudo apt-get install python-setuptools - - - name: Set Versions - run: | - bash ./scripts/set_versions_ga.sh - - - name: Set release - run: | - if [[ "$BRANCH" == "stable" ]]; then - export PRERELEASE=false - else - export PRERELEASE=true - fi - echo "PRERELEASE=$PRERELEASE" >> $GITHUB_ENV - - - name: Create Release - id: create_release - uses: actions/create-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - tag_name: ${{ env.VERSION }} - release_name: ${{ env.VERSION }} - draft: false - prerelease: ${{ env.PRERELEASE }} - - - name: Export outputs - id: export_outputs - run: | - echo "::set-output name=version::$VERSION" - echo "::set-output name=branch::$BRANCH" - - build_and_publish: - if: github.event.pull_request.merged - needs: create_release - name: Build and publish ${{ matrix.build_type }} for ${{ matrix.os }} - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-22.04] - build_type: [normal, sync, fair] - steps: - - name: Checkout code - uses: actions/checkout@v4 + - name: Set up Python 3.13 + uses: actions/setup-python@v5 with: - submodules: true + python-version: 3.13 - - name: Set up Python 3.11 - uses: actions/setup-python@v5 - with: - python-version: 3.11 - - - name: Install ubuntu dependencies - if: matrix.os == 'ubuntu-22.04' + - name: Calculate version + id: versioning run: | - sudo apt-get update - - - name: Ensure submodules are updated - run: git submodule update --init --recursive - - - name: Define Asset Name - id: asset_details + bash ./scripts/set_versions_ga.sh + + - name: Determine prerelease status + id: release_info run: | - ASSET_BASE_NAME="skale-${{ needs.create_release.outputs.version }}-Linux-x86_64" - if [[ "${{ matrix.build_type }}" == "normal" ]]; then - echo "FINAL_ASSET_NAME=${ASSET_BASE_NAME}" >> $GITHUB_OUTPUT + if [[ "${{ env.BRANCH }}" == "stable" ]]; then + echo "prerelease=false" >> $GITHUB_OUTPUT else - echo "FINAL_ASSET_NAME=${ASSET_BASE_NAME}-${{ matrix.build_type }}" >> $GITHUB_OUTPUT + echo "prerelease=true" >> $GITHUB_OUTPUT fi - - name: Build ${{ matrix.build_type }} release binary + - name: Build binaries + id: build run: | mkdir -p ${{ github.workspace }}/dist docker build . -t node-cli-builder + docker run --rm -v ${{ github.workspace }}/dist:/app/dist node-cli-builder \ - bash scripts/build.sh ${{ needs.create_release.outputs.version }} ${{ needs.create_release.outputs.branch }} ${{ matrix.build_type }} - echo "Contents of dist directory:" - ls -altr ${{ github.workspace }}/dist/ - docker rm -f $(docker ps -aq) || true - - - name: Save sha512sum for ${{ steps.asset_details.outputs.FINAL_ASSET_NAME }} + bash scripts/build.sh ${{ env.VERSION }} ${{ env.BRANCH }} skale + + docker run --rm -v ${{ github.workspace }}/dist:/app/dist node-cli-builder \ + bash scripts/build.sh ${{ env.VERSION }} ${{ env.BRANCH }} fair + + echo "dist_path=${{ github.workspace }}/dist" >> $GITHUB_OUTPUT + + - name: Generate checksums run: | - cd ${{ github.workspace }}/dist - sha512sum ${{ steps.asset_details.outputs.FINAL_ASSET_NAME }} > ${{ steps.asset_details.outputs.FINAL_ASSET_NAME }}.sha512sum - echo "Checksum file created: ${{ steps.asset_details.outputs.FINAL_ASSET_NAME }}.sha512sum" - cat ${{ steps.asset_details.outputs.FINAL_ASSET_NAME }}.sha512sum - - - name: Upload release binary (${{ matrix.build_type }}) - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + cd ${{ steps.build.outputs.dist_path }} + for file in skale-*; do + sha512sum "$file" > "$file.sha512" + done + echo "Checksums generated:" + ls -l *.sha512 + + - name: Create GitHub release + uses: softprops/action-gh-release@v2 with: - upload_url: ${{ needs.create_release.outputs.upload_url }} - asset_path: ${{ github.workspace }}/dist/${{ steps.asset_details.outputs.FINAL_ASSET_NAME }} - asset_name: ${{ steps.asset_details.outputs.FINAL_ASSET_NAME }} - asset_content_type: application/octet-stream - - - name: Upload release checksum (${{ matrix.build_type }}) - uses: actions/upload-release-asset@v1 + tag_name: ${{ env.VERSION }} + name: Release ${{ env.VERSION }} + draft: false + prerelease: ${{ steps.release_info.outputs.prerelease }} + generate_release_notes: true + files: | + ${{ steps.build.outputs.dist_path }}/skale-* env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ needs.create_release.outputs.upload_url }} - asset_path: ${{ github.workspace }}/dist/${{ steps.asset_details.outputs.FINAL_ASSET_NAME }}.sha512sum - asset_name: ${{ steps.asset_details.outputs.FINAL_ASSET_NAME }}.sha512 - asset_content_type: text/plain \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1a230126..5f589d47 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -6,9 +6,9 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: - python-version: [3.11] + python-version: ['3.13'] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: submodules: true @@ -16,10 +16,19 @@ jobs: run: git submodule update --init - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + - name: Install uv + run: curl -LsSf https://astral.sh/uv/install.sh | sh + + - name: Cache uv + uses: actions/cache@v4 + with: + path: ~/.cache/uv + key: ${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('pyproject.toml') }} + - name: Install ubuntu dependencies run: | sudo apt-get update @@ -27,40 +36,31 @@ jobs: - name: Install python dependencies run: | - python -m pip install --upgrade pip - pip install -e ".[dev]" + uv venv + uv pip install -e ".[dev]" - name: Generate info - run: bash ./scripts/generate_info.sh 1.0.0 my-branch normal + run: bash ./scripts/generate_info.sh 1.0.0 my-branch skale - name: Check with ruff run: | - ruff check + uv run ruff check - - name: Build binary - normal - run: | - mkdir -p ./dist - docker build . -t node-cli-builder - docker run -v /home/ubuntu/dist:/app/dist node-cli-builder bash scripts/build.sh test test normal - docker rm -f $(docker ps -aq) - - - name: Check build - normal - run: sudo /home/ubuntu/dist/skale-test-Linux-x86_64 + - name: Build docker image + run: docker build . -t node-cli-builder - - name: Build binary - sync + - name: Build binary - skale run: | mkdir -p ./dist - docker build . -t node-cli-builder - docker run -v /home/ubuntu/dist:/app/dist node-cli-builder bash scripts/build.sh test test sync + docker run -v /home/ubuntu/dist:/app/dist node-cli-builder bash scripts/build.sh test test skale docker rm -f $(docker ps -aq) - - name: Check build - sync - run: sudo /home/ubuntu/dist/skale-test-Linux-x86_64-sync + - name: Check build - skale + run: sudo /home/ubuntu/dist/skale-test-Linux-x86_64 - name: Build binary - fair run: | mkdir -p ./dist - docker build . -t node-cli-builder docker run -v /home/ubuntu/dist:/app/dist node-cli-builder bash scripts/build.sh test test fair docker rm -f $(docker ps -aq) @@ -69,7 +69,7 @@ jobs: - name: Run prepare test build run: | - bash scripts/build.sh test test normal + uv run bash scripts/build.sh test test skale - name: Run redis run: | @@ -78,8 +78,8 @@ jobs: - name: Run tests run: | export PYTHONPATH=${PYTHONPATH}:/usr/lib/python3/dist-packages/ - bash ./scripts/run_tests.sh + uv run bash ./scripts/run_tests.sh - name: Run nftables tests run: | - scripts/run_nftables_test.sh + uv run scripts/run_nftables_test.sh diff --git a/.gitignore b/.gitignore index c1b8fee2..65232943 100644 --- a/.gitignore +++ b/.gitignore @@ -113,7 +113,7 @@ node_cli/cli/info.py meta.json -disk_mountpoint.txt +block_device.txt sgx_server_url.txt resource_allocation.json conf.json @@ -125,3 +125,4 @@ tests/.skale/node_data/node_options.json tests/.skale/config/nginx.conf.j2 .zed +uv.lock \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index c2fc6972..7fc37545 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,29 +1,32 @@ -FROM python:3.11-bookworm - -ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt install -y \ - git \ - build-essential \ - software-properties-common \ - zlib1g-dev \ - libssl-dev \ - libffi-dev \ - swig \ - iptables \ - nftables \ - python3-nftables \ - libxslt-dev \ - kmod - - -RUN mkdir /app +FROM python:3.13-slim-bookworm AS builder + +COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv + WORKDIR /app -COPY . . +COPY pyproject.toml ./ + +RUN uv pip install --system --no-cache ".[dev]" + +FROM python:3.13-slim-bookworm -ENV PATH=/app/buildvenv/bin:$PATH -ENV PYTHONPATH="{PYTHONPATH}:/usr/lib/python3/dist-packages" +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + git \ + iptables \ + nftables \ + python3-nftables \ + kmod \ + wget \ + binutils && \ + rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY --from=builder /usr/local/lib/python3.13/site-packages /usr/local/lib/python3.13/site-packages +COPY --from=builder /usr/local/bin /usr/local/bin + +COPY . . -RUN pip install --upgrade pip && \ - pip install wheel setuptools==63.2.0 && \ - pip install -e '.[dev]' +ENV PYTHONPATH="/app:/usr/lib/python3/dist-packages" +ENV COLUMNS=80 diff --git a/README.md b/README.md index 9336f431..5a2ef37f 100644 --- a/README.md +++ b/README.md @@ -4,13 +4,13 @@ ![Test](https://github.com/skalenetwork/node-cli/workflows/Test/badge.svg) [![Discord](https://img.shields.io/discord/534485763354787851.svg)](https://discord.gg/vvUtWJB) -SKALE Node CLI, part of the SKALE suite of validator tools, is the command line interface to setup, register and maintain your SKALE node. It comes in three distinct build types: Standard (for validator nodes), Sync (for dedicated sChain synchronization), and Fair. +SKALE Node CLI, part of the SKALE suite of validator tools, is the command line interface to setup, register and maintain your SKALE node. It comes in three distinct build types: Standard (for validator nodes), Passive (for dedicated sChain synchronization), and Fair. ## Table of Contents 1. [Installation](#installation) 1. [Standard Node Binary](#standard-node-binary) - 2. [Sync Node Binary](#sync-node-binary) + 2. [Passive Node Binary](#passive-node-binary) 3. [Fair Node Binary](#fair-node-binary) 4. [Permissions and Testing](#permissions-and-testing) 2. [Standard Node Usage (`skale` - Normal Build)](#standard-node-usage-skale---normal-build) @@ -21,14 +21,19 @@ SKALE Node CLI, part of the SKALE suite of validator tools, is the command line 5. [Health commands (Standard)](#health-commands-standard) 6. [SSL commands (Standard)](#ssl-commands-standard) 7. [Logs commands (Standard)](#logs-commands-standard) - 8. [Resources allocation commands (Standard)](#resources-allocation-commands-standard) -3. [Sync Node Usage (`skale` - Sync Build)](#sync-node-usage-skale---sync-build) - 1. [Top level commands (Sync)](#top-level-commands-sync) - 2. [Sync node commands](#sync-node-commands) +3. [Passive Node Usage (`skale` - Passive Build)](#passive-node-usage-skale---passive-build) + 1. [Top level commands (Passive)](#top-level-commands-passive) + 2. [Passive node commands](#passive-node-commands) 4. [Fair Node Usage (`fair`)](#fair-node-usage-fair) 1. [Top level commands (Fair)](#top-level-commands-fair) 2. [Fair Boot commands](#fair-boot-commands) 3. [Fair Node commands](#fair-node-commands) + 4. [Fair Chain commands](#fair-chain-commands) + 5. [Fair Wallet commands](#fair-wallet-commands) + 6. [Fair Logs commands](#fair-logs-commands) + 7. [Fair SSL commands](#fair-ssl-commands) + 8. [Fair Staking commands](#fair-staking-commands) + 9. [Passive Fair Node commands](#passive-fair-node-commands) 5. [Exit codes](#exit-codes) 6. [Development](#development) @@ -40,34 +45,24 @@ SKALE Node CLI, part of the SKALE suite of validator tools, is the command line Ensure that the following packages are installed: **docker**, **docker-compose** (1.27.4+) -### Standard Node Binary +### SKALE Node Binary -This binary (`skale-VERSION-OS`) is used for managing standard SKALE validator nodes. +This binary (`skale-VERSION-OS`) is used for managing SKALE validator nodes. ```shell -# Replace {version} with the desired release version (e.g., 2.6.0) -VERSION_NUM={version} && \ -sudo -E bash -c "curl -L https://github.com/skalenetwork/node-cli/releases/download/$VERSION_NUM/skale-$VERSION_NUM-`uname -s`-`uname -m` > /usr/local/bin/skale" -``` - -### Sync Node Binary - -This binary (`skale-VERSION-OS-sync`) is used for managing dedicated Sync nodes. **Ensure you download the correct `-sync` suffixed binary for Sync node operations.** - -```shell -# Replace {version} with the desired release version (e.g., 2.6.0) -VERSION_NUM={version} && \ -sudo -E bash -c "curl -L https://github.com/skalenetwork/node-cli/releases/download/$VERSION_NUM/skale-$VERSION_NUM-`uname -s`-`uname -m`-sync > /usr/local/bin/skale" +# Replace {version} with the desired release version (e.g., 3.0.0) +CLI_VERSION={version} && \ +sudo -E bash -c "curl -L https://github.com/skalenetwork/node-cli/releases/download/$CLI_VERSION/skale-$CLI_VERSION-`uname -s`-`uname -m` > /usr/local/bin/skale" ``` ### Fair Node Binary -This binary (`skale-VERSION-OS-fair`) is used specifically for managing nodes on the Fair network. It is named `fair`. +This binary (`skale-VERSION-OS-fair`) is used for managing nodes on the Fair network. ```shell -# Replace {version} with the desired release version (e.g., 2.6.0) -VERSION_NUM={version} && \ -sudo -E bash -c "curl -L https://github.com/skalenetwork/node-cli/releases/download/$VERSION_NUM/skale-$VERSION_NUM-`uname -s`-`uname -m`-fair > /usr/local/bin/fair" +# Replace {version} with the desired release version (e.g., 3.0.0) +CLI_VERSION={version} && \ +sudo -E bash -c "curl -L https://github.com/skalenetwork/node-cli/releases/download/$CLI_VERSION/skale-$CLI_VERSION-`uname -s`-`uname -m`-fair > /usr/local/bin/fair" ``` ### Permissions and Testing @@ -75,7 +70,7 @@ sudo -E bash -c "curl -L https://github.com/skalenetwork/node-cli/releases/downl Apply executable permissions to the downloaded binary (adjust name accordingly): ```shell -# For Standard or Sync binary +# For Standard or Passive binary sudo chmod +x /usr/local/bin/skale # For Fair binary @@ -85,7 +80,7 @@ sudo chmod +x /usr/local/bin/fair Test the installation: ```shell -# Standard or Sync build +# Standard or Passive build skale --help # Fair build @@ -153,9 +148,9 @@ Arguments: Required environment variables in `ENV_FILE`: * `SGX_SERVER_URL` - SGX server URL. -* `DISK_MOUNTPOINT` - Mount point for storing sChains data. -* `DOCKER_LVMPY_STREAM` - Stream of `docker-lvmpy` to use. -* `CONTAINER_CONFIGS_STREAM` - Stream of `skale-node` to use. +* `BLOCK_DEVICE` - Absolute path to a dedicated raw block device (e.g. /dev/sdc) +* `DOCKER_LVMPY_VERSION` - Version of `docker-lvmpy`. +* `NODE_VERSION` - Version of `skale-node`. * `ENDPOINT` - RPC endpoint of the network where SKALE Manager is deployed. * `MANAGER_CONTRACTS` - SKALE Manager `message_proxy_mainnet` contract alias or address. * `IMA_CONTRACTS` - IMA `skale_manager` contract alias or address. @@ -515,57 +510,26 @@ Options: * `--container`, `-c` - Dump logs only from specified container. -### Resources allocation commands (Standard) - -> Prefix: `skale resources-allocation` - -Manage the resources allocation file for the standard node. - -#### Show allocation file - -Show resources allocation file: - -```shell -skale resources-allocation show -``` - -#### Generate/update allocation file - -Generate/update allocation file: - -```shell -skale resources-allocation generate [ENV_FILE] [--yes] [-f/--force] -``` - -Arguments: - -* `ENV_FILE` - path to .env file (required parameters are listed in the `skale node init` command). - -Options: - -* `--yes` - generate without additional confirmation. -* `-f/--force` - rewrite allocation file if it exists. - *** -## Sync Node Usage (`skale` - Sync Build) +## Passive Node Usage (`skale` - Passive Build) -Commands available in the **sync `skale` binary** for managing dedicated Sync nodes. +Commands available in the **passive `skale` binary** for managing dedicated Passive nodes. Note that this binary contains a **different set of commands** compared to the standard build. -### Top level commands (Sync) +### Top level commands (Passive) -#### Info (Sync) +#### Info (Passive) -Print build info for the `skale` (sync) binary. +Print build info for the `skale` (passive) binary. ```shell skale info ``` -#### Version (Sync) +#### Version (Passive) -Print version number for the `skale` (sync) binary. +Print version number for the `skale` (passive) binary. ```shell skale version @@ -575,16 +539,16 @@ Options: * `--short` - prints version only, without additional text. -### Sync node commands +### Passive node commands -> Prefix: `skale sync-node` +> Prefix: `skale passive-node` -#### Sync node initialization +#### Passive node initialization -Initialize a dedicated Sync node on the current machine. +Initialize a dedicated Passive node on the current machine. ```shell -skale sync-node init [ENV_FILE] [--indexer | --archive] [--snapshot] [--snapshot-from ] [--yes] +skale passive-node init [ENV_FILE] [--indexer | --archive] [--snapshot] [--snapshot-from ] [--yes] ``` Arguments: @@ -593,9 +557,9 @@ Arguments: Required environment variables in `ENV_FILE`: -* `DISK_MOUNTPOINT` - Mount point for storing sChain data. -* `DOCKER_LVMPY_STREAM` - Stream of `docker-lvmpy`. -* `CONTAINER_CONFIGS_STREAM` - Stream of `skale-node`. +* `BLOCK_DEVICE` - Absolute path to a dedicated raw block device (e.g. /dev/sdc). +* `DOCKER_LVMPY_VERSION` - Version of `docker-lvmpy`. +* `NODE_VERSION` - Version of `skale-node`. * `ENDPOINT` - RPC endpoint of the network where SKALE Manager is deployed. * `MANAGER_CONTRACTS` - SKALE Manager alias or address. * `IMA_CONTRACTS` - IMA alias or address. @@ -613,12 +577,12 @@ Options: * `--snapshot-from ` - Specify the IP of another node to download a snapshot from. * `--yes` - Initialize without additional confirmation. -#### Sync node update +#### Passive node update -Update the Sync node software and configuration. +Update the Passive node software and configuration. ```shell -skale sync-node update [ENV_FILEPATH] [--yes] +skale passive-node update [ENV_FILEPATH] [--yes] ``` Arguments: @@ -629,21 +593,21 @@ Options: * `--yes` - Update without additionalconfirmation. -> NOTE: You can just update a file with environment variables used during `skale sync-node init`. +> NOTE: You can just update a file with environment variables used during `skale passive-node init`. -#### Sync node cleanup +#### Passive node cleanup -Remove all data and containers for the Sync node. +Remove all data and containers for the Passive node. ```shell -skale sync-node cleanup [--yes] +skale passive-node cleanup [--yes] ``` Options: * `--yes` - Cleanup without confirmation. -> WARNING: This command removes all Sync node data. +> WARNING: This command removes all Passive node data. *** @@ -679,23 +643,35 @@ Options: Commands for a Fair node in the Boot phase. +#### Fair Boot Info + +Get information about the Fair node during boot phase. + +```shell +fair boot info [--format FORMAT] +``` + +Options: + +* `--format`/`-f` - Output format (`json` or `text`). + #### Fair Boot Initialization Initialize the Fair node boot phase. ```shell -fair boot init [ENV_FILE] +fair boot init ``` Arguments: -* `ENV_FILE` - path to .env file (required). +* `ENV_FILE` - Path to the environment file containing configuration. Required environment variables in `ENV_FILE`: * `SGX_SERVER_URL` - SGX server URL. -* `DISK_MOUNTPOINT` - Mount point for storing data (BTRFS recommended). -* `CONTAINER_CONFIGS_STREAM` - Stream of `skale-node` configs. +* `BLOCK_DEVICE` - Absolute path to a dedicated raw block device (e.g. /dev/sdc). +* `NODE_VERSION` - Version of `skale-node`. * `ENDPOINT` - RPC endpoint of the network where Fair Manager is deployed. * `MANAGER_CONTRACTS` - SKALE Manager alias or address. * `IMA_CONTRACTS` - IMA alias or address (*Note: Required by boot service, may not be used by Fair itself*). @@ -714,19 +690,16 @@ Register the Fair node with Fair Manager *during* the boot phase. fair boot register --name --ip --domain [--port ] ``` -Required arguments: - -* `--name`/`-n` - Fair node name. -* `--ip` - Public IP for RPC connections and consensus. -* `--domain`/`-d` - Fair node domain name (e.g., `fair1.example.com`). - -Optional arguments: +Options: -* `--port`/`-p` - Base port for node sChains (default: `10000`). +* `--name`/`-n` - Fair node name (required). +* `--ip` - Public IP for RPC connections & consensus (required). +* `--domain`/`-d` - Fair node domain name (e.g., `fair1.example.com`, required). +* `--port`/`-p` - Base port for node sChains (default: from configuration). #### Fair Boot Signature -Get the node signature for a validator ID *during* the boot phase. +Get the node signature for a validator ID during boot phase. ```shell fair boot signature @@ -736,21 +709,37 @@ Arguments: * `VALIDATOR_ID` - The ID of the validator requesting the signature. -#### Fair Boot Migrate +#### Fair Boot Update -Migrate the Fair node from the boot phase to the main phase (regular operation). +Update the Fair node software during boot phase. ```shell -fair boot migrate [ENV_FILEPATH] [--yes] +fair boot update [--yes] [--pull-config SCHAIN] ``` Arguments: -* `ENV_FILEPATH` - Path to the .env file. +* `ENV_FILE` - Path to the environment file for node configuration. + +Required environment variables in `ENV_FILE`: + +* `SGX_SERVER_URL` - SGX server URL. +* `BLOCK_DEVICE` - Absolute path to a dedicated raw block device (e.g. /dev/sdc). +* `NODE_VERSION` - Version of `skale-node`. +* `ENDPOINT` - RPC endpoint of the network where Fair Manager is deployed. +* `MANAGER_CONTRACTS` - SKALE Manager alias or address. +* `IMA_CONTRACTS` - IMA alias or address (*Note: Required by boot service, may not be used by Fair itself*). +* `FILEBEAT_HOST` - URL/IP:Port of the Filebeat log server. +* `ENV_TYPE` - Environment type (e.g., 'mainnet', 'devnet'). + +Optional variables: + +* `MONITORING_CONTAINERS` - Enable monitoring containers (`cadvisor`, `node-exporter`). Options: -* `--yes` - Migrate without confirmation. +* `--yes` - Update without confirmation prompt. +* `--pull-config` - Pull configuration for specific sChain (hidden option). ### Fair Node commands @@ -758,47 +747,157 @@ Options: Commands for managing a Fair node during its regular operation (main phase). -#### Fair Node Initialization (Placeholder) +#### Fair Node Info + +Get information about the Fair node. + +```shell +fair node info [--format FORMAT] +``` + +Options: + +* `--format`/`-f` - Output format (`json` or `text`). + +#### Fair Node Initialization Initialize the regular operation phase of the Fair node. ```shell -fair node init +fair node init ``` -> **Note:** This command is currently a placeholder and not implemented. +Arguments: + +* `ENV_FILEPATH` - Path to the environment file for node configuration. + +Required environment variables in `ENV_FILEPATH`: + +* `FAIR_CONTRACTS` - Fair contracts alias or address (e.g., `mainnet`). +* `NODE_VERSION` - Version of `skale-node`. +* `BOOT_ENDPOINT` - RPC endpoint of the Fair network (e.g., `https://rpc.fair.cloud/`). +* `SGX_SERVER_URL` - SGX server URL (e.g., `https://127.0.0.1:1026/`). +* `BLOCK_DEVICE` - Absolute path to a dedicated raw block device (e.g., `/dev/sdc`). +* `ENV_TYPE` - Environment type (e.g., `mainnet`). + +Optional variables: + +* `ENFORCE_BTRFS` - Format existing filesystem on attached disk (`True`/`False`). +* `FILEBEAT_HOST` - URL of the Filebeat log server to send logs. -#### Fair Node Registration (Placeholder) +#### Fair Node Registration -Register the node during regular operation. +Register the Fair node with the specified IP address. ```shell -fair node register +fair node register --ip ``` -> **Note:** This command is currently a placeholder and not implemented. +Options: + +* `--ip` - Public IP address for the Fair node (required). -#### Fair Node Update (Placeholder) +#### Fair Node Update -Update the Fair node during regular operation. +Update the Fair node software. ```shell -fair node update [ENV_FILEPATH] [--yes] [--unsafe] +fair node update [--yes] [--force-skaled-start] ``` -> **Note:** This command is currently a placeholder and not implemented. +Arguments: -#### Fair Node Signature +* `ENV_FILEPATH` - Path to the environment file for node configuration. -Get the node signature for a validator ID during regular operation. +Required environment variables in `ENV_FILEPATH`: + +* `FAIR_CONTRACTS` - Fair contracts alias or address (e.g., `mainnet`). +* `NODE_VERSION` - Version of `skale-node`. +* `BOOT_ENDPOINT` - RPC endpoint of the Fair network (e.g., `https://rpc.fair.cloud/`). +* `SGX_SERVER_URL` - SGX server URL (e.g., `https://127.0.0.1:1026/`). +* `BLOCK_DEVICE` - Absolute path to a dedicated raw block device (e.g., `/dev/sdc`). +* `ENV_TYPE` - Environment type (e.g., `mainnet`). + +Optional variables: + +* `ENFORCE_BTRFS` - Format existing filesystem on attached disk (`True`/`False`). +* `FILEBEAT_HOST` - URL of the Filebeat log server to send logs. + +Options: + +* `--yes` - Update without confirmation prompt. +* `--force-skaled-start` - Force skaled container to start (hidden option). + +#### Fair Node turn-off + +Turn off the Fair node containers. ```shell -fair node signature +fair node turn-off [--yes] +``` + +Options: + +* `--yes` - Turn off without confirmation. + +#### Fair Node turn-on + +Turn on the Fair node containers. + +```shell +fair node turn-on [ENV_FILEPATH] [--yes] ``` Arguments: -* `VALIDATOR_ID` - The ID of the validator requesting the signature. +* `ENV_FILEPATH` - Path to the .env file. + +Options: + +* `--yes` - Turn on without additional confirmation. + +#### Fair Node Migrate + +Switch from boot phase to regular Fair node operation. + +```shell +fair node migrate [--yes] +``` + +Arguments: + +* `ENV_FILEPATH` - Path to the environment file for node configuration. + +Required environment variables in `ENV_FILEPATH`: + +* `FAIR_CONTRACTS` - Fair contracts alias or address (e.g., `mainnet`). +* `NODE_VERSION` - Version of `skale-node`. +* `BOOT_ENDPOINT` - RPC endpoint of the Fair network (e.g., `https://rpc.fair.cloud/`). +* `SGX_SERVER_URL` - SGX server URL (e.g., `https://127.0.0.1:1026/`). +* `BLOCK_DEVICE` - Absolute path to a dedicated raw block device (e.g., `/dev/sdc`). +* `ENV_TYPE` - Environment type (e.g., `mainnet`). + +Optional variables: + +* `ENFORCE_BTRFS` - Format existing filesystem on attached disk (`True`/`False`). +* `FILEBEAT_HOST` - URL of the Filebeat log server to send logs. + +Options: + +* `--yes` - Migrate without confirmation prompt. + +#### Fair Node Repair + +Toggle fair chain repair mode. + +```shell +fair node repair [--snapshot-from SOURCE] [--yes] +``` + +Options: + +* `--snapshot-from` - Source for snapshots (`any` by default, hidden option). +* `--yes` - Proceed without confirmation prompt. #### Fair Node Backup @@ -822,13 +921,358 @@ fair node restore [--config-only] Arguments: -* `BACKUP_PATH` - Path to the archive. +* `BACKUP_PATH` - Path to the backup archive. * `ENV_FILE` - Path to the .env file for the restored node configuration. Options: * `--config-only` - Only restore configuration files. +#### Fair Node Cleanup + +Cleanup Fair node data and configuration. + +```shell +fair node cleanup [--yes] +``` + +Options: + +* `--yes` - Cleanup without confirmation prompt. + +#### Fair Node Change IP + +Change the IP address of the Fair node. + +```shell +fair node change-ip +``` + +Arguments: + +* `IP_ADDRESS` - New public IP address for the Fair node. + +### Fair Chain commands + +> Prefix: `fair chain` + +Commands for managing and monitoring the Fair chain state and configuration. + +#### Fair Chain Record + +Get information about the Fair chain record, including chain name, configuration status, DKG status, and operational metadata. + +```shell +fair chain record [--json] +``` + +Options: + +* `--json` - Output in JSON format instead of formatted table. + +#### Fair Chain Checks + +Get the status of Fair chain checks, including configuration checks and skaled checks. + +```shell +fair chain checks [--json] +``` + +Options: + +* `--json` - Output in JSON format instead of formatted table. + +### Fair Wallet commands + +> Prefix: `fair wallet` + +Commands for managing the node wallet. + +#### Fair Wallet Info + +Get information about the SKALE node wallet. + +```shell +fair wallet info [--format FORMAT] +``` + +Options: + +* `--format`/`-f` - Output format (`json` or `text`). + +#### Fair Wallet Send + +Send ETH from SKALE node wallet to an address. + +```shell +fair wallet send
[--yes] +``` + +Arguments: + +* `ADDRESS` - Destination address for ETH transfer. +* `AMOUNT` - Amount of ETH to send (as float). + +Options: + +* `--yes` - Send without confirmation prompt. + +### Fair Logs commands + +> Prefix: `fair logs` + +Commands for managing and accessing node logs. + +#### Fair CLI Logs + +Fetch the logs of the node-cli. + +```shell +fair logs cli [--debug] +``` + +Options: + +* `--debug` - Show debug logs instead of regular logs. + +#### Fair Logs Dump + +Dump all logs from the connected node. + +```shell +fair logs dump [--container CONTAINER] +``` + +Arguments: + +* `PATH` - Path where the logs dump will be saved. + +Options: + +* `--container`/`-c` - Dump logs only from specified container. + +### Fair SSL commands + +> Prefix: `fair ssl` + +Commands for managing SSL certificates for sChains. + +#### Fair SSL Status + +Check the status of SSL certificates on the node. + +```shell +fair ssl status +``` + +#### Fair SSL Upload + +Upload SSL certificate files to the node. + +```shell +fair ssl upload --cert-path --key-path [--force] +``` + +Options: + +* `--cert-path`/`-c` - Path to the SSL certificate file (required). +* `--key-path`/`-k` - Path to the SSL private key file (required). +* `--force`/`-f` - Overwrite existing certificates. + +#### Fair SSL Check + +Check SSL certificate validity and connectivity. + +```shell +fair ssl check [--cert-path CERT_PATH] [--key-path KEY_PATH] [--port PORT] [--type TYPE] [--no-client] [--no-wss] +``` + +Options: + +* `--cert-path`/`-c` - Path to the certificate file (default: system default). +* `--key-path`/`-k` - Path to the key file (default: system default). +* `--port`/`-p` - Port to start SSL health check server (default: from configuration). +* `--type`/`-t` - Check type: `all`, `openssl`, or `skaled` (default: `all`). +* `--no-client` - Skip client connection for openssl check. +* `--no-wss` - Skip WSS server starting for skaled check. + +### Fair Staking commands + +> Prefix: `fair staking` + +Commands for interacting with the Fair staking functionality. + +#### Add allowed receiver + +Allow an address to receive staking fees. + +```shell +fair staking add-receiver +``` + +Arguments: + +* `RECEIVER_ADDRESS` - Address to add to the allowed receivers list. + +#### Remove allowed receiver + +Remove an address from the allowed receivers list. + +```shell +fair staking remove-receiver +``` + +Arguments: + +* `RECEIVER_ADDRESS` - Address to remove from the allowed receivers list. + +Workflow (fees): request fees -> review exit requests -> claim request. + +#### Request fees + +Create a request to claim a specific amount of earned fees (FAIR). Use `--all` to request all. + +```shell +fair staking request-fees +fair staking request-fees --all +``` + +#### Request send fees + +Create a request to send a specific amount (or all) of earned fees to an address. + +```shell +fair staking request-send-fees +fair staking request-send-fees --all +``` + +Arguments: + +* `TO_ADDRESS` - Destination address for the fee transfer. +* `AMOUNT` - Amount of fees to include in the request (FAIR). + +#### Claim request + +Claim a previously created request by its request ID once it is unlocked. + +```shell +fair staking claim-request +``` + +#### Get exit requests + +List exit (fee withdrawal) requests for the current wallet. Use `--json` for raw JSON output. + +```shell +fair staking exit-requests +fair staking exit-requests --json +``` + +Default output (non-JSON) shows: `request_id`, `user`, `node_id`, `amount_wei`, `amount_fair`, `unlock_date (ISO)`. + +#### Get earned fee amount + +Get the currently earned (unrequested) fee amount. + +```shell +fair staking earned-fee-amount +``` + +#### Set fee rate + +Set the fee rate (uint16 value) used by the staking logic. + +```shell +fair staking set-fee-rate +``` + +Arguments: + +* `FEE_RATE` - Fee rate value as integer (uint16). + +### Passive Fair Node commands + +> Prefix: `fair passive-node` (passive Fair build) + +Commands for operating a passive Fair node (sync/indexer/archive). + +#### Passive Fair Node Initialization + +Initialize a passive Fair node. + +```shell +fair passive-node init --id [--indexer | --archive] [--snapshot ] +``` + +Arguments: + +* `ENV_FILEPATH` - Path to the environment file with configuration. + +Required environment variables in `ENV_FILEPATH`: + +* `FAIR_CONTRACTS` - Fair Manager contracts alias or address. +* `NODE_VERSION` - Version of `skale-node`. +* `BOOT_ENDPOINT` - RPC endpoint of Fair network. +* `BLOCK_DEVICE` - Absolute path to a dedicated raw block device (e.g., `/dev/sdc`). +* `ENV_TYPE` - Environment type (e.g., `mainnet`, `devnet`). + +Options: + +* `--id` - Numerical node identifier (required). +* `--indexer` - Run in indexer mode (no block rotation). +* `--archive` - Run in archive mode (historical state kept; disables block rotation). Mutually exclusive with `--indexer`. +* `--snapshot ` - Start from provided snapshot URL or from any available source (not allowed together with `--indexer` or `--archive`). + +By default runs a regular sync node. + +#### Passive Fair Node Update + +Update software / configs for passive Fair node. + +```shell +fair passive-node update [--yes] +``` + +#### Passive Fair Node turn-off + +Turn off the Fair passive node containers. + +```shell +fair passive-node turn-off [--yes] +``` + +Options: + +* `--yes` - Turn off without confirmation. + +#### Passive Fair Node turn-on + +Turn on the Fair passive node containers. + +```shell +fair passive-node turn-on [ENV_FILEPATH] [--yes] +``` + +Arguments: + +* `ENV_FILEPATH` - Path to the .env file. + +Options: + +* `--yes` - Turn on without additional confirmation. + +#### Passive Fair Node Cleanup + +Remove all passive Fair node data and containers. + +```shell +fair passive-node cleanup [--yes] +``` + +Options: + +* `--yes` - Proceed without confirmation. + *** ## Exit codes @@ -887,14 +1331,14 @@ pip install -e ".[dev]" #### Generate info.py locally -Specify the build type (`normal`, `sync`, or `fair`): +Specify the build type (`normal`, `passive`, or `fair`): ```shell # Example for Standard build ./scripts/generate_info.sh 1.0.0 my-branch normal -# Example for Sync build -./scripts/generate_info.sh 1.0.0 my-branch sync +# Example for Passive build +./scripts/generate_info.sh 1.0.0 my-branch passive # Example for Fair build ./scripts/generate_info.sh 1.0.0 my-branch fair diff --git a/helper-scripts b/helper-scripts index 808c768f..c1f67269 160000 --- a/helper-scripts +++ b/helper-scripts @@ -1 +1 @@ -Subproject commit 808c768feebfa99d9148e076b5b6b24b1b340734 +Subproject commit c1f67269955126ac400c2445d2aa81f1a387d964 diff --git a/main.spec b/main.spec index e3844bc1..8b7da407 100644 --- a/main.spec +++ b/main.spec @@ -1,16 +1,15 @@ # -*- mode: python -*- -import importlib.util - +import os block_cipher = None a = Analysis( ['node_cli/main.py'], - pathex=['.'], + pathex=[SPECPATH], datas=[ - ("./text.yml", "data"), - ("./datafiles/skaled-ssl-test", "data/datafiles") + (os.path.join(SPECPATH, "text.yml"), "data"), + (os.path.join(SPECPATH, "datafiles/skaled-ssl-test"), "data/datafiles") ], hiddenimports=[], hookspath=[], diff --git a/node_cli/cli/__init__.py b/node_cli/cli/__init__.py index b0105764..6f899042 100644 --- a/node_cli/cli/__init__.py +++ b/node_cli/cli/__init__.py @@ -1,4 +1,6 @@ -__version__ = '3.0.0' +from importlib.metadata import version + +__version__ = version('node-cli') if __name__ == '__main__': print(__version__) diff --git a/node_cli/cli/chain.py b/node_cli/cli/chain.py new file mode 100644 index 00000000..10826236 --- /dev/null +++ b/node_cli/cli/chain.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +# +# This file is part of node-cli +# +# Copyright (C) 2025-Present SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import click + +from node_cli.fair.chain import get_chain_record, get_chain_checks + + +@click.group() +def chain_cli(): + pass + + +@chain_cli.group(help='Fair chain commands') +def chain(): + pass + + +@chain.command('record', help='Get Fair chain record information') +@click.option('--json', 'raw', is_flag=True, help='Output in JSON format') +def chain_record(raw: bool) -> None: + get_chain_record(raw=raw) + + +@chain.command('checks', help='Get Fair chain checks status') +@click.option('--json', 'raw', is_flag=True, help='Output in JSON format') +def chain_checks(raw: bool) -> None: + get_chain_checks(raw=raw) diff --git a/node_cli/cli/fair_boot.py b/node_cli/cli/fair_boot.py index 703bea88..d5ff6039 100644 --- a/node_cli/cli/fair_boot.py +++ b/node_cli/cli/fair_boot.py @@ -22,7 +22,7 @@ from node_cli.configs import DEFAULT_NODE_BASE_PORT from node_cli.core.node import get_node_info, get_node_signature from node_cli.core.node import register_node as register -from node_cli.fair.fair_boot import init, update +from node_cli.fair.boot import init, update from node_cli.utils.helper import IP_TYPE, abort_if_false, error_exit, streamed_cmd @@ -50,9 +50,7 @@ def init_boot(env_file): @boot.command('register', help='Register Fair node in SKALE Manager (during Boot Phase).') -@click.option( - '--name', '-n', required=True, prompt='Enter fair node name', help='Fair node name' -) +@click.option('--name', '-n', required=True, prompt='Enter fair node name', help='Fair node name') @click.option( '--ip', prompt='Enter node public IP', @@ -90,6 +88,6 @@ def signature_boot(validator_id): @streamed_cmd def update_node(env_file, pull_config_for_schain): update( - env_filepath=env_file, + config_file=env_file, pull_config_for_schain=pull_config_for_schain, ) diff --git a/node_cli/cli/fair_node.py b/node_cli/cli/fair_node.py index b6461871..efdf3149 100644 --- a/node_cli/cli/fair_node.py +++ b/node_cli/cli/fair_node.py @@ -19,18 +19,22 @@ import click +from node_cli.cli.info import TYPE from node_cli.core.node import backup -from node_cli.fair.fair_node import cleanup as fair_cleanup -from node_cli.fair.fair_node import init as init_fair -from node_cli.fair.fair_node import ( - migrate_from_boot, - request_repair, - restore_fair, - get_node_info, -) -from node_cli.fair.fair_node import register as register_fair -from node_cli.fair.fair_node import update as update_fair -from node_cli.utils.helper import IP_TYPE, URL_TYPE, abort_if_false, streamed_cmd +from node_cli.fair.active import change_ip as change_ip_fair +from node_cli.fair.active import exit as exit_fair +from node_cli.fair.active import get_node_info, migrate_from_boot +from node_cli.fair.active import register as register_fair +from node_cli.fair.active import restore as restore_fair +from node_cli.fair.active import set_domain_name as set_domain_name_fair +from node_cli.fair.common import cleanup as cleanup_fair +from node_cli.fair.common import init as init_fair +from node_cli.fair.common import repair_chain +from node_cli.fair.common import turn_off as turn_off_fair +from node_cli.fair.common import turn_on as turn_on_fair +from node_cli.fair.common import update as update_fair +from node_cli.utils.helper import IP_TYPE, URL_OR_ANY_TYPE, abort_if_false, streamed_cmd +from node_cli.utils.node_type import NodeMode from node_cli.utils.texts import safe_load_texts TEXTS = safe_load_texts() @@ -53,10 +57,10 @@ def fair_node_info(format): @node.command('init', help='Initialize regular Fair node') -@click.argument('env_filepath') +@click.argument('config_file') @streamed_cmd -def init_node(env_filepath: str): - init_fair(env_filepath=env_filepath) +def init_node(config_file: str): + init_fair(node_mode=NodeMode.ACTIVE, config_file=config_file) @node.command('register', help=TEXTS['fair']['node']['register']['help']) @@ -66,7 +70,7 @@ def register(ip: str) -> None: @node.command('update', help='Update Fair node') -@click.argument('env_filepath') +@click.argument('config_file') @click.option( '--yes', is_flag=True, @@ -75,9 +79,22 @@ def register(ip: str) -> None: prompt='Are you sure you want to update Fair node software?', ) @click.option('--pull-config', 'pull_config_for_schain', hidden=True, type=str) +@click.option( + '--force-skaled-start', + 'force_skaled_start', + hidden=True, + type=bool, + default=False, + is_flag=True, +) @streamed_cmd -def update_node(env_filepath: str, pull_config_for_schain): - update_fair(env_filepath=env_filepath, pull_config_for_schain=pull_config_for_schain) +def update_node(config_file: str, pull_config_for_schain, force_skaled_start: bool): + update_fair( + node_mode=NodeMode.ACTIVE, + config_file=config_file, + pull_config_for_schain=pull_config_for_schain, + force_skaled_start=force_skaled_start, + ) @node.command('backup', help='Generate backup file for the Fair node.') @@ -89,7 +106,7 @@ def backup_node(backup_folder_path): @node.command('restore', help='Restore Fair node from a backup file.') @click.argument('backup_path') -@click.argument('env_file') +@click.argument('config_file') @click.option( '--config-only', help='Only restore configuration files in .skale and artifacts', @@ -97,12 +114,12 @@ def backup_node(backup_folder_path): hidden=True, ) @streamed_cmd -def restore_node(backup_path, env_file, config_only): - restore_fair(backup_path, env_file, config_only) +def restore_node(backup_path, config_file, config_only): + restore_fair(backup_path, config_file, config_only) @node.command('migrate', help='Switch from boot to regular Fair node.') -@click.argument('env_filepath') +@click.argument('config_file') @click.option( '--yes', is_flag=True, @@ -111,17 +128,17 @@ def restore_node(backup_path, env_file, config_only): prompt='Are you sure you want to migrate to regular Fair node? The action cannot be undone', ) @streamed_cmd -def migrate_node(env_filepath: str) -> None: - migrate_from_boot(env_filepath=env_filepath) +def migrate_node(config_file: str) -> None: + migrate_from_boot(config_file=config_file) @node.command('repair', help='Toggle fair chain repair mode') @click.option( - '--snapshot-from', - type=URL_TYPE, - default='', + '--snapshot', + type=URL_OR_ANY_TYPE, + default='any', hidden=True, - help=TEXTS['fair']['node']['repair']['snapshot_from'], + help=TEXTS['fair']['node']['repair']['snapshot'], ) @click.option( '--yes', @@ -130,18 +147,81 @@ def migrate_node(env_filepath: str) -> None: expose_value=False, prompt=TEXTS['fair']['node']['repair']['warning'], ) -def repair(snapshot_from: str = '') -> None: - request_repair(snapshot_from=snapshot_from) +@streamed_cmd +def repair(snapshot: str = 'any') -> None: + repair_chain(snapshot_from=snapshot) + + +@node.command('cleanup', help='Remove all FAIR node data and containers.') +@click.option( + '--yes', + is_flag=True, + callback=abort_if_false, + expose_value=False, + prompt='Are you sure you want to remove all FAIR node data and containers?', +) +@click.option('--prune', is_flag=True, help='Prune docker system.') +@streamed_cmd +def cleanup_node(prune): + cleanup_fair(node_mode=NodeMode.ACTIVE, prune=prune) + + +@node.command('change-ip', help=TEXTS['fair']['node']['change-ip']['help']) +@click.argument('ip', type=IP_TYPE) +@streamed_cmd +def change_ip(ip: str) -> None: + change_ip_fair(ip=ip) + + +@node.command('exit', help=TEXTS['fair']['node']['exit']['help']) +@click.option( + '--yes', + is_flag=True, + callback=abort_if_false, + expose_value=False, + prompt=TEXTS['fair']['node']['exit']['prompt'], +) +@streamed_cmd +def exit_node() -> None: + exit_fair() + + +@node.command('set-domain', help='Set node domain name') +@click.option('--domain', '-d', prompt='Enter node domain name', type=str, help='Node domain name') +@click.option( + '--yes', + is_flag=True, + callback=abort_if_false, + expose_value=False, + prompt='Are you sure you want to set domain name?', +) +@streamed_cmd +def set_domain_name(domain): + set_domain_name_fair(domain) + + +@node.command('turn-off', help='Turn off the node') +@click.option( + '--yes', + is_flag=True, + callback=abort_if_false, + expose_value=False, + prompt='Are you sure you want to turn off the node?', +) +@streamed_cmd +def turn_off_node() -> None: + turn_off_fair(node_type=TYPE) -@node.command('cleanup', help='Cleanup Fair node.') +@node.command('turn-on', help='Turn on the node') @click.option( '--yes', is_flag=True, callback=abort_if_false, expose_value=False, - prompt='Are you sure you want to cleanup Fair node?', + prompt='Are you sure you want to turn on the node?', ) +@click.argument('config_file') @streamed_cmd -def cleanup_node(): - fair_cleanup() +def turn_on_node(config_file: str) -> None: + turn_on_fair(env_file=config_file, node_type=TYPE) diff --git a/node_cli/cli/node.py b/node_cli/cli/node.py index 61d71af7..b34fd155 100644 --- a/node_cli/cli/node.py +++ b/node_cli/cli/node.py @@ -17,10 +17,14 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +from typing import get_args + import click +from skale_core.types import EnvType from node_cli.cli.info import TYPE from node_cli.core.node import ( + cleanup as cleanup_skale, configure_firewall_rules, get_node_signature, init, @@ -37,9 +41,10 @@ run_checks, ) from node_cli.configs import DEFAULT_NODE_BASE_PORT -from node_cli.configs.user import ALLOWED_ENV_TYPES +from node_cli.core.node_options import upsert_node_mode from node_cli.utils.decorators import check_inited from node_cli.utils.helper import abort_if_false, streamed_cmd, IP_TYPE +from node_cli.utils.node_type import NodeMode from node_cli.utils.texts import safe_load_texts from node_cli.utils.meta import CliMetaManager from node_cli.utils.print_formatters import print_meta_info @@ -82,10 +87,10 @@ def register_node(name, ip, port, domain): @node.command('init', help='Initialize SKALE node') -@click.argument('env_file') +@click.argument('config_file') @streamed_cmd -def init_node(env_file): - init(env_filepath=env_file, node_type=TYPE) +def init_node(config_file): + init(config_file=config_file, node_type=TYPE) @node.command('update', help='Update node from .env file') @@ -98,11 +103,12 @@ def init_node(env_file): ) @click.option('--pull-config', 'pull_config_for_schain', hidden=True, type=str) @click.option('--unsafe', 'unsafe_ok', help='Allow unsafe update', hidden=True, is_flag=True) -@click.argument('env_file') +@click.argument('config_file') @streamed_cmd -def update_node(env_file, pull_config_for_schain, unsafe_ok): +def update_node(config_file, pull_config_for_schain, unsafe_ok): update( - env_filepath=env_file, + node_mode=NodeMode.ACTIVE, + config_file=config_file, pull_config_for_schain=pull_config_for_schain, node_type=TYPE, unsafe_ok=unsafe_ok, @@ -139,7 +145,7 @@ def backup_node(backup_folder_path): def restore_node(backup_path, env_file, no_snapshot, config_only): restore( backup_path=backup_path, - env_filepath=env_file, + config_file=env_file, no_snapshot=no_snapshot, config_only=config_only, node_type=TYPE, @@ -223,12 +229,13 @@ def _set_domain_name(domain): @click.option( '--network', '-n', - type=click.Choice(ALLOWED_ENV_TYPES), + type=click.Choice(get_args(EnvType)), default='mainnet', help='Network to check', ) def check(network): - run_checks(node_type=TYPE, network=network) + node_mode = upsert_node_mode() + run_checks(node_type=TYPE, node_mode=node_mode, network=network) @node.command(help='Reconfigure nftables rules') @@ -253,3 +260,17 @@ def version(raw: bool) -> None: print(meta_info) else: print_meta_info(meta_info) + + +@node.command('cleanup', help='Remove all SKALE node data and containers.') +@click.option( + '--yes', + is_flag=True, + callback=abort_if_false, + expose_value=False, + prompt='Are you sure you want to remove all SKALE node data and containers?', +) +@click.option('--prune', is_flag=True, help='Prune docker system.') +@streamed_cmd +def cleanup_node(prune): + cleanup_skale(node_mode=NodeMode.ACTIVE, prune=prune) diff --git a/node_cli/cli/passive_fair_node.py b/node_cli/cli/passive_fair_node.py new file mode 100644 index 00000000..7bac3581 --- /dev/null +++ b/node_cli/cli/passive_fair_node.py @@ -0,0 +1,152 @@ +# -*- coding: utf-8 -*- +# +# This file is part of node-cli +# +# Copyright (C) 2025-Present SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import click + +from node_cli.cli.info import TYPE +from node_cli.fair.common import cleanup as cleanup_fair +from node_cli.fair.common import init as init_fair +from node_cli.fair.common import turn_off as turn_off_fair +from node_cli.fair.common import turn_on as turn_on_fair +from node_cli.fair.common import update as update_fair +from node_cli.fair.passive import setup_fair_passive +from node_cli.utils.helper import ( + URL_OR_ANY_TYPE, + abort_if_false, + error_exit, + streamed_cmd, +) +from node_cli.utils.node_type import NodeMode +from node_cli.utils.texts import safe_load_texts + +TEXTS = safe_load_texts() + + +@click.group() +def passive_fair_node_cli(): + pass + + +@passive_fair_node_cli.group(help='Commands for passive Fair Node operations.') +def passive_node(): + pass + + +@passive_node.command('init', help='Initialize a passive Fair node') +@click.argument('config_file') +@click.option('--id', required=True, type=int, help=TEXTS['fair']['node']['setup']['id']) +@click.option('--indexer', help=TEXTS['passive_node']['init']['indexer'], is_flag=True) +@click.option('--archive', help=TEXTS['passive_node']['init']['archive'], is_flag=True) +@click.option( + '--snapshot', + type=URL_OR_ANY_TYPE, + default=None, + help=TEXTS['passive_node']['init']['snapshot_from'], +) +@streamed_cmd +def init_passive_node( + config_file: str, id: int, indexer: bool, archive: bool, snapshot: str | None +): + if indexer and archive: + error_exit('Cannot use both --indexer and --archive options') + if (indexer or archive) and snapshot == 'any': + error_exit('Cannot use any for indexer/archive node') + init_fair( + node_mode=NodeMode.PASSIVE, + config_file=config_file, + node_id=id, + indexer=indexer, + archive=archive, + snapshot=snapshot, + ) + + +@passive_node.command('update', help='Update Fair node') +@click.argument('config_file') +@click.option( + '--yes', + is_flag=True, + callback=abort_if_false, + expose_value=False, + prompt='Are you sure you want to update Fair node software?', +) +@click.option('--pull-config', 'pull_config_for_schain', hidden=True, type=str) +@click.option( + '--force-skaled-start', + 'force_skaled_start', + hidden=True, + type=bool, + default=False, + is_flag=True, +) +@streamed_cmd +def update_node(config_file: str, pull_config_for_schain, force_skaled_start: bool): + update_fair( + node_mode=NodeMode.PASSIVE, + config_file=config_file, + pull_config_for_schain=pull_config_for_schain, + force_skaled_start=force_skaled_start, + ) + + +@passive_node.command('cleanup', help='Remove all FAIR node data and containers.') +@click.option( + '--yes', + is_flag=True, + callback=abort_if_false, + expose_value=False, + prompt='Are you sure you want to remove all FAIR node data and containers?', +) +@click.option('--prune', is_flag=True, help='Prune docker system.') +@streamed_cmd +def cleanup_node(prune): + cleanup_fair(node_mode=NodeMode.PASSIVE, prune=prune) + + +@passive_node.command('setup', help=TEXTS['fair']['node']['setup']['help']) +@click.option('--id', required=True, type=int, help=TEXTS['fair']['node']['setup']['id']) +def _setup(id: int) -> None: + setup_fair_passive(node_id=id) + + +@passive_node.command('turn-off', help='Turn off the node') +@click.option( + '--yes', + is_flag=True, + callback=abort_if_false, + expose_value=False, + prompt='Are you sure you want to turn off the node?', +) +@streamed_cmd +def turn_off_node() -> None: + turn_off_fair(node_type=TYPE) + + +@passive_node.command('turn-on', help='Turn on the node') +@click.option( + '--yes', + is_flag=True, + callback=abort_if_false, + expose_value=False, + prompt='Are you sure you want to turn on the node?', +) +@click.argument('config_file') +@streamed_cmd +def turn_on_node(config_file: str) -> None: + turn_on_fair(env_file=config_file, node_type=TYPE) diff --git a/node_cli/cli/sync_node.py b/node_cli/cli/passive_node.py similarity index 65% rename from node_cli/cli/sync_node.py rename to node_cli/cli/passive_node.py index 5f0a5217..f86ba99b 100644 --- a/node_cli/cli/sync_node.py +++ b/node_cli/cli/passive_node.py @@ -21,31 +21,27 @@ import click -from node_cli.core.node import init_sync, update_sync, cleanup_sync -from node_cli.utils.helper import ( - abort_if_false, - error_exit, - streamed_cmd, - URL_TYPE, -) +from node_cli.core.node import init_passive, update_passive, cleanup as cleanup_skale +from node_cli.utils.helper import abort_if_false, error_exit, streamed_cmd, URL_TYPE +from node_cli.utils.node_type import NodeMode from node_cli.utils.texts import safe_load_texts G_TEXTS = safe_load_texts() -TEXTS = G_TEXTS['sync_node'] +TEXTS = G_TEXTS['passive_node'] @click.group() -def sync_node_cli(): +def passive_node_cli(): pass -@sync_node_cli.group(help='SKALE sync node commands') -def sync_node(): +@passive_node_cli.group(help='SKALE passive node commands') +def passive_node(): pass -@sync_node.command('init', help=TEXTS['init']['help']) +@passive_node.command('init', help=TEXTS['init']['help']) @click.argument('env_file') @click.option('--indexer', help=TEXTS['init']['indexer'], is_flag=True) @click.option('--archive', help=TEXTS['init']['archive'], is_flag=True) @@ -54,15 +50,15 @@ def sync_node(): '--snapshot-from', type=URL_TYPE, default=None, hidden=True, help=TEXTS['init']['snapshot_from'] ) @streamed_cmd -def _init_sync( +def _init_passive( env_file, indexer: bool, archive: bool, snapshot: bool, snapshot_from: Optional[str] ) -> None: if indexer and archive: error_exit('Cannot use both --indexer and --archive options') - init_sync(env_file, indexer, archive, snapshot, snapshot_from) + init_passive(env_file, indexer, archive, snapshot, snapshot_from) -@sync_node.command('update', help='Update sync node from .env file') +@passive_node.command('update', help='Update passive node from .env file') @click.option( '--yes', is_flag=True, @@ -70,21 +66,21 @@ def _init_sync( expose_value=False, prompt='Are you sure you want to update SKALE node software?', ) -@click.option('--unsafe', 'unsafe_ok', help='Allow unsafe update', hidden=True, is_flag=True) @click.argument('env_file') @streamed_cmd -def _update_sync(env_file, unsafe_ok): - update_sync(env_file) +def _update_passive(env_file): + update_passive(env_file) -@sync_node.command('cleanup', help='Remove sync node data and containers') +@passive_node.command('cleanup', help='Remove all SKALE node data and containers.') @click.option( '--yes', is_flag=True, callback=abort_if_false, expose_value=False, - prompt='Are you sure you want to remove all node containers and data?', + prompt='Are you sure you want to remove all SKALE node data and containers?', ) +@click.option('--prune', is_flag=True, help='Prune docker system.') @streamed_cmd -def _cleanup_sync() -> None: - cleanup_sync() +def cleanup_node(prune): + cleanup_skale(node_mode=NodeMode.PASSIVE, prune=prune) \ No newline at end of file diff --git a/node_cli/cli/resources_allocation.py b/node_cli/cli/resources_allocation.py deleted file mode 100644 index 01825350..00000000 --- a/node_cli/cli/resources_allocation.py +++ /dev/null @@ -1,64 +0,0 @@ -# -*- coding: utf-8 -*- -# -# This file is part of node-cli -# -# Copyright (C) 2019 SKALE Labs -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -import json -import click - -from node_cli.core.resources import ( - get_resource_allocation_info, - generate_resource_allocation_config, -) -from node_cli.utils.helper import abort_if_false -from node_cli.utils.texts import safe_load_texts -from node_cli.utils.node_type import NodeType - -TEXTS = safe_load_texts() - - -@click.group() -def resources_allocation_cli(): - pass - - -@resources_allocation_cli.group(help='Resources allocation commands') -def resources_allocation(): - pass - - -@resources_allocation.command('show', help='Show resources allocation file') -def show(): - resource_allocation_info = get_resource_allocation_info() - if resource_allocation_info: - print(json.dumps(resource_allocation_info, indent=4)) - else: - print('No resources allocation file on this machine') - - -@resources_allocation.command('generate', help='Generate/update resources allocation file') -@click.argument('env_file') -@click.option( - '--yes', - is_flag=True, - callback=abort_if_false, - expose_value=False, - prompt='Are you sure you want to generate/update resource allocation file?', -) -@click.option('--force', '-f', is_flag=True, help='Rewrite if already exists') -def generate(env_file, force): - generate_resource_allocation_config(node_type=NodeType.REGULAR, env_file=env_file, force=force) diff --git a/node_cli/cli/schains.py b/node_cli/cli/schains.py index 803ea754..6bb1fce2 100644 --- a/node_cli/cli/schains.py +++ b/node_cli/cli/schains.py @@ -21,6 +21,8 @@ import click +from skale_core.settings import get_settings + from node_cli.utils.helper import abort_if_false, URL_TYPE from node_cli.core.schains import ( describe, @@ -104,8 +106,12 @@ def info_(schain_name: str, json_format: bool) -> None: @click.argument('schain_name') @click.argument('snapshot_path') @click.option('--schain-type', default='medium') -@click.option('--env-type', default=None) -def restore( - schain_name: str, snapshot_path: str, schain_type: str, env_type: Optional[str] -) -> None: - restore_schain_from_snapshot(schain_name, snapshot_path, node_type=TYPE) +def restore(schain_name: str, snapshot_path: str, schain_type: str) -> None: + settings = get_settings() + restore_schain_from_snapshot( + schain_name, + snapshot_path, + node_type=TYPE, + env_type=settings.env_type, + schain_type=schain_type, + ) diff --git a/node_cli/cli/ssl.py b/node_cli/cli/ssl.py index e371e8ce..c22ac226 100644 --- a/node_cli/cli/ssl.py +++ b/node_cli/cli/ssl.py @@ -82,7 +82,7 @@ def upload(key_path, cert_path, force): @click.option( '--port', '-p', - help='Port to start ssl healtcheck server', + help='Port to start ssl healthcheck server', type=int, default=DEFAULT_SSL_CHECK_PORT, ) diff --git a/node_cli/cli/staking.py b/node_cli/cli/staking.py new file mode 100644 index 00000000..8cb08d2d --- /dev/null +++ b/node_cli/cli/staking.py @@ -0,0 +1,141 @@ +# -*- coding: utf-8 -*- +# +# This file is part of node-cli +# +# Copyright (C) 2025-Present SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import click + +from node_cli.fair.staking import ( + add_allowed_receiver, + remove_allowed_receiver, + set_fee_rate, + request_fees, + request_send_fees, + claim_request, + get_earned_fee_amount, + get_exit_requests, +) +from node_cli.utils.helper import abort_if_false + + +@click.group() +def staking_cli(): + pass + + +@staking_cli.group(help='Staking commands') +def staking(): + pass + + +@staking.command('add-receiver', help='Add allowed receiver') +@click.argument('receiver') +@click.option( + '--yes', + is_flag=True, + callback=abort_if_false, + expose_value=False, + prompt='Are you sure you want to add allowed receiver?', +) +def _add_allowed_receiver(receiver: str) -> None: + add_allowed_receiver(receiver) + + +@staking.command('remove-receiver', help='Remove allowed receiver') +@click.argument('receiver') +@click.option( + '--yes', + is_flag=True, + callback=abort_if_false, + expose_value=False, + prompt='Are you sure you want to remove allowed receiver?', +) +def _remove_allowed_receiver(receiver: str) -> None: + remove_allowed_receiver(receiver) + + +@staking.command('set-fee-rate', help='Set fee rate (uint16, basis points; 25 = 2.5%)') +@click.argument('fee_rate', type=int) +@click.option( + '--yes', + is_flag=True, + callback=abort_if_false, + expose_value=False, + prompt='Are you sure you want to set fee rate?', +) +def _set_fee_rate(fee_rate: int) -> None: + set_fee_rate(fee_rate) + + +@staking.command('request-fees', help='Create a request to claim fees (FAIR) or all with --all') +@click.argument('amount', type=float, required=False) +@click.option('--all', 'request_all', is_flag=True, help='Request all fees') +@click.option( + '--yes', + is_flag=True, + callback=abort_if_false, + expose_value=False, + prompt='Are you sure you want to request fees?', +) +def _request_fees(amount: float | None, request_all: bool) -> None: + if amount is None and not request_all: + raise click.UsageError('Provide or use --all') + request_fees(None if request_all else amount) + + +@staking.command( + 'request-send-fees', + help='Create a request to send fees to address (or all with --all)', +) +@click.argument('to') +@click.argument('amount', type=float, required=False) +@click.option('--all', 'send_all', is_flag=True, help='Request to send all fees to address') +@click.option( + '--yes', + is_flag=True, + callback=abort_if_false, + expose_value=False, + prompt='Are you sure you want to request to send fees?', +) +def _request_send_fees(to: str, amount: float | None, send_all: bool) -> None: + if amount is None and not send_all: + raise click.UsageError('Provide or use --all') + request_send_fees(to, None if send_all else amount) + + +@staking.command('claim-request', help='Claim previously created request by request ID') +@click.argument('request_id', type=int) +@click.option( + '--yes', + is_flag=True, + callback=abort_if_false, + expose_value=False, + prompt='Are you sure you want to claim this request?', +) +def _claim_request(request_id: int) -> None: + claim_request(request_id) + + +@staking.command('earned-fee-amount', help='Get earned fee amount') +def _get_earned_fee_amount() -> None: + get_earned_fee_amount() + + +@staking.command('exit-requests', help='Get exit requests for current wallet') +@click.option('--json', 'raw', is_flag=True, help='Output in JSON format') +def _get_exit_requests(raw: bool) -> None: + get_exit_requests(raw=raw) diff --git a/node_cli/configs/__init__.py b/node_cli/configs/__init__.py index 6709ee75..7ed6afee 100644 --- a/node_cli/configs/__init__.py +++ b/node_cli/configs/__init__.py @@ -19,6 +19,7 @@ import os import sys +from pathlib import Path from node_cli.utils.global_config import read_g_config @@ -43,26 +44,27 @@ NODE_DATA_PATH = os.path.join(SKALE_DIR, 'node_data') SCHAIN_NODE_DATA_PATH = os.path.join(NODE_DATA_PATH, 'schains') NODE_CLI_STATUS_FILENAME = 'node_cli.status' + +SETTINGS_DIR = Path(NODE_DATA_PATH) / 'settings' +NODE_SETTINGS_PATH = SETTINGS_DIR / 'node.toml' +INTERNAL_SETTINGS_PATH = SETTINGS_DIR / 'internal.toml' + NODE_CONFIG_PATH = os.path.join(NODE_DATA_PATH, 'node_config.json') CONTAINER_CONFIG_PATH = os.path.join(SKALE_DIR, 'config') CONTAINER_CONFIG_TMP_PATH = os.path.join(SKALE_TMP_DIR, 'config') CONTRACTS_PATH = os.path.join(SKALE_DIR, 'contracts_info') REPORTS_PATH = os.path.join(SKALE_DIR, 'reports') BACKUP_CONTRACTS_PATH = os.path.join(SKALE_DIR, '.old_contracts_info') -INIT_ENV_FILEPATH = os.path.join(SKALE_DIR, '.env') SKALE_RUN_DIR = '/var/run/skale' -SGX_CERTIFICATES_DIR_NAME = 'sgx_certs' - COMPOSE_PATH = os.path.join(CONTAINER_CONFIG_PATH, 'docker-compose.yml') -SYNC_COMPOSE_PATH = os.path.join(CONTAINER_CONFIG_PATH, 'docker-compose-sync.yml') FAIR_COMPOSE_PATH = os.path.join(CONTAINER_CONFIG_PATH, 'docker-compose-fair.yml') STATIC_PARAMS_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, 'static_params.yaml') FAIR_STATIC_PARAMS_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, 'fair_static_params.yaml') NGINX_TEMPLATE_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, 'nginx.conf.j2') NGINX_CONFIG_FILEPATH = os.path.join(NODE_DATA_PATH, 'nginx.conf') -NGINX_CONTAINER_NAME = 'skale_nginx' +NGINX_CONTAINER_NAME = 'sk_nginx' LOG_PATH = os.path.join(NODE_DATA_PATH, 'log') REMOVED_CONTAINERS_FOLDER_NAME = '.removed_containers' @@ -74,9 +76,6 @@ SGX_CERTS_PATH = os.path.join(NODE_DATA_PATH, 'sgx_certs') SCHAINS_DATA_PATH = os.path.join(NODE_DATA_PATH, 'schains') -CURRENT_FILE_LOCATION = os.path.dirname(os.path.realpath(__file__)) -DOTENV_FILEPATH = os.path.join(os.path.dirname(CURRENT_FILE_LOCATION), '.env') - SRC_FILEBEAT_CONFIG_PATH = os.path.join(CONTAINER_CONFIG_PATH, 'filebeat.yml') FILEBEAT_CONFIG_PATH = os.path.join(NODE_DATA_PATH, 'filebeat.yml') @@ -96,9 +95,6 @@ IPTABLES_RULES_STATE_FILEPATH = os.path.join(IPTABLES_DIR, 'rules.v4') DEFAULT_SSH_PORT = 22 -FLASK_SECRET_KEY_FILENAME = 'flask_db_key.txt' -FLASK_SECRET_KEY_FILE = os.path.join(NODE_DATA_PATH, FLASK_SECRET_KEY_FILENAME) - DOCKER_CONFIG_FILEPATH = '/etc/docker/daemon.json' HIDE_STREAM_LOG = os.getenv('HIDE_STREAM_LOG') @@ -140,6 +136,7 @@ def _get_env(): TM_INIT_TIMEOUT = 20 RESTORE_SLEEP_TIMEOUT = 20 +INIT_TIMEOUT = 20 META_FILEPATH = os.path.join(NODE_DATA_PATH, 'meta.json') diff --git a/node_cli/configs/alias_address_validation.py b/node_cli/configs/alias_address_validation.py deleted file mode 100644 index ffdbb2ee..00000000 --- a/node_cli/configs/alias_address_validation.py +++ /dev/null @@ -1,118 +0,0 @@ -# -*- coding: utf-8 -*- -# -# This file is part of node-cli -# -# Copyright (C) 2025-Present SKALE Labs -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -from enum import Enum -from typing import Dict, Optional - -import requests - -from node_cli.utils.helper import error_exit, is_contract_address - - -METADATA_URL: str = ( - 'https://raw.githubusercontent.com/skalenetwork/skale-contracts/' - 'refs/heads/deployments/metadata.json' -) - - -class ContractType(Enum): - """Contract types supported by the system using skale-contracts library.""" - - IMA = 'mainnet-ima' - MANAGER = 'skale-manager' - - -def validate_alias_or_address( - alias_or_address: str, contract_type: ContractType, endpoint: str -) -> None: - if is_contract_address(alias_or_address): - validate_contract_address(alias_or_address, endpoint) - else: - validate_contract_alias(alias_or_address, contract_type, endpoint) - - -def validate_contract_address(contract_address: str, endpoint: str) -> None: - try: - response = requests.post( - endpoint, - json={ - 'jsonrpc': '2.0', - 'method': 'eth_getCode', - 'params': [contract_address, 'latest'], - 'id': 1, - }, - ) - if response.status_code != 200: - error_exit(f'Failed to verify contract at address {contract_address}') - result = response.json().get('result') - if not result or result in ['0x', '0x0']: - error_exit(f'No contract code found at address {contract_address}') - except requests.RequestException as e: - error_exit(f'Failed to validate contract address: {str(e)}') - - -def get_deployment_url(alias: str, contract_type: ContractType, network_path: str) -> str: - return ( - f'https://raw.githubusercontent.com/skalenetwork/skale-contracts/' - f'refs/heads/deployments/{network_path}/{contract_type.value}/{alias}.json' - ) - - -def validate_contract_alias(alias: str, contract_type: ContractType, endpoint: str) -> None: - try: - chain_id = get_chain_id(endpoint) - metadata = get_network_metadata() - networks = metadata.get('networks', []) - network_path: Optional[str] = None - for net in networks: - if net.get('chainId') == chain_id: - network_path = net.get('path') - break - if not network_path: - error_exit(f'Network with chain ID {chain_id} not found in metadata') - if not isinstance(network_path, str): - error_exit(f'Invalid network path type: {network_path}') - deployment_url = get_deployment_url(alias, contract_type, network_path) - if requests.get(deployment_url).status_code != 200: - error_exit(f"Contract alias '{alias}' not found for {contract_type.value}") - except requests.RequestException as e: - error_exit(f"Failed to validate contract alias '{alias}': {str(e)}") - - -def get_chain_id(endpoint: str) -> int: - try: - response = requests.post( - endpoint, - json={'jsonrpc': '2.0', 'method': 'eth_chainId', 'params': [], 'id': 1}, - ) - if response.status_code != 200: - error_exit('Failed to get chain ID from endpoint') - return int(response.json()['result'], 16) - except requests.RequestException as e: - error_exit(f'Failed to get chain ID: {str(e)}') - - -def get_network_metadata() -> Dict: - try: - response = requests.get(METADATA_URL) - if response.status_code != 200: - error_exit('Failed to fetch networks metadata') - return response.json() - except requests.RequestException as e: - error_exit(f'Failed to fetch networks metadata: {str(e)}') diff --git a/node_cli/configs/routes.py b/node_cli/configs/routes.py index 8afc10d9..336fea3e 100644 --- a/node_cli/configs/routes.py +++ b/node_cli/configs/routes.py @@ -40,7 +40,19 @@ 'schains': ['config', 'list', 'dkg-statuses', 'firewall-rules', 'repair', 'get'], 'ssl': ['status', 'upload'], 'wallet': ['info', 'send-eth'], - 'fair-node': ['info', 'register'], + 'fair-node': ['info', 'register', 'set-domain-name', 'change-ip', 'exit'], + 'fair-chain': ['record', 'checks'], + 'fair-node-passive': ['setup'], + 'fair-staking': [ + 'add-receiver', + 'remove-receiver', + 'set-fee-rate', + 'request-fees', + 'request-send-fees', + 'claim-request', + 'get-earned-fee-amount', + 'get-exit-requests', + ], } } diff --git a/node_cli/configs/user.py b/node_cli/configs/user.py deleted file mode 100644 index 49f44d58..00000000 --- a/node_cli/configs/user.py +++ /dev/null @@ -1,188 +0,0 @@ -# -*- coding: utf-8 -*- -# -# This file is part of node-cli -# -# Copyright (C) 2019-Present SKALE Labs -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -import inspect -import os -from abc import ABC -from dataclasses import dataclass -from typing import Dict, NamedTuple - -from dotenv.main import DotEnv - -from node_cli.configs import CONTAINER_CONFIG_PATH, SKALE_DIR -from node_cli.configs.alias_address_validation import ContractType, validate_alias_or_address -from node_cli.utils.helper import error_exit -from node_cli.utils.node_type import NodeType - -SKALE_DIR_ENV_FILEPATH = os.path.join(SKALE_DIR, '.env') -CONFIGS_ENV_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, '.env') - -ALLOWED_ENV_TYPES = ['mainnet', 'testnet', 'qanet', 'devnet'] - - -class ValidationResult(NamedTuple): - result: bool - missing: set - extra: set - - -@dataclass(kw_only=True) -class BaseUserConfig(ABC): - container_configs_stream: str - env_type: str - filebeat_host: str - disk_mountpoint: str - - container_configs_dir: str = '' - skip_docker_config: str = '' - skip_docker_cleanup: str = '' - - def to_env(self) -> Dict[str, str]: - result = {} - for field_name, field_value in self.__dict__.items(): - upper_key = field_name.upper() - result[upper_key] = str(field_value) if field_value is not None else '' - return result - - @classmethod - def validate_params(cls, params: Dict) -> ValidationResult: - parameters = inspect.signature(cls.__init__).parameters - missing = [] - keys = params.keys() - expected_keys = { - name.upper() - for name, value in parameters.items() - if name != 'self' and value.default == inspect._empty - } - optional_keys = { - name.upper() - for name, value in parameters.items() - if name != 'self' and value.default != inspect._empty - } - missing = expected_keys - keys - extra = keys - expected_keys - optional_keys - return ValidationResult(missing == set() and extra == set(), missing, extra) - - -@dataclass -class FairUserConfig(BaseUserConfig): - fair_contracts: str - boot_endpoint: str - sgx_server_url: str - enforce_btrfs: str = '' - - -@dataclass -class FairBootUserConfig(BaseUserConfig): - endpoint: str - manager_contracts: str - ima_contracts: str - sgx_server_url: str - enforce_btrfs: str = '' - - -@dataclass -class SkaleUserConfig(BaseUserConfig): - endpoint: str - manager_contracts: str - ima_contracts: str - docker_lvmpy_stream: str - sgx_server_url: str - monitoring_containers: str = '' - telegraf: str = '' - influx_token: str = '' - influx_url: str = '' - tg_api_key: str = '' - tg_chat_id: str = '' - disable_dry_run: str = '' - default_gas_limit: str = '' - default_gas_price_wei: str = '' - - -@dataclass -class SyncUserConfig(BaseUserConfig): - endpoint: str - manager_contracts: str - schain_name: str = '' - ima_contracts: str = '' - enforce_btrfs: str = '' - - -def get_validated_user_config( - node_type: NodeType, - env_filepath: str = SKALE_DIR_ENV_FILEPATH, - is_fair_boot: bool = False, -) -> BaseUserConfig: - params = parse_env_file(env_filepath) - user_config_class = get_user_config_class(node_type, is_fair_boot) - _, missing_params, extra_params = user_config_class.validate_params(params) - - if len(missing_params) > 0: - error_exit(f'Missing required parameters: {missing_params}') - - if len(extra_params) > 0: - error_exit(f'Extra parameters: {extra_params}') - - params = to_lower_keys(params) - user_config = user_config_class(**params) - validate_user_config(user_config) - - return user_config - - -def validate_user_config(user_config: BaseUserConfig) -> None: - validate_env_type(env_type=user_config.env_type) - - if not isinstance(user_config, FairUserConfig): - validate_alias_or_address( - user_config.manager_contracts, ContractType.MANAGER, user_config.endpoint - ) - - if isinstance(user_config, (SkaleUserConfig, FairBootUserConfig)): - validate_alias_or_address(user_config.ima_contracts, ContractType.IMA, user_config.endpoint) - - -def to_lower_keys(params: Dict[str, str]) -> Dict[str, str]: - return {key.lower(): value for key, value in params.items()} - - -def parse_env_file(env_filepath: str) -> Dict: - if not os.path.isfile(env_filepath): - error_exit(f'Failed to load environment from {env_filepath}') - return DotEnv(env_filepath).dict() - - -def get_user_config_class( - node_type: NodeType, - is_fair_boot: bool = False, -) -> type[BaseUserConfig]: - if node_type == NodeType.FAIR and is_fair_boot: - user_config_class = FairBootUserConfig - elif node_type == NodeType.FAIR: - user_config_class = FairUserConfig - elif node_type == NodeType.SYNC: - user_config_class = SyncUserConfig - else: - user_config_class = SkaleUserConfig - return user_config_class - - -def validate_env_type(env_type: str) -> None: - if env_type not in ALLOWED_ENV_TYPES: - error_exit(f'Allowed ENV_TYPE values are {ALLOWED_ENV_TYPES}. Actual: "{env_type}"') diff --git a/node_cli/core/checks.py b/node_cli/core/checks.py index 929cc60b..0424577f 100644 --- a/node_cli/core/checks.py +++ b/node_cli/core/checks.py @@ -48,6 +48,8 @@ from debian import debian_support from packaging.version import parse as version_parse +from skale_core.types import EnvType + from node_cli.configs import ( CHECK_REPORT_PATH, CONTAINER_CONFIG_PATH, @@ -60,6 +62,7 @@ from node_cli.core.static_config import get_static_params from node_cli.utils.docker_utils import NodeType from node_cli.utils.helper import run_cmd, safe_mkdir +from node_cli.utils.node_type import NodeMode logger = logging.getLogger(__name__) @@ -272,7 +275,7 @@ class PackageChecker(BaseChecker): def __init__(self, requirements: Dict) -> None: super().__init__(requirements=requirements) - def _check_apt_package(self, package_name: str, version: str = None) -> CheckResult: + def _check_apt_package(self, package_name: str, version: str | None = None) -> CheckResult: # TODO: check versions dpkg_cmd_result = run_cmd(['dpkg', '-s', package_name], check_code=False) output = dpkg_cmd_result.stdout.decode('utf-8').strip() @@ -384,7 +387,6 @@ def docker_compose(self) -> CheckResult: ) output = v_cmd_result.stdout.decode('utf-8').rstrip() if v_cmd_result.returncode != 0: - info = f'Checking docker compose version failed with: {output}' return self._failed(name=name, info=output) actual_version = output.split(',')[0].split()[-1].strip() @@ -457,24 +459,28 @@ def get_checks(checkers: List[BaseChecker], check_type: CheckType = CheckType.AL ) -def get_all_checkers(disk: str, requirements: Dict) -> List[BaseChecker]: - return [ - MachineChecker(requirements['server'], disk), +def get_all_checkers(disk: str, requirements: Dict, node_mode: NodeMode) -> List[BaseChecker]: + checkers = [ PackageChecker(requirements['package']), DockerChecker(requirements['docker']), ] + if node_mode == NodeMode.ACTIVE: + checkers.append(MachineChecker(requirements['server'], disk)) + return checkers def run_checks( disk: str, node_type: NodeType, - env_type: str = 'mainnet', + node_mode: NodeMode, + env_type: EnvType = 'mainnet', config_path: str = CONTAINER_CONFIG_PATH, check_type: CheckType = CheckType.ALL, ) -> ResultList: logger.info('Executing checks. Type: %s', check_type) requirements = get_static_params(node_type, env_type, config_path) - checkers = get_all_checkers(disk, requirements) + + checkers = get_all_checkers(disk, requirements, node_mode) checks = get_checks(checkers, check_type) results = [check() for check in checks] diff --git a/node_cli/core/host.py b/node_cli/core/host.py index 8c684455..a040b8c4 100644 --- a/node_cli/core/host.py +++ b/node_cli/core/host.py @@ -19,9 +19,11 @@ import logging import os -from shutil import copyfile +from shutil import chown from urllib.parse import urlparse +from skale_core.types import EnvType + from node_cli.core.resources import update_resource_allocation from node_cli.utils.helper import error_exit @@ -39,6 +41,7 @@ SGX_CERTS_PATH, REPORTS_PATH, REDIS_DATA_PATH, + SETTINGS_DIR, SCHAINS_DATA_PATH, LOG_PATH, REMOVED_CONTAINERS_FOLDER_PATH, @@ -50,7 +53,6 @@ NGINX_CONFIG_FILEPATH, ) from node_cli.configs.cli_logger import LOG_DATA_PATH -from node_cli.configs.user import SKALE_DIR_ENV_FILEPATH, CONFIGS_ENV_FILEPATH from node_cli.core.nftables import NFTablesManager from node_cli.utils.helper import safe_mkdir @@ -73,28 +75,11 @@ def fix_url(url): return False -def get_flask_secret_key() -> str: - secret_key_filepath = os.path.join(NODE_DATA_PATH, 'flask_db_key.txt') - - if not os.path.exists(secret_key_filepath): - error_exit(f'Flask secret key file not found at {secret_key_filepath}') - - try: - with open(secret_key_filepath, 'r') as key_file: - secret_key = key_file.read().strip() - return secret_key - except (IOError, OSError) as e: - error_exit(f'Failed to read Flask secret key: {e}') - - -def prepare_host(env_filepath: str, env_type: str, allocation: bool = False) -> None: - if not env_filepath or not env_type: - error_exit('Missing required parameters for host initialization') - +def prepare_host(env_type: EnvType, allocation: bool = False) -> None: try: logger.info('Preparing host started') make_dirs() - save_env_params(env_filepath) + chown(REDIS_DATA_PATH, user=999, group=1000) if allocation: update_resource_allocation(env_type) @@ -120,6 +105,7 @@ def make_dirs(): LOG_PATH, REPORTS_PATH, REDIS_DATA_PATH, + SETTINGS_DIR, SKALE_RUN_DIR, SKALE_STATE_DIR, SKALE_TMP_DIR, @@ -127,16 +113,6 @@ def make_dirs(): safe_mkdir(dir_path) -def save_env_params(env_filepath: str) -> None: - copyfile(env_filepath, SKALE_DIR_ENV_FILEPATH) - - -def link_env_file(): - if not (os.path.islink(CONFIGS_ENV_FILEPATH) or os.path.isfile(CONFIGS_ENV_FILEPATH)): - logger.info('Creating symlink %s → %s', SKALE_DIR_ENV_FILEPATH, CONFIGS_ENV_FILEPATH) - os.symlink(SKALE_DIR_ENV_FILEPATH, CONFIGS_ENV_FILEPATH) - - def init_logs_dir(): safe_mkdir(LOG_DATA_PATH) safe_mkdir(REMOVED_CONTAINERS_FOLDER_PATH) diff --git a/node_cli/core/logs.py b/node_cli/core/logs.py index 67472e96..1563ddfc 100644 --- a/node_cli/core/logs.py +++ b/node_cli/core/logs.py @@ -42,7 +42,7 @@ def create_logs_dump(path, filter_container=None): if filter_container: containers = get_containers(filter_container) else: - containers = get_containers('skale') + containers = get_containers('sk_*') for container in containers: log_filepath = os.path.join(containers_logs_path, f'{container.name}.log') diff --git a/node_cli/core/nftables.py b/node_cli/core/nftables.py index 50afff82..e0059fff 100644 --- a/node_cli/core/nftables.py +++ b/node_cli/core/nftables.py @@ -22,9 +22,9 @@ import os import shutil import sys +from dataclasses import dataclass from pathlib import Path from typing import Optional -from dataclasses import dataclass from node_cli.configs import ( ENV, @@ -44,7 +44,8 @@ class ServicePort: DNS: int = 53 CADVISOR: int = 9100 EXPORTER: int = 8080 - WATCHDOG: int = 3009 + WATCHDOG_HTTP: int = 3009 + WATCHDOG_HTTPS: int = 311 HTTPS: int = 443 HTTP: int = 80 @@ -528,7 +529,6 @@ def add_loopback_rule(self, chain) -> None: def get_base_ruleset(self) -> str: self.nft.set_json_output(False) - output = '' try: cmd = f'list chain {self.family} {self.table} {self.chain}' rc, output, error = self.nft.cmd(cmd) @@ -538,7 +538,6 @@ def get_base_ruleset(self) -> str: finally: self.nft.set_json_output(True) - return output def setup_firewall(self, enable_monitoring: bool = False) -> None: """Setup firewall rules.""" @@ -561,7 +560,8 @@ def setup_firewall(self, enable_monitoring: bool = False) -> None: ServicePort.DNS, ServicePort.HTTPS, ServicePort.HTTP, - ServicePort.WATCHDOG, + ServicePort.WATCHDOG_HTTP, + ServicePort.WATCHDOG_HTTPS, ] if enable_monitoring: tcp_ports.extend([ServicePort.EXPORTER, ServicePort.CADVISOR]) @@ -601,7 +601,8 @@ def cleanup_legacy_rules(self, ssh: bool = False, dns: bool = False) -> None: self.remove_drop_rule('udp') tcp_ports = [ ServicePort.HTTPS, - ServicePort.WATCHDOG, + ServicePort.WATCHDOG_HTTP, + ServicePort.WATCHDOG_HTTPS, ServicePort.EXPORTER, ServicePort.CADVISOR, ServicePort.DNS, # tcp is redundant, making sure it's removed diff --git a/node_cli/core/nginx.py b/node_cli/core/nginx.py index 97ea4844..340e3299 100644 --- a/node_cli/core/nginx.py +++ b/node_cli/core/nginx.py @@ -35,12 +35,12 @@ def generate_nginx_config() -> None: ssl_on = check_ssl_certs() - regular_node = is_regular_node_nginx() + skale_node = is_skale_node_nginx() template_data = { 'ssl': ssl_on, - 'regular_node': regular_node, + 'skale_node': skale_node, } - logger.info(f'Processing nginx template. ssl: {ssl_on}, regular_node: {regular_node}') + logger.info(f'Processing nginx template. ssl: {ssl_on}, skale_node: {skale_node}') process_template(NGINX_TEMPLATE_FILEPATH, NGINX_CONFIG_FILEPATH, template_data) @@ -50,8 +50,8 @@ def check_ssl_certs(): return os.path.exists(crt_path) and os.path.exists(key_path) -def is_regular_node_nginx() -> bool: - return TYPE in [NodeType.REGULAR, NodeType.SYNC] +def is_skale_node_nginx() -> bool: + return TYPE == NodeType.SKALE def reload_nginx() -> None: diff --git a/node_cli/core/node.py b/node_cli/core/node.py index f5e0a16b..7c9e2409 100644 --- a/node_cli/core/node.py +++ b/node_cli/core/node.py @@ -28,11 +28,11 @@ import docker +from node_cli.cli import __version__ from node_cli.configs import ( BACKUP_ARCHIVE_NAME, CONTAINER_CONFIG_PATH, FILESTORAGE_MAPPING, - INIT_ENV_FILEPATH, LOG_PATH, RESTORE_SLEEP_TIMEOUT, SCHAINS_MNT_DIR_REGULAR, @@ -41,54 +41,59 @@ SKALE_STATE_DIR, TM_INIT_TIMEOUT, ) -from node_cli.cli import __version__ -from node_cli.configs.user import get_validated_user_config, SKALE_DIR_ENV_FILEPATH from node_cli.configs.cli_logger import LOG_DATA_PATH as CLI_LOG_DATA_PATH - -from node_cli.core.host import is_node_inited, save_env_params, get_flask_secret_key from node_cli.core.checks import run_checks as run_host_checks +from node_cli.core.host import is_node_inited from node_cli.core.resources import update_resource_allocation +from node_cli.core.node_options import ( + active_fair, + active_skale, + upsert_node_mode, + passive_skale, + passive_fair, +) +from node_cli.migrations.focal_to_jammy import migrate as migrate_2_6 from node_cli.operations import ( + cleanup_skale_op, configure_nftables, - update_op, init_op, + init_passive_op, + restore_op, turn_off_op, turn_on_op, - restore_op, - init_sync_op, - update_sync_op, - cleanup_sync_op, + update_op, + update_passive_op, ) -from node_cli.utils.print_formatters import ( - print_failed_requirements_checks, - print_node_cmd_error, - print_node_info, +from node_cli.utils.decorators import check_inited, check_not_inited, check_user +from node_cli.utils.docker_utils import ( + BASE_FAIR_BOOT_COMPOSE_SERVICES, + BASE_FAIR_COMPOSE_SERVICES, + BASE_SKALE_COMPOSE_SERVICES, + BASE_PASSIVE_COMPOSE_SERVICES, + BASE_PASSIVE_FAIR_COMPOSE_SERVICES, + is_admin_running, + is_api_running, ) +from node_cli.utils.exit_codes import CLIExitCodes from node_cli.utils.helper import ( error_exit, get_request, post_request, ) from node_cli.utils.meta import CliMetaManager -from node_cli.utils.texts import safe_load_texts -from node_cli.utils.exit_codes import CLIExitCodes -from node_cli.utils.decorators import check_not_inited, check_inited, check_user -from node_cli.utils.docker_utils import ( - is_admin_running, - is_api_running, - BASE_SKALE_COMPOSE_SERVICES, - BASE_SYNC_COMPOSE_SERVICES, - BASE_FAIR_COMPOSE_SERVICES, - BASE_FAIR_BOOT_COMPOSE_SERVICES, +from node_cli.utils.node_type import NodeType, NodeMode +from node_cli.utils.print_formatters import ( + print_failed_requirements_checks, + print_node_cmd_error, + print_node_info, ) -from node_cli.utils.node_type import NodeType -from node_cli.migrations.focal_to_jammy import migrate as migrate_2_6 - +from node_cli.utils.settings import validate_and_save_node_settings +from skale_core.settings import get_settings +from node_cli.utils.texts import safe_load_texts logger = logging.getLogger(__name__) TEXTS = safe_load_texts() -BASE_CONTAINERS_AMOUNT = 5 BLUEPRINT_NAME = 'node' @@ -103,18 +108,20 @@ class NodeStatuses(Enum): NOT_CREATED = 5 -def is_update_safe(node_type: NodeType) -> bool: - if not is_admin_running(node_type): - if node_type == NodeType.SYNC: +def is_update_safe(node_mode: NodeMode) -> bool: + if not is_admin_running(): + if node_mode == NodeMode.PASSIVE: return True - elif not is_api_running(node_type): + elif not is_api_running(): return True status, payload = get_request(BLUEPRINT_NAME, 'update-safe') if status == 'error': return False - safe = payload['update_safe'] + if not isinstance(payload, dict): + return False + safe = bool(payload.get('update_safe')) if not safe: - logger.info('Locked schains: %s', payload['unsafe_chains']) + logger.info('Locked schains: %s', payload.get('unsafe_chains')) return safe @@ -145,68 +152,85 @@ def register_node(name, p2p_ip, public_ip, port, domain_name): @check_not_inited -def init(env_filepath: str, node_type: NodeType) -> None: - env = compose_node_env(env_filepath=env_filepath, node_type=node_type) +def init(config_file: str, node_type: NodeType) -> None: + node_mode = NodeMode.ACTIVE + settings = validate_and_save_node_settings(config_file, node_type, node_mode) + compose_env = compose_node_env(node_type=node_type, node_mode=node_mode) - init_op(env_filepath=env_filepath, env=env, node_type=node_type) + init_op(settings=settings, compose_env=compose_env, node_mode=node_mode) logger.info('Waiting for containers initialization') time.sleep(TM_INIT_TIMEOUT) - if not is_base_containers_alive(node_type=node_type): + if not is_base_containers_alive(node_type=node_type, node_mode=node_mode): error_exit('Containers are not running', exit_code=CLIExitCodes.OPERATION_EXECUTION_ERROR) logger.info('Generating resource allocation file ...') - update_resource_allocation(env['ENV_TYPE']) + update_resource_allocation(settings.env_type) logger.info('Init procedure finished') @check_not_inited -def restore(backup_path, env_filepath, node_type: NodeType, no_snapshot=False, config_only=False): - env = compose_node_env(env_filepath=env_filepath, node_type=node_type) - if env is None: - return - save_env_params(env_filepath) - env['SKALE_DIR'] = SKALE_DIR - - if not no_snapshot: - logger.info('Adding BACKUP_RUN to env ...') - env['BACKUP_RUN'] = 'True' # should be str - - restored_ok = restore_op(env, backup_path, node_type=node_type, config_only=config_only) +def restore( + backup_path: str, + config_file: str, + node_type: NodeType, + no_snapshot: bool = False, + config_only: bool = False, +): + node_mode = NodeMode.ACTIVE + settings = validate_and_save_node_settings(config_file, node_type, node_mode) + compose_env = compose_node_env(node_type=node_type, node_mode=node_mode) + + restored_ok = restore_op( + settings=settings, + compose_env=compose_env, + backup_path=backup_path, + node_type=node_type, + config_only=config_only, + backup_run=not no_snapshot, + ) if not restored_ok: error_exit('Restore operation failed', exit_code=CLIExitCodes.OPERATION_EXECUTION_ERROR) time.sleep(RESTORE_SLEEP_TIMEOUT) logger.info('Generating resource allocation file ...') - update_resource_allocation(env['ENV_TYPE']) + update_resource_allocation(settings.env_type) print('Node is restored from backup') @check_not_inited -def init_sync( - env_filepath: str, indexer: bool, archive: bool, snapshot: bool, snapshot_from: Optional[str] +def init_passive( + config_file: str, indexer: bool, archive: bool, snapshot: bool, snapshot_from: Optional[str] ) -> None: - env = compose_node_env(env_filepath, node_type=NodeType.SYNC) - if env is None: - return - init_sync_op(env_filepath, env, indexer, archive, snapshot, snapshot_from) + node_mode = NodeMode.PASSIVE + settings = validate_and_save_node_settings(config_file, NodeType.SKALE, node_mode) + compose_env = compose_node_env(node_type=NodeType.SKALE, node_mode=node_mode) + init_passive_op( + settings=settings, + compose_env=compose_env, + indexer=indexer, + archive=archive, + snapshot=snapshot, + snapshot_from=snapshot_from, + ) logger.info('Waiting for containers initialization') time.sleep(TM_INIT_TIMEOUT) - if not is_base_containers_alive(node_type=NodeType.SYNC): + if not is_base_containers_alive(node_type=NodeType.SKALE, node_mode=node_mode): error_exit('Containers are not running', exit_code=CLIExitCodes.OPERATION_EXECUTION_ERROR) - logger.info('Sync node initialized successfully') + logger.info('Passive node initialized successfully') @check_inited @check_user -def update_sync(env_filepath: str, unsafe_ok: bool = False) -> None: +def update_passive(config_file: str) -> None: logger.info('Node update started') prev_version = CliMetaManager().get_meta_info().version if (__version__ == 'test' or __version__.startswith('2.6')) and prev_version == '2.5.0': migrate_2_6() - env = compose_node_env(env_filepath, node_type=NodeType.SYNC) - update_ok = update_sync_op(env_filepath, env) + settings = validate_and_save_node_settings(config_file, NodeType.SKALE, NodeMode.PASSIVE) + compose_env = compose_node_env(node_type=NodeType.SKALE, node_mode=NodeMode.PASSIVE) + update_ok = update_passive_op(settings=settings, compose_env=compose_env) if update_ok: logger.info('Waiting for containers initialization') time.sleep(TM_INIT_TIMEOUT) - alive = is_base_containers_alive(node_type=NodeType.SYNC) + alive = is_base_containers_alive(node_type=NodeType.SKALE, node_mode=NodeMode.PASSIVE) if not update_ok or not alive: print_node_cmd_error() return @@ -214,73 +238,42 @@ def update_sync(env_filepath: str, unsafe_ok: bool = False) -> None: logger.info('Node update finished') -@check_inited @check_user -def cleanup_sync() -> None: - env = compose_node_env(SKALE_DIR_ENV_FILEPATH, save=False, node_type=NodeType.SYNC) - schain_name = env['SCHAIN_NAME'] - cleanup_sync_op(env, schain_name) - logger.info('Sync node was cleaned up, all containers and data removed') +def cleanup(node_mode: NodeMode, prune: bool = False) -> None: + node_mode = upsert_node_mode(node_mode=node_mode) + env = compose_node_env(NodeType.SKALE, node_mode) + cleanup_skale_op(node_mode=node_mode, compose_env=env, prune=prune) + logger.info('SKALE node was cleaned up, all containers and data removed') -def compose_node_env( - env_filepath: str, - node_type: NodeType, - inited_node: bool = False, - sync_schains: Optional[bool] = None, - pull_config_for_schain: Optional[str] = None, - save: bool = True, - is_fair_boot: bool = False, -) -> dict[str, str]: - if env_filepath is not None: - user_config = get_validated_user_config( - node_type=node_type, - env_filepath=env_filepath, - is_fair_boot=is_fair_boot, - ) - if save: - save_env_params(env_filepath) - else: - user_config = get_validated_user_config( - node_type=node_type, - env_filepath=INIT_ENV_FILEPATH, - is_fair_boot=is_fair_boot, - ) - - if node_type == NodeType.SYNC or node_type == NodeType.FAIR: +def compose_node_env(node_type: NodeType, node_mode: NodeMode) -> dict[str, str]: + st = get_settings() + if node_mode == NodeMode.PASSIVE or node_type == NodeType.FAIR: mnt_dir = SCHAINS_MNT_DIR_SINGLE_CHAIN else: mnt_dir = SCHAINS_MNT_DIR_REGULAR - env = { 'SKALE_DIR': SKALE_DIR, 'SCHAINS_MNT_DIR': mnt_dir, 'FILESTORAGE_MAPPING': FILESTORAGE_MAPPING, 'SKALE_LIB_PATH': SKALE_STATE_DIR, - **user_config.to_env(), + 'FILEBEAT_HOST': st.filebeat_host, } - - if inited_node and not node_type == NodeType.SYNC: - env['FLASK_SECRET_KEY'] = get_flask_secret_key() - - if sync_schains and not node_type == NodeType.SYNC: - env['BACKUP_RUN'] = 'True' - - if pull_config_for_schain: - env['PULL_CONFIG_FOR_SCHAIN'] = pull_config_for_schain - return {k: v for k, v in env.items() if v != ''} @check_inited @check_user def update( - env_filepath: str, + config_file: str, pull_config_for_schain: Optional[str], node_type: NodeType, + node_mode: NodeMode, unsafe_ok: bool = False, ) -> None: - if not unsafe_ok and not is_update_safe(node_type=node_type): + node_mode = upsert_node_mode(node_mode=node_mode) + + if not unsafe_ok and not is_update_safe(node_mode=node_mode): error_msg = 'Cannot update safely' error_exit(error_msg, exit_code=CLIExitCodes.UNSAFE_UPDATE) @@ -288,18 +281,13 @@ def update( if (__version__ == 'test' or __version__.startswith('2.6')) and prev_version == '2.5.0': migrate_2_6() logger.info('Node update started') - env = compose_node_env( - env_filepath, - inited_node=True, - sync_schains=False, - pull_config_for_schain=pull_config_for_schain, - node_type=node_type, - ) - update_ok = update_op(env_filepath, env, node_type=node_type) + settings = validate_and_save_node_settings(config_file, node_type, node_mode) + compose_env = compose_node_env(node_type=node_type, node_mode=node_mode) + update_ok = update_op(settings=settings, compose_env=compose_env, node_mode=node_mode) if update_ok: logger.info('Waiting for containers initialization') time.sleep(TM_INIT_TIMEOUT) - alive = is_base_containers_alive(node_type=node_type) + alive = is_base_containers_alive(node_type=node_type, node_mode=node_mode) if not update_ok or not alive: print_node_cmd_error() return @@ -402,25 +390,33 @@ def set_maintenance_mode_off(): @check_inited @check_user def turn_off(node_type: NodeType, maintenance_on: bool = False, unsafe_ok: bool = False) -> None: - if not unsafe_ok and not is_update_safe(node_type=node_type): + node_mode = upsert_node_mode() + if not unsafe_ok and not is_update_safe(node_mode=node_mode): error_msg = 'Cannot turn off safely' error_exit(error_msg, exit_code=CLIExitCodes.UNSAFE_UPDATE) if maintenance_on: set_maintenance_mode_on() - env = compose_node_env(SKALE_DIR_ENV_FILEPATH, save=False, node_type=node_type) - turn_off_op(node_type=node_type, env=env) + compose_env = compose_node_env(node_type=node_type, node_mode=node_mode) + turn_off_op(compose_env=compose_env, node_type=node_type, node_mode=node_mode) @check_inited @check_user -def turn_on(maintenance_off, sync_schains, env_file, node_type: NodeType) -> None: - env = compose_node_env( - env_file, inited_node=True, sync_schains=sync_schains, node_type=node_type +def turn_on(maintenance_off: bool, sync_schains: bool, env_file: str, node_type: NodeType) -> None: + node_mode = upsert_node_mode() + settings = validate_and_save_node_settings(env_file, node_type, node_mode) + compose_env = compose_node_env(node_type=node_type, node_mode=node_mode) + backup_run = sync_schains and node_mode != NodeMode.PASSIVE + turn_on_op( + settings=settings, + compose_env=compose_env, + node_type=node_type, + node_mode=node_mode, + backup_run=backup_run, ) - turn_on_op(env=env, node_type=node_type) logger.info('Waiting for containers initialization') time.sleep(TM_INIT_TIMEOUT) - if not is_base_containers_alive(node_type=node_type): + if not is_base_containers_alive(node_type=node_type, node_mode=node_mode): print_node_cmd_error() return logger.info('Node turned on') @@ -428,21 +424,30 @@ def turn_on(maintenance_off, sync_schains, env_file, node_type: NodeType) -> Non set_maintenance_mode_off() -def get_expected_container_names(node_type: NodeType, is_fair_boot: bool) -> list[str]: +def get_expected_container_names( + node_type: NodeType, + node_mode: NodeMode, + is_fair_boot: bool, +) -> list[str]: if node_type == NodeType.FAIR and is_fair_boot: services = BASE_FAIR_BOOT_COMPOSE_SERVICES - elif node_type == NodeType.FAIR and not is_fair_boot: + elif active_fair(node_type, node_mode): services = BASE_FAIR_COMPOSE_SERVICES - elif node_type == NodeType.SYNC: - services = BASE_SYNC_COMPOSE_SERVICES - else: + elif passive_fair(node_type, node_mode): + services = BASE_PASSIVE_FAIR_COMPOSE_SERVICES + elif active_skale(node_type, node_mode): services = BASE_SKALE_COMPOSE_SERVICES - + elif passive_skale(node_type, node_mode): + services = BASE_PASSIVE_COMPOSE_SERVICES return list(services.values()) -def is_base_containers_alive(node_type: NodeType, is_fair_boot: bool = False) -> bool: - base_container_names = get_expected_container_names(node_type, is_fair_boot) +def is_base_containers_alive( + node_type: NodeType, + node_mode: NodeMode, + is_fair_boot: bool = False, +) -> bool: + base_container_names = get_expected_container_names(node_type, node_mode, is_fair_boot) dclient = docker.from_env() running_container_names = set(container.name for container in dclient.containers.list()) @@ -493,6 +498,7 @@ def set_domain_name(domain_name): def run_checks( node_type: NodeType, + node_mode: NodeMode, network: str = 'mainnet', container_config_path: str = CONTAINER_CONFIG_PATH, disk: Optional[str] = None, @@ -502,9 +508,9 @@ def run_checks( return if disk is None: - env_config = get_validated_user_config(node_type=node_type) - disk = env_config.disk_mountpoint - failed_checks = run_host_checks(disk, node_type, network, container_config_path) + settings = get_settings() + disk = settings.block_device + failed_checks = run_host_checks(disk, node_type, node_mode, network, container_config_path) if not failed_checks: print('Requirements checking successfully finished!') else: diff --git a/node_cli/core/node_options.py b/node_cli/core/node_options.py index 49a0ea05..764e3b5f 100644 --- a/node_cli/core/node_options.py +++ b/node_cli/core/node_options.py @@ -19,9 +19,11 @@ import logging +from node_cli.utils.node_type import NodeMode, NodeType from node_cli.utils.helper import read_json, write_json, init_file from node_cli.configs.node_options import NODE_OPTIONS_FILEPATH + logger = logging.getLogger(__name__) @@ -41,7 +43,7 @@ def _set(self, field_name: str, field_value) -> None: @property def archive(self) -> bool: - return self._get('archive') + return self._get('archive') or False @archive.setter def archive(self, archive: bool) -> None: @@ -49,7 +51,7 @@ def archive(self, archive: bool) -> None: @property def catchup(self) -> bool: - return self._get('catchup') + return self._get('catchup') or False @catchup.setter def catchup(self, catchup: bool) -> None: @@ -57,11 +59,79 @@ def catchup(self, catchup: bool) -> None: @property def historic_state(self) -> bool: - return self._get('historic_state') + return self._get('historic_state') or False @historic_state.setter def historic_state(self, historic_state: bool) -> None: return self._set('historic_state', historic_state) + @property + def node_mode(self) -> NodeMode: + return NodeMode(self._get('node_mode')) + + @node_mode.setter + def node_mode(self, node_mode: NodeMode) -> None: + return self._set('node_mode', node_mode.value) + def all(self) -> dict: return read_json(self.filepath) + + +def mark_active_node() -> None: + node_options = NodeOptions() + node_options.node_mode = NodeMode.ACTIVE + logger.info('Node marked as active.') + + +def mark_passive_node() -> None: + node_options = NodeOptions() + node_options.node_mode = NodeMode.PASSIVE + logger.info('Node marked as passive.') + + +def set_passive_node_options( + archive: bool, + indexer: bool, +) -> None: + node_options = NodeOptions() + node_options.node_mode = NodeMode.PASSIVE + node_options.archive = archive or indexer + node_options.catchup = archive or indexer + node_options.historic_state = archive + logger.info('Node options set for passive mode.') + + +class NodeModeMismatchError(Exception): + pass + + +def upsert_node_mode(node_mode: NodeMode | None = None) -> NodeMode: + node_options = NodeOptions() + try: + options_mode = node_options.node_mode + if node_mode is not None and options_mode != node_mode: + raise NodeModeMismatchError( + f'Cannot change node mode from {options_mode} to {node_mode}' + ) + return options_mode + except ValueError: + if node_mode is None: + raise NodeModeMismatchError('Node mode is not set') + node_options.node_mode = node_mode + return node_mode + + +def active_skale(node_type: NodeType, node_mode: NodeMode) -> bool: + return node_mode == NodeMode.ACTIVE and node_type == NodeType.SKALE + + +def active_fair(node_type: NodeType, node_mode: NodeMode) -> bool: + return node_mode == NodeMode.ACTIVE and node_type == NodeType.FAIR + + +def passive_skale(node_type: NodeType, node_mode: NodeMode) -> bool: + return node_mode == NodeMode.PASSIVE and node_type == NodeType.SKALE + + +def passive_fair(node_type: NodeType, node_mode: NodeMode) -> bool: + return node_mode == NodeMode.PASSIVE and node_type == NodeType.FAIR diff --git a/node_cli/core/resources.py b/node_cli/core/resources.py index 216aa657..34f4cac5 100644 --- a/node_cli/core/resources.py +++ b/node_cli/core/resources.py @@ -24,11 +24,13 @@ import psutil -from node_cli.configs.user import get_validated_user_config +from skale_core.types import EnvType + +from node_cli.utils.settings import validate_and_save_node_settings from node_cli.utils.docker_utils import ensure_volume from node_cli.utils.schain_types import SchainTypes from node_cli.utils.helper import write_json, read_json, run_cmd, safe_load_yml -from node_cli.utils.node_type import NodeType +from node_cli.utils.node_type import NodeType, NodeMode from node_cli.configs import ALLOCATION_FILEPATH, STATIC_PARAMS_FILEPATH, SNAPSHOTS_SHARED_VOLUME from node_cli.configs.resource_allocation import ( RESOURCE_ALLOCATION_FILEPATH, @@ -73,7 +75,7 @@ def get_resource_allocation_info(): return None -def compose_resource_allocation_config(env_type: str, params_by_env_type: Dict = None) -> Dict: +def compose_resource_allocation_config(env_type: EnvType, params_by_env_type: Dict = None) -> Dict: params_by_env_type = params_by_env_type or safe_load_yml(STATIC_PARAMS_FILEPATH) common_config = params_by_env_type['common'] schain_cpu_alloc, ima_cpu_alloc = get_cpu_alloc(common_config) @@ -81,7 +83,7 @@ def compose_resource_allocation_config(env_type: str, params_by_env_type: Dict = schain_allocation_data = safe_load_yml(ALLOCATION_FILEPATH) return { - 'schain': { + 'skaled': { 'cpu_shares': schain_cpu_alloc.dict(), 'mem': schain_mem_alloc.dict(), 'disk': schain_allocation_data[env_type]['disk'], @@ -93,19 +95,20 @@ def compose_resource_allocation_config(env_type: str, params_by_env_type: Dict = def generate_resource_allocation_config( - env_file, + env_file: str, node_type: NodeType, - force=False, + node_mode: NodeMode, + force: bool = False, ) -> None: if not force and os.path.isfile(RESOURCE_ALLOCATION_FILEPATH): msg = 'Resource allocation file already exists' logger.debug(msg) print(msg) return - user_config = get_validated_user_config(node_type=node_type, env_filepath=env_file) + settings = validate_and_save_node_settings(env_file, node_type, node_mode) logger.info('Generating resource allocation file ...') try: - update_resource_allocation(user_config.env_type) + update_resource_allocation(settings.env_type) except Exception as e: logger.exception(e) print("Can't generate resource allocation file, check out CLI logs") @@ -113,7 +116,7 @@ def generate_resource_allocation_config( print(f'Resource allocation file generated: {RESOURCE_ALLOCATION_FILEPATH}') -def update_resource_allocation(env_type: str) -> None: +def update_resource_allocation(env_type: EnvType) -> None: resource_allocation_config = compose_resource_allocation_config(env_type) write_json(RESOURCE_ALLOCATION_FILEPATH, resource_allocation_config) @@ -148,13 +151,10 @@ def get_cpu_alloc(common_config: Dict) -> ResourceAlloc: cpu_proportions = common_config['schain']['cpu'] schain_max_cpu_shares = int(cpu_proportions['skaled'] * MAX_CPU_SHARES) ima_max_cpu_shares = int(cpu_proportions['ima'] * MAX_CPU_SHARES) - return (ResourceAlloc(schain_max_cpu_shares), ResourceAlloc(ima_max_cpu_shares)) + return ResourceAlloc(schain_max_cpu_shares), ResourceAlloc(ima_max_cpu_shares) -def verify_disk_size( - disk_device: str, - env_configs: dict, -) -> Dict: +def verify_disk_size(disk_device: str, env_configs: dict): disk_size = get_disk_size(disk_device) env_disk_size = env_configs['server']['disk'] check_disk_size(disk_size, env_disk_size) diff --git a/node_cli/core/schains.py b/node_cli/core/schains.py index eb420fdb..8aac42f1 100644 --- a/node_cli/core/schains.py +++ b/node_cli/core/schains.py @@ -26,6 +26,8 @@ from pathlib import Path from typing import Dict, Optional +from skale_core.types import EnvType + from lvmpy.src.core import mount, volume_mountpoint from node_cli.configs import ( ALLOCATION_FILEPATH, @@ -34,12 +36,13 @@ SCHAIN_NODE_DATA_PATH, SCHAINS_MNT_DIR_SINGLE_CHAIN, ) -from node_cli.configs.user import get_validated_user_config from node_cli.utils.docker_utils import ensure_volume, is_volume_exists from node_cli.utils.exit_codes import CLIExitCodes from node_cli.utils.helper import ( + cleanup_dir_content, error_exit, get_request, + is_btrfs_subvolume, read_json, run_cmd, safe_load_yml, @@ -212,12 +215,9 @@ def restore_schain_from_snapshot( schain: str, snapshot_path: str, node_type: NodeType, - env_type: Optional[str] = None, + env_type: EnvType, schain_type: str = 'medium', ) -> None: - if env_type is None: - user_config = get_validated_user_config(node_type=node_type) - env_type = user_config.env_type ensure_schain_volume(schain, schain_type, env_type) block_number = get_block_number_from_path(snapshot_path) if block_number == -1: @@ -240,12 +240,12 @@ def get_schains_by_artifacts() -> str: return '\n'.join(os.listdir(SCHAIN_NODE_DATA_PATH)) -def get_schain_volume_size(schain_type: str, env_type: str) -> int: +def get_schain_volume_size(schain_type: str, env_type: EnvType) -> int: alloc = safe_load_yml(ALLOCATION_FILEPATH) return alloc[env_type]['disk'][schain_type] -def ensure_schain_volume(schain: str, schain_type: str, env_type: str) -> None: +def ensure_schain_volume(schain: str, schain_type: str, env_type: EnvType) -> None: if not is_volume_exists(schain): size = get_schain_volume_size(schain_type, env_type) ensure_volume(schain, size) @@ -253,35 +253,62 @@ def ensure_schain_volume(schain: str, schain_type: str, env_type: str) -> None: logger.warning('Volume %s already exists', schain) -def cleanup_datadir_for_single_chain_node( +def cleanup_datadir_content(datadir_path: str) -> None: + regular_folders_pattern = f'{datadir_path}/[!snapshots]*' + logger.info('Removing regular folders of %s', datadir_path) + for path in glob.glob(regular_folders_pattern): + logger.debug('Removing recursively %s', path) + if os.path.isfile(path): + logger.debug('Deleting file in datadir: %s', path) + os.remove(path) + if os.path.isdir(path): + logger.debug('Deleting folder in datadir: %s', path) + shutil.rmtree(path) + + logger.info('Removing subvolumes of %s', datadir_path) + subvolumes_pattern = f'{datadir_path}/snapshots/*/*' + for path in glob.glob(subvolumes_pattern): + if is_btrfs_subvolume(path): + logger.debug('Deleting subvolume %s', path) + rm_btrfs_subvolume(path) + if os.path.isfile(path): + logger.debug('Deleting file in snapshots directory: %s', path) + os.remove(path) + if os.path.isdir(path): + logger.debug('Deleting folder in snapshots directory %s', path) + shutil.rmtree(path) + + shutil.rmtree(os.path.join(datadir_path, 'snapshots'), ignore_errors=True) + + +def cleanup_no_lvm_datadir( chain_name: str = '', base_path: str = SCHAINS_MNT_DIR_SINGLE_CHAIN ) -> None: - if not chain_name: + if chain_name: + folders = [chain_name] + else: folders = [f for f in os.listdir(base_path) if os.path.isdir(os.path.join(base_path, f))] if not folders: raise NoDataDirForChainError( f'No data directory found in {base_path}. ' 'Please check the path or specify a chain name.' ) - chain_name = folders[0] - base_path = os.path.join(base_path, chain_name) - regular_folders_pattern = f'{base_path}/[!snapshots]*' - logger.info('Removing regular folders') - for filepath in glob.glob(regular_folders_pattern): - if os.path.isdir(filepath): - logger.debug('Removing recursively %s', filepath) - shutil.rmtree(filepath) - if os.path.isfile(filepath): - os.remove(filepath) - - logger.info('Removing subvolumes') - subvolumes_pattern = f'{base_path}/snapshots/*/*' - for filepath in glob.glob(subvolumes_pattern): - logger.debug('Deleting subvolume %s', filepath) - if os.path.isdir(filepath): - rm_btrfs_subvolume(filepath) - else: - os.remove(filepath) - logger.info('Cleaning up snapshots folder') - if os.path.isdir(base_path): - shutil.rmtree(base_path) + for folder_name in folders: + folder_path = os.path.join(base_path, folder_name) + if folder_name != 'shared-space': + logger.info('Removing datadir content for %s', folder_path) + cleanup_datadir_content(folder_path) + if os.path.isdir(folder_path): + shutil.rmtree(folder_path) + run_cmd(['umount', base_path]) + + +def cleanup_lvm_datadir(): + logger.info('Starting cleanup for active node...') + logger.info('Unmounting /mnt/schains-shared-space...') + run_cmd(['sudo', 'umount', '/mnt/schains-shared-space'], check_code=False) + logger.info('Cleaning up /mnt directory content...') + cleanup_dir_content('/mnt/') + logger.info('Removing LVM volume group "schains"...') + run_cmd(['sudo', 'lvremove', '-f', 'schains'], check_code=False) + logger.info('Active node cleanup finished.') diff --git a/node_cli/core/ssl/check.py b/node_cli/core/ssl/check.py index 289bc77e..327f72a6 100644 --- a/node_cli/core/ssl/check.py +++ b/node_cli/core/ssl/check.py @@ -20,6 +20,7 @@ import time import socket import logging +import subprocess from contextlib import contextmanager from node_cli.core.ssl.utils import detached_subprocess @@ -196,8 +197,15 @@ def check_ssl_connection(host, port, silent=False): ] expose_output = not silent with detached_subprocess(ssl_check_cmd, expose_output=expose_output) as dp: - time.sleep(1) - code = dp.poll() - if code is not None: - logger.error('Healthcheck connection failed') - raise SSLHealthcheckError('OpenSSL connection verification failed') + timeout = 20 + try: + dp.wait(timeout=timeout) + except subprocess.TimeoutExpired: + logger.error('Healthcheck timed-out after %s s', timeout) + raise SSLHealthcheckError('OpenSSL connection verification timed-out') + + if dp.returncode == 0: # success + return + + logger.error('Healthcheck connection failed (code %s)', dp.returncode) + raise SSLHealthcheckError('OpenSSL connection verification failed') diff --git a/node_cli/core/ssl/utils.py b/node_cli/core/ssl/utils.py index a5a329dd..c80b3a8a 100644 --- a/node_cli/core/ssl/utils.py +++ b/node_cli/core/ssl/utils.py @@ -17,13 +17,13 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import logging import os import shutil -import logging import subprocess from contextlib import contextmanager -from node_cli.configs.ssl import SSL_CERT_FILEPATH, SSL_KEY_FILEPATH, SSL_FOLDER_PATH +from node_cli.configs.ssl import SSL_CERT_FILEPATH, SSL_FOLDER_PATH, SSL_KEY_FILEPATH logger = logging.getLogger(__name__) @@ -48,7 +48,13 @@ def is_ssl_folder_empty(ssl_path=SSL_FOLDER_PATH): @contextmanager def detached_subprocess(cmd, expose_output=False): logger.debug(f'Starting detached subprocess: {cmd}') - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, encoding='utf-8') + p = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + stdin=subprocess.DEVNULL, + encoding='utf-8', + ) try: yield p finally: diff --git a/node_cli/core/static_config.py b/node_cli/core/static_config.py index a93a7a60..06e0e7f4 100644 --- a/node_cli/core/static_config.py +++ b/node_cli/core/static_config.py @@ -28,10 +28,12 @@ ) from node_cli.utils.node_type import NodeType +from skale_core.types import EnvType + def get_static_params( node_type: NodeType, - env_type: str = 'mainnet', + env_type: EnvType = 'mainnet', config_path: str = CONTAINER_CONFIG_PATH, ) -> dict: if node_type == NodeType.FAIR: @@ -46,7 +48,7 @@ def get_static_params( return ydata['envs'][env_type] -def get_fair_chain_name(env: dict) -> str: +def get_fair_chain_name(env_type: EnvType) -> str: node_type = NodeType.FAIR - params = get_static_params(node_type, env['ENV_TYPE']) + params = get_static_params(node_type, env_type) return params['info']['chain_name'] diff --git a/node_cli/fair/fair_node.py b/node_cli/fair/active.py similarity index 54% rename from node_cli/fair/fair_node.py rename to node_cli/fair/active.py index e32149e9..2ec42bde 100644 --- a/node_cli/fair/fair_node.py +++ b/node_cli/fair/active.py @@ -22,24 +22,20 @@ import time from typing import cast -from node_cli.configs import DEFAULT_SKALED_BASE_PORT, RESTORE_SLEEP_TIMEOUT, SKALE_DIR -from node_cli.configs.user import SKALE_DIR_ENV_FILEPATH -from node_cli.core.docker_config import cleanup_docker_configuration -from node_cli.core.host import is_node_inited, save_env_params +from node_cli.configs import DEFAULT_SKALED_BASE_PORT, RESTORE_SLEEP_TIMEOUT +from node_cli.core.host import is_node_inited from node_cli.core.node import compose_node_env, is_base_containers_alive -from node_cli.fair.record.chain_record import get_fair_chain_record from node_cli.operations import ( FairUpdateType, - cleanup_fair_op, - init_fair_op, restore_fair_op, update_fair_op, ) from node_cli.utils.decorators import check_inited, check_not_inited, check_user from node_cli.utils.exit_codes import CLIExitCodes from node_cli.utils.helper import error_exit, get_request, post_request -from node_cli.utils.node_type import NodeType +from node_cli.utils.node_type import NodeMode, NodeType from node_cli.utils.print_formatters import print_node_cmd_error, print_node_info_fair +from node_cli.utils.settings import validate_and_save_node_settings from node_cli.utils.texts import safe_load_texts logger = logging.getLogger(__name__) @@ -65,35 +61,22 @@ def get_node_info(format): print_node_info_fair(node_info) -@check_not_inited -def restore_fair(backup_path, env_filepath, config_only=False): - env = compose_node_env(env_filepath, node_type=NodeType.FAIR) - if env is None: - return - save_env_params(env_filepath) - env['SKALE_DIR'] = SKALE_DIR - - restored_ok = restore_fair_op(env, backup_path, config_only=config_only) - if not restored_ok: - error_exit('Restore operation failed', exit_code=CLIExitCodes.OPERATION_EXECUTION_ERROR) - time.sleep(RESTORE_SLEEP_TIMEOUT) - print('Fair node is restored from backup') - - @check_inited @check_user def migrate_from_boot( - env_filepath: str, + config_file: str, ) -> None: logger.info('Migrating from boot to fair node...') - env = compose_node_env( - env_filepath, - inited_node=True, - sync_schains=False, - node_type=NodeType.FAIR, + settings = validate_and_save_node_settings(config_file, NodeType.FAIR, NodeMode.ACTIVE) + compose_env = compose_node_env(node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE) + migrate_ok = update_fair_op( + settings=settings, + compose_env=compose_env, + node_mode=NodeMode.ACTIVE, + update_type=FairUpdateType.FROM_BOOT, + force_skaled_start=False, ) - migrate_ok = update_fair_op(env_filepath, env, update_type=FairUpdateType.FROM_BOOT) - alive = is_base_containers_alive(node_type=NodeType.FAIR) + alive = is_base_containers_alive(node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE) if not migrate_ok or not alive: print_node_cmd_error() return @@ -103,70 +86,94 @@ def migrate_from_boot( @check_inited @check_user -def update(env_filepath: str, pull_config_for_schain: str | None = None) -> None: - logger.info('Updating fair node...') - env = compose_node_env( - env_filepath, - inited_node=True, - sync_schains=False, - node_type=NodeType.FAIR, - pull_config_for_schain=pull_config_for_schain, - ) - update_ok = update_fair_op(env_filepath, env, update_type=FairUpdateType.REGULAR) - alive = is_base_containers_alive(node_type=NodeType.FAIR) - if not update_ok or not alive: - print_node_cmd_error() +def register(ip: str) -> None: + if not is_node_inited(): + print(TEXTS['fair']['node']['not_inited']) return + + json_data = {'ip': ip, 'port': DEFAULT_SKALED_BASE_PORT} + status, payload = post_request(blueprint=BLUEPRINT_NAME, method='register', json=json_data) + if status == 'ok': + msg = TEXTS['fair']['node']['registered'] + logger.info(msg) + print(msg) else: - logger.info('Fair update completed successfully') + error_msg = payload + logger.error(f'Registration error {error_msg}') + error_exit(error_msg, exit_code=CLIExitCodes.BAD_API_RESPONSE) -def request_repair(snapshot_from: str = '') -> None: - env = compose_node_env(SKALE_DIR_ENV_FILEPATH, save=False, node_type=NodeType.FAIR) - record = get_fair_chain_record(env) - record.set_repair_ts(int(time.time())) - record.set_snapshot_from(snapshot_from) - print(TEXTS['fair']['node']['repair']['repair_requested']) +@check_inited +@check_user +def change_ip(ip: str) -> None: + if not is_node_inited(): + print(TEXTS['fair']['node']['not_inited']) + return + + json_data = {'ip': ip, 'port': DEFAULT_SKALED_BASE_PORT} + status, payload = post_request(blueprint=BLUEPRINT_NAME, method='change-ip', json=json_data) + if status == 'ok': + msg = TEXTS['fair']['node']['ip_changed'] + logger.info(msg) + print(msg) + else: + error_msg = payload + logger.error(f'Change IP error {error_msg}') + error_exit(error_msg, exit_code=CLIExitCodes.BAD_API_RESPONSE) @check_inited @check_user -def cleanup() -> None: - env = compose_node_env(SKALE_DIR_ENV_FILEPATH, save=False, node_type=NodeType.FAIR) - cleanup_fair_op(env) - logger.info('Fair node was cleaned up, all containers and data removed') - cleanup_docker_configuration() +def exit() -> None: + if not is_node_inited(): + print(TEXTS['fair']['node']['not_inited']) + return + status, payload = post_request(blueprint=BLUEPRINT_NAME, method='exit', json={}) + if status == 'ok': + msg = TEXTS['fair']['node']['exited'] + logger.info(msg) + print(msg) + else: + error_msg = payload + logger.error(f'Node exit error {error_msg}') + error_exit(error_msg, exit_code=CLIExitCodes.BAD_API_RESPONSE) -@check_not_inited -def init(env_filepath: str) -> None: - env = compose_node_env(env_filepath, node_type=NodeType.FAIR) - if env is None: - return - save_env_params(env_filepath) - env['SKALE_DIR'] = SKALE_DIR - init_ok = init_fair_op(env_filepath, env) - if not init_ok: - error_exit('Init operation failed', exit_code=CLIExitCodes.OPERATION_EXECUTION_ERROR) +@check_not_inited +def restore(backup_path: str, config_file: str, config_only: bool = False): + node_mode = NodeMode.ACTIVE + settings = validate_and_save_node_settings(config_file, NodeType.FAIR, node_mode) + compose_env = compose_node_env(node_type=NodeType.FAIR, node_mode=node_mode) + + restored_ok = restore_fair_op( + node_mode=node_mode, + settings=settings, + compose_env=compose_env, + backup_path=backup_path, + config_only=config_only, + ) + if not restored_ok: + error_exit('Restore operation failed', exit_code=CLIExitCodes.OPERATION_EXECUTION_ERROR) time.sleep(RESTORE_SLEEP_TIMEOUT) - print('Fair node is initialized') + print('Fair node is restored from backup') @check_inited @check_user -def register(ip: str) -> None: +def set_domain_name(domain_name): if not is_node_inited(): print(TEXTS['fair']['node']['not_inited']) return - json_data = {'ip': ip, 'port': DEFAULT_SKALED_BASE_PORT} - status, payload = post_request(blueprint=BLUEPRINT_NAME, method='register', json=json_data) + status, payload = post_request( + blueprint=BLUEPRINT_NAME, method='set-domain-name', json={'domain_name': domain_name} + ) if status == 'ok': - msg = TEXTS['fair']['node']['registered'] + msg = TEXTS['node']['domain_name_changed'] logger.info(msg) print(msg) else: error_msg = payload - logger.error(f'Registration error {error_msg}') + logger.error(f'Setting domain name error {error_msg}') error_exit(error_msg, exit_code=CLIExitCodes.BAD_API_RESPONSE) diff --git a/node_cli/fair/fair_boot.py b/node_cli/fair/boot.py similarity index 62% rename from node_cli/fair/fair_boot.py rename to node_cli/fair/boot.py index 290bbb1e..d9da98fa 100644 --- a/node_cli/fair/fair_boot.py +++ b/node_cli/fair/boot.py @@ -23,49 +23,51 @@ from node_cli.configs import TM_INIT_TIMEOUT from node_cli.core.node import compose_node_env, is_base_containers_alive +from node_cli.core.node_options import upsert_node_mode from node_cli.operations import init_fair_boot_op, update_fair_boot_op from node_cli.utils.decorators import check_inited, check_not_inited, check_user from node_cli.utils.exit_codes import CLIExitCodes from node_cli.utils.helper import error_exit -from node_cli.utils.node_type import NodeType +from node_cli.utils.node_type import NodeMode, NodeType from node_cli.utils.print_formatters import print_node_cmd_error +from node_cli.utils.settings import validate_and_save_node_settings logger = logging.getLogger(__name__) @check_not_inited -def init(env_filepath: str) -> None: - env = compose_node_env( - env_filepath, - node_type=NodeType.FAIR, - is_fair_boot=True, - ) +def init(config_file: str) -> None: + node_mode = NodeMode.ACTIVE + node_type = NodeType.FAIR + settings = validate_and_save_node_settings(config_file, node_type, node_mode) + compose_env = compose_node_env(node_type=node_type, node_mode=node_mode) - init_fair_boot_op(env_filepath, env) + init_fair_boot_op(settings=settings, compose_env=compose_env, node_mode=node_mode) logger.info('Waiting for fair containers initialization') time.sleep(TM_INIT_TIMEOUT) - if not is_base_containers_alive(node_type=NodeType.FAIR, is_fair_boot=True): + if not is_base_containers_alive(node_type=node_type, node_mode=node_mode, is_fair_boot=True): error_exit('Containers are not running', exit_code=CLIExitCodes.OPERATION_EXECUTION_ERROR) logger.info('Init fair procedure finished') @check_inited @check_user -def update(env_filepath: str, pull_config_for_schain: str) -> None: +def update(config_file: str, pull_config_for_schain: str) -> None: logger.info('Fair boot node update started') - env = compose_node_env( - env_filepath, - inited_node=True, - sync_schains=False, - pull_config_for_schain=pull_config_for_schain, - node_type=NodeType.FAIR, - is_fair_boot=True, + node_mode = upsert_node_mode(node_mode=NodeMode.ACTIVE) + settings = validate_and_save_node_settings(config_file, NodeType.FAIR, node_mode) + compose_env = compose_node_env(node_type=NodeType.FAIR, node_mode=node_mode) + migrate_ok = update_fair_boot_op( + settings=settings, + compose_env=compose_env, + node_mode=NodeMode.ACTIVE, ) - migrate_ok = update_fair_boot_op(env_filepath, env) if migrate_ok: logger.info('Waiting for containers initialization') time.sleep(TM_INIT_TIMEOUT) - alive = is_base_containers_alive(node_type=NodeType.FAIR, is_fair_boot=True) + alive = is_base_containers_alive( + node_type=NodeType.FAIR, node_mode=node_mode, is_fair_boot=True + ) if not migrate_ok or not alive: print_node_cmd_error() return diff --git a/node_cli/fair/chain.py b/node_cli/fair/chain.py new file mode 100644 index 00000000..8ca380ae --- /dev/null +++ b/node_cli/fair/chain.py @@ -0,0 +1,65 @@ +# -*- coding: utf-8 -*- +# +# This file is part of node-cli +# +# Copyright (C) 2025-Present SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import json +from typing import Any, Dict + +from node_cli.utils.exit_codes import CLIExitCodes +from node_cli.utils.helper import error_exit, get_request +from node_cli.utils.print_formatters import print_chain_record, print_chain_checks + +BLUEPRINT_NAME = 'fair-chain' + + +def get_chain_record_plain() -> Dict[str, Any]: + status, payload = get_request(blueprint=BLUEPRINT_NAME, method='record') + if status == 'ok': + if isinstance(payload, dict): + return payload.get('record', {}) + else: + error_exit('Invalid response format', exit_code=CLIExitCodes.BAD_API_RESPONSE) + else: + error_exit(payload, exit_code=CLIExitCodes.BAD_API_RESPONSE) + + +def get_chain_record(raw: bool = False) -> None: + record = get_chain_record_plain() + if raw: + print(json.dumps(record, indent=4)) + else: + print_chain_record(record) + + +def get_chain_checks_plain() -> Dict[str, Any]: + status, payload = get_request(blueprint=BLUEPRINT_NAME, method='checks') + if status == 'ok': + if isinstance(payload, dict): + return payload + else: + error_exit('Invalid response format', exit_code=CLIExitCodes.BAD_API_RESPONSE) + else: + error_exit(payload, exit_code=CLIExitCodes.BAD_API_RESPONSE) + + +def get_chain_checks(raw: bool = False) -> None: + checks = get_chain_checks_plain() + if raw: + print(json.dumps(checks, indent=4)) + else: + print_chain_checks(checks) diff --git a/node_cli/fair/common.py b/node_cli/fair/common.py new file mode 100644 index 00000000..98f10a7f --- /dev/null +++ b/node_cli/fair/common.py @@ -0,0 +1,146 @@ +# -*- coding: utf-8 -*- +# +# This file is part of node-cli +# +# Copyright (C) 2025-Present SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import logging +import time + +from node_cli.configs import INIT_TIMEOUT, TM_INIT_TIMEOUT +from node_cli.core.node import compose_node_env, is_base_containers_alive +from node_cli.core.node_options import upsert_node_mode +from node_cli.fair.passive import setup_fair_passive +from node_cli.operations import ( + FairUpdateType, + cleanup_fair_op, + init_fair_op, + repair_fair_op, + turn_off_op, + turn_on_op, + update_fair_op, +) +from node_cli.utils.decorators import check_inited, check_not_inited, check_user +from node_cli.utils.exit_codes import CLIExitCodes +from node_cli.utils.helper import error_exit +from node_cli.utils.node_type import NodeMode, NodeType +from node_cli.utils.print_formatters import print_node_cmd_error +from node_cli.utils.settings import validate_and_save_node_settings +from node_cli.utils.texts import safe_load_texts +from skale_core.settings import get_settings + +logger = logging.getLogger(__name__) +TEXTS = safe_load_texts() + + +@check_not_inited +def init( + node_mode: NodeMode, + config_file: str, + node_id: int | None = None, + indexer: bool = False, + archive: bool = False, + snapshot: str | None = None, +) -> None: + settings = validate_and_save_node_settings(config_file, NodeType.FAIR, node_mode) + compose_env = compose_node_env(node_type=NodeType.FAIR, node_mode=node_mode) + + init_ok = init_fair_op( + settings=settings, + compose_env=compose_env, + node_mode=node_mode, + indexer=indexer, + archive=archive, + snapshot=snapshot, + ) + if not init_ok: + error_exit('Init operation failed', exit_code=CLIExitCodes.OPERATION_EXECUTION_ERROR) + time.sleep(INIT_TIMEOUT) + + if node_mode == NodeMode.PASSIVE and node_id is not None: + setup_fair_passive(node_id) + + print('Fair node is initialized') + + +@check_inited +@check_user +def cleanup(node_mode: NodeMode, prune: bool = False) -> None: + node_mode = upsert_node_mode(node_mode=node_mode) + compose_env = compose_node_env(node_type=NodeType.FAIR, node_mode=node_mode) + cleanup_fair_op(node_mode=node_mode, compose_env=compose_env, prune=prune) + logger.info('Fair node was cleaned up, all containers and data removed') + + +@check_inited +@check_user +def update( + node_mode: NodeMode, + config_file: str, + pull_config_for_schain: str | None = None, + force_skaled_start: bool = False, +) -> None: + logger.info( + 'Updating fair node: %s, pull_config_for_schain: %s, force_skaled_start: %s', + config_file, + pull_config_for_schain, + force_skaled_start, + ) + node_mode = upsert_node_mode(node_mode=node_mode) + + settings = validate_and_save_node_settings(config_file, NodeType.FAIR, node_mode) + compose_env = compose_node_env(node_type=NodeType.FAIR, node_mode=node_mode) + update_ok = update_fair_op( + settings=settings, + compose_env=compose_env, + node_mode=node_mode, + update_type=FairUpdateType.REGULAR, + force_skaled_start=force_skaled_start, + ) + alive = is_base_containers_alive(node_type=NodeType.FAIR, node_mode=node_mode) + if not update_ok or not alive: + print_node_cmd_error() + return + else: + logger.info('Fair update completed successfully') + + +def repair_chain(snapshot_from: str = 'any') -> None: + settings = get_settings() + repair_fair_op(env_type=settings.env_type, snapshot_from=snapshot_from) + + +@check_inited +@check_user +def turn_off(node_type: NodeType) -> None: + node_mode = upsert_node_mode() + compose_env = compose_node_env(node_type=node_type, node_mode=node_mode) + turn_off_op(compose_env=compose_env, node_type=node_type, node_mode=node_mode) + + +@check_inited +@check_user +def turn_on(env_file: str, node_type: NodeType) -> None: + node_mode = upsert_node_mode() + settings = validate_and_save_node_settings(env_file, node_type, node_mode) + compose_env = compose_node_env(node_type=node_type, node_mode=node_mode) + turn_on_op(settings=settings, compose_env=compose_env, node_type=node_type, node_mode=node_mode) + logger.info('Waiting for containers initialization') + time.sleep(TM_INIT_TIMEOUT) + if not is_base_containers_alive(node_type=node_type, node_mode=node_mode): + print_node_cmd_error() + return + logger.info('Node turned on') diff --git a/node_cli/fair/passive.py b/node_cli/fair/passive.py new file mode 100644 index 00000000..dcd175d9 --- /dev/null +++ b/node_cli/fair/passive.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- +# +# This file is part of node-cli +# +# Copyright (C) 2025-Present SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import logging + +from node_cli.core.host import is_node_inited +from node_cli.utils.texts import safe_load_texts +from node_cli.utils.helper import error_exit, post_request +from node_cli.utils.exit_codes import CLIExitCodes + +logger = logging.getLogger(__name__) +TEXTS = safe_load_texts() +BLUEPRINT_NAME = 'fair-node-passive' + + +def setup_fair_passive(node_id: int) -> None: + if not is_node_inited(): + print(TEXTS['fair']['node']['not_inited']) + return + + json_data = {'id': node_id} + status, payload = post_request(blueprint=BLUEPRINT_NAME, method='setup', json=json_data) + if status == 'ok': + msg = TEXTS['fair']['node']['setup_complete'] + logger.info(msg) + print(msg) + else: + error_msg = payload + logger.error(f'Setup error {error_msg}') + error_exit(error_msg, exit_code=CLIExitCodes.BAD_API_RESPONSE) diff --git a/node_cli/fair/record/chain_record.py b/node_cli/fair/record/chain_record.py index a97ca9fe..b6d48886 100644 --- a/node_cli/fair/record/chain_record.py +++ b/node_cli/fair/record/chain_record.py @@ -22,6 +22,8 @@ from typing import cast from datetime import datetime +from skale_core.types import EnvType + from node_cli.core.static_config import get_fair_chain_name from node_cli.fair.record.redis_record import FlatRedisRecord, FieldInfo @@ -34,6 +36,7 @@ 'repair_date': FieldInfo('repair_date', datetime, datetime.fromtimestamp(0)), 'repair_ts': FieldInfo('repair_ts', int, None), 'snapshot_from': FieldInfo('snapshot_from', str, None), + 'force_skaled_start': FieldInfo('force_skaled_start', bool, False), } @@ -57,6 +60,10 @@ def snapshot_from(self) -> str | None: def repair_ts(self) -> int | None: return cast(int | None, self._get_field('repair_ts')) + @property + def force_skaled_start(self) -> bool: + return cast(bool, self._get_field('force_skaled_start')) + def set_config_version(self, version: str) -> None: self._set_field('config_version', version) @@ -69,13 +76,21 @@ def set_snapshot_from(self, value: str | None) -> None: def set_repair_ts(self, value: int | None) -> None: self._set_field('repair_ts', value) + def set_force_skaled_start(self, value: bool) -> None: + self._set_field('force_skaled_start', value) + + +def get_fair_chain_record(env_type: EnvType) -> ChainRecord: + return ChainRecord(get_fair_chain_name(env_type)) + -def get_fair_chain_record(env: dict) -> ChainRecord: - return ChainRecord(get_fair_chain_name(env)) +def migrate_chain_record(env_type: EnvType, node_version: str) -> None: + logger.info('Migrating fair chain record, setting config version to %s', node_version) + record = get_fair_chain_record(env_type) + record.set_config_version(node_version) -def migrate_chain_record(env: dict) -> None: - version = env['CONTAINER_CONFIGS_STREAM'] - logger.info('Migrating fair chain record, setting config version to %s', version) - record = get_fair_chain_record(env) - record.set_config_version(version) +def update_chain_record(env_type: EnvType, force_skaled_start: bool) -> None: + record = get_fair_chain_record(env_type) + record.set_force_skaled_start(force_skaled_start) + logger.info('Updated fair chain record with force_skaled_start=%s', force_skaled_start) diff --git a/node_cli/fair/record/redis_record.py b/node_cli/fair/record/redis_record.py index b71aa9e4..f40aa898 100644 --- a/node_cli/fair/record/redis_record.py +++ b/node_cli/fair/record/redis_record.py @@ -18,6 +18,7 @@ # along with this program. If not, see . import abc +import logging from dataclasses import dataclass from datetime import datetime from typing import Any @@ -26,6 +27,8 @@ from node_cli.configs import REDIS_URI +logger = logging.getLogger(__name__) + cpool: redis.ConnectionPool = redis.ConnectionPool.from_url(REDIS_URI) rs: redis.Redis = redis.Redis(connection_pool=cpool) @@ -83,6 +86,7 @@ def _get_field(self, field_name: str): def _set_field(self, field_name: str, value) -> None: key = self._get_field_key(field_name) serialized_value = self._serialize_field(value, self._record_fields()[field_name].type) + logger.info('Setting field %s to value %s', field_name, serialized_value) rs.set(key, serialized_value) def _deserialize_field(self, value, field_type: type): diff --git a/node_cli/fair/staking.py b/node_cli/fair/staking.py new file mode 100644 index 00000000..95b93adf --- /dev/null +++ b/node_cli/fair/staking.py @@ -0,0 +1,137 @@ +# -*- coding: utf-8 -*- +# +# This file is part of node-cli +# +# Copyright (C) 2025-Present SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from typing import Any +import json +from datetime import datetime, timezone + +from node_cli.utils.decorators import check_inited +from node_cli.utils.exit_codes import CLIExitCodes +from node_cli.utils.helper import error_exit, post_request + +BLUEPRINT_NAME = 'fair-staking' + + +def _handle_response(status: str, payload: Any, success: str | None = None) -> None: + if status == 'ok': + print(success if success is not None else 'OK') + else: + error_exit(payload, exit_code=CLIExitCodes.BAD_API_RESPONSE) + + +@check_inited +def add_allowed_receiver(receiver: str) -> None: + status, payload = post_request( + blueprint=BLUEPRINT_NAME, method='add-receiver', json={'receiver': receiver} + ) + _handle_response(status, payload, success=f'Allowed receiver added: {receiver}') + + +@check_inited +def remove_allowed_receiver(receiver: str) -> None: + status, payload = post_request( + blueprint=BLUEPRINT_NAME, method='remove-receiver', json={'receiver': receiver} + ) + _handle_response(status, payload, success=f'Allowed receiver removed: {receiver}') + + +@check_inited +def request_fees(amount: float | None) -> None: + json_data: dict[str, Any] = {} + if amount is not None: + json_data['amount'] = amount + status, payload = post_request(blueprint=BLUEPRINT_NAME, method='request-fees', json=json_data) + _handle_response( + status, + payload, + success='All fees requested' if amount is None else f'Fees requested: {amount}', + ) + + +@check_inited +def request_send_fees(to: str, amount: float | None) -> None: + json_data: dict[str, Any] = {'to': to} + if amount is not None: + json_data['amount'] = amount + status, payload = post_request( + blueprint=BLUEPRINT_NAME, method='request-send-fees', json=json_data + ) + _handle_response(status, payload, success=f'Fees request to send to {to} created') + + +@check_inited +def set_fee_rate(fee_rate: int) -> None: + status, payload = post_request( + blueprint=BLUEPRINT_NAME, method='set-fee-rate', json={'feeRate': fee_rate} + ) + _handle_response(status, payload, success=f'Fee rate set to {fee_rate}') + + +@check_inited +def claim_request(request_id: int) -> None: + status, payload = post_request( + blueprint=BLUEPRINT_NAME, method='claim-request', json={'requestId': request_id} + ) + _handle_response(status, payload, success=f'Request claimed: {request_id}') + + +@check_inited +def get_earned_fee_amount() -> None: + status, payload = post_request(blueprint=BLUEPRINT_NAME, method='get-earned-fee-amount') + if status == 'ok' and isinstance(payload, dict): + amount_wei = payload.get('amount_wei') + amount_ether = payload.get('amount_ether') + print(f'Earned fee amount: {amount_wei} wei ({amount_ether} FAIR)') + return + error_exit(payload, exit_code=CLIExitCodes.BAD_API_RESPONSE) + + +@check_inited +def get_exit_requests(raw: bool = False) -> None: + status, payload = post_request(blueprint=BLUEPRINT_NAME, method='get-exit-requests') + if status == 'ok' and isinstance(payload, dict): + exit_requests = payload.get('exit_requests') + if not isinstance(exit_requests, list): + error_exit(payload, exit_code=CLIExitCodes.BAD_API_RESPONSE) + if raw: + print(json.dumps(exit_requests, indent=2)) + return + for req in exit_requests: + try: + request_id = req.get('request_id') + user = req.get('user') + node_id = req.get('node_id') + amount = req.get('amount') + unlock_date = req.get('unlock_date') + amount_fair = None + if isinstance(amount, int): + amount_fair = amount / 10**18 + unlock_iso = None + if isinstance(unlock_date, int): + unlock_iso = datetime.fromtimestamp(unlock_date, tz=timezone.utc).isoformat() + base = ( + f'request_id: {request_id} | user: {user} | node_id: {node_id} | ' + f'amount_wei: {amount} | amount_fair: {amount_fair} | ' + f'unlock_date: {unlock_date}' + ) + print(base + (f' ({unlock_iso})' if unlock_iso else '')) + except Exception: # noqa: BLE001 + print(req) + return + error_exit(payload, exit_code=CLIExitCodes.BAD_API_RESPONSE) diff --git a/node_cli/main.py b/node_cli/main.py index 28d87fe0..9016f79d 100644 --- a/node_cli/main.py +++ b/node_cli/main.py @@ -36,10 +36,12 @@ from node_cli.cli.schains import schains_cli from node_cli.cli.wallet import wallet_cli from node_cli.cli.ssl import ssl_cli -from node_cli.cli.resources_allocation import resources_allocation_cli -from node_cli.cli.sync_node import sync_node_cli +from node_cli.cli.passive_node import passive_node_cli from node_cli.cli.fair_boot import fair_boot_cli from node_cli.cli.fair_node import fair_node_cli +from node_cli.cli.passive_fair_node import passive_fair_node_cli +from node_cli.cli.chain import chain_cli +from node_cli.cli.staking import staking_cli from node_cli.core.host import init_logs_dir from node_cli.utils.node_type import NodeType from node_cli.configs import LONG_LINE @@ -52,9 +54,20 @@ logger = logging.getLogger(__name__) -@click.group() -def cli(): - pass +@click.group(invoke_without_command=True) +@click.pass_context +def cli(ctx): + if ctx.invoked_subcommand is None: + print(ctx.get_help()) + ctx.exit(0) + + start_time = time.time() + init_logs_dir() + init_default_logger() + args = sys.argv + # todo: hide secret variables (passwords, private keys) + logger.debug(f'cmd: {" ".join(str(x) for x in args)}, v.{__version__}') + ctx.call_on_close(lambda: logger.debug('Execution time: %d seconds', time.time() - start_time)) @cli.command('version', help='Show SKALE node CLI version') @@ -82,27 +95,25 @@ def info(): ) -def get_sources_list() -> List[click.MultiCommand]: - if TYPE == NodeType.SYNC: - return [cli, sync_node_cli, ssl_cli] - elif TYPE == NodeType.FAIR: - return [ - cli, +def get_command_groups() -> List[click.Group]: + if TYPE == NodeType.FAIR: + return [ # type: ignore logs_cli, fair_boot_cli, fair_node_cli, + passive_fair_node_cli, + chain_cli, + staking_cli, wallet_cli, ssl_cli, ] else: - return [ - cli, + return [ # type: ignore health_cli, schains_cli, logs_cli, - resources_allocation_cli, node_cli, - sync_node_cli, + passive_node_cli, wallet_cli, ssl_cli, exit_cli, @@ -120,19 +131,12 @@ def handle_exception(exc_type, exc_value, exc_traceback): sys.excepthook = handle_exception if __name__ == '__main__': - start_time = time.time() - init_logs_dir() - init_default_logger() - args = sys.argv - # todo: hide secret variables (passwords, private keys) - logger.debug(f'cmd: {" ".join(str(x) for x in args)}, v.{__version__}') - sources = get_sources_list() - cmd_collection = click.CommandCollection(sources=sources) + for group in get_command_groups(): + for cmd_name, cmd_obj in group.commands.items(): + cli.add_command(cmd_obj, cmd_name) try: - cmd_collection() + cli() except Exception as err: traceback.print_exc() - logger.debug('Execution time: %d seconds', time.time() - start_time) error_exit(err) - logger.debug('Execution time: %d seconds', time.time() - start_time) diff --git a/node_cli/migrations/fair/from_boot.py b/node_cli/migrations/fair/from_boot.py index 6affdff3..784908f3 100644 --- a/node_cli/migrations/fair/from_boot.py +++ b/node_cli/migrations/fair/from_boot.py @@ -1,4 +1,3 @@ -import glob import logging import os from pathlib import Path @@ -21,6 +20,7 @@ def rename_chain_file(old_filepath: str, new_filepath: str) -> None: new_path = Path(new_filepath) if not old_path.exists(): raise NoLegacyNFTChainConfigError(f'File {old_filepath} does not exists') + old_path.rename(Path(new_path)) @@ -35,24 +35,28 @@ def rename_chain_in_config(config_path: str, old_chain_name: str, new_chain_name f.write(updated_content) -def migrate_nft_chain() -> None: - after_boot_chain_path = glob.glob(os.path.join(NFT_CHAIN_BASE_PATH, '*'))[0] - old_chain_name = Path(after_boot_chain_path).name.removesuffix('.conf') +def migrate_nft_chain(chain_name: str) -> None: + after_boot_chain_path = os.path.join(NFT_CHAIN_BASE_PATH, f'skale-{chain_name}.conf') new_chain_name = NFT_COMMITTEE_SCOPE_CHAIN_NAME - rename_chain_in_config(after_boot_chain_path, old_chain_name, new_chain_name) after_migration_chain_path = os.path.join( NFT_CHAIN_BASE_PATH, f'{NFT_COMMITTEE_SCOPE_CHAIN_NAME}.conf' ) - rename_chain_file(after_boot_chain_path, after_migration_chain_path) + logger.debug('Renaming %s to %s', after_boot_chain_path, after_migration_chain_path) + if os.path.isfile(after_boot_chain_path): + rename_chain_in_config(after_boot_chain_path, f'skale-{chain_name}', new_chain_name) + if os.path.isfile(after_migration_chain_path): + os.remove(after_boot_chain_path) + else: + rename_chain_file(after_boot_chain_path, after_migration_chain_path) def reload_nft(): run_cmd(['nft', '-f', '/etc/nftables.conf']) -def migrate_nftables_from_boot(): +def migrate_nftables_from_boot(chain_name: str): logger.info('Starting nftables migration from boot') - migrate_nft_chain() + migrate_nft_chain(chain_name=chain_name) logger.info('Reloading nftables rules') reload_nft() logger.info('Restart docker service') diff --git a/node_cli/operations/__init__.py b/node_cli/operations/__init__.py index 4f3aa410..7a72b442 100644 --- a/node_cli/operations/__init__.py +++ b/node_cli/operations/__init__.py @@ -20,20 +20,21 @@ from node_cli.operations.base import ( # noqa update as update_op, init as init_op, - init_sync as init_sync_op, - init_fair_boot as init_fair_boot_op, - update_fair_boot as update_fair_boot_op, - update_sync as update_sync_op, + init_passive as init_passive_op, + update_passive as update_passive_op, turn_off as turn_off_op, turn_on as turn_on_op, restore as restore_op, - cleanup_sync as cleanup_sync_op, + cleanup as cleanup_skale_op, configure_nftables, ) from node_cli.operations.fair import ( # noqa + init_fair_boot as init_fair_boot_op, init as init_fair_op, - update_fair as update_fair_op, + update_fair_boot as update_fair_boot_op, + update as update_fair_op, FairUpdateType, - restore_fair as restore_fair_op, + restore as restore_fair_op, + repair as repair_fair_op, cleanup as cleanup_fair_op, ) diff --git a/node_cli/operations/base.py b/node_cli/operations/base.py index a5e29a4c..803fe299 100644 --- a/node_cli/operations/base.py +++ b/node_cli/operations/base.py @@ -20,34 +20,41 @@ import functools import logging import time -from typing import Dict, Optional +from typing import Optional import distro +from skale_core.settings import BaseNodeSettings, SkalePassiveSettings, SkaleSettings, get_settings + from node_cli.cli.info import TYPE, VERSION from node_cli.configs import ( CONTAINER_CONFIG_PATH, CONTAINER_CONFIG_TMP_PATH, GLOBAL_SKALE_DIR, + NFTABLES_CHAIN_FOLDER_PATH, SKALE_DIR, ) from node_cli.core.checks import CheckType from node_cli.core.checks import run_checks as run_host_checks -from node_cli.core.docker_config import configure_docker +from node_cli.core.docker_config import cleanup_docker_configuration, configure_docker from node_cli.core.host import ( ensure_btrfs_kernel_module_autoloaded, - link_env_file, prepare_host, ) from node_cli.core.nftables import configure_nftables from node_cli.core.nginx import generate_nginx_config -from node_cli.core.node_options import NodeOptions +from node_cli.core.node_options import ( + mark_active_node, + set_passive_node_options, + upsert_node_mode, +) from node_cli.core.resources import init_shared_space_volume, update_resource_allocation from node_cli.core.schains import ( - cleanup_datadir_for_single_chain_node, + cleanup_lvm_datadir, + cleanup_no_lvm_datadir, update_node_cli_schain_status, ) -from node_cli.operations.common import configure_filebeat, configure_flask, unpack_backup_archive +from node_cli.operations.common import configure_filebeat, unpack_backup_archive from node_cli.operations.config_repo import ( download_skale_node, sync_skale_node, @@ -64,23 +71,28 @@ compose_up, docker_cleanup, remove_dynamic_containers, + system_prune, ) -from node_cli.utils.helper import rm_dir, str_to_bool +from node_cli.utils.helper import cleanup_dir_content, rm_dir from node_cli.utils.meta import CliMetaManager, FairCliMetaManager -from node_cli.utils.node_type import NodeType +from node_cli.utils.node_type import NodeMode, NodeType from node_cli.utils.print_formatters import print_failed_requirements_checks +from node_cli.utils.settings import save_internal_settings logger = logging.getLogger(__name__) def checked_host(func): @functools.wraps(func) - def wrapper(env_filepath: str, env: Dict, *args, **kwargs): - download_skale_node(env.get('CONTAINER_CONFIGS_STREAM'), env.get('CONTAINER_CONFIGS_DIR')) + def wrapper( + settings: BaseNodeSettings, compose_env: dict, node_mode: NodeMode, *args, **kwargs + ): + download_skale_node(settings.node_version, settings.container_configs_dir or None) failed_checks = run_host_checks( - env['DISK_MOUNTPOINT'], + settings.block_device, TYPE, - env['ENV_TYPE'], + node_mode, + settings.env_type, CONTAINER_CONFIG_TMP_PATH, check_type=CheckType.PREINSTALL, ) @@ -88,14 +100,15 @@ def wrapper(env_filepath: str, env: Dict, *args, **kwargs): print_failed_requirements_checks(failed_checks) return False - result = func(env_filepath, env, *args, **kwargs) + result = func(settings, compose_env, node_mode, *args, **kwargs) if not result: return result failed_checks = run_host_checks( - env['DISK_MOUNTPOINT'], + settings.block_device, TYPE, - env['ENV_TYPE'], + node_mode, + settings.env_type, CONTAINER_CONFIG_PATH, check_type=CheckType.POSTINSTALL, ) @@ -108,278 +121,268 @@ def wrapper(env_filepath: str, env: Dict, *args, **kwargs): @checked_host -def update(env_filepath: str, env: Dict, node_type: NodeType) -> bool: - compose_rm(node_type=node_type, env=env) +def update(settings: BaseNodeSettings, compose_env: dict, node_mode: NodeMode) -> bool: + compose_rm(node_type=NodeType.SKALE, node_mode=node_mode, env=compose_env) remove_dynamic_containers() sync_skale_node() ensure_btrfs_kernel_module_autoloaded() - if env.get('SKIP_DOCKER_CONFIG') != 'True': + if not settings.skip_docker_config: configure_docker() - enable_monitoring = str_to_bool(env.get('MONITORING_CONTAINERS', 'False')) - configure_nftables(enable_monitoring=enable_monitoring) + configure_nftables(enable_monitoring=settings.monitoring_containers) - lvmpy_install(env) + lvmpy_install(settings.block_device) generate_nginx_config() - prepare_host(env_filepath, env['ENV_TYPE'], allocation=True) - init_shared_space_volume(env['ENV_TYPE']) + prepare_host(settings.env_type, allocation=True) + save_internal_settings(node_type=NodeType.SKALE, node_mode=node_mode) + init_shared_space_volume(settings.env_type) meta_manager = CliMetaManager() current_stream = meta_manager.get_meta_info().config_stream - skip_cleanup = env.get('SKIP_DOCKER_CLEANUP') == 'True' - if not skip_cleanup and current_stream != env['CONTAINER_CONFIGS_STREAM']: + if not settings.skip_docker_cleanup and current_stream != settings.node_version: logger.info( 'Stream version was changed from %s to %s', current_stream, - env['CONTAINER_CONFIGS_STREAM'], + settings.node_version, ) docker_cleanup() + skale_settings = get_settings(SkaleSettings) meta_manager.update_meta( VERSION, - env['CONTAINER_CONFIGS_STREAM'], - env['DOCKER_LVMPY_STREAM'], + settings.node_version, + skale_settings.docker_lvmpy_version, distro.id(), distro.version(), ) - update_images(env=env, node_type=node_type) - compose_up(env=env, node_type=node_type) - return True - - -@checked_host -def update_fair_boot(env_filepath: str, env: Dict) -> bool: - compose_rm(node_type=NodeType.FAIR, env=env) - remove_dynamic_containers() - cleanup_volume_artifacts(env['DISK_MOUNTPOINT']) - - sync_skale_node() - ensure_btrfs_kernel_module_autoloaded() - - if env.get('SKIP_DOCKER_CONFIG') != 'True': - configure_docker() - - enable_monitoring = str_to_bool(env.get('MONITORING_CONTAINERS', 'False')) - configure_nftables(enable_monitoring=enable_monitoring) - - generate_nginx_config() - prepare_block_device(env['DISK_MOUNTPOINT'], force=env['ENFORCE_BTRFS'] == 'True') - - prepare_host(env_filepath, env['ENV_TYPE']) - - meta_manager = FairCliMetaManager() - current_stream = meta_manager.get_meta_info().config_stream - skip_cleanup = env.get('SKIP_DOCKER_CLEANUP') == 'True' - if not skip_cleanup and current_stream != env['CONTAINER_CONFIGS_STREAM']: - logger.info( - 'Stream version was changed from %s to %s', - current_stream, - env['CONTAINER_CONFIGS_STREAM'], - ) - docker_cleanup() - - meta_manager.update_meta( - VERSION, - env['CONTAINER_CONFIGS_STREAM'], - distro.id(), - distro.version(), + update_images( + compose_env=compose_env, + container_configs_dir=settings.container_configs_dir, + node_type=NodeType.SKALE, + node_mode=node_mode, ) - update_images(env=env, node_type=NodeType.FAIR) - compose_up(env=env, node_type=NodeType.FAIR, is_fair_boot=True) + compose_up(env=compose_env, settings=settings, node_type=NodeType.SKALE, node_mode=node_mode) return True @checked_host -def init(env_filepath: str, env: dict, node_type: NodeType) -> None: +def init(settings: BaseNodeSettings, compose_env: dict, node_mode: NodeMode) -> None: sync_skale_node() - ensure_btrfs_kernel_module_autoloaded() - if env.get('SKIP_DOCKER_CONFIG') != 'True': + if not settings.skip_docker_config: configure_docker() - enable_monitoring = str_to_bool(env.get('MONITORING_CONTAINERS', 'False')) - configure_nftables(enable_monitoring=enable_monitoring) + configure_nftables(enable_monitoring=settings.monitoring_containers) + + prepare_host(env_type=settings.env_type) + save_internal_settings(node_type=NodeType.SKALE, node_mode=node_mode) - prepare_host(env_filepath, env_type=env['ENV_TYPE']) - link_env_file() + mark_active_node() configure_filebeat() - configure_flask() generate_nginx_config() - lvmpy_install(env) - init_shared_space_volume(env['ENV_TYPE']) + lvmpy_install(settings.block_device) + init_shared_space_volume(settings.env_type) + skale_settings = get_settings(SkaleSettings) meta_manager = CliMetaManager() meta_manager.update_meta( VERSION, - env['CONTAINER_CONFIGS_STREAM'], - env['DOCKER_LVMPY_STREAM'], + settings.node_version, + skale_settings.docker_lvmpy_version, distro.id(), distro.version(), ) - update_resource_allocation(env_type=env['ENV_TYPE']) - update_images(env=env, node_type=node_type) - - compose_up(env=env, node_type=node_type) - - -@checked_host -def init_fair_boot(env_filepath: str, env: dict) -> None: - sync_skale_node() - cleanup_volume_artifacts(env['DISK_MOUNTPOINT']) - - ensure_btrfs_kernel_module_autoloaded() - if env.get('SKIP_DOCKER_CONFIG') != 'True': - configure_docker() - - enable_monitoring = str_to_bool(env.get('MONITORING_CONTAINERS', 'False')) - configure_nftables(enable_monitoring=enable_monitoring) - - prepare_host(env_filepath, env_type=env['ENV_TYPE']) - link_env_file() - - configure_filebeat() - configure_flask() - generate_nginx_config() - prepare_block_device(env['DISK_MOUNTPOINT'], force=env['ENFORCE_BTRFS'] == 'True') - - meta_manager = FairCliMetaManager() - meta_manager.update_meta( - VERSION, - env['CONTAINER_CONFIGS_STREAM'], - distro.id(), - distro.version(), + update_resource_allocation(env_type=settings.env_type) + update_images( + compose_env=compose_env, + container_configs_dir=settings.container_configs_dir, + node_type=NodeType.SKALE, + node_mode=node_mode, ) - update_images(env=env, node_type=NodeType.FAIR) - - compose_up(env=env, node_type=NodeType.FAIR, is_fair_boot=True) + compose_up(env=compose_env, settings=settings, node_type=NodeType.SKALE, node_mode=node_mode) -def init_sync( - env_filepath: str, - env: dict, +def init_passive( + settings: BaseNodeSettings, + compose_env: dict, indexer: bool, archive: bool, snapshot: bool, snapshot_from: Optional[str], ) -> None: - cleanup_volume_artifacts(env['DISK_MOUNTPOINT']) - download_skale_node(env.get('CONTAINER_CONFIGS_STREAM'), env.get('CONTAINER_CONFIGS_DIR')) + cleanup_volume_artifacts(settings.block_device) + download_skale_node(settings.node_version, settings.container_configs_dir or None) sync_skale_node() - if env.get('SKIP_DOCKER_CONFIG') != 'True': + if not settings.skip_docker_config: configure_docker() - enable_monitoring = str_to_bool(env.get('MONITORING_CONTAINERS', 'False')) - configure_nftables(enable_monitoring=enable_monitoring) + configure_nftables(enable_monitoring=settings.monitoring_containers) - prepare_host( - env_filepath, - env_type=env['ENV_TYPE'], + prepare_host(env_type=settings.env_type) + save_internal_settings(node_type=NodeType.SKALE, node_mode=NodeMode.PASSIVE) + failed_checks = run_host_checks( + settings.block_device, + TYPE, + NodeMode.PASSIVE, + settings.env_type, + CONTAINER_CONFIG_PATH, + check_type=CheckType.PREINSTALL, ) + if failed_checks: + print_failed_requirements_checks(failed_checks) - node_options = NodeOptions() - node_options.archive = archive or indexer - node_options.catchup = archive or indexer - node_options.historic_state = archive + set_passive_node_options(archive=archive, indexer=indexer) ensure_filestorage_mapping() - link_env_file() generate_nginx_config() - prepare_block_device(env['DISK_MOUNTPOINT'], force=env['ENFORCE_BTRFS'] == 'True') + passive_settings = get_settings(SkalePassiveSettings) + prepare_block_device(settings.block_device, force=passive_settings.enforce_btrfs) meta_manager = CliMetaManager() meta_manager.update_meta( VERSION, - env['CONTAINER_CONFIGS_STREAM'], + settings.node_version, None, distro.id(), distro.version(), ) - update_resource_allocation(env_type=env['ENV_TYPE']) + update_resource_allocation(env_type=settings.env_type) - schain_name = env['SCHAIN_NAME'] - if snapshot or snapshot_from: + if passive_settings.schain_name and (snapshot or snapshot_from): ts = int(time.time()) - update_node_cli_schain_status(schain_name, repair_ts=ts, snapshot_from=snapshot_from) - - update_images(env=env, node_type=NodeType.SYNC) + update_node_cli_schain_status( + passive_settings.schain_name, repair_ts=ts, snapshot_from=snapshot_from + ) - compose_up(env=env, node_type=NodeType.SYNC) + update_images( + compose_env=compose_env, + container_configs_dir=settings.container_configs_dir, + node_type=NodeType.SKALE, + node_mode=NodeMode.PASSIVE, + ) + compose_up( + env=compose_env, settings=settings, node_type=NodeType.SKALE, node_mode=NodeMode.PASSIVE + ) -def update_sync(env_filepath: str, env: Dict) -> bool: - compose_rm(env=env, node_type=NodeType.SYNC) +def update_passive(settings: BaseNodeSettings, compose_env: dict) -> bool: + compose_rm(env=compose_env, node_type=NodeType.SKALE, node_mode=NodeMode.PASSIVE) remove_dynamic_containers() - cleanup_volume_artifacts(env['DISK_MOUNTPOINT']) - download_skale_node(env['CONTAINER_CONFIGS_STREAM'], env.get('CONTAINER_CONFIGS_DIR')) + cleanup_volume_artifacts(settings.block_device) + download_skale_node(settings.node_version, settings.container_configs_dir or None) sync_skale_node() - if env.get('SKIP_DOCKER_CONFIG') != 'True': + if not settings.skip_docker_config: configure_docker() - enable_monitoring = str_to_bool(env.get('MONITORING_CONTAINERS', 'False')) - configure_nftables(enable_monitoring=enable_monitoring) + configure_nftables(enable_monitoring=settings.monitoring_containers) ensure_filestorage_mapping() - prepare_block_device(env['DISK_MOUNTPOINT'], force=env['ENFORCE_BTRFS'] == 'True') + passive_settings = get_settings(SkalePassiveSettings) + prepare_block_device(settings.block_device, force=passive_settings.enforce_btrfs) generate_nginx_config() - prepare_host(env_filepath, env['ENV_TYPE'], allocation=True) + prepare_host(settings.env_type, allocation=True) + save_internal_settings(node_type=NodeType.SKALE, node_mode=NodeMode.PASSIVE) + + failed_checks = run_host_checks( + settings.block_device, + TYPE, + NodeMode.PASSIVE, + settings.env_type, + CONTAINER_CONFIG_PATH, + check_type=CheckType.PREINSTALL, + ) + if failed_checks: + print_failed_requirements_checks(failed_checks) meta_manager = CliMetaManager() meta_manager.update_meta( VERSION, - env['CONTAINER_CONFIGS_STREAM'], - env['DOCKER_LVMPY_STREAM'], + settings.node_version, + None, distro.id(), distro.version(), ) - update_images(env=env, node_type=NodeType.SYNC) - - compose_up(env=env, node_type=NodeType.SYNC) + update_images( + compose_env=compose_env, + container_configs_dir=settings.container_configs_dir, + node_type=NodeType.SKALE, + node_mode=NodeMode.PASSIVE, + ) + compose_up( + env=compose_env, settings=settings, node_type=NodeType.SKALE, node_mode=NodeMode.PASSIVE + ) return True -def turn_off(env: dict, node_type: NodeType) -> None: +def turn_off(compose_env: dict, node_type: NodeType, node_mode: NodeMode) -> None: logger.info('Turning off the node...') - compose_rm(env=env, node_type=node_type) + compose_rm(env=compose_env, node_type=node_type, node_mode=node_mode) remove_dynamic_containers() logger.info('Node was successfully turned off') -def turn_on(env: dict, node_type: NodeType) -> None: +def turn_on( + settings: BaseNodeSettings, + compose_env: dict, + node_type: NodeType, + node_mode: NodeMode, + backup_run: bool = False, +) -> None: logger.info('Turning on the node...') - meta_manager = CliMetaManager() - meta_manager.update_meta( - VERSION, - env['CONTAINER_CONFIGS_STREAM'], - env['DOCKER_LVMPY_STREAM'], - distro.id(), - distro.version(), - ) - if env.get('SKIP_DOCKER_CONFIG') != 'True': + if node_type == NodeType.FAIR: + meta_manager = FairCliMetaManager() + meta_manager.update_meta( + VERSION, + settings.node_version, + distro.id(), + distro.version(), + ) + else: + skale_settings = get_settings((SkaleSettings, SkalePassiveSettings)) + docker_lvmpy_version = ( + skale_settings.docker_lvmpy_version + if isinstance(skale_settings, SkaleSettings) + else None + ) + meta_manager = CliMetaManager() + meta_manager.update_meta( + VERSION, settings.node_version, docker_lvmpy_version, distro.id(), distro.version() + ) + if not settings.skip_docker_config: configure_docker() - enable_monitoring = str_to_bool(env.get('MONITORING_CONTAINERS', 'False')) - configure_nftables(enable_monitoring=enable_monitoring) + configure_nftables(enable_monitoring=settings.monitoring_containers) + save_internal_settings(node_type=node_type, node_mode=node_mode, backup_run=backup_run) logger.info('Launching containers on the node...') - compose_up(env=env, node_type=node_type) - - -def restore(env, backup_path, node_type: NodeType, config_only=False): + compose_up(env=compose_env, settings=settings, node_type=node_type, node_mode=node_mode) + + +def restore( + settings: BaseNodeSettings, + compose_env: dict, + backup_path: str, + node_type: NodeType, + config_only: bool = False, + backup_run: bool = False, +) -> bool: + node_mode = upsert_node_mode(node_mode=NodeMode.ACTIVE) unpack_backup_archive(backup_path) failed_checks = run_host_checks( - env['DISK_MOUNTPOINT'], + settings.block_device, TYPE, - env['ENV_TYPE'], + node_mode, + settings.env_type, CONTAINER_CONFIG_PATH, check_type=CheckType.PREINSTALL, ) @@ -389,31 +392,32 @@ def restore(env, backup_path, node_type: NodeType, config_only=False): ensure_btrfs_kernel_module_autoloaded() - if env.get('SKIP_DOCKER_CONFIG') != 'True': + if not settings.skip_docker_config: configure_docker() - enable_monitoring = str_to_bool(env.get('MONITORING_CONTAINERS', 'False')) - configure_nftables(enable_monitoring=enable_monitoring) + configure_nftables(enable_monitoring=settings.monitoring_containers) - link_env_file() - lvmpy_install(env) - init_shared_space_volume(env['ENV_TYPE']) + lvmpy_install(settings.block_device) + init_shared_space_volume(settings.env_type) + skale_settings = get_settings(SkaleSettings) meta_manager = CliMetaManager() meta_manager.update_meta( VERSION, - env['CONTAINER_CONFIGS_STREAM'], - env['DOCKER_LVMPY_STREAM'], + settings.node_version, + skale_settings.docker_lvmpy_version, distro.id(), distro.version(), ) + save_internal_settings(node_type=node_type, node_mode=node_mode, backup_run=backup_run) if not config_only: - compose_up(env=env, node_type=node_type) + compose_up(env=compose_env, settings=settings, node_type=node_type, node_mode=node_mode) failed_checks = run_host_checks( - env['DISK_MOUNTPOINT'], + settings.block_device, TYPE, - env['ENV_TYPE'], + node_mode, + settings.env_type, CONTAINER_CONFIG_PATH, check_type=CheckType.POSTINSTALL, ) @@ -423,8 +427,24 @@ def restore(env, backup_path, node_type: NodeType, config_only=False): return True -def cleanup_sync(env, schain_name: str) -> None: - turn_off(env, node_type=NodeType.SYNC) - cleanup_datadir_for_single_chain_node(schain_name=schain_name) +def cleanup_passive(compose_env: dict, schain_name: str) -> None: + turn_off(compose_env, node_type=NodeType.SKALE, node_mode=NodeMode.PASSIVE) + cleanup_no_lvm_datadir(chain_name=schain_name) + rm_dir(GLOBAL_SKALE_DIR) + rm_dir(SKALE_DIR) + + +def cleanup( + node_mode: NodeMode, compose_env: dict, schain_name: Optional[str] = None, prune: bool = False +) -> None: + turn_off(compose_env, node_type=NodeType.SKALE, node_mode=node_mode) + if prune: + system_prune() + if node_mode == NodeMode.PASSIVE: + cleanup_no_lvm_datadir(chain_name=schain_name) + else: + cleanup_lvm_datadir() rm_dir(GLOBAL_SKALE_DIR) rm_dir(SKALE_DIR) + cleanup_dir_content(NFTABLES_CHAIN_FOLDER_PATH) + cleanup_docker_configuration() diff --git a/node_cli/operations/common.py b/node_cli/operations/common.py index 7c876fa8..c595a3b8 100644 --- a/node_cli/operations/common.py +++ b/node_cli/operations/common.py @@ -19,7 +19,6 @@ import logging import os -import secrets import shutil import stat import tarfile @@ -27,7 +26,6 @@ from node_cli.configs import ( FILEBEAT_CONFIG_PATH, - FLASK_SECRET_KEY_FILE, G_CONF_HOME, SRC_FILEBEAT_CONFIG_PATH, ) @@ -43,17 +41,6 @@ def configure_filebeat(): logger.info('Filebeat configured') -def configure_flask(): - if os.path.isfile(FLASK_SECRET_KEY_FILE): - logger.info('Flask secret key already exists') - else: - logger.info('Generating Flask secret key...') - flask_secret_key = secrets.token_urlsafe(16) - with open(FLASK_SECRET_KEY_FILE, 'w') as f: - f.write(flask_secret_key) - logger.info('Flask secret key generated and saved') - - def unpack_backup_archive(backup_path: str) -> None: logger.info('Unpacking backup archive...') with tarfile.open(backup_path) as tar: diff --git a/node_cli/operations/config_repo.py b/node_cli/operations/config_repo.py index 656e4cb1..19cf284a 100644 --- a/node_cli/operations/config_repo.py +++ b/node_cli/operations/config_repo.py @@ -27,18 +27,19 @@ from node_cli.utils.git_utils import clone_repo from node_cli.utils.docker_utils import compose_pull, compose_build from node_cli.configs import CONTAINER_CONFIG_PATH, CONTAINER_CONFIG_TMP_PATH, SKALE_NODE_REPO_URL -from node_cli.utils.node_type import NodeType +from node_cli.utils.node_type import NodeType, NodeMode logger = logging.getLogger(__name__) -def update_images(env: dict, node_type: NodeType) -> None: - local = env.get('CONTAINER_CONFIGS_DIR') != '' - if local: - compose_build(env=env, node_type=node_type) +def update_images( + compose_env: dict, container_configs_dir: str, node_type: NodeType, node_mode: NodeMode +) -> None: + if container_configs_dir: + compose_build(env=compose_env, node_type=node_type, node_mode=node_mode) else: - compose_pull(env=env, node_type=node_type) + compose_pull(env=compose_env, node_type=node_type, node_mode=node_mode) def download_skale_node(stream: Optional[str] = None, src: Optional[str] = None) -> None: diff --git a/node_cli/operations/docker_lvmpy.py b/node_cli/operations/docker_lvmpy.py index 6e28d58c..eb307d55 100644 --- a/node_cli/operations/docker_lvmpy.py +++ b/node_cli/operations/docker_lvmpy.py @@ -35,13 +35,14 @@ SCHAINS_MNT_DIR_REGULAR, VOLUME_GROUP, ) +from node_cli.operations.volume import ensure_filestorage_mapping from lvmpy.src.install import setup as setup_lvmpy logger = logging.getLogger(__name__) def update_docker_lvmpy_env(env): - env['PHYSICAL_VOLUME'] = env['DISK_MOUNTPOINT'] + env['PHYSICAL_VOLUME'] = env['BLOCK_DEVICE'] env['VOLUME_GROUP'] = 'schains' env['FILESTORAGE_MAPPING'] = FILESTORAGE_MAPPING env['MNT_DIR'] = SCHAINS_MNT_DIR_REGULAR @@ -49,23 +50,16 @@ def update_docker_lvmpy_env(env): return env -def ensure_filestorage_mapping(mapping_dir=FILESTORAGE_MAPPING): - if not os.path.isdir(FILESTORAGE_MAPPING): - os.makedirs(FILESTORAGE_MAPPING) - - def sync_docker_lvmpy_repo(env): if os.path.isdir(DOCKER_LVMPY_PATH): shutil.rmtree(DOCKER_LVMPY_PATH) - sync_repo(DOCKER_LVMPY_REPO_URL, DOCKER_LVMPY_PATH, env['DOCKER_LVMPY_STREAM']) + sync_repo(DOCKER_LVMPY_REPO_URL, DOCKER_LVMPY_PATH, env['DOCKER_LVMPY_VERSION']) -def lvmpy_install(env): +def lvmpy_install(block_device: str) -> None: ensure_filestorage_mapping() logging.info('Configuring and starting lvmpy') - setup_lvmpy( - block_device=env['DISK_MOUNTPOINT'], volume_group=VOLUME_GROUP, exec_start=LVMPY_RUN_CMD - ) + setup_lvmpy(block_device=block_device, volume_group=VOLUME_GROUP, exec_start=LVMPY_RUN_CMD) init_healing_cron() logger.info('docker-lvmpy is configured and started') diff --git a/node_cli/operations/fair.py b/node_cli/operations/fair.py index 0471c399..3a48fdbd 100644 --- a/node_cli/operations/fair.py +++ b/node_cli/operations/fair.py @@ -23,41 +23,57 @@ import distro +from skale_core.settings import BaseNodeSettings, FairBaseSettings, FairSettings, get_settings +from skale_core.types import EnvType + from node_cli.cli.info import TYPE, VERSION from node_cli.configs import ( CONTAINER_CONFIG_PATH, GLOBAL_SKALE_DIR, + NFTABLES_CHAIN_FOLDER_PATH, SKALE_DIR, ) from node_cli.core.checks import CheckType from node_cli.core.checks import run_checks as run_host_checks from node_cli.core.docker_config import cleanup_docker_configuration, configure_docker -from node_cli.core.host import ensure_btrfs_kernel_module_autoloaded, link_env_file, prepare_host +from node_cli.core.host import ensure_btrfs_kernel_module_autoloaded, prepare_host from node_cli.core.nftables import configure_nftables from node_cli.core.nginx import generate_nginx_config -from node_cli.core.schains import cleanup_datadir_for_single_chain_node +from node_cli.core.schains import cleanup_no_lvm_datadir +from node_cli.core.static_config import get_fair_chain_name +from node_cli.core.node_options import mark_active_node, set_passive_node_options, upsert_node_mode +from node_cli.fair.record.chain_record import ( + get_fair_chain_record, + migrate_chain_record, + update_chain_record, +) from node_cli.migrations.fair.from_boot import migrate_nftables_from_boot -from node_cli.fair.record.chain_record import migrate_chain_record from node_cli.operations.base import checked_host, turn_off -from node_cli.operations.common import configure_filebeat, configure_flask, unpack_backup_archive +from node_cli.operations.common import configure_filebeat, unpack_backup_archive from node_cli.operations.config_repo import ( sync_skale_node, update_images, ) from node_cli.operations.volume import cleanup_volume_artifacts, prepare_block_device from node_cli.utils.docker_utils import ( + BASE_PASSIVE_FAIR_COMPOSE_SERVICES, REDIS_SERVICE_DICT, REDIS_START_TIMEOUT, - NodeType, compose_rm, compose_up, docker_cleanup, + is_admin_running, remove_dynamic_containers, + start_container_by_name, + stop_container_by_name, + system_prune, wait_for_container, ) -from node_cli.utils.helper import rm_dir, str_to_bool +from node_cli.utils.helper import cleanup_dir_content, rm_dir from node_cli.utils.meta import FairCliMetaManager from node_cli.utils.print_formatters import print_failed_requirements_checks +from node_cli.utils.node_type import NodeMode, NodeType +from node_cli.utils.settings import save_internal_settings logger = logging.getLogger(__name__) @@ -69,135 +85,252 @@ class FairUpdateType(Enum): @checked_host -def init(env_filepath: str, env: dict) -> bool: +def init_fair_boot( + settings: BaseNodeSettings, + compose_env: dict, + node_mode: NodeMode, +) -> None: sync_skale_node() + cleanup_volume_artifacts(settings.block_device) + ensure_btrfs_kernel_module_autoloaded() - cleanup_volume_artifacts(env['DISK_MOUNTPOINT']) + if not settings.skip_docker_config: + configure_docker() + + configure_nftables(enable_monitoring=settings.monitoring_containers) + + prepare_host(env_type=settings.env_type) + save_internal_settings(node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE) + mark_active_node() - if env.get('SKIP_DOCKER_CONFIG') != 'True': + configure_filebeat() + generate_nginx_config() + fair_settings = get_settings((FairSettings, FairBaseSettings)) + prepare_block_device(settings.block_device, force=fair_settings.enforce_btrfs) + + meta_manager = FairCliMetaManager() + meta_manager.update_meta( + VERSION, + settings.node_version, + distro.id(), + distro.version(), + ) + update_images( + compose_env=compose_env, + container_configs_dir=settings.container_configs_dir, + node_type=NodeType.FAIR, + node_mode=NodeMode.ACTIVE, + ) + + compose_up( + env=compose_env, + settings=settings, + node_type=NodeType.FAIR, + node_mode=NodeMode.ACTIVE, + is_fair_boot=True, + ) + + +@checked_host +def init( + settings: BaseNodeSettings, + compose_env: dict, + node_mode: NodeMode, + indexer: bool, + archive: bool, + snapshot: str | None, +) -> bool: + sync_skale_node() + ensure_btrfs_kernel_module_autoloaded() + cleanup_volume_artifacts(settings.block_device) + + if not settings.skip_docker_config: configure_docker() configure_nftables() configure_filebeat() - configure_flask() generate_nginx_config() - prepare_host(env_filepath, env_type=env['ENV_TYPE']) - link_env_file() + prepare_host(env_type=settings.env_type) + save_internal_settings(node_type=NodeType.FAIR, node_mode=node_mode) + + fair_settings = get_settings((FairSettings, FairBaseSettings)) + prepare_block_device(settings.block_device, force=fair_settings.enforce_btrfs) + + update_images( + compose_env=compose_env, + container_configs_dir=settings.container_configs_dir, + node_type=NodeType.FAIR, + node_mode=node_mode, + ) + compose_up( + env=compose_env, + settings=settings, + node_type=NodeType.FAIR, + node_mode=node_mode, + services=list(REDIS_SERVICE_DICT), + ) - prepare_block_device(env['DISK_MOUNTPOINT'], force=env['ENFORCE_BTRFS'] == 'True') + upsert_node_mode(node_mode=node_mode) + if node_mode == NodeMode.PASSIVE: + logger.info('Setting passive node options') + set_passive_node_options(archive=archive, indexer=indexer) + if snapshot: + logger.info('Waiting %s seconds for redis to start', REDIS_START_TIMEOUT) + time.sleep(REDIS_START_TIMEOUT) + trigger_skaled_snapshot_mode(env_type=settings.env_type, snapshot_from=snapshot) meta_manager = FairCliMetaManager() meta_manager.update_meta( VERSION, - env['CONTAINER_CONFIGS_STREAM'], + settings.node_version, distro.id(), distro.version(), ) - update_images(env=env, node_type=NodeType.FAIR) - compose_up(env=env, node_type=NodeType.FAIR) - wait_for_container(REDIS_SERVICE_DICT['redis']) + + compose_up(env=compose_env, settings=settings, node_type=NodeType.FAIR, node_mode=node_mode) + wait_for_container(BASE_PASSIVE_FAIR_COMPOSE_SERVICES['api']) time.sleep(REDIS_START_TIMEOUT) return True @checked_host -def update_fair_boot(env_filepath: str, env: dict) -> bool: - compose_rm(node_type=NodeType.FAIR, env=env) +def update_fair_boot( + settings: BaseNodeSettings, + compose_env: dict, + node_mode: NodeMode = NodeMode.ACTIVE, +) -> bool: + compose_rm(node_type=NodeType.FAIR, node_mode=node_mode, env=compose_env) remove_dynamic_containers() - cleanup_volume_artifacts(env['DISK_MOUNTPOINT']) + cleanup_volume_artifacts(settings.block_device) sync_skale_node() ensure_btrfs_kernel_module_autoloaded() - if env.get('SKIP_DOCKER_CONFIG') != 'True': + if not settings.skip_docker_config: configure_docker() - enable_monitoring = str_to_bool(env.get('MONITORING_CONTAINERS', 'False')) - configure_nftables(enable_monitoring=enable_monitoring) + configure_nftables(enable_monitoring=settings.monitoring_containers) generate_nginx_config() - prepare_block_device(env['DISK_MOUNTPOINT'], force=env['ENFORCE_BTRFS'] == 'True') + fair_settings = get_settings((FairSettings, FairBaseSettings)) + prepare_block_device(settings.block_device, force=fair_settings.enforce_btrfs) - prepare_host(env_filepath, env['ENV_TYPE']) + prepare_host(settings.env_type) + save_internal_settings(node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE) meta_manager = FairCliMetaManager() current_stream = meta_manager.get_meta_info().config_stream - skip_cleanup = env.get('SKIP_DOCKER_CLEANUP') == 'True' - if not skip_cleanup and current_stream != env['CONTAINER_CONFIGS_STREAM']: + if not settings.skip_docker_cleanup and current_stream != settings.node_version: logger.info( 'Stream version was changed from %s to %s', current_stream, - env['CONTAINER_CONFIGS_STREAM'], + settings.node_version, ) docker_cleanup() meta_manager.update_meta( VERSION, - env['CONTAINER_CONFIGS_STREAM'], + settings.node_version, distro.id(), distro.version(), ) - update_images(env=env, node_type=NodeType.FAIR) - compose_up(env=env, node_type=NodeType.FAIR, is_fair_boot=True) + update_images( + compose_env=compose_env, + container_configs_dir=settings.container_configs_dir, + node_type=NodeType.FAIR, + node_mode=NodeMode.ACTIVE, + ) + compose_up( + env=compose_env, + settings=settings, + node_type=NodeType.FAIR, + node_mode=NodeMode.ACTIVE, + is_fair_boot=True, + ) return True @checked_host -def update_fair(env_filepath: str, env: dict, update_type: FairUpdateType) -> bool: - compose_rm(node_type=NodeType.FAIR, env=env) +def update( + settings: BaseNodeSettings, + compose_env: dict, + node_mode: NodeMode, + update_type: FairUpdateType, + force_skaled_start: bool, +) -> bool: + compose_rm(node_type=NodeType.FAIR, node_mode=node_mode, env=compose_env) if update_type not in (FairUpdateType.INFRA_ONLY, FairUpdateType.FROM_BOOT): remove_dynamic_containers() sync_skale_node() ensure_btrfs_kernel_module_autoloaded() - if env.get('SKIP_DOCKER_CONFIG') != 'True': + if not settings.skip_docker_config: configure_docker() configure_nftables() generate_nginx_config() - prepare_host(env_filepath, env['ENV_TYPE'], allocation=True) + prepare_host(settings.env_type, allocation=True) + save_internal_settings(node_type=NodeType.FAIR, node_mode=node_mode) meta_manager = FairCliMetaManager() current_stream = meta_manager.get_meta_info().config_stream - skip_cleanup = env.get('SKIP_DOCKER_CLEANUP') == 'True' - if not skip_cleanup and current_stream != env['CONTAINER_CONFIGS_STREAM']: + if not settings.skip_docker_cleanup and current_stream != settings.node_version: logger.info( 'Stream version was changed from %s to %s', current_stream, - env['CONTAINER_CONFIGS_STREAM'], + settings.node_version, ) docker_cleanup() meta_manager.update_meta( VERSION, - env['CONTAINER_CONFIGS_STREAM'], + settings.node_version, distro.id(), distro.version(), ) + fair_chain_name = get_fair_chain_name(settings.env_type) if update_type == FairUpdateType.FROM_BOOT: - migrate_nftables_from_boot() + migrate_nftables_from_boot(chain_name=fair_chain_name) - update_images(env=env, node_type=NodeType.FAIR) + update_images( + compose_env=compose_env, + container_configs_dir=settings.container_configs_dir, + node_type=NodeType.FAIR, + node_mode=node_mode, + ) - compose_up(env=env, node_type=NodeType.FAIR, services=list(REDIS_SERVICE_DICT)) + compose_up( + env=compose_env, + settings=settings, + node_type=NodeType.FAIR, + node_mode=node_mode, + services=list(REDIS_SERVICE_DICT), + ) wait_for_container(REDIS_SERVICE_DICT['redis']) time.sleep(REDIS_START_TIMEOUT) if update_type == FairUpdateType.FROM_BOOT: - migrate_chain_record(env) - - compose_up(env=env, node_type=NodeType.FAIR) + migrate_chain_record(settings.env_type, settings.node_version) + update_chain_record(settings.env_type, force_skaled_start=force_skaled_start) + compose_up(env=compose_env, settings=settings, node_type=NodeType.FAIR, node_mode=node_mode) return True -def restore_fair(env, backup_path, config_only=False): +def restore( + node_mode: NodeMode, + settings: BaseNodeSettings, + compose_env: dict, + backup_path: str, + config_only: bool = False, +) -> bool: unpack_backup_archive(backup_path) failed_checks = run_host_checks( - env['DISK_MOUNTPOINT'], + settings.block_device, TYPE, - env['ENV_TYPE'], + node_mode, + settings.env_type, CONTAINER_CONFIG_PATH, check_type=CheckType.PREINSTALL, ) @@ -207,29 +340,27 @@ def restore_fair(env, backup_path, config_only=False): ensure_btrfs_kernel_module_autoloaded() - if env.get('SKIP_DOCKER_CONFIG') != 'True': + if not settings.skip_docker_config: configure_docker() - enable_monitoring = str_to_bool(env.get('MONITORING_CONTAINERS', 'False')) - configure_nftables(enable_monitoring=enable_monitoring) - - link_env_file() + configure_nftables(enable_monitoring=settings.monitoring_containers) meta_manager = FairCliMetaManager() meta_manager.update_meta( VERSION, - env['CONTAINER_CONFIGS_STREAM'], + settings.node_version, distro.id(), distro.version(), ) if not config_only: - compose_up(env=env, node_type=NodeType.FAIR) + compose_up(env=compose_env, settings=settings, node_type=NodeType.FAIR, node_mode=node_mode) failed_checks = run_host_checks( - env['DISK_MOUNTPOINT'], + settings.block_device, TYPE, - env['ENV_TYPE'], + node_mode, + settings.env_type, CONTAINER_CONFIG_PATH, check_type=CheckType.POSTINSTALL, ) @@ -239,9 +370,37 @@ def restore_fair(env, backup_path, config_only=False): return True -def cleanup(env) -> None: - turn_off(env, node_type=NodeType.FAIR) - cleanup_datadir_for_single_chain_node() +def cleanup(node_mode: NodeMode, compose_env: dict, prune: bool = False) -> None: + turn_off(compose_env, node_type=NodeType.FAIR, node_mode=node_mode) + if prune: + system_prune() + cleanup_no_lvm_datadir() rm_dir(GLOBAL_SKALE_DIR) rm_dir(SKALE_DIR) + cleanup_dir_content(NFTABLES_CHAIN_FOLDER_PATH) cleanup_docker_configuration() + + +def trigger_skaled_snapshot_mode(env_type: EnvType, snapshot_from: str = 'any') -> None: + record = get_fair_chain_record(env_type) + if not snapshot_from: + snapshot_from = 'any' + logger.info('Triggering skaled snapshot mode, snapshot_from: %s', snapshot_from) + record.set_snapshot_from(snapshot_from) + + +def repair(env_type: EnvType, snapshot_from: str = 'any') -> None: + logger.info('Starting fair node repair') + container_name = 'sk_admin' + if is_admin_running(): + logger.info('Stopping admin container') + stop_container_by_name(container_name=container_name) + logger.info('Removing chain container') + remove_dynamic_containers() + logger.info('Cleaning up datadir') + cleanup_no_lvm_datadir() + logger.info('Requesting fair node repair') + trigger_skaled_snapshot_mode(env_type=env_type, snapshot_from=snapshot_from) + logger.info('Starting admin') + start_container_by_name(container_name=container_name) + logger.info('Fair node repair completed successfully') diff --git a/node_cli/operations/volume.py b/node_cli/operations/volume.py index 1595a442..e944547a 100644 --- a/node_cli/operations/volume.py +++ b/node_cli/operations/volume.py @@ -42,7 +42,7 @@ class FilesystemExistsError(Exception): def update_docker_lvmpy_env(env): - env['PHYSICAL_VOLUME'] = env['DISK_MOUNTPOINT'] + env['PHYSICAL_VOLUME'] = env['BLOCK_DEVICE'] env['VOLUME_GROUP'] = 'schains' env['FILESTORAGE_MAPPING'] = FILESTORAGE_MAPPING env['SCHAINS_MNT_DIR'] = SCHAINS_MNT_DIR_REGULAR @@ -58,7 +58,7 @@ def ensure_filestorage_mapping(mapping_dir=FILESTORAGE_MAPPING): def sync_docker_lvmpy_repo(env): if os.path.isdir(DOCKER_LVMPY_PATH): shutil.rmtree(DOCKER_LVMPY_PATH) - sync_repo(DOCKER_LVMPY_REPO_URL, DOCKER_LVMPY_PATH, env['DOCKER_LVMPY_STREAM']) + sync_repo(DOCKER_LVMPY_REPO_URL, DOCKER_LVMPY_PATH, env['DOCKER_LVMPY_VERSION']) def docker_lvmpy_update(env): diff --git a/node_cli/utils/docker_utils.py b/node_cli/utils/docker_utils.py index 6986d44e..98a4946c 100644 --- a/node_cli/utils/docker_utils.py +++ b/node_cli/utils/docker_utils.py @@ -29,16 +29,17 @@ from docker.errors import NotFound from docker.models.containers import Container +from skale_core.settings import BaseNodeSettings + from node_cli.configs import ( COMPOSE_PATH, FAIR_COMPOSE_PATH, NGINX_CONTAINER_NAME, REMOVED_CONTAINERS_FOLDER_PATH, - SGX_CERTIFICATES_DIR_NAME, - SYNC_COMPOSE_PATH, ) -from node_cli.utils.helper import run_cmd, str_to_bool -from node_cli.utils.node_type import NodeType +from node_cli.core.node_options import active_fair, active_skale, passive_fair, passive_skale +from node_cli.utils.helper import run_cmd +from node_cli.utils.node_type import NodeMode, NodeType logger = logging.getLogger(__name__) @@ -47,38 +48,50 @@ TELEGRAF_REMOVE_TIMEOUT = 20 REDIS_START_TIMEOUT = 10 -REDIS_SERVICE_DICT = {'redis': 'skale_redis'} +REDIS_SERVICE_DICT = {'redis': 'sk_redis'} CORE_COMMON_COMPOSE_SERVICES = { - 'transaction-manager': 'skale_transaction-manager', - 'redis': 'skale_redis', - 'watchdog': 'skale_watchdog', - 'nginx': 'skale_nginx', - 'filebeat': 'skale_filebeat', + 'transaction-manager': 'sk_tm', + 'redis': 'sk_redis', + 'watchdog': 'sk_watchdog', + 'nginx': 'sk_nginx', + 'filebeat': 'sk_filebeat', } BASE_SKALE_COMPOSE_SERVICES = { **CORE_COMMON_COMPOSE_SERVICES, - 'skale-admin': 'skale_admin', - 'skale-api': 'skale_api', - 'bounty': 'skale_bounty', + 'admin': 'sk_admin', + 'api': 'sk_api', + 'bounty': 'sk_bounty', } BASE_FAIR_COMPOSE_SERVICES = { **CORE_COMMON_COMPOSE_SERVICES, - 'fair-admin': 'fair_admin', - 'fair-api': 'fair_api', + 'admin': 'sk_admin', + 'api': 'sk_api', } BASE_FAIR_BOOT_COMPOSE_SERVICES = { **CORE_COMMON_COMPOSE_SERVICES, - 'fair-boot': 'fair_boot_admin', - 'fair-boot-api': 'fair_boot_api', + 'boot-admin': 'sk_boot_admin', + 'boot-api': 'sk_boot_api', +} + +BASE_PASSIVE_COMPOSE_SERVICES = { + 'admin': 'sk_admin', + 'nginx': 'sk_nginx', + 'api': 'sk_api', + 'watchdog': 'sk_watchdog', + **REDIS_SERVICE_DICT, } -BASE_SYNC_COMPOSE_SERVICES = { - 'skale-sync-admin': 'skale_sync_admin', - 'nginx': 'skale_nginx', +BASE_PASSIVE_FAIR_COMPOSE_SERVICES = { + 'admin': 'sk_admin', + 'api': 'sk_api', + 'nginx': 'sk_nginx', + 'watchdog': 'sk_watchdog', + 'filebeat': 'sk_filebeat', + **REDIS_SERVICE_DICT, } MONITORING_COMPOSE_SERVICES = { @@ -106,15 +119,18 @@ def get_sanitized_container_name(container_info: dict) -> str: def get_containers(container_name_filter=None, _all=True) -> list: - return docker_client().containers.list(all=_all) + filters = {} + if container_name_filter: + filters['name'] = container_name_filter + return docker_client().containers.list(all=_all, filters=filters) def get_all_schain_containers(_all=True) -> list: - return docker_client().containers.list(all=_all, filters={'name': 'skale_schain_*'}) + return docker_client().containers.list(all=_all, filters={'name': 'sk_skaled_*'}) def get_all_ima_containers(_all=True) -> list: - return docker_client().containers.list(all=_all, filters={'name': 'skale_ima_*'}) + return docker_client().containers.list(all=_all, filters={'name': 'sk_ima_*'}) def remove_dynamic_containers() -> None: @@ -160,7 +176,7 @@ def safe_rm(container: Container, timeout=DOCKER_DEFAULT_STOP_TIMEOUT, **kwargs) logger.info(f'Container removed: {container_name}') -def stop_container( +def stop_container_by_name( container_name: str, timeout: int = DOCKER_DEFAULT_STOP_TIMEOUT, dclient: Optional[DockerClient] = None, @@ -171,7 +187,7 @@ def stop_container( container.stop(timeout=timeout) -def rm_container( +def remove_container_by_name( container_name: str, timeout: int = DOCKER_DEFAULT_STOP_TIMEOUT, dclient: Optional[DockerClient] = None, @@ -180,29 +196,30 @@ def rm_container( container_names = [container.name for container in get_containers()] if container_name in container_names: container = dc.containers.get(container_name) - safe_rm(container) + safe_rm(container, timeout=timeout) -def start_container(container_name: str, dclient: Optional[DockerClient] = None) -> None: +def start_container_by_name(container_name: str, dclient: Optional[DockerClient] = None) -> None: dc = dclient or docker_client() container = dc.containers.get(container_name) logger.info('Starting container %s', container_name) container.start() -def remove_schain_container(schain_name: str, dclient: Optional[DockerClient] = None) -> None: - container_name = f'skale_schain_{schain_name}' - rm_container(container_name, timeout=SCHAIN_REMOVE_TIMEOUT, dclient=dclient) +def remove_schain_container_by_name( + schain_name: str, dclient: Optional[DockerClient] = None +) -> None: + container_name = f'sk_skaled_{schain_name}' + remove_container_by_name(container_name, timeout=SCHAIN_REMOVE_TIMEOUT, dclient=dclient) def backup_container_logs( container: Container, - head: int = DOCKER_DEFAULT_HEAD_LINES, - tail: int = DOCKER_DEFAULT_TAIL_LINES, + tail: int | str = DOCKER_DEFAULT_TAIL_LINES, ) -> None: logger.info(f'Going to backup container logs: {container.name}') logs_backup_filepath = get_logs_backup_filepath(container) - save_container_logs(container, logs_backup_filepath, tail) + save_container_logs(container, logs_backup_filepath, tail=tail) logger.info(f'Old container logs saved to {logs_backup_filepath}, tail: {tail}') @@ -210,7 +227,7 @@ def save_container_logs( container: Container, log_filepath: str, head: int = DOCKER_DEFAULT_HEAD_LINES, - tail: int = DOCKER_DEFAULT_TAIL_LINES, + tail: int | str = DOCKER_DEFAULT_TAIL_LINES, ) -> None: separator = b'=' * 80 + b'\n' tail_lines = container.logs(tail=tail) @@ -252,9 +269,9 @@ def is_volume_exists(name: str, dutils=None): return True -def compose_rm(node_type: NodeType, env={}): +def compose_rm(node_type: NodeType, node_mode: NodeMode, env={}): logger.info('Removing compose containers') - compose_path = get_compose_path(node_type) + compose_path = get_compose_path(node_type, node_mode) run_cmd( cmd=( 'docker', @@ -270,90 +287,104 @@ def compose_rm(node_type: NodeType, env={}): logger.info('Compose containers removed') -def compose_pull(env: dict, node_type: NodeType): +def compose_pull(env: dict, node_type: NodeType, node_mode: NodeMode): logger.info('Pulling compose containers') - compose_path = get_compose_path(node_type) + compose_path = get_compose_path(node_type, node_mode) run_cmd(cmd=('docker', 'compose', '-f', compose_path, 'pull'), env=env) -def compose_build(env: dict, node_type: NodeType): +def compose_build(env: dict, node_type: NodeType, node_mode: NodeMode): logger.info('Building compose containers') - compose_path = get_compose_path(node_type) + compose_path = get_compose_path(node_type, node_mode) run_cmd(cmd=('docker', 'compose', '-f', compose_path, 'build'), env=env) -def get_compose_path(node_type: NodeType) -> str: - if node_type == NodeType.SYNC: - return SYNC_COMPOSE_PATH - elif node_type == NodeType.FAIR: +def get_compose_path(node_type: NodeType, node_mode: NodeMode) -> str: + if node_type == NodeType.FAIR: return FAIR_COMPOSE_PATH - else: - return COMPOSE_PATH + return COMPOSE_PATH -def get_compose_services(node_type: NodeType) -> list[str]: - if node_type == NodeType.SYNC: - result = list(BASE_SYNC_COMPOSE_SERVICES) - elif node_type == NodeType.FAIR: - result = list(BASE_FAIR_COMPOSE_SERVICES) - else: - result = list(BASE_SKALE_COMPOSE_SERVICES) +def get_compose_services(node_type: NodeType, node_mode: NodeMode) -> list[str]: + if passive_skale(node_type, node_mode): + return list(BASE_PASSIVE_COMPOSE_SERVICES) + elif active_fair(node_type, node_mode): + return list(BASE_FAIR_COMPOSE_SERVICES) + elif passive_fair(node_type, node_mode): + return list(BASE_PASSIVE_FAIR_COMPOSE_SERVICES) + return list(BASE_SKALE_COMPOSE_SERVICES) - return result - -def get_up_compose_cmd(node_type: NodeType, services: list[str] | None = None) -> tuple: - compose_path = get_compose_path(node_type) +def get_up_compose_cmd( + node_type: NodeType, node_mode: NodeMode, services: list[str] | None = None +) -> tuple: + compose_path = get_compose_path(node_type, node_mode) if services is None: - services = get_compose_services(node_type) + services = get_compose_services(node_type, node_mode) return ('docker', 'compose', '-f', compose_path, 'up', '-d', *services) def compose_up( - env, node_type: NodeType, is_fair_boot: bool = False, services: list[str] | None = None + env, + settings: BaseNodeSettings, + node_type: NodeType, + node_mode: NodeMode, + is_fair_boot: bool = False, + services: list[str] | None = None, ): - if node_type == NodeType.SYNC: - logger.info('Running containers for sync node') - run_cmd(cmd=get_up_compose_cmd(node_type=NodeType.SYNC), env=env) + env['PASSIVE_NODE'] = str(node_mode == NodeMode.PASSIVE) + if passive_skale(node_type, node_mode) or passive_fair(node_type, node_mode): + logger.info('Running containers for passive node') + run_cmd(cmd=get_up_compose_cmd(node_type=node_type, node_mode=node_mode), env=env) return - if 'SGX_CERTIFICATES_DIR_NAME' not in env: - env['SGX_CERTIFICATES_DIR_NAME'] = SGX_CERTIFICATES_DIR_NAME - - if node_type == NodeType.FAIR: + if active_fair(node_type, node_mode): logger.info('Running fair base set of containers') if is_fair_boot: logger.debug('Launching fair boot containers with env %s', env) run_cmd( cmd=get_up_compose_cmd( - node_type=NodeType.FAIR, services=list(BASE_FAIR_BOOT_COMPOSE_SERVICES) + node_type=node_type, + node_mode=node_mode, + services=list(BASE_FAIR_BOOT_COMPOSE_SERVICES), ), env=env, ) else: logger.debug('Launching fair containers with env %s', env) - run_cmd(cmd=get_up_compose_cmd(node_type=NodeType.FAIR, services=services), env=env) - else: + run_cmd( + cmd=get_up_compose_cmd( + node_type=node_type, + node_mode=node_mode, + services=services, + ), + env=env, + ) + elif active_skale(node_type, node_mode): logger.info('Running skale node base set of containers') logger.debug('Launching skale node containers with env %s', env) - run_cmd(cmd=get_up_compose_cmd(node_type=NodeType.REGULAR), env=env) + run_cmd(cmd=get_up_compose_cmd(node_type=node_type, node_mode=node_mode), env=env) - if 'TG_API_KEY' in env and 'TG_CHAT_ID' in env: + if settings.tg_api_key and settings.tg_chat_id: logger.info('Running containers for Telegram notifications') run_cmd( cmd=get_up_compose_cmd( - node_type=NodeType.REGULAR, services=list(NOTIFICATION_COMPOSE_SERVICES) + node_type=NodeType.SKALE, + node_mode=node_mode, + services=list(NOTIFICATION_COMPOSE_SERVICES), ), env=env, ) - if str_to_bool(env.get('MONITORING_CONTAINERS', 'False')): + if settings.monitoring_containers: logger.info('Running monitoring containers') run_cmd( cmd=get_up_compose_cmd( - node_type=NodeType.REGULAR, services=list(MONITORING_COMPOSE_SERVICES) + node_type=NodeType.SKALE, + node_mode=node_mode, + services=list(MONITORING_COMPOSE_SERVICES), ), env=env, ) @@ -393,22 +424,12 @@ def is_container_running(name: str, dclient: Optional[DockerClient] = None) -> b return False -def is_api_running(node_type: NodeType, dclient: Optional[DockerClient] = None) -> bool: - if node_type == NodeType.FAIR: - return is_container_running(name='fair_api', dclient=dclient) - else: - return is_container_running(name='skale_api', dclient=dclient) - +def is_api_running(dclient: Optional[DockerClient] = None) -> bool: + return is_container_running(name='sk_api', dclient=dclient) -def is_admin_running(node_type: NodeType, client: Optional[DockerClient] = None) -> bool: - if node_type == NodeType.FAIR: - result = is_container_running(name='fair_admin', dclient=client) - elif node_type == NodeType.SYNC: - result = is_container_running(name='skale_sync_admin', dclient=client) - else: - result = is_container_running(name='skale_admin', dclient=client) - return result +def is_admin_running(dclient: Optional[DockerClient] = None) -> bool: + return is_container_running(name='sk_admin', dclient=dclient) def system_prune(): diff --git a/node_cli/utils/global_config.py b/node_cli/utils/global_config.py index 4347c6b0..974c573a 100644 --- a/node_cli/utils/global_config.py +++ b/node_cli/utils/global_config.py @@ -52,6 +52,10 @@ def generate_g_config_file(g_skale_dir: str, g_skale_conf_filepath: str) -> dict json.dump(g_config, outfile, indent=4) except PermissionError as e: logger.exception(e) - print('No permissions to write into /etc directory') + print(f'No permissions to write into {g_skale_dir} directory') + sys.exit(7) + except OSError as e: + logger.exception(e) + print(f'Error writing to {g_skale_conf_filepath}: {e}') sys.exit(7) return g_config diff --git a/node_cli/utils/helper.py b/node_cli/utils/helper.py index 7adfba3b..8a642aee 100644 --- a/node_cli/utils/helper.py +++ b/node_cli/utils/helper.py @@ -17,8 +17,6 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import distutils -import distutils.util import ipaddress import json import logging @@ -32,6 +30,7 @@ import urllib.parse import urllib.request import uuid +from pathlib import Path from functools import wraps from logging import Formatter, StreamHandler from typing import Any, NoReturn, Optional @@ -69,7 +68,7 @@ DEFAULT_ERROR_DATA = { 'status': 'error', - 'payload': 'Request failed. Check skale_api container logs', + 'payload': 'Request failed. Check API container logs', } @@ -87,13 +86,13 @@ def write_json(path: str, content: dict) -> None: json.dump(content, outfile, indent=4) -def save_json(path: str, content: dict) -> None: +def save_json(path: str | Path, content: dict) -> None: tmp_path = get_tmp_path(path) write_json(tmp_path, content) shutil.move(tmp_path, path) -def init_file(path, content=None): +def init_file(path: str | Path, content=None): if not os.path.exists(path): write_json(path, content) @@ -145,10 +144,6 @@ def get_username(): return os.environ.get('USERNAME') or os.environ.get('USER') -def str_to_bool(val): - return bool(distutils.util.strtobool(val)) - - def error_exit(error_payload: Any, exit_code: CLIExitCodes = CLIExitCodes.FAILURE) -> NoReturn: """Print error message and exit the program with specified exit code. @@ -203,7 +198,7 @@ def post_request(blueprint, method, json=None, files=None): response = requests.post(url, json=json, files=files) data = response.json() except Exception as err: - logger.error('Request failed', exc_info=err) + logger.exception('Request failed', exc_info=err) data = DEFAULT_ERROR_DATA status = data['status'] payload = data['payload'] @@ -324,7 +319,18 @@ def rm_dir(folder: str) -> None: logger.info(f"{folder} doesn't exist, skipping...") -def safe_mkdir(path: str, print_res: bool = False) -> None: +def cleanup_dir_content(folder: str) -> None: + if os.path.exists(folder): + logger.info('Removing contents of %s', folder) + for filename in os.listdir(folder): + file_path = os.path.join(folder, filename) + if os.path.isfile(file_path) or os.path.islink(file_path): + os.unlink(file_path) + elif os.path.isdir(file_path): + shutil.rmtree(file_path) + + +def safe_mkdir(path: str | Path, print_res: bool = False) -> None: if os.path.exists(path): logger.debug(f'Directory {path} already exists') return @@ -371,6 +377,15 @@ def convert(self, value, param, ctx): return value +class UrlOrAnyType(UrlType): + name = 'url' + + def convert(self, value, param, ctx): + if value == 'any': + return value + return super().convert(value, param, ctx) + + class IpType(click.ParamType): name = 'ip' @@ -383,11 +398,12 @@ def convert(self, value, param, ctx): URL_TYPE = UrlType() +URL_OR_ANY_TYPE = UrlOrAnyType() IP_TYPE = IpType() -def get_tmp_path(path: str) -> str: - base, ext = os.path.splitext(path) +def get_tmp_path(path: str | Path) -> str: + base, ext = os.path.splitext(str(path)) salt = uuid.uuid4().hex[:5] return base + salt + '.tmp' + ext @@ -400,5 +416,10 @@ def get_ssh_port(ssh_service_name='ssh'): return DEFAULT_SSH_PORT -def is_contract_address(value: str) -> bool: - return bool(re.fullmatch(r'0x[a-fA-F0-9]{40}', value)) +def is_btrfs_subvolume(path: str) -> bool: + """Check if the given path is a Btrfs subvolume.""" + try: + output = run_cmd(['btrfs', 'subvolume', 'show', path], check_code=False) + return output.returncode == 0 + except subprocess.CalledProcessError: + return False diff --git a/node_cli/utils/meta.py b/node_cli/utils/meta.py index 0dafd54d..651efc1e 100644 --- a/node_cli/utils/meta.py +++ b/node_cli/utils/meta.py @@ -7,7 +7,7 @@ DEFAULT_VERSION = '1.0.0' DEFAULT_CONFIG_STREAM = '1.1.0' -DEFAULT_DOCKER_LVMPY_STREAM = '1.0.0' +DEFAULT_DOCKER_LVMPY_VERSION = '1.0.0' DEFAULT_OS_ID = 'ubuntu' DEFAULT_OS_VERSION = '18.04' @@ -26,13 +26,13 @@ def asdict(self) -> dict: @dataclass class CliMeta(CliMetaBase): - docker_lvmpy_stream: str = DEFAULT_DOCKER_LVMPY_STREAM + docker_lvmpy_version: str | None = DEFAULT_DOCKER_LVMPY_VERSION def asdict(self) -> dict: return { 'version': self.version, 'config_stream': self.config_stream, - 'docker_lvmpy_stream': self.docker_lvmpy_stream, + 'docker_lvmpy_version': self.docker_lvmpy_version, 'os_id': self.os_id, 'os_version': self.os_version, } @@ -96,7 +96,7 @@ def get_meta_info(self, raw: bool = False) -> CliMeta | dict | None: def compose_default_meta(self) -> CliMeta: return CliMeta( version=DEFAULT_VERSION, - docker_lvmpy_stream=DEFAULT_DOCKER_LVMPY_STREAM, + docker_lvmpy_version=DEFAULT_DOCKER_LVMPY_VERSION, config_stream=DEFAULT_CONFIG_STREAM, os_id=DEFAULT_OS_ID, os_version=DEFAULT_OS_VERSION, @@ -106,7 +106,7 @@ def update_meta( self, version: str, config_stream: str, - docker_lvmpy_stream: str | None, + docker_lvmpy_version: str | None, os_id: str, os_version: str, ) -> None: @@ -116,7 +116,7 @@ def update_meta( config_stream, os_id, os_version, - docker_lvmpy_stream, + docker_lvmpy_version, ) self.save_meta(meta) diff --git a/node_cli/utils/node_type.py b/node_cli/utils/node_type.py index f35d4640..754a1d69 100644 --- a/node_cli/utils/node_type.py +++ b/node_cli/utils/node_type.py @@ -20,7 +20,11 @@ from enum import Enum -class NodeType(Enum): - REGULAR = 0 - SYNC = 1 - FAIR = 2 +class NodeType(str, Enum): + SKALE = 'skale' + FAIR = 'fair' + + +class NodeMode(str, Enum): + ACTIVE = 'active' + PASSIVE = 'passive' diff --git a/node_cli/utils/print_formatters.py b/node_cli/utils/print_formatters.py index 1fd1ca33..1da07d51 100644 --- a/node_cli/utils/print_formatters.py +++ b/node_cli/utils/print_formatters.py @@ -319,7 +319,61 @@ def print_meta_info(meta_info: CliMeta) -> None: {LONG_LINE} Version: {meta_info.version} Config Stream: {meta_info.config_stream} - Lvmpy stream: {meta_info.docker_lvmpy_stream} + Lvmpy stream: {meta_info.docker_lvmpy_version} {LONG_LINE} """) ) + + +def format_timestamp(value): + if value is None or value == 'N/A' or value == 0 or value == 0.0: + return 'N/A' + try: + timestamp = float(value) + if timestamp == 0: + return 'N/A' + dt = datetime.datetime.fromtimestamp(timestamp) + human_date = dt.strftime('%Y-%m-%d %H:%M:%S') + return f'{human_date} ({timestamp})' + except (ValueError, TypeError): + return str(value) + + +def print_chain_record(record): + print( + inspect.cleandoc(f""" + {LONG_LINE} + Fair Chain Record + Chain Name: {record.get('name', 'N/A')} + Config Version: {record.get('config_version', 'N/A')} + Sync Config Run: {record.get('sync_config_run', 'N/A')} + First Run: {record.get('first_run', 'N/A')} + Backup Run: {record.get('backup_run', 'N/A')} + Restart Count: {record.get('restart_count', 'N/A')} + Failed RPC Count: {record.get('failed_rpc_count', 'N/A')} + Monitor Last Seen: {format_timestamp(record.get('monitor_last_seen', 'N/A'))} + SSL Change Date: {format_timestamp(record.get('ssl_change_date', 'N/A'))} + Repair Date: {format_timestamp(record.get('repair_date', 'N/A'))} + DKG Status: {record.get('dkg_status', 'N/A')} + Repair Timestamp: {format_timestamp(record.get('repair_ts', 'N/A'))} + Snapshot From: {record.get('snapshot_from', 'N/A')} + Restart Timestamp: {format_timestamp(record.get('restart_ts', 'N/A'))} + Force Skaled Start: {record.get('force_skaled_start', 'N/A')} + {LONG_LINE} + """) + ) + + +def print_chain_checks(checks): + def format_checks(check_dict, title): + print(f'\n{title}:') + for name, result in check_dict.items(): + status = 'PASS' if result else 'FAIL' + print(f' {name}: {status}') + + print(f'{LONG_LINE}') + print('Fair Chain Checks') + print(f'{LONG_LINE}') + format_checks(checks['config_checks'], 'Config Checks') + format_checks(checks['skaled_checks'], 'Skaled Checks') + print(f'{LONG_LINE}') diff --git a/node_cli/utils/settings.py b/node_cli/utils/settings.py new file mode 100644 index 00000000..8e7f957c --- /dev/null +++ b/node_cli/utils/settings.py @@ -0,0 +1,77 @@ +# -*- coding: utf-8 -*- +# +# This file is part of node-cli +# +# Copyright (C) 2026 SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import tomllib + +from dotenv.main import DotEnv + +from skale_core.settings import ( + SETTINGS_MAP, + BaseNodeSettings, + FairBaseSettings, + FairSettings, + InternalSettings, + SkalePassiveSettings, + SkaleSettings, + write_internal_settings_file, + write_node_settings_file, +) + +from node_cli.configs import INTERNAL_SETTINGS_PATH, NODE_SETTINGS_PATH, SKALE_DIR +from node_cli.utils.node_type import NodeMode, NodeType + +InternalSettings.model_config['toml_file'] = INTERNAL_SETTINGS_PATH +SkaleSettings.model_config['toml_file'] = NODE_SETTINGS_PATH +SkalePassiveSettings.model_config['toml_file'] = NODE_SETTINGS_PATH +FairSettings.model_config['toml_file'] = NODE_SETTINGS_PATH +FairBaseSettings.model_config['toml_file'] = NODE_SETTINGS_PATH + + +def load_config_file(filepath: str) -> dict: + if filepath.endswith('.toml'): + with open(filepath, 'rb') as f: + return tomllib.load(f) + return {k.lower(): v for k, v in DotEnv(filepath).dict().items()} + + +def validate_and_save_node_settings( + config_filepath: str, + node_type: NodeType, + node_mode: NodeMode, +) -> BaseNodeSettings: + data = load_config_file(config_filepath) + settings_type = SETTINGS_MAP[(node_type.value, node_mode.value)] + write_node_settings_file(path=NODE_SETTINGS_PATH, settings_type=settings_type, data=data) + return settings_type() + + +def save_internal_settings( + node_type: NodeType, + node_mode: NodeMode, + backup_run: bool = False, + pull_config_for_schain: str | None = None, +) -> None: + data = { + 'node_type': node_type.value, + 'node_mode': node_mode.value, + 'skale_dir_host': str(SKALE_DIR), + 'backup_run': backup_run, + 'pull_config_for_schain': pull_config_for_schain, + } + write_internal_settings_file(path=INTERNAL_SETTINGS_PATH, data=data) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..4ab3258d --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,89 @@ +[build-system] +requires = ["setuptools>=75", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "node-cli" +version = "3.2.0" +description = "Node CLI tools" +readme = "README.md" +requires-python = ">=3.13" +license = { file = "LICENSE" } +keywords = ["skale", "cli"] +authors = [{ name = "SKALE Labs", email = "support@skalelabs.com" }] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "License :: OSI Approved :: GNU Affero General Public License v3", + "Natural Language :: English", + "Programming Language :: Python :: 3.13", +] + +dependencies = [ + "click==8.3.1", + "distro==1.9.0", + "docker==7.1.0", + "texttable==1.7.0", + "python-dateutil==2.9.0.post0", + "Jinja2==3.1.6", + "psutil==7.1.3", + "python-dotenv==1.2.1", + "terminaltables==3.1.10", + "requests==2.32.5", + "GitPython==3.1.45", + "packaging==25.0", + "python-debian==1.0.1", + "PyYAML==6.0.3", + "pyOpenSSL==25.3.0", + "MarkupSafe==3.0.3", + "Flask==3.1.2", + "itsdangerous==2.2.0", + "cryptography==46.0.5", + "filelock==3.20.0", + "sh==2.2.2", + "python-crontab==3.3.0", + "requests-mock==1.12.1", + "redis==7.1.1", + "PyInstaller==6.18.0", + "skale.py-core==7.13.dev1", +] + +[project.urls] +Homepage = "https://github.com/skalenetwork/node-cli" + +[project.optional-dependencies] +dev = [ + "ruff==0.14.6", + "bumpversion==0.6.0", + "pytest==9.0.1", + "pytest-cov==7.0.0", + "twine==6.2.0", + "mock==5.2.0", + "freezegun==1.5.5", +] + +[tool.setuptools] +package-dir = { "" = "." } +include-package-data = true + +[tool.setuptools.packages.find] +where = ["."] +exclude = ["tests"] + +[tool.ruff] +line-length = 100 +target-version = "py313" + +[tool.ruff.format] +quote-style = "single" + +[tool.uv] +prerelease = "allow" + + +[tool.pytest.ini_options] +log_cli = false +log_cli_level = "INFO" +log_cli_format = "%(asctime)s [%(levelname)8s] %(message)s (%(filename)s:%(lineno)s)" +log_cli_date_format = "%Y-%m-%d %H:%M:%S" +filterwarnings = ["ignore::DeprecationWarning"] diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index 5785bf3f..00000000 --- a/pytest.ini +++ /dev/null @@ -1,6 +0,0 @@ -[pytest] -log_cli = 0 -log_cli_level = INFO -log_cli_format = %(asctime)s [%(levelname)8s] %(message)s (%(filename)s:%(lineno)s) -log_cli_date_format=%Y-%m-%d %H:%M:%S -filterwarnings = ignore::DeprecationWarning diff --git a/ruff.toml b/ruff.toml deleted file mode 100644 index 9653b675..00000000 --- a/ruff.toml +++ /dev/null @@ -1,8 +0,0 @@ -line-length = 100 - -[format] -quote-style = "single" - -[lint] -# Add the `line-too-long` rule to the enforced rule set. -extend-select = ["E501"] \ No newline at end of file diff --git a/scripts/build.sh b/scripts/build.sh old mode 100644 new mode 100755 index a1adc516..8e26577a --- a/scripts/build.sh +++ b/scripts/build.sh @@ -24,7 +24,7 @@ fi if [ -z "$3" ] then - (>&2 echo 'You should provide type: normal, sync or fair') + (>&2 echo 'You should provide type: skale, or fair') echo $USAGE_MSG exit 1 fi @@ -37,9 +37,8 @@ OS=`uname -s`-`uname -m` # Use the new generate_info.sh script bash "${DIR}/generate_info.sh" "$VERSION" "$BRANCH" "$TYPE" -if [ "$TYPE" = "sync" ]; then - EXECUTABLE_NAME=skale-$VERSION-$OS-sync -elif [ "$TYPE" = "fair" ]; then + +if [ "$TYPE" = "fair" ]; then EXECUTABLE_NAME=skale-$VERSION-$OS-fair else EXECUTABLE_NAME=skale-$VERSION-$OS diff --git a/scripts/export_env.sh b/scripts/export_env.sh new file mode 100644 index 00000000..af30b4ab --- /dev/null +++ b/scripts/export_env.sh @@ -0,0 +1,8 @@ +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +PROJECT_DIR=$(dirname $DIR) + +export LVMPY_LOG_DIR="$PROJECT_DIR/tests/" +export HIDE_STREAM_LOG=true +export TEST_HOME_DIR="$PROJECT_DIR/tests/" +export GLOBAL_SKALE_DIR="$PROJECT_DIR/tests/etc/skale" +export DOTENV_FILEPATH='tests/test-env' \ No newline at end of file diff --git a/scripts/generate_info.sh b/scripts/generate_info.sh index f4993b7e..67ee6aef 100644 --- a/scripts/generate_info.sh +++ b/scripts/generate_info.sh @@ -18,7 +18,7 @@ if [ -z "$BRANCH" ]; then exit 1 fi if [ -z "$TYPE_STR" ]; then - (>&2 echo 'You should provide type: normal, sync or fair') + (>&2 echo 'You should provide type: skale or fair') echo $USAGE_MSG exit 1 fi @@ -32,17 +32,14 @@ CURRENT_DATETIME="$(date "+%Y-%m-%d %H:%M:%S")" OS="$(uname -s)-$(uname -m)" case "$TYPE_STR" in - normal) - TYPE_ENUM="NodeType.REGULAR" - ;; - sync) - TYPE_ENUM="NodeType.SYNC" + skale) + TYPE_ENUM="NodeType.SKALE" ;; fair) TYPE_ENUM="NodeType.FAIR" ;; *) - (>&2 echo "Error: Invalid type '$TYPE_STR'. Must be 'normal', 'sync', or 'fair'") + (>&2 echo "Error: Invalid type '$TYPE_STR'. Must be 'skale', or 'fair'") exit 1 ;; esac diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh index efc72c6c..1d3a3fe6 100755 --- a/scripts/run_tests.sh +++ b/scripts/run_tests.sh @@ -3,9 +3,6 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" PROJECT_DIR=$(dirname $DIR) -LVMPY_LOG_DIR="$PROJECT_DIR/tests/" \ - HIDE_STREAM_LOG=true \ - TEST_HOME_DIR="$PROJECT_DIR/tests/" \ - GLOBAL_SKALE_DIR="$PROJECT_DIR/tests/etc/skale" \ - DOTENV_FILEPATH='tests/test-env' \ - py.test --cov=$PROJECT_DIR/ --ignore=tests/core/nftables_test.py --ignore=tests/core/migration_test.py tests/ $@ +. "$DIR/export_env.sh" + +py.test --cov=$PROJECT_DIR/ --ignore=tests/core/nftables_test.py --ignore=tests/core/migration_test.py tests/ $@ diff --git a/scripts/set_versions_ga.sh b/scripts/set_versions_ga.sh index ddcc7aaa..7333357f 100644 --- a/scripts/set_versions_ga.sh +++ b/scripts/set_versions_ga.sh @@ -7,7 +7,7 @@ echo PROJECT_DIR: $GITHUB_WORKSPACE export BRANCH=${GITHUB_REF##*/} echo "Branch $BRANCH" -export VERSION=$(python setup.py --version) +export VERSION=$(python -c "import tomllib; print(tomllib.load(open('pyproject.toml', 'rb'))['project']['version'])") export VERSION=$(bash ./helper-scripts/calculate_version.sh) echo "VERSION=$VERSION" >> $GITHUB_ENV diff --git a/setup.py b/setup.py deleted file mode 100644 index 5ea3e6ce..00000000 --- a/setup.py +++ /dev/null @@ -1,87 +0,0 @@ -import os -import re -from setuptools import find_packages, setup - - -def read(*parts): - path = os.path.join(os.path.dirname(__file__), *parts) - f = open(path, 'r') - return f.read() - - -def find_version(*file_paths): - version_file = read(*file_paths) - version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", version_file, re.M) - if version_match: - return version_match.group(1) - raise RuntimeError("Couldn't parse version from file.") - - -extras_require = { - 'linter': [ - 'isort>=4.2.15,<5.10.2', - 'ruff==0.9.9', - ], - 'dev': [ - 'bumpversion==0.6.0', - 'pytest==8.3.2', - 'pytest-cov==5.0.0', - 'twine==4.0.2', - 'mock==4.0.3', - 'freezegun==1.2.2', - ], -} - -extras_require['dev'] = extras_require['linter'] + extras_require['dev'] - - -setup( - name='node-cli', - # *IMPORTANT*: Don't manually change the version here. - # Use the 'bumpversion' utility instead. - version=find_version('node_cli', 'cli', '__init__.py'), - include_package_data=True, - description='SKALE client tools', - long_description_markdown_filename='README.md', - author='SKALE Labs', - author_email='support@skalelabs.com', - url='https://github.com/skalenetwork/node-cli', - install_requires=[ - 'click==8.1.7', - 'PyInstaller==5.12.0', - 'distro==1.9.0', - 'docker==6.0.1', - 'texttable==1.6.7', - 'python-dateutil==2.8.2', - 'Jinja2==3.1.4', - 'psutil==5.9.4', - 'python-dotenv==0.21.0', - 'terminaltables==3.1.10', - 'requests==2.28.1', - 'GitPython==3.1.41', - 'packaging==23.0', - 'python-debian==0.1.49', - 'PyYAML==6.0', - 'pyOpenSSL==24.2.1', - 'MarkupSafe==3.0.2', - 'Flask==2.3.3', - 'itsdangerous==2.1.2', - 'cryptography==42.0.4', - 'filelock==3.0.12', - 'sh==1.14.2', - 'python-crontab==2.6.0', - 'requests-mock==1.12.1', - 'redis==6.2.0', - ], - python_requires='>=3.8,<4', - extras_require=extras_require, - keywords=['skale', 'cli'], - packages=find_packages(exclude=['tests']), - classifiers=[ - 'Development Status :: 5 - Production/Stable', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: GNU Affero General Public License v3', - 'Natural Language :: English', - 'Programming Language :: Python :: 3.11', - ], -) diff --git a/tests/.skale/config/docker-compose-fair.yml b/tests/.skale/config/docker-compose-fair.yml new file mode 100644 index 00000000..c09f2c08 --- /dev/null +++ b/tests/.skale/config/docker-compose-fair.yml @@ -0,0 +1,5 @@ +services: + test: + container_name: test + image: alpine:latest + network_mode: host diff --git a/tests/cli/fair_cli_test.py b/tests/cli/fair_cli_test.py index d29ae357..14e2aee9 100644 --- a/tests/cli/fair_cli_test.py +++ b/tests/cli/fair_cli_test.py @@ -2,7 +2,6 @@ from unittest import mock from click.testing import CliRunner - from node_cli.cli.fair_boot import ( init_boot, register_boot, @@ -10,9 +9,16 @@ ) from node_cli.cli.fair_node import ( backup_node, + cleanup_node, migrate_node, + exit_node, restore_node, ) +from node_cli.configs import SKALE_DIR +from node_cli.utils.node_type import NodeMode +from node_cli.utils.meta import CliMeta +from tests.helper import run_command, subprocess_run_mock +from tests.resources_test import BIG_DISK_SIZE @mock.patch('node_cli.cli.fair_node.restore_fair') @@ -100,4 +106,36 @@ def test_fair_node_migrate(mock_migrate_core, valid_env_file): result = runner.invoke(migrate_node, ['--yes', valid_env_file]) assert result.exit_code == 0, f'Output: {result.output}\nException: {result.exception}' - mock_migrate_core.assert_called_once_with(env_filepath=valid_env_file) + mock_migrate_core.assert_called_once_with(config_file=valid_env_file) + + +@mock.patch('node_cli.cli.fair_node.exit_fair') +def test_fair_node_exit(mock_exit_core): + runner = CliRunner() + result = runner.invoke(exit_node, ['--yes']) + + assert result.exit_code == 0, f'Output: {result.output}\nException: {result.exception}' + mock_exit_core.assert_called_once() + + +def test_cleanup_node(mocked_g_config, inited_node): + pathlib.Path(SKALE_DIR).mkdir(parents=True, exist_ok=True) + + with ( + mock.patch('subprocess.run', new=subprocess_run_mock), + mock.patch('node_cli.fair.common.cleanup_fair_op') as cleanup_mock, + mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), + mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), + mock.patch('node_cli.operations.base.configure_nftables'), + mock.patch('node_cli.utils.decorators.is_node_inited', return_value=True), + mock.patch('node_cli.fair.common.compose_node_env', return_value={'SCHAIN_NAME': 'test'}), + mock.patch( + 'node_cli.core.node.CliMetaManager.get_meta_info', + return_value=CliMeta(version='2.6.0', config_stream='3.0.2'), + ), + ): + result = run_command(cleanup_node, ['--yes']) + assert result.exit_code == 0 + cleanup_mock.assert_called_once_with( + node_mode=NodeMode.ACTIVE, prune=False, compose_env={'SCHAIN_NAME': 'test'} + ) diff --git a/tests/cli/fair_passive_node_test.py b/tests/cli/fair_passive_node_test.py new file mode 100644 index 00000000..00c61ca0 --- /dev/null +++ b/tests/cli/fair_passive_node_test.py @@ -0,0 +1,107 @@ +import logging +import pathlib + +import mock + +from node_cli.cli.passive_fair_node import cleanup_node, init_passive_node, update_node +from node_cli.configs import NODE_DATA_PATH, SKALE_DIR +from node_cli.utils.helper import init_default_logger +from node_cli.utils.meta import CliMeta +from node_cli.utils.node_type import NodeMode +from tests.helper import run_command, subprocess_run_mock +from tests.resources_test import BIG_DISK_SIZE + +logger = logging.getLogger(__name__) +init_default_logger() + + +def test_init_fair_passive(mocked_g_config, fair_passive_settings, fair_passive_user_conf): + pathlib.Path(SKALE_DIR).mkdir(parents=True, exist_ok=True) + with ( + mock.patch('subprocess.run', new=subprocess_run_mock), + mock.patch('node_cli.fair.common.init_fair_op', return_value=True), + mock.patch('node_cli.fair.common.compose_node_env', return_value={}), + mock.patch('node_cli.fair.passive.setup_fair_passive'), + mock.patch('node_cli.fair.common.time.sleep'), + mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), + mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), + mock.patch('node_cli.operations.base.configure_nftables'), + mock.patch('node_cli.utils.decorators.is_node_inited', return_value=False), + ): + result = run_command( + init_passive_node, + [ + fair_passive_user_conf.as_posix(), + '--id', + '1', + ], + ) + assert result.exit_code == 0 + + +def test_init_fair_passive_snapshot_any( + mocked_g_config, fair_passive_settings, fair_passive_user_conf +): + pathlib.Path(SKALE_DIR).mkdir(parents=True, exist_ok=True) + with ( + mock.patch('subprocess.run', new=subprocess_run_mock), + mock.patch('node_cli.fair.common.init_fair_op', return_value=True), + mock.patch('node_cli.fair.common.compose_node_env', return_value={}), + mock.patch('node_cli.fair.passive.setup_fair_passive'), + mock.patch('node_cli.fair.common.time.sleep'), + mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), + mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), + mock.patch('node_cli.operations.base.configure_nftables'), + mock.patch('node_cli.utils.decorators.is_node_inited', return_value=False), + ): + result = run_command( + init_passive_node, + [ + fair_passive_user_conf.as_posix(), + '--id', + '2', + '--snapshot', + 'any', + ], + ) + assert result.exit_code == 0 + + +def test_update_fair_passive( + mocked_g_config, fair_passive_settings, fair_passive_user_conf, clean_node_options +): + pathlib.Path(NODE_DATA_PATH).mkdir(parents=True, exist_ok=True) + with ( + mock.patch('subprocess.run', new=subprocess_run_mock), + mock.patch('node_cli.fair.common.update_fair_op', return_value=True), + mock.patch('node_cli.fair.common.compose_node_env', return_value={}), + mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), + mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), + mock.patch('node_cli.operations.base.configure_nftables'), + mock.patch('node_cli.utils.decorators.is_node_inited', return_value=True), + ): + result = run_command(update_node, [fair_passive_user_conf.as_posix(), '--yes']) + assert result.exit_code == 0 + + +def test_cleanup_node(mocked_g_config): + pathlib.Path(SKALE_DIR).mkdir(parents=True, exist_ok=True) + + with ( + mock.patch('subprocess.run', new=subprocess_run_mock), + mock.patch('node_cli.fair.common.cleanup_fair_op') as cleanup_mock, + mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), + mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), + mock.patch('node_cli.operations.base.configure_nftables'), + mock.patch('node_cli.utils.decorators.is_node_inited', return_value=True), + mock.patch('node_cli.fair.common.compose_node_env', return_value={'SCHAIN_NAME': 'test'}), + mock.patch( + 'node_cli.core.node.CliMetaManager.get_meta_info', + return_value=CliMeta(version='2.6.0', config_stream='3.0.2'), + ), + ): + result = run_command(cleanup_node, ['--yes']) + assert result.exit_code == 0 + cleanup_mock.assert_called_once_with( + node_mode=NodeMode.PASSIVE, compose_env={'SCHAIN_NAME': 'test'}, prune=False + ) diff --git a/tests/cli/health_test.py b/tests/cli/health_test.py index de14c7c2..404d9a73 100644 --- a/tests/cli/health_test.py +++ b/tests/cli/health_test.py @@ -9,7 +9,7 @@ 'payload': [ { 'image': 'skalenetwork/schain:1.46-develop.21', - 'name': 'skale_schain_shapely-alfecca-meridiana', + 'name': 'sk_skaled_shapely-alfecca-meridiana', 'state': { 'Status': 'running', 'Running': True, @@ -26,7 +26,7 @@ }, { 'image': 'skale-admin:latest', - 'name': 'skale_api', + 'name': 'sk_api', 'state': { 'Status': 'running', 'Running': True, @@ -51,7 +51,7 @@ def test_containers(): assert result.exit_code == 0 assert ( result.output - == ' Name Status Started At Image \n-------------------------------------------------------------------------------------------------------------\nskale_schain_shapely-alfecca-meridiana Running Jul 31 2020 11:56:35 skalenetwork/schain:1.46-develop.21\nskale_api Running Jul 31 2020 11:55:17 skale-admin:latest \n' # noqa + == ' Name Status Started At Image \n----------------------------------------------------------------------------------------------------------\nsk_skaled_shapely-alfecca-meridiana Running Jul 31 2020 11:56:35 skalenetwork/schain:1.46-develop.21\nsk_api Running Jul 31 2020 11:55:17 skale-admin:latest \n' # noqa ) diff --git a/tests/cli/node_test.py b/tests/cli/node_test.py index 0db1d97a..a0c064f0 100644 --- a/tests/cli/node_test.py +++ b/tests/cli/node_test.py @@ -30,6 +30,7 @@ _turn_off, _turn_on, backup_node, + cleanup_node, node_info, register_node, remove_node_from_maintenance, @@ -42,7 +43,7 @@ from node_cli.utils.exit_codes import CLIExitCodes from node_cli.utils.helper import init_default_logger from node_cli.utils.meta import CliMeta -from node_cli.utils.node_type import NodeType +from node_cli.utils.node_type import NodeType, NodeMode from tests.helper import ( response_mock, run_command, @@ -322,14 +323,13 @@ def test_backup(): @pytest.mark.parametrize( - 'node_type,test_user_conf', + 'node_type,node_mode,test_user_conf', [ - (NodeType.REGULAR, 'regular_user_conf'), - (NodeType.FAIR, 'fair_user_conf'), - (NodeType.SYNC, 'sync_user_conf'), + (NodeType.SKALE, NodeMode.ACTIVE, 'regular_user_conf'), + (NodeType.FAIR, NodeMode.ACTIVE, 'fair_user_conf'), ], ) -def test_restore(request, node_type, test_user_conf, mocked_g_config, tmp_path): +def test_restore(request, node_type, node_mode, test_user_conf, mocked_g_config, tmp_path): pathlib.Path(SKALE_DIR).mkdir(parents=True, exist_ok=True) result = run_command(backup_node, [tmp_path]) backup_path = result.output.replace('Backup archive successfully created: ', '').replace( @@ -347,19 +347,17 @@ def test_restore(request, node_type, test_user_conf, mocked_g_config, tmp_path): return_value=CliMeta(version='2.4.0', config_stream='3.0.2'), ), patch('node_cli.operations.base.configure_nftables'), - patch('node_cli.configs.user.validate_alias_or_address'), ): user_conf_path = request.getfixturevalue(test_user_conf).as_posix() - result = run_command(restore_node, [backup_path, user_conf_path]) assert result.exit_code == 0 assert 'Node is restored from backup\n' in result.output # noqa - assert mock_restore_op.call_args[0][0].get('BACKUP_RUN') == 'True' + assert mock_restore_op.call_args.kwargs.get('backup_run') is True result = run_command(restore_node, [backup_path, user_conf_path, '--no-snapshot']) assert result.exit_code == 0 assert 'Node is restored from backup\n' in result.output # noqa - assert mock_restore_op.call_args[0][0].get('BACKUP_RUN') is None + assert mock_restore_op.call_args.kwargs.get('backup_run') is False def test_maintenance_on(): @@ -386,15 +384,13 @@ def test_maintenance_off(mocked_g_config): ) -def test_turn_off_maintenance_on(mocked_g_config, regular_user_conf): +def test_turn_off_maintenance_on(mocked_g_config, regular_user_conf, active_node_option, skale_active_settings): resp_mock = response_mock(requests.codes.ok, {'status': 'ok', 'payload': None}) with ( mock.patch('subprocess.run', new=subprocess_run_mock), - mock.patch('node_cli.core.node.SKALE_DIR_ENV_FILEPATH', regular_user_conf.as_posix()), mock.patch('node_cli.core.node.turn_off_op'), mock.patch('node_cli.utils.decorators.is_node_inited', return_value=True), - mock.patch('node_cli.configs.user.validate_alias_or_address'), - mock.patch('node_cli.cli.node.TYPE', NodeType.REGULAR), + mock.patch('node_cli.cli.node.TYPE', NodeType.SKALE), ): result = run_command_mock( 'node_cli.utils.helper.requests.post', @@ -402,6 +398,7 @@ def test_turn_off_maintenance_on(mocked_g_config, regular_user_conf): _turn_off, ['--maintenance-on', '--yes'], ) + assert ( result.output == 'Setting maintenance mode on...\nNode is successfully set in maintenance mode\n' @@ -418,16 +415,14 @@ def test_turn_off_maintenance_on(mocked_g_config, regular_user_conf): assert result.exit_code == CLIExitCodes.UNSAFE_UPDATE -def test_turn_on_maintenance_off(mocked_g_config, regular_user_conf): +def test_turn_on_maintenance_off(mocked_g_config, regular_user_conf, active_node_option, skale_active_settings): resp_mock = response_mock(requests.codes.ok, {'status': 'ok', 'payload': None}) with ( mock.patch('subprocess.run', new=subprocess_run_mock), - mock.patch('node_cli.core.node.get_flask_secret_key'), mock.patch('node_cli.core.node.turn_on_op'), mock.patch('node_cli.core.node.is_base_containers_alive'), mock.patch('node_cli.utils.decorators.is_node_inited', return_value=True), - mock.patch('node_cli.configs.user.validate_alias_or_address'), - mock.patch('node_cli.cli.node.TYPE', NodeType.REGULAR), + mock.patch('node_cli.cli.node.TYPE', NodeType.SKALE), ): result = run_command_mock( 'node_cli.utils.helper.requests.post', @@ -472,5 +467,26 @@ def test_node_version(meta_file_v2): assert result.exit_code == 0 assert ( result.output - == "{'version': '0.1.1', 'config_stream': 'develop', 'docker_lvmpy_stream': '1.1.2'}\n" + == "{'version': '0.1.1', 'config_stream': 'develop', 'docker_lvmpy_version': '1.1.2'}\n" ) + + +def test_cleanup_node(mocked_g_config): + pathlib.Path(SKALE_DIR).mkdir(parents=True, exist_ok=True) + + with ( + mock.patch('subprocess.run', new=subprocess_run_mock), + mock.patch('node_cli.core.node.cleanup_skale_op') as cleanup_mock, + mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), + mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), + mock.patch('node_cli.operations.base.configure_nftables'), + mock.patch('node_cli.utils.decorators.is_node_inited', return_value=True), + mock.patch('node_cli.core.node.compose_node_env', return_value={}), + mock.patch( + 'node_cli.core.node.CliMetaManager.get_meta_info', + return_value=CliMeta(version='2.6.0', config_stream='3.0.2'), + ), + ): + result = run_command(cleanup_node, ['--yes']) + assert result.exit_code == 0 + cleanup_mock.assert_called_once_with(node_mode=NodeMode.ACTIVE, prune=False, compose_env={}) diff --git a/tests/cli/sync_node_test.py b/tests/cli/passive_node_test.py similarity index 78% rename from tests/cli/sync_node_test.py rename to tests/cli/passive_node_test.py index 014510e1..e8419220 100644 --- a/tests/cli/sync_node_test.py +++ b/tests/cli/passive_node_test.py @@ -22,12 +22,12 @@ import mock -from node_cli.cli.sync_node import _cleanup_sync, _init_sync, _update_sync +from node_cli.cli.passive_node import cleanup_node, _init_passive, _update_passive from node_cli.configs import NODE_DATA_PATH, SKALE_DIR from node_cli.core.node_options import NodeOptions from node_cli.utils.helper import init_default_logger from node_cli.utils.meta import CliMeta -from node_cli.utils.node_type import NodeType +from node_cli.utils.node_type import NodeType, NodeMode from tests.conftest import set_env_var from tests.helper import run_command, subprocess_run_mock from tests.resources_test import BIG_DISK_SIZE @@ -36,18 +36,17 @@ init_default_logger() -def test_init_sync(mocked_g_config, clean_node_options, sync_user_conf): +def test_init_passive(mocked_g_config, clean_node_options, passive_user_conf): pathlib.Path(SKALE_DIR).mkdir(parents=True, exist_ok=True) with ( mock.patch('subprocess.run', new=subprocess_run_mock), - mock.patch('node_cli.core.node.init_sync_op'), + mock.patch('node_cli.core.node.init_passive_op'), mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), mock.patch('node_cli.operations.base.configure_nftables'), mock.patch('node_cli.utils.decorators.is_node_inited', return_value=False), - mock.patch('node_cli.configs.user.validate_alias_or_address'), ): - result = run_command(_init_sync, [sync_user_conf.as_posix()]) + result = run_command(_init_passive, [passive_user_conf.as_posix()]) node_options = NodeOptions() assert not node_options.archive @@ -57,7 +56,7 @@ def test_init_sync(mocked_g_config, clean_node_options, sync_user_conf): assert result.exit_code == 0 -def test_init_sync_archive(mocked_g_config, clean_node_options, sync_user_conf): +def test_init_passive_archive(mocked_g_config, clean_node_options, passive_user_conf): pathlib.Path(NODE_DATA_PATH).mkdir(parents=True, exist_ok=True) with ( mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), @@ -66,9 +65,11 @@ def test_init_sync_archive(mocked_g_config, clean_node_options, sync_user_conf): mock.patch('node_cli.operations.base.sync_skale_node'), mock.patch('node_cli.operations.base.configure_docker'), mock.patch('node_cli.operations.base.prepare_host'), + mock.patch('node_cli.operations.base.save_internal_settings'), + mock.patch('node_cli.operations.base.run_host_checks', return_value=[]), mock.patch('node_cli.operations.base.ensure_filestorage_mapping'), - mock.patch('node_cli.operations.base.link_env_file'), mock.patch('node_cli.operations.base.generate_nginx_config'), + mock.patch('node_cli.operations.base.get_settings'), mock.patch('node_cli.operations.base.prepare_block_device'), mock.patch('node_cli.operations.base.CliMetaManager.update_meta'), mock.patch('node_cli.operations.base.update_resource_allocation'), @@ -77,10 +78,9 @@ def test_init_sync_archive(mocked_g_config, clean_node_options, sync_user_conf): mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), mock.patch('node_cli.operations.base.configure_nftables'), mock.patch('node_cli.utils.decorators.is_node_inited', return_value=False), - mock.patch('node_cli.configs.user.validate_alias_or_address'), - mock.patch('node_cli.cli.node.TYPE', NodeType.SYNC), + mock.patch('node_cli.cli.node.TYPE', NodeType.SKALE), ): - result = run_command(_init_sync, [sync_user_conf.as_posix(), '--archive']) + result = run_command(_init_passive, [passive_user_conf.as_posix(), '--archive']) node_options = NodeOptions() assert node_options.archive @@ -94,7 +94,7 @@ def test_init_archive_indexer_fail(mocked_g_config, clean_node_options): pathlib.Path(SKALE_DIR).mkdir(parents=True, exist_ok=True) with ( mock.patch('subprocess.run', new=subprocess_run_mock), - mock.patch('node_cli.core.node.init_sync_op'), + mock.patch('node_cli.core.node.init_passive_op'), mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), mock.patch('node_cli.operations.base.configure_nftables'), @@ -102,17 +102,17 @@ def test_init_archive_indexer_fail(mocked_g_config, clean_node_options): mock.patch('node_cli.core.node.compose_node_env', return_value={}), set_env_var('ENV_TYPE', 'devnet'), ): - result = run_command(_init_sync, ['./tests/test-env', '--archive', '--indexer']) + result = run_command(_init_passive, ['./tests/test-env', '--archive', '--indexer']) assert result.exit_code == 1 assert 'Cannot use both' in result.output -def test_update_sync(sync_user_conf, mocked_g_config): +def test_update_passive(passive_user_conf, mocked_g_config): pathlib.Path(SKALE_DIR).mkdir(parents=True, exist_ok=True) with ( mock.patch('subprocess.run', new=subprocess_run_mock), - mock.patch('node_cli.core.node.update_sync_op'), + mock.patch('node_cli.core.node.update_passive_op'), mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), mock.patch('node_cli.operations.base.configure_nftables'), @@ -121,18 +121,17 @@ def test_update_sync(sync_user_conf, mocked_g_config): 'node_cli.core.node.CliMetaManager.get_meta_info', return_value=CliMeta(version='2.6.0', config_stream='3.0.2'), ), - mock.patch('node_cli.configs.user.validate_alias_or_address'), ): - result = run_command(_update_sync, [sync_user_conf.as_posix(), '--yes']) + result = run_command(_update_passive, [passive_user_conf.as_posix(), '--yes']) assert result.exit_code == 0 -def test_cleanup_sync(mocked_g_config): +def test_cleanup_node(mocked_g_config): pathlib.Path(SKALE_DIR).mkdir(parents=True, exist_ok=True) with ( mock.patch('subprocess.run', new=subprocess_run_mock), - mock.patch('node_cli.core.node.cleanup_sync_op'), + mock.patch('node_cli.core.node.cleanup_skale_op') as cleanup_mock, mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), mock.patch('node_cli.operations.base.configure_nftables'), @@ -143,5 +142,8 @@ def test_cleanup_sync(mocked_g_config): return_value=CliMeta(version='2.6.0', config_stream='3.0.2'), ), ): - result = run_command(_cleanup_sync, ['--yes']) + result = run_command(cleanup_node, ['--yes']) assert result.exit_code == 0 + cleanup_mock.assert_called_once_with( + node_mode=NodeMode.PASSIVE, prune=False, compose_env={'SCHAIN_NAME': 'test'} + ) diff --git a/tests/cli/resources_allocation_test.py b/tests/cli/resources_allocation_test.py deleted file mode 100644 index 4f59daea..00000000 --- a/tests/cli/resources_allocation_test.py +++ /dev/null @@ -1,96 +0,0 @@ -# -*- coding: utf-8 -*- -# -# This file is part of node-cli -# -# Copyright (C) 2019 SKALE Labs -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -import json -import os - -import mock -import pytest -import requests - -from node_cli.cli.resources_allocation import generate, show -from node_cli.configs.resource_allocation import NODE_DATA_PATH, RESOURCE_ALLOCATION_FILEPATH -from node_cli.utils.helper import safe_mkdir, write_json -from node_cli.utils.node_type import NodeType -from tests.helper import response_mock, run_command_mock -from tests.resources_test import BIG_DISK_SIZE - -TEST_CONFIG = {'test': 1} - - -@pytest.fixture -def resource_alloc_config(): - write_json(RESOURCE_ALLOCATION_FILEPATH, TEST_CONFIG) - yield RESOURCE_ALLOCATION_FILEPATH - os.remove(RESOURCE_ALLOCATION_FILEPATH) - - -def test_show(resource_alloc_config): - resp_mock = response_mock(requests.codes.created) - write_json(RESOURCE_ALLOCATION_FILEPATH, TEST_CONFIG) - result = run_command_mock('node_cli.utils.helper.post_request', resp_mock, show) - assert result.output == json.dumps(TEST_CONFIG, indent=4) + '\n' - assert result.exit_code == 0 - - -def test_generate(regular_user_conf): - safe_mkdir(NODE_DATA_PATH) - resp_mock = response_mock(requests.codes.created) - with ( - mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), - mock.patch('node_cli.configs.user.validate_alias_or_address'), - ): - result = run_command_mock( - 'node_cli.utils.helper.post_request', - resp_mock, - generate, - [regular_user_conf.as_posix(), '--yes'], - ) - assert result.output == ( - f'Resource allocation file generated: {RESOURCE_ALLOCATION_FILEPATH}\n' - ) - assert result.exit_code == 0 - - -def test_generate_already_exists(regular_user_conf, resource_alloc_config): - resp_mock = response_mock(requests.codes.created) - with ( - mock.patch('node_cli.core.resources.get_disk_size', return_value=BIG_DISK_SIZE), - mock.patch('node_cli.cli.node.TYPE', NodeType.REGULAR), - mock.patch('node_cli.configs.user.validate_alias_or_address'), - ): - result = run_command_mock( - 'node_cli.utils.helper.post_request', - resp_mock, - generate, - [regular_user_conf.as_posix(), '--yes'], - ) - assert result.output == 'Resource allocation file already exists\n' - assert result.exit_code == 0 - - result = run_command_mock( - 'node_cli.utils.helper.post_request', - resp_mock, - generate, - [regular_user_conf.as_posix(), '--yes', '--force'], - ) - assert result.output == ( - f'Resource allocation file generated: {RESOURCE_ALLOCATION_FILEPATH}\n' - ) - assert result.exit_code == 0 diff --git a/tests/configs/configs_env_validate_test.py b/tests/configs/configs_env_validate_test.py deleted file mode 100644 index be7d1eb8..00000000 --- a/tests/configs/configs_env_validate_test.py +++ /dev/null @@ -1,181 +0,0 @@ -import os -from typing import Optional - -import pytest -import requests - -from node_cli.configs.alias_address_validation import ( - ContractType, - get_chain_id, - get_network_metadata, - validate_alias_or_address, - validate_contract_address, - validate_contract_alias, -) -from node_cli.configs.user import ( - ALLOWED_ENV_TYPES, - FairBootUserConfig, - FairUserConfig, - SkaleUserConfig, - SyncUserConfig, - get_user_config_class, - get_validated_user_config, - validate_env_type, -) -from node_cli.utils.node_type import NodeType - -ENDPOINT = 'http://localhost:8545' - - -class FakeResponse: - def __init__(self, status_code: int, json_data: Optional[dict] = None): - self.status_code = status_code - self._json_data = json_data or {} - - def json(self): - return self._json_data - - -@pytest.mark.parametrize( - 'node_type, is_fair_boot, expected_type', - [ - (NodeType.REGULAR, False, SkaleUserConfig), - (NodeType.SYNC, False, SyncUserConfig), - (NodeType.FAIR, True, FairBootUserConfig), - (NodeType.FAIR, False, FairUserConfig), - ], - ids=['regular', 'sync', 'fair_boot', 'fair_regular'], -) -def test_build_env_params_keys(node_type, is_fair_boot, expected_type): - env_type = get_user_config_class(node_type=node_type, is_fair_boot=is_fair_boot) - assert env_type == expected_type - - -@pytest.mark.parametrize( - 'env_types, should_fail', - [ - (ALLOWED_ENV_TYPES, False), - (['invalid'], True), - ], - ids=[ - 'correct_env', - 'invalid_env', - ], -) -def test_env_types(env_types, should_fail): - for env_type in env_types: - if should_fail: - with pytest.raises(SystemExit): - validate_env_type(env_type=env_type) - else: - validate_env_type(env_type=env_type) - - -def test_get_chain_id_success(monkeypatch): - fake_response = FakeResponse(200, {'result': '0x1'}) - - def fake_post(url, json): - return fake_response - - monkeypatch.setattr(requests, 'post', fake_post) - assert get_chain_id(ENDPOINT) == 1 - - -def test_get_chain_id_failure(monkeypatch): - fake_response = FakeResponse(404) - - def fake_post(url, json): - return fake_response - - monkeypatch.setattr(requests, 'post', fake_post) - with pytest.raises(SystemExit): - get_chain_id(ENDPOINT) - - -@pytest.mark.parametrize( - 'metadata,status_code,should_raise', - [ - ({'networks': [{'chainId': 1, 'path': 'mainnet'}]}, 200, False), - (None, 404, True), - ], -) -def test_get_network_metadata(requests_mock, metadata, status_code, should_raise): - metadata_url = 'https://raw.githubusercontent.com/skalenetwork/skale-contracts/refs/heads/deployments/metadata.json' - requests_mock.get(metadata_url, json=metadata, status_code=status_code) - - if should_raise: - with pytest.raises(SystemExit): - get_network_metadata() - else: - assert get_network_metadata() == metadata - - -@pytest.mark.parametrize( - 'code,should_raise', - [ - ('0x123', False), - ('0x', True), - ], -) -def test_validate_contract_address(requests_mock, code, should_raise): - requests_mock.post(ENDPOINT, json={'result': code}) - addr = '0x' + 'a' * 40 - if should_raise: - with pytest.raises(SystemExit): - validate_contract_address(addr, ENDPOINT) - else: - validate_contract_address(addr, ENDPOINT) - - -@pytest.mark.parametrize( - 'networks,should_raise', - [ - ([{'chainId': 1, 'path': 'mainnet'}], False), - ([], True), - ], -) -def test_validate_contract_alias(requests_mock, networks, should_raise): - requests_mock.post(ENDPOINT, json={'result': '0x1'}) - metadata_url = 'https://raw.githubusercontent.com/skalenetwork/skale-contracts/refs/heads/deployments/metadata.json' - requests_mock.get(metadata_url, json={'networks': networks}, status_code=200) - - if not should_raise: - alias_url = 'https://raw.githubusercontent.com/skalenetwork/skale-contracts/refs/heads/deployments/mainnet/skale-manager/test-alias.json' - requests_mock.get(alias_url, status_code=200) - validate_contract_alias('test-alias', ContractType.MANAGER, ENDPOINT) - else: - with pytest.raises(SystemExit): - validate_contract_alias('test-alias', ContractType.MANAGER, ENDPOINT) - - -def test_validate_env_alias_or_address_with_address(requests_mock): - addr = '0x' + 'b' * 40 - requests_mock.post(ENDPOINT, json={'result': '0x1'}) - validate_alias_or_address(addr, ContractType.IMA, ENDPOINT) - - -def test_validate_env_alias_or_address_with_alias(requests_mock): - requests_mock.post(ENDPOINT, json={'result': '0x1'}) - metadata_url = 'https://raw.githubusercontent.com/skalenetwork/skale-contracts/refs/heads/deployments/metadata.json' - metadata = {'networks': [{'chainId': 1, 'path': 'mainnet'}]} - requests_mock.get(metadata_url, json=metadata, status_code=200) - alias_url = 'https://raw.githubusercontent.com/skalenetwork/skale-contracts/refs/heads/deployments/mainnet/mainnet-ima/test-alias.json' - requests_mock.get(alias_url, status_code=200) - validate_alias_or_address('test-alias', ContractType.IMA, ENDPOINT) - - -def test_get_validated_env_config_missing_file(): - with pytest.raises(SystemExit): - get_validated_user_config(env_filepath='nonexistent.env', node_type=NodeType.REGULAR) - - -def test_get_validated_env_config_unreadable_file(tmp_path): - env_file = tmp_path / 'unreadable.env' - env_file.touch() - original_mode = env_file.stat().st_mode - try: - os.chmod(env_file, 0o000) - with pytest.raises(PermissionError): - get_validated_user_config(env_filepath=str(env_file), node_type=NodeType.REGULAR) - finally: - os.chmod(env_file, original_mode) diff --git a/tests/conftest.py b/tests/conftest.py index 05da0fae..8b610d96 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -36,6 +36,7 @@ META_FILEPATH, NGINX_CONFIG_FILEPATH, NGINX_CONTAINER_NAME, + NODE_DATA_PATH, REDIS_URI, REMOVED_CONTAINERS_FOLDER_PATH, SCHAIN_NODE_DATA_PATH, @@ -43,10 +44,44 @@ from node_cli.configs.node_options import NODE_OPTIONS_FILEPATH from node_cli.configs.resource_allocation import RESOURCE_ALLOCATION_FILEPATH from node_cli.configs.ssl import SSL_FOLDER_PATH +from node_cli.core.node_options import NodeOptions from node_cli.utils.docker_utils import docker_client from node_cli.utils.global_config import generate_g_config_file +from node_cli.utils.node_type import NodeMode +from tests.fixtures.settings import ( # noqa: F401 + INTERNAL_FAIR_ACTIVE, + INTERNAL_FAIR_PASSIVE, + INTERNAL_SKALE_ACTIVE, + INTERNAL_SKALE_PASSIVE, + NODE_FAIR_ACTIVE, + NODE_FAIR_PASSIVE, + NODE_SKALE_ACTIVE, + NODE_SKALE_PASSIVE, + _cleanup_settings, + _write_settings, + fair_active_settings, + fair_passive_settings, + skale_active_settings, + skale_passive_settings, +) from tests.helper import TEST_META_V1, TEST_META_V2, TEST_META_V3, TEST_SCHAINS_MNT_DIR_SINGLE_CHAIN +TIMEOUT_PATCHES = [ + 'node_cli.configs.TM_INIT_TIMEOUT', + 'node_cli.configs.RESTORE_SLEEP_TIMEOUT', + 'node_cli.configs.INIT_TIMEOUT', +] + + +@pytest.fixture(autouse=True, scope='session') +def _fast_timeouts(): + patchers = [mock.patch(target, 1) for target in TIMEOUT_PATCHES] + for p in patchers: + p.start() + yield + for p in patchers: + p.stop() + @pytest.fixture() def tmp_dir_path(): @@ -118,7 +153,10 @@ def resource_alloc(): with open(RESOURCE_ALLOCATION_FILEPATH, 'w') as alloc_file: json.dump({}, alloc_file) yield RESOURCE_ALLOCATION_FILEPATH - os.remove(RESOURCE_ALLOCATION_FILEPATH) + try: + os.remove(RESOURCE_ALLOCATION_FILEPATH) + except FileNotFoundError: + pass @pytest.fixture @@ -129,7 +167,10 @@ def inited_node(): try: yield finally: - os.remove(NGINX_CONFIG_FILEPATH) + try: + os.remove(NGINX_CONFIG_FILEPATH) + except FileNotFoundError: + pass @pytest.fixture @@ -144,6 +185,46 @@ def ssl_folder(): shutil.rmtree(SSL_FOLDER_PATH) +@pytest.fixture +def active_node_option(): + if os.path.isdir(NODE_DATA_PATH): + shutil.rmtree(NODE_DATA_PATH) + path = pathlib.Path(NODE_DATA_PATH) + path.mkdir(parents=True, exist_ok=True) + node_options = NodeOptions() + node_options.node_mode = NodeMode.ACTIVE + try: + yield + finally: + try: + if os.path.isdir(NODE_OPTIONS_FILEPATH): + shutil.rmtree(NODE_OPTIONS_FILEPATH) + elif os.path.isfile(NODE_OPTIONS_FILEPATH): + os.remove(NODE_OPTIONS_FILEPATH) + except FileNotFoundError: + pass + + +@pytest.fixture +def passive_node_option(): + if os.path.isdir(NODE_DATA_PATH): + shutil.rmtree(NODE_DATA_PATH) + path = pathlib.Path(NODE_DATA_PATH) + path.mkdir(parents=True, exist_ok=True) + node_options = NodeOptions() + node_options.node_mode = NodeMode.PASSIVE + try: + yield + finally: + try: + if os.path.isdir(NODE_OPTIONS_FILEPATH): + shutil.rmtree(NODE_OPTIONS_FILEPATH) + elif os.path.isfile(NODE_OPTIONS_FILEPATH): + os.remove(NODE_OPTIONS_FILEPATH) + except FileNotFoundError: + pass + + @pytest.fixture def dutils(): return docker_client() @@ -195,7 +276,10 @@ def meta_file_v3(): try: yield META_FILEPATH finally: - os.remove(META_FILEPATH) + try: + os.remove(META_FILEPATH) + except FileNotFoundError: + pass @pytest.fixture @@ -226,7 +310,7 @@ def tmp_schains_dir(): @pytest.fixture -def tmp_sync_datadir(): +def tmp_passive_datadir(): os.makedirs(TEST_SCHAINS_MNT_DIR_SINGLE_CHAIN, exist_ok=True) try: yield TEST_SCHAINS_MNT_DIR_SINGLE_CHAIN @@ -238,20 +322,13 @@ def tmp_sync_datadir(): def valid_env_params(): return { 'ENDPOINT': 'http://localhost:8545', - 'IMA_ENDPOINT': 'http://127.0.01', - 'DB_USER': 'user', - 'DB_PASSWORD': 'pass', - 'DB_PORT': '3307', - 'CONTAINER_CONFIGS_STREAM': 'master', + 'NODE_VERSION': 'master', 'FILEBEAT_HOST': '127.0.0.1:3010', - 'SGX_SERVER_URL': 'http://127.0.0.1', - 'DISK_MOUNTPOINT': '/dev/sss', - 'DOCKER_LVMPY_STREAM': 'master', + 'SGX_URL': 'http://127.0.0.1', + 'BLOCK_DEVICE': '/dev/sss', 'ENV_TYPE': 'devnet', - 'SCHAIN_NAME': 'test', 'ENFORCE_BTRFS': 'False', - 'MANAGER_CONTRACTS': 'test-manager', - 'IMA_CONTRACTS': 'test-ima', + 'FAIR_CONTRACTS': 'test-fair', } @@ -267,6 +344,7 @@ def valid_env_file(valid_env_params): finally: if file_name: os.unlink(file_name) + _cleanup_settings() @pytest.fixture @@ -304,14 +382,15 @@ def set_env_var(name, value): @pytest.fixture def regular_user_conf(tmp_path): test_env_path = pathlib.Path(tmp_path / 'test-env') + _write_settings(INTERNAL_SKALE_ACTIVE, NODE_SKALE_ACTIVE) try: test_env = """ ENDPOINT=http://localhost:8545 - CONTAINER_CONFIGS_STREAM='main' + NODE_VERSION='main' FILEBEAT_HOST=127.0.0.1:3010 - SGX_SERVER_URL=http://127.0.0.1 - DISK_MOUNTPOINT=/dev/sss - DOCKER_LVMPY_STREAM='master' + SGX_URL=http://127.0.0.1 + BLOCK_DEVICE=/dev/sss + DOCKER_LVMPY_VERSION='master' ENV_TYPE='devnet' MANAGER_CONTRACTS='test-manager' IMA_CONTRACTS='test-ima' @@ -321,18 +400,20 @@ def regular_user_conf(tmp_path): yield test_env_path finally: test_env_path.unlink() + _cleanup_settings() @pytest.fixture def fair_user_conf(tmp_path): test_env_path = pathlib.Path(tmp_path / 'test-env') + _write_settings(INTERNAL_FAIR_ACTIVE, NODE_FAIR_ACTIVE) try: test_env = """ - BOOT_ENDPOINT=http://localhost:8545 - CONTAINER_CONFIGS_STREAM='main' + ENDPOINT=http://localhost:8545 + NODE_VERSION='main' FILEBEAT_HOST=127.0.0.1:3010 - SGX_SERVER_URL=http://127.0.0.1 - DISK_MOUNTPOINT=/dev/sss + SGX_URL=http://127.0.0.1 + BLOCK_DEVICE=/dev/sss ENV_TYPE='devnet' ENFORCE_BTRFS=False FAIR_CONTRACTS='test-fair' @@ -342,48 +423,75 @@ def fair_user_conf(tmp_path): yield test_env_path finally: test_env_path.unlink() + _cleanup_settings() @pytest.fixture def fair_boot_user_conf(tmp_path): test_env_path = pathlib.Path(tmp_path / 'test-env') + _write_settings(INTERNAL_FAIR_ACTIVE, NODE_FAIR_ACTIVE) try: test_env = """ ENDPOINT=http://localhost:8545 - CONTAINER_CONFIGS_STREAM='main' + NODE_VERSION='main' FILEBEAT_HOST=127.0.0.1:3010 - SGX_SERVER_URL=http://127.0.0.1 - DISK_MOUNTPOINT=/dev/sss + SGX_URL=http://127.0.0.1 + BLOCK_DEVICE=/dev/sss ENV_TYPE='devnet' - MANAGER_CONTRACTS='test-manager' - IMA_CONTRACTS='test-ima' + FAIR_CONTRACTS='test-fair' """ with open(test_env_path, 'w') as env_file: env_file.write(test_env) yield test_env_path finally: test_env_path.unlink() + _cleanup_settings() @pytest.fixture -def sync_user_conf(tmp_path): +def fair_passive_user_conf(tmp_path): test_env_path = pathlib.Path(tmp_path / 'test-env') + _write_settings(INTERNAL_FAIR_PASSIVE, NODE_FAIR_PASSIVE) try: test_env = """ ENDPOINT=http://localhost:8545 - CONTAINER_CONFIGS_STREAM='main' + NODE_VERSION='main' FILEBEAT_HOST=127.0.0.1:3010 - DISK_MOUNTPOINT=/dev/sss + BLOCK_DEVICE=/dev/sss + ENV_TYPE='devnet' + ENFORCE_BTRFS=False + FAIR_CONTRACTS='test-fair' + """ + with open(test_env_path, 'w') as env_file: + env_file.write(test_env) + yield test_env_path + finally: + test_env_path.unlink() + _cleanup_settings() + + +@pytest.fixture +def passive_user_conf(tmp_path): + test_env_path = pathlib.Path(tmp_path / 'test-env') + _write_settings(INTERNAL_SKALE_PASSIVE, NODE_SKALE_PASSIVE) + try: + test_env = """ + ENDPOINT=http://localhost:8545 + NODE_VERSION='main' + FILEBEAT_HOST=127.0.0.1:3010 + BLOCK_DEVICE=/dev/sss ENV_TYPE='devnet' SCHAIN_NAME='test-schain' ENFORCE_BTRFS=False MANAGER_CONTRACTS='test-manager' + IMA_CONTRACTS='test-ima' """ with open(test_env_path, 'w') as env_file: env_file.write(test_env) yield test_env_path finally: test_env_path.unlink() + _cleanup_settings() @pytest.fixture diff --git a/tests/core/core_checks_test.py b/tests/core/core_checks_test.py index a4f7d437..aa986cc7 100644 --- a/tests/core/core_checks_test.py +++ b/tests/core/core_checks_test.py @@ -21,7 +21,7 @@ save_report, ) -from node_cli.utils.node_type import NodeType +from node_cli.utils.node_type import NodeMode, NodeType @pytest.fixture @@ -318,18 +318,26 @@ def run_cmd_mock(*args, **kwargs): assert r.status == 'ok' -def test_get_all_checkers(requirements_data): +def test_get_all_checkers(requirements_data, active_node_option): disk = 'test-disk' - checkers = get_all_checkers(disk, requirements_data) + checkers = get_all_checkers(disk, requirements_data, node_mode=NodeMode.ACTIVE) assert len(checkers) == 3 - assert isinstance(checkers[0], MachineChecker) - assert isinstance(checkers[1], PackageChecker) - assert isinstance(checkers[2], DockerChecker) + assert isinstance(checkers[0], PackageChecker) + assert isinstance(checkers[1], DockerChecker) + assert isinstance(checkers[2], MachineChecker) -def test_get_checks(requirements_data): +def test_get_all_checkers_passive(requirements_data, passive_node_option): disk = 'test-disk' - checkers = get_all_checkers(disk, requirements_data) + checkers = get_all_checkers(disk, requirements_data, node_mode=NodeMode.PASSIVE) + assert len(checkers) == 2 + assert isinstance(checkers[0], PackageChecker) + assert isinstance(checkers[1], DockerChecker) + + +def test_get_checks(requirements_data, active_node_option): + disk = 'test-disk' + checkers = get_all_checkers(disk, requirements_data, node_mode=NodeMode.ACTIVE) checks = get_checks(checkers) assert len(checks) == 16 checks = get_checks(checkers, check_type=CheckType.PREINSTALL) @@ -338,9 +346,9 @@ def test_get_checks(requirements_data): assert len(checks) == 2 -def test_get_checks_fair(fair_requirements_data): +def test_get_checks_fair(fair_requirements_data, active_node_option): disk = 'test-disk' - fair_checkers = get_all_checkers(disk, fair_requirements_data) + fair_checkers = get_all_checkers(disk, fair_requirements_data, node_mode=NodeMode.ACTIVE) fair_all_checks = get_checks(fair_checkers, CheckType.ALL) fair_all_names = {f.func.__name__ for f in fair_all_checks} @@ -379,8 +387,8 @@ def test_merge_report(): def test_get_static_params(tmp_config_dir): - params = get_static_params(NodeType.REGULAR) + params = get_static_params(NodeType.SKALE) shutil.copy(STATIC_PARAMS_FILEPATH, tmp_config_dir) - tmp_params = get_static_params(NodeType.REGULAR, config_path=tmp_config_dir) + tmp_params = get_static_params(NodeType.SKALE, config_path=tmp_config_dir) assert params['server']['cpu_total'] == 8 assert params == tmp_params diff --git a/tests/core/core_node_test.py b/tests/core/core_node_test.py index 894e07b6..b28719d4 100644 --- a/tests/core/core_node_test.py +++ b/tests/core/core_node_test.py @@ -5,13 +5,20 @@ from pathlib import Path import docker +from docker import errors as docker_errors import mock import pytest import requests -from node_cli.configs import NODE_DATA_PATH, SCHAINS_MNT_DIR_REGULAR, SCHAINS_MNT_DIR_SINGLE_CHAIN +from node_cli.configs import ( + NODE_DATA_PATH, + SCHAINS_MNT_DIR_REGULAR, + SCHAINS_MNT_DIR_SINGLE_CHAIN, +) from node_cli.configs.resource_allocation import RESOURCE_ALLOCATION_FILEPATH + from node_cli.core.node import ( + cleanup, compose_node_env, get_expected_container_names, init, @@ -21,7 +28,7 @@ update, ) from node_cli.utils.meta import CliMeta -from node_cli.utils.node_type import NodeType +from node_cli.utils.node_type import NodeType, NodeMode from tests.helper import response_mock, safe_update_api_response, subprocess_run_mock from tests.resources_test import BIG_DISK_SIZE @@ -34,46 +41,49 @@ 'WRONG_CONTAINER_1', 'skale_WRONG_CONTAINER_4', 'fair_WRONG_CONTAINER_6', - 'sync_WRONG_CONTAINER_8', + 'passive_WRONG_CONTAINER_8', ] -NODE_TYPE_BOOT_COMBINATIONS: list[tuple[NodeType, bool]] = [ - (NodeType.REGULAR, False), - (NodeType.SYNC, False), - (NodeType.FAIR, True), - (NodeType.FAIR, False), +NODE_TYPE_MODE_BOOT_COMBINATIONS: list[tuple[NodeType, NodeMode, bool]] = [ + (NodeType.SKALE, NodeMode.ACTIVE, False), + (NodeType.SKALE, NodeMode.PASSIVE, False), + (NodeType.FAIR, NodeMode.ACTIVE, True), + (NodeType.FAIR, NodeMode.ACTIVE, False), ] alive_test_params = [ pytest.param( node_type, + node_mode, is_boot, - get_expected_container_names(node_type, is_boot), - id=f'{node_type.name}-boot_{is_boot}-correct_containers', + get_expected_container_names(node_type, node_mode, is_boot), + id=f'{node_type.name}-{node_mode.name}-boot_{is_boot}-correct_containers', ) - for node_type, is_boot in NODE_TYPE_BOOT_COMBINATIONS + for node_type, node_mode, is_boot in NODE_TYPE_MODE_BOOT_COMBINATIONS ] wrong_test_params = [ pytest.param( node_type, + node_mode, is_boot, WRONG_CONTAINERS, - id=f'{node_type.name}-boot_{is_boot}-wrong_containers', + id=f'{node_type.name}-{node_mode.name}-boot_{is_boot}-wrong_containers', ) - for node_type, is_boot in NODE_TYPE_BOOT_COMBINATIONS + for node_type, node_mode, is_boot in NODE_TYPE_MODE_BOOT_COMBINATIONS ] missing_test_params = [] -for node_type, is_boot in NODE_TYPE_BOOT_COMBINATIONS: - expected_names = get_expected_container_names(node_type, is_boot) +for node_type, node_mode, is_boot in NODE_TYPE_MODE_BOOT_COMBINATIONS: + expected_names = get_expected_container_names(node_type, node_mode, is_boot) containers_to_create = expected_names[1:] missing_test_params.append( pytest.param( node_type, + node_mode, is_boot, containers_to_create, - id=f'{node_type.name}-boot_{is_boot}-missing_containers', + id=f'{node_type.name}-{node_mode.name}-boot_{is_boot}-missing_containers', ) ) @@ -87,7 +97,7 @@ def manage_node_containers(request): try: existing_container = dclient.containers.get(name) existing_container.remove(force=True) - except docker.errors.NotFound: + except docker_errors.NotFound: pass container = dclient.containers.run( ALPINE_IMAGE_NAME, @@ -110,142 +120,87 @@ def manage_node_containers(request): try: container_obj.remove(force=True) cleaned_count += 1 - except docker.errors.NotFound: + except docker_errors.NotFound: pass @pytest.mark.parametrize( - 'node_type, is_boot, manage_node_containers', + 'node_type, node_mode, is_boot, manage_node_containers', alive_test_params, indirect=['manage_node_containers'], ) -def test_is_base_containers_alive(manage_node_containers, node_type, is_boot): - assert is_base_containers_alive(node_type=node_type, is_fair_boot=is_boot) is True +def test_is_base_containers_alive(manage_node_containers, node_type, node_mode, is_boot): + assert ( + is_base_containers_alive(node_type=node_type, node_mode=node_mode, is_fair_boot=is_boot) + is True + ) @pytest.mark.parametrize( - 'node_type, is_boot, manage_node_containers', + 'node_type, node_mode, is_boot, manage_node_containers', wrong_test_params, indirect=['manage_node_containers'], ) -def test_is_base_containers_alive_wrong(manage_node_containers, node_type, is_boot): - assert is_base_containers_alive(node_type=node_type, is_fair_boot=is_boot) is False +def test_is_base_containers_alive_wrong(manage_node_containers, node_type, node_mode, is_boot): + assert ( + is_base_containers_alive(node_type=node_type, node_mode=node_mode, is_fair_boot=is_boot) + is False + ) @pytest.mark.parametrize( - 'node_type, is_boot, manage_node_containers', + 'node_type, node_mode, is_boot, manage_node_containers', missing_test_params, indirect=['manage_node_containers'], ) -def test_is_base_containers_alive_missing(manage_node_containers, node_type, is_boot): - assert is_base_containers_alive(node_type=node_type, is_fair_boot=is_boot) is False +def test_is_base_containers_alive_missing(manage_node_containers, node_type, node_mode, is_boot): + assert ( + is_base_containers_alive(node_type=node_type, node_mode=node_mode, is_fair_boot=is_boot) + is False + ) -@pytest.mark.parametrize('node_type, is_boot', NODE_TYPE_BOOT_COMBINATIONS) -def test_is_base_containers_alive_empty(node_type, is_boot): - assert is_base_containers_alive(node_type=node_type, is_fair_boot=is_boot) is False +@pytest.mark.parametrize('node_type, node_mode, is_boot', NODE_TYPE_MODE_BOOT_COMBINATIONS) +def test_is_base_containers_alive_empty(node_type, node_mode, is_boot): + assert ( + is_base_containers_alive(node_type=node_type, node_mode=node_mode, is_fair_boot=is_boot) + is False + ) @pytest.mark.parametrize( - ( - 'node_type, test_user_conf, is_boot, inited_node, sync_schains, expected_mnt_dir,' - 'expect_flask_key, expect_backup_run' - ), + 'node_type, node_mode, expected_mnt_dir', [ ( - NodeType.REGULAR, - 'regular_user_conf', - False, - True, - False, + NodeType.SKALE, + NodeMode.ACTIVE, SCHAINS_MNT_DIR_REGULAR, - True, - False, ), ( - NodeType.REGULAR, - 'regular_user_conf', - False, - True, - True, - SCHAINS_MNT_DIR_REGULAR, - True, - True, - ), - ( - NodeType.SYNC, - 'sync_user_conf', - False, - False, - False, - SCHAINS_MNT_DIR_SINGLE_CHAIN, - False, - False, - ), - ( - NodeType.FAIR, - 'fair_boot_user_conf', - True, - True, - False, + NodeType.SKALE, + NodeMode.PASSIVE, SCHAINS_MNT_DIR_SINGLE_CHAIN, - True, - False, ), ( NodeType.FAIR, - 'fair_user_conf', - False, - True, - False, + NodeMode.ACTIVE, SCHAINS_MNT_DIR_SINGLE_CHAIN, - True, - False, ), ], ids=[ 'regular', - 'regular_sync_flag', - 'sync', - 'fair_boot', - 'fair_regular', + 'passive', + 'fair', ], ) -def test_compose_node_env( - request, - node_type, - test_user_conf, - is_boot, - inited_node, - sync_schains, - expected_mnt_dir, - expect_flask_key, - expect_backup_run, -): - user_config_path = request.getfixturevalue(test_user_conf) - - with ( - mock.patch('node_cli.configs.user.validate_alias_or_address'), - mock.patch('node_cli.core.node.save_env_params'), - mock.patch('node_cli.core.node.get_flask_secret_key', return_value='mock_secret'), - ): - result_env = compose_node_env( - env_filepath=user_config_path.as_posix(), - inited_node=inited_node, - sync_schains=sync_schains, - node_type=node_type, - is_fair_boot=is_boot, - save=True, - ) +def test_compose_node_env(node_type, node_mode, expected_mnt_dir, regular_user_conf): + result_env = compose_node_env( + node_type=node_type, + node_mode=node_mode, + ) assert result_env['SCHAINS_MNT_DIR'] == expected_mnt_dir - assert ( - 'FLASK_SECRET_KEY' in result_env and result_env['FLASK_SECRET_KEY'] is not None - ) == expect_flask_key - if expect_flask_key: - assert result_env['FLASK_SECRET_KEY'] == 'mock_secret' - should_have_backup = sync_schains and node_type != NodeType.SYNC - assert ('BACKUP_RUN' in result_env and result_env['BACKUP_RUN'] == 'True') == should_have_backup + assert 'BACKUP_RUN' not in result_env @pytest.fixture @@ -321,9 +276,8 @@ def test_init_node(regular_user_conf, no_resource_file): # todo: write new init mock.patch('node_cli.core.node.init_op'), mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), mock.patch('node_cli.utils.helper.post_request', resp_mock), - mock.patch('node_cli.configs.user.validate_alias_or_address'), ): - init(env_filepath=regular_user_conf.as_posix(), node_type=NodeType.REGULAR) + init(config_file=regular_user_conf.as_posix(), node_type=NodeType.SKALE) assert os.path.isfile(RESOURCE_ALLOCATION_FILEPATH) @@ -333,8 +287,6 @@ def test_update_node(regular_user_conf, mocked_g_config, resource_file, inited_n with ( mock.patch('subprocess.run', new=subprocess_run_mock), mock.patch('node_cli.core.node.update_op'), - mock.patch('node_cli.core.node.get_flask_secret_key'), - mock.patch('node_cli.core.node.save_env_params'), mock.patch('node_cli.operations.base.configure_nftables'), mock.patch('node_cli.core.host.prepare_host'), mock.patch('node_cli.core.node.is_base_containers_alive', return_value=True), @@ -345,7 +297,6 @@ def test_update_node(regular_user_conf, mocked_g_config, resource_file, inited_n 'node_cli.core.node.CliMetaManager.get_meta_info', return_value=CliMeta(version='2.6.0', config_stream='3.0.2'), ), - mock.patch('node_cli.configs.user.validate_alias_or_address'), ): with mock.patch( 'node_cli.utils.helper.requests.get', return_value=safe_update_api_response() @@ -353,33 +304,48 @@ def test_update_node(regular_user_conf, mocked_g_config, resource_file, inited_n result = update( regular_user_conf.as_posix(), pull_config_for_schain=None, - node_type=NodeType.REGULAR, + node_type=NodeType.SKALE, + node_mode=NodeMode.ACTIVE, ) assert result is None -@pytest.mark.parametrize('node_type', [NodeType.REGULAR, NodeType.SYNC, NodeType.FAIR]) +@pytest.mark.parametrize( + 'node_type,node_mode', + [ + (NodeType.SKALE, NodeMode.ACTIVE), + (NodeType.SKALE, NodeMode.PASSIVE), + (NodeType.FAIR, NodeMode.ACTIVE), + ], +) @mock.patch('node_cli.core.node.is_admin_running', return_value=False) @mock.patch('node_cli.core.node.is_api_running', return_value=False) @mock.patch('node_cli.utils.helper.requests.get') def test_is_update_safe_when_admin_and_api_not_running( - mock_requests_get, mock_is_api_running, mock_is_admin_running, node_type + mock_requests_get, mock_is_api_running, mock_is_admin_running, node_type, node_mode ): - assert is_update_safe(node_type=node_type) is True + assert is_update_safe(node_mode=node_mode) is True mock_requests_get.assert_not_called() @mock.patch('node_cli.core.node.is_admin_running', return_value=False) @mock.patch('node_cli.core.node.is_api_running', return_value=True) @mock.patch('node_cli.utils.helper.requests.get') -def test_is_update_safe_when_admin_not_running_for_sync( +def test_is_update_safe_when_admin_not_running_for_passive( mock_requests_get, mock_is_api_running, mock_is_admin_running ): - assert is_update_safe(node_type=NodeType.SYNC) is True + assert is_update_safe(node_mode=NodeMode.PASSIVE) is True mock_requests_get.assert_not_called() -@pytest.mark.parametrize('node_type', [NodeType.REGULAR, NodeType.SYNC, NodeType.FAIR]) +@pytest.mark.parametrize( + 'node_type,node_mode', + [ + (NodeType.SKALE, NodeMode.ACTIVE), + (NodeType.SKALE, NodeMode.PASSIVE), + (NodeType.FAIR, NodeMode.ACTIVE), + ], +) @pytest.mark.parametrize( 'api_is_safe, expected_result', [(True, True), (False, False)], @@ -388,14 +354,14 @@ def test_is_update_safe_when_admin_not_running_for_sync( @mock.patch('node_cli.core.node.is_admin_running', return_value=True) @mock.patch('node_cli.utils.helper.requests.get') def test_is_update_safe_when_admin_running( - mock_requests_get, mock_is_admin_running, api_is_safe, expected_result, node_type + mock_requests_get, mock_is_admin_running, api_is_safe, expected_result, node_type, node_mode ): mock_requests_get.return_value = safe_update_api_response(safe=api_is_safe) - assert is_update_safe(node_type=node_type) is expected_result + assert is_update_safe(node_mode=node_mode) is expected_result mock_requests_get.assert_called_once() -@pytest.mark.parametrize('node_type', [NodeType.REGULAR, NodeType.FAIR]) +@pytest.mark.parametrize('node_type', [NodeType.SKALE, NodeType.FAIR]) @pytest.mark.parametrize( 'api_is_safe, expected_result', [(True, True), (False, False)], @@ -413,14 +379,46 @@ def test_is_update_safe_when_only_api_running_for_regular( node_type, ): mock_requests_get.return_value = safe_update_api_response(safe=api_is_safe) - assert is_update_safe(node_type=node_type) is expected_result + assert is_update_safe(node_mode=NodeMode.ACTIVE) is expected_result mock_requests_get.assert_called_once() -@pytest.mark.parametrize('node_type', [NodeType.REGULAR, NodeType.SYNC, NodeType.FAIR]) +@pytest.mark.parametrize( + 'node_type,node_mode', + [ + (NodeType.SKALE, NodeMode.ACTIVE), + (NodeType.SKALE, NodeMode.PASSIVE), + (NodeType.FAIR, NodeMode.ACTIVE), + ], +) @mock.patch('node_cli.core.node.is_admin_running', return_value=True) @mock.patch('node_cli.utils.helper.requests.get') -def test_is_update_safe_when_api_call_fails(mock_requests_get, mock_is_admin_running, node_type): +def test_is_update_safe_when_api_call_fails( + mock_requests_get, mock_is_admin_running, node_type, node_mode +): mock_requests_get.side_effect = requests.exceptions.ConnectionError('Test connection error') - assert is_update_safe(node_type=node_type) is False + assert is_update_safe(node_mode=node_mode) is False mock_requests_get.assert_called_once() + + +@mock.patch('node_cli.utils.decorators.is_user_valid', return_value=True) +@mock.patch('node_cli.core.node.cleanup_skale_op') +@mock.patch('node_cli.core.node.compose_node_env') +def test_cleanup_success( + mock_compose_env, + mock_cleanup_skale_op, + mock_is_user_valid, + inited_node, + resource_alloc, + meta_file_v3, + active_node_option, +): + mock_env = {'ENV_TYPE': 'devnet'} + mock_compose_env.return_value = mock_env + + cleanup(node_mode=NodeMode.ACTIVE) + + mock_compose_env.assert_called_once_with(NodeType.SKALE, NodeMode.ACTIVE) + mock_cleanup_skale_op.assert_called_once_with( + node_mode=NodeMode.ACTIVE, compose_env=mock_env, prune=False + ) diff --git a/tests/core/core_schains_test.py b/tests/core/core_schains_test.py index c6ce426b..7016a262 100644 --- a/tests/core/core_schains_test.py +++ b/tests/core/core_schains_test.py @@ -4,7 +4,7 @@ import freezegun -from node_cli.core.schains import cleanup_datadir_for_single_chain_node, toggle_schain_repair_mode +from node_cli.core.schains import cleanup_no_lvm_datadir, toggle_schain_repair_mode from node_cli.utils.helper import read_json from tests.helper import CURRENT_DATETIME, CURRENT_TIMESTAMP @@ -34,9 +34,9 @@ def test_toggle_repair_mode(tmp_schains_dir): @freezegun.freeze_time(CURRENT_DATETIME) -def test_cleanup_sync_datadir(tmp_sync_datadir): +def test_cleanup_passive_datadir(tmp_passive_datadir): schain_name = 'test_schain' - base_folder = Path(tmp_sync_datadir).joinpath(schain_name) + base_folder = Path(tmp_passive_datadir).joinpath(schain_name) base_folder.mkdir() folders = [ '28e07f34', @@ -80,6 +80,9 @@ def test_cleanup_sync_datadir(tmp_sync_datadir): hash_path = snapshot_folder.joinpath('snapshot_hash.txt') hash_path.touch() - with mock.patch('node_cli.core.schains.rm_btrfs_subvolume'): - cleanup_datadir_for_single_chain_node(schain_name, base_path=tmp_sync_datadir) + with ( + mock.patch('node_cli.core.schains.rm_btrfs_subvolume'), + mock.patch('node_cli.core.schains.run_cmd'), + ): + cleanup_no_lvm_datadir(schain_name, base_path=tmp_passive_datadir) assert not os.path.isdir(base_folder) diff --git a/tests/core/nginx_test.py b/tests/core/nginx_test.py index 56b6eb49..a867b0b0 100644 --- a/tests/core/nginx_test.py +++ b/tests/core/nginx_test.py @@ -7,11 +7,11 @@ from node_cli.core.nginx import ( generate_nginx_config, check_ssl_certs, - is_regular_node_nginx, + is_skale_node_nginx, SSL_KEY_NAME, SSL_CRT_NAME, ) -from node_cli.utils.node_type import NodeType +from node_cli.utils.node_type import NodeType, NodeMode from node_cli.configs import NGINX_TEMPLATE_FILEPATH, NGINX_CONFIG_FILEPATH, NODE_CERTS_PATH TEST_NGINX_TEMPLATE = """ @@ -24,7 +24,7 @@ {% endif %} } -{% if regular_node %} +{% if skale_node %} server { listen 80; {% if ssl %} @@ -57,14 +57,14 @@ def nginx_template(): @pytest.mark.parametrize( - 'node_type, ssl_exists, expected_regular_flag, expected_ssl_flag', + 'node_type, node_mode, ssl_exists, expected_regular_flag, expected_ssl_flag', [ - (NodeType.REGULAR, True, True, True), - (NodeType.REGULAR, False, True, False), - (NodeType.SYNC, True, True, True), - (NodeType.SYNC, False, True, False), - (NodeType.FAIR, True, False, True), - (NodeType.FAIR, False, False, False), + (NodeType.SKALE, NodeMode.ACTIVE, True, True, True), + (NodeType.SKALE, NodeMode.ACTIVE, False, True, False), + (NodeType.SKALE, NodeMode.PASSIVE, True, True, True), + (NodeType.SKALE, NodeMode.PASSIVE, False, True, False), + (NodeType.FAIR, NodeMode.ACTIVE, True, False, True), + (NodeType.FAIR, NodeMode.ACTIVE, False, False, False), ], ids=[ 'regular_ssl_on', @@ -81,6 +81,7 @@ def test_generate_nginx_config( mock_type, mock_check_ssl, node_type, + node_mode, ssl_exists, expected_regular_flag, expected_ssl_flag, @@ -130,16 +131,16 @@ def test_check_ssl_certs_missing_both(ssl_folder): @pytest.mark.parametrize( - 'node_type, expected_result', + 'node_type, node_mode, expected_result', [ - (NodeType.REGULAR, True), - (NodeType.SYNC, True), - (NodeType.FAIR, False), + (NodeType.SKALE, NodeMode.ACTIVE, True), + (NodeType.SKALE, NodeMode.PASSIVE, True), + (NodeType.FAIR, NodeMode.ACTIVE, False), ], ) @mock.patch('node_cli.core.nginx.TYPE') -def test_is_regular_node_nginx(mock_type, node_type, expected_result): +def test_is_skale_node_nginx(mock_type, node_type, node_mode, expected_result): mock_type.__eq__.side_effect = lambda other: node_type == other mock_type.__ne__.side_effect = lambda other: node_type != other - assert is_regular_node_nginx() is expected_result + assert is_skale_node_nginx() is expected_result diff --git a/tests/fair/fair_node_test.py b/tests/fair/fair_node_test.py index fab00451..6554cd3f 100644 --- a/tests/fair/fair_node_test.py +++ b/tests/fair/fair_node_test.py @@ -1,48 +1,48 @@ from unittest import mock -import freezegun import pytest -from node_cli.configs import SKALE_DIR -from node_cli.configs.user import SKALE_DIR_ENV_FILEPATH -from node_cli.fair.fair_boot import init as init_boot -from node_cli.fair.fair_boot import update -from node_cli.fair.fair_node import cleanup, migrate_from_boot, request_repair, restore_fair +from node_cli.fair.boot import init as init_boot +from node_cli.fair.boot import update +from node_cli.fair.common import cleanup +from node_cli.fair.active import migrate_from_boot, restore from node_cli.operations.fair import FairUpdateType -from node_cli.utils.node_type import NodeType -from tests.helper import CURRENT_DATETIME, CURRENT_TIMESTAMP +from node_cli.utils.node_type import NodeMode, NodeType -@mock.patch('node_cli.fair.fair_node.time.sleep') -@mock.patch('node_cli.fair.fair_node.restore_fair_op') -@mock.patch('node_cli.fair.fair_node.save_env_params') -@mock.patch('node_cli.fair.fair_node.compose_node_env') +@mock.patch('node_cli.fair.active.time.sleep') +@mock.patch('node_cli.fair.active.restore_fair_op') +@mock.patch('node_cli.fair.active.compose_node_env') def test_restore_fair( mock_compose_env, - mock_save_env, mock_restore_op, mock_sleep, valid_env_file, ensure_meta_removed, + active_node_option, ): mock_env = {'ENV_TYPE': 'devnet'} mock_compose_env.return_value = mock_env mock_restore_op.return_value = True backup_path = '/fake/backup' - restore_fair(backup_path, valid_env_file) + restore(backup_path, valid_env_file) - mock_compose_env.assert_called_once_with(valid_env_file, node_type=NodeType.FAIR) - mock_save_env.assert_called_once_with(valid_env_file) - expected_env = {**mock_env, 'SKALE_DIR': SKALE_DIR} - mock_restore_op.assert_called_once_with(expected_env, backup_path, config_only=False) + mock_compose_env.assert_called_once_with(node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE) + mock_restore_op.assert_called_once_with( + node_mode=NodeMode.ACTIVE, + settings=mock.ANY, + compose_env=mock_env, + backup_path=backup_path, + config_only=False, + ) mock_sleep.assert_called_once() -@mock.patch('node_cli.fair.fair_boot.is_base_containers_alive', return_value=True) -@mock.patch('node_cli.fair.fair_boot.time.sleep') -@mock.patch('node_cli.fair.fair_boot.init_fair_boot_op') -@mock.patch('node_cli.fair.fair_boot.compose_node_env') +@mock.patch('node_cli.fair.boot.is_base_containers_alive', return_value=True) +@mock.patch('node_cli.fair.boot.time.sleep') +@mock.patch('node_cli.fair.boot.init_fair_boot_op') +@mock.patch('node_cli.fair.boot.compose_node_env') def test_init_fair_boot( mock_compose_env, mock_init_op, @@ -56,21 +56,23 @@ def test_init_fair_boot( init_boot(valid_env_file) - mock_compose_env.assert_called_once_with( - valid_env_file, - node_type=NodeType.FAIR, - is_fair_boot=True, + mock_compose_env.assert_called_once_with(node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE) + mock_init_op.assert_called_once_with( + settings=mock.ANY, + compose_env=mock_env, + node_mode=NodeMode.ACTIVE, ) - mock_init_op.assert_called_once_with(valid_env_file, mock_env) mock_sleep.assert_called_once() - mock_is_alive.assert_called_once_with(node_type=NodeType.FAIR, is_fair_boot=True) + mock_is_alive.assert_called_once_with( + node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE, is_fair_boot=True + ) @mock.patch('node_cli.utils.decorators.is_user_valid', return_value=True) -@mock.patch('node_cli.fair.fair_boot.is_base_containers_alive', return_value=True) -@mock.patch('node_cli.fair.fair_boot.time.sleep') -@mock.patch('node_cli.fair.fair_boot.update_fair_boot_op') -@mock.patch('node_cli.fair.fair_boot.compose_node_env') +@mock.patch('node_cli.fair.boot.is_base_containers_alive', return_value=True) +@mock.patch('node_cli.fair.boot.time.sleep') +@mock.patch('node_cli.fair.boot.update_fair_boot_op') +@mock.patch('node_cli.fair.boot.compose_node_env') def test_update_fair_boot( mock_compose_env, mock_update_op, @@ -89,21 +91,20 @@ def test_update_fair_boot( update(valid_env_file, pull_config_for_schain) - mock_compose_env.assert_called_once_with( - valid_env_file, - inited_node=True, - sync_schains=False, - pull_config_for_schain=pull_config_for_schain, - node_type=NodeType.FAIR, - is_fair_boot=True, + mock_compose_env.assert_called_once_with(node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE) + mock_update_op.assert_called_once_with( + settings=mock.ANY, + compose_env=mock_env, + node_mode=NodeMode.ACTIVE, ) - mock_update_op.assert_called_once_with(valid_env_file, mock_env) mock_sleep.assert_called_once() - mock_is_alive.assert_called_once_with(node_type=NodeType.FAIR, is_fair_boot=True) + mock_is_alive.assert_called_once_with( + node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE, is_fair_boot=True + ) -@mock.patch('node_cli.fair.fair_node.update_fair_op') -@mock.patch('node_cli.fair.fair_node.compose_node_env') +@mock.patch('node_cli.fair.active.update_fair_op') +@mock.patch('node_cli.fair.active.compose_node_env') @mock.patch('node_cli.utils.decorators.is_user_valid', return_value=True) def test_migrate_from_boot( mock_is_user_valid, @@ -120,68 +121,52 @@ def test_migrate_from_boot( migrate_from_boot(valid_env_file) - mock_compose_env.assert_called_once_with( - valid_env_file, - inited_node=True, - sync_schains=False, - node_type=NodeType.FAIR, - ) + mock_compose_env.assert_called_once_with(node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE) mock_migrate_op.assert_called_once_with( - valid_env_file, mock_env, update_type=FairUpdateType.FROM_BOOT + settings=mock.ANY, + compose_env=mock_env, + node_mode=NodeMode.ACTIVE, + update_type=FairUpdateType.FROM_BOOT, + force_skaled_start=False, ) -@freezegun.freeze_time(CURRENT_DATETIME) -@mock.patch('node_cli.fair.fair_node.compose_node_env', return_value={'ENV_TYPE': 'devnet'}) -@mock.patch('node_cli.fair.record.chain_record.get_fair_chain_name', return_value='test') -def test_fair_repair(compose_node_env_mock, get_static_params_mock, redis_client, inited_node): - request_repair() - assert redis_client.get('test_repair_ts') == f'{CURRENT_TIMESTAMP}'.encode('utf-8') - assert redis_client.get('test_snapshot_from') == b'' - request_repair(snapshot_from='127.0.0.1') - assert redis_client.get('test_repair_ts') == f'{CURRENT_TIMESTAMP}'.encode('utf-8') - assert redis_client.get('test_snapshot_from') == b'127.0.0.1' - - @mock.patch('node_cli.utils.decorators.is_user_valid', return_value=True) -@mock.patch('node_cli.fair.fair_node.cleanup_docker_configuration') -@mock.patch('node_cli.fair.fair_node.cleanup_fair_op') -@mock.patch('node_cli.fair.fair_node.compose_node_env') +@mock.patch('node_cli.fair.common.cleanup_fair_op') +@mock.patch('node_cli.fair.common.compose_node_env') def test_cleanup_success( mock_compose_env, mock_cleanup_fair_op, - mock_cleanup_docker_config, mock_is_user_valid, - inited_node, resource_alloc, meta_file_v3, + active_node_option, + inited_node, ): mock_env = {'ENV_TYPE': 'devnet'} mock_compose_env.return_value = mock_env - cleanup() + cleanup(node_mode=NodeMode.ACTIVE) - mock_compose_env.assert_called_once_with( - SKALE_DIR_ENV_FILEPATH, save=False, node_type=NodeType.FAIR + mock_compose_env.assert_called_once_with(node_type=NodeType.FAIR, node_mode=NodeMode.ACTIVE) + mock_cleanup_fair_op.assert_called_once_with( + node_mode=NodeMode.ACTIVE, compose_env=mock_env, prune=False ) - mock_cleanup_fair_op.assert_called_once_with(mock_env) - mock_cleanup_docker_config.assert_called_once() @mock.patch('node_cli.utils.decorators.is_user_valid', return_value=True) -@mock.patch('node_cli.fair.fair_node.cleanup_docker_configuration') -@mock.patch('node_cli.fair.fair_node.cleanup_fair_op') -@mock.patch('node_cli.fair.fair_node.compose_node_env') +@mock.patch('node_cli.fair.common.cleanup_fair_op') +@mock.patch('node_cli.fair.common.compose_node_env') def test_cleanup_calls_operations_in_correct_order( mock_compose_env, mock_cleanup_fair_op, - mock_cleanup_docker_config, mock_is_user_valid, - inited_node, resource_alloc, meta_file_v3, + active_node_option, + inited_node, ): - from node_cli.fair.fair_node import cleanup + from node_cli.fair.common import cleanup mock_env = {'ENV_TYPE': 'devnet'} mock_compose_env.return_value = mock_env @@ -189,87 +174,148 @@ def test_cleanup_calls_operations_in_correct_order( manager = mock.Mock() manager.attach_mock(mock_compose_env, 'compose_env') manager.attach_mock(mock_cleanup_fair_op, 'cleanup_fair_op') - manager.attach_mock(mock_cleanup_docker_config, 'cleanup_docker_config') - cleanup() + cleanup(node_mode=NodeMode.ACTIVE) expected_calls = [ - mock.call.compose_env(mock.ANY, save=False, node_type=mock.ANY), - mock.call.cleanup_fair_op(mock_env), - mock.call.cleanup_docker_config(), + mock.call.compose_env( + node_type=NodeType.FAIR, + node_mode=NodeMode.ACTIVE, + ), + mock.call.cleanup_fair_op(node_mode=NodeMode.ACTIVE, compose_env=mock_env, prune=False), ] manager.assert_has_calls(expected_calls, any_order=False) @mock.patch('node_cli.utils.decorators.is_user_valid', return_value=True) -@mock.patch('node_cli.fair.fair_node.cleanup_docker_configuration') -@mock.patch( - 'node_cli.fair.fair_node.cleanup_fair_op', side_effect=Exception('Cleanup failed') -) -@mock.patch('node_cli.fair.fair_node.compose_node_env') +@mock.patch('node_cli.fair.common.cleanup_fair_op', side_effect=Exception('Cleanup failed')) +@mock.patch('node_cli.fair.common.compose_node_env') def test_cleanup_continues_after_fair_op_error( mock_compose_env, mock_cleanup_fair_op, - mock_cleanup_docker_config, mock_is_user_valid, - inited_node, resource_alloc, meta_file_v3, + active_node_option, + inited_node, ): mock_env = {'ENV_TYPE': 'devnet'} mock_compose_env.return_value = mock_env with pytest.raises(Exception, match='Cleanup failed'): - cleanup() + cleanup(node_mode=NodeMode.ACTIVE) mock_compose_env.assert_called_once() - mock_cleanup_fair_op.assert_called_once_with(mock_env) - mock_cleanup_docker_config.assert_not_called() + mock_cleanup_fair_op.assert_called_once_with( + node_mode=NodeMode.ACTIVE, compose_env=mock_env, prune=False + ) @mock.patch('node_cli.utils.decorators.is_user_valid', return_value=False) def test_cleanup_fails_when_user_invalid( mock_is_user_valid, - inited_node, resource_alloc, meta_file_v3, + inited_node, ): """Test that cleanup fails when user validation fails""" import pytest - from node_cli.fair.fair_node import cleanup + from node_cli.fair.common import cleanup with pytest.raises(SystemExit): - cleanup() + cleanup(node_mode=NodeMode.ACTIVE) -def test_cleanup_fails_when_not_inited(ensure_meta_removed): +def test_cleanup_fails_when_not_inited(ensure_meta_removed, active_node_option, fair_user_conf): import pytest - with pytest.raises(SystemExit): - cleanup() + with mock.patch('node_cli.operations.cleanup_fair_op', return_value=None): + with pytest.raises(SystemExit): + cleanup(node_mode=NodeMode.ACTIVE) @mock.patch('node_cli.utils.decorators.is_user_valid', return_value=True) -@mock.patch('node_cli.fair.fair_node.cleanup_docker_configuration') -@mock.patch('node_cli.fair.fair_node.cleanup_fair_op') -@mock.patch('node_cli.fair.fair_node.compose_node_env') -@mock.patch('node_cli.fair.fair_node.logger') +@mock.patch('node_cli.fair.common.cleanup_fair_op') +@mock.patch('node_cli.fair.common.compose_node_env') +@mock.patch('node_cli.fair.common.logger') def test_cleanup_logs_success_message( mock_logger, mock_compose_env, mock_cleanup_fair_op, - mock_cleanup_docker_config, mock_is_user_valid, - inited_node, resource_alloc, meta_file_v3, + active_node_option, + inited_node, ): mock_env = {'ENV_TYPE': 'devnet'} mock_compose_env.return_value = mock_env - cleanup() + cleanup(node_mode=NodeMode.ACTIVE) mock_logger.info.assert_called_once_with( 'Fair node was cleaned up, all containers and data removed' ) + + +@mock.patch('node_cli.utils.decorators.is_user_valid', return_value=True) +@mock.patch('node_cli.fair.active.post_request') +@mock.patch('node_cli.fair.active.is_node_inited', return_value=True) +def test_exit_success( + mock_is_inited, + mock_post_request, + mock_is_user_valid, + inited_node, + resource_alloc, + meta_file_v3, +): + from node_cli.fair.active import exit + + mock_post_request.return_value = ('ok', {}) + + exit() + + mock_post_request.assert_called_once_with(blueprint='fair-node', method='exit', json={}) + + +@mock.patch('node_cli.utils.decorators.is_user_valid', return_value=True) +@mock.patch('node_cli.fair.active.error_exit') +@mock.patch('node_cli.fair.active.post_request') +@mock.patch('node_cli.fair.active.is_node_inited', return_value=True) +def test_exit_error( + mock_is_inited, + mock_post_request, + mock_error_exit, + mock_is_user_valid, + inited_node, + resource_alloc, + meta_file_v3, +): + from node_cli.fair.active import exit + + error_msg = 'Exit failed' + mock_post_request.return_value = ('error', error_msg) + + exit() + + mock_post_request.assert_called_once_with(blueprint='fair-node', method='exit', json={}) + mock_error_exit.assert_called_once_with(error_msg, exit_code=mock.ANY) + + +@mock.patch('node_cli.utils.decorators.is_user_valid', return_value=True) +@mock.patch('node_cli.fair.active.is_node_inited', return_value=False) +def test_exit_not_inited( + mock_is_inited, + mock_is_user_valid, + inited_node, + resource_alloc, + meta_file_v3, + capsys, +): + from node_cli.fair.active import exit + + exit() + + captured = capsys.readouterr() + assert 'Node should be initialized to proceed with operation' in captured.out diff --git a/tests/.skale/node_data/.gitkeep b/tests/fixtures/__init__.py similarity index 100% rename from tests/.skale/node_data/.gitkeep rename to tests/fixtures/__init__.py diff --git a/tests/fixtures/settings.py b/tests/fixtures/settings.py new file mode 100644 index 00000000..55b080b2 --- /dev/null +++ b/tests/fixtures/settings.py @@ -0,0 +1,131 @@ +# -*- coding: utf-8 -*- +# +# This file is part of node-cli +# +# Copyright (C) 2026-Present SKALE Labs +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import pytest +import tomli_w + +from skale_core.settings import get_internal_settings + +from node_cli.configs import INTERNAL_SETTINGS_PATH, NODE_SETTINGS_PATH + +SKALE_DIR_HOST = './skale-data/' + +INTERNAL_SKALE_ACTIVE = { + 'node_type': 'skale', + 'node_mode': 'active', + 'skale_dir_host': SKALE_DIR_HOST, +} + +INTERNAL_SKALE_PASSIVE = { + 'node_type': 'skale', + 'node_mode': 'passive', + 'skale_dir_host': SKALE_DIR_HOST, +} + +INTERNAL_FAIR_ACTIVE = { + 'node_type': 'fair', + 'node_mode': 'active', + 'skale_dir_host': SKALE_DIR_HOST, +} + +INTERNAL_FAIR_PASSIVE = { + 'node_type': 'fair', + 'node_mode': 'passive', + 'skale_dir_host': SKALE_DIR_HOST, +} + +_BASE_NODE = { + 'env_type': 'devnet', + 'endpoint': 'http://127.0.0.1:8545', + 'container_stop_timeout': 1, + 'tg_api_key': '123', + 'tg_chat_id': '-1231232', + 'node_version': '0.0.0', + 'block_device': '/dev/sda', +} + +NODE_SKALE_ACTIVE = { + **_BASE_NODE, + 'sgx_url': 'https://localhost:1026', + 'docker_lvmpy_version': '0.0.0', + 'manager_contracts': 'test-manager', + 'ima_contracts': 'test-ima', +} + +NODE_SKALE_PASSIVE = { + **_BASE_NODE, + 'manager_contracts': 'test-manager', + 'ima_contracts': 'test-ima', + 'schain_name': 'test-schain', + 'enforce_btrfs': False, +} + +NODE_FAIR_ACTIVE = { + **_BASE_NODE, + 'sgx_url': 'https://localhost:1026', + 'fair_contracts': 'test-fair', + 'enforce_btrfs': False, +} + +NODE_FAIR_PASSIVE = { + **_BASE_NODE, + 'fair_contracts': 'test-fair', + 'enforce_btrfs': False, +} + + +def _write_settings(internal: dict, node: dict) -> None: + INTERNAL_SETTINGS_PATH.parent.mkdir(parents=True, exist_ok=True) + INTERNAL_SETTINGS_PATH.write_bytes(tomli_w.dumps(internal).encode()) + NODE_SETTINGS_PATH.write_bytes(tomli_w.dumps(node).encode()) + get_internal_settings.cache_clear() + + +def _cleanup_settings() -> None: + INTERNAL_SETTINGS_PATH.unlink(missing_ok=True) + NODE_SETTINGS_PATH.unlink(missing_ok=True) + get_internal_settings.cache_clear() + + +@pytest.fixture +def skale_active_settings(): + _write_settings(INTERNAL_SKALE_ACTIVE, NODE_SKALE_ACTIVE) + yield + _cleanup_settings() + + +@pytest.fixture +def skale_passive_settings(): + _write_settings(INTERNAL_SKALE_PASSIVE, NODE_SKALE_PASSIVE) + yield + _cleanup_settings() + + +@pytest.fixture +def fair_active_settings(): + _write_settings(INTERNAL_FAIR_ACTIVE, NODE_FAIR_ACTIVE) + yield + _cleanup_settings() + + +@pytest.fixture +def fair_passive_settings(): + _write_settings(INTERNAL_FAIR_PASSIVE, NODE_FAIR_PASSIVE) + yield + _cleanup_settings() diff --git a/tests/helper.py b/tests/helper.py index 7bf18a98..6af0af14 100644 --- a/tests/helper.py +++ b/tests/helper.py @@ -34,12 +34,12 @@ TEST_META_V1 = {'version': '0.1.1', 'config_stream': 'develop'} -TEST_META_V2 = {'version': '0.1.1', 'config_stream': 'develop', 'docker_lvmpy_stream': '1.1.2'} +TEST_META_V2 = {'version': '0.1.1', 'config_stream': 'develop', 'docker_lvmpy_version': '1.1.2'} TEST_META_V3 = { 'version': '0.1.1', 'config_stream': 'develop', - 'docker_lvmpy_stream': '1.1.2', + 'docker_lvmpy_version': '1.1.2', 'os_id': 'ubuntu', 'os_version': '18.04', } diff --git a/tests/resources_test.py b/tests/resources_test.py index 39effaf7..1f545080 100644 --- a/tests/resources_test.py +++ b/tests/resources_test.py @@ -81,23 +81,23 @@ def test_generate_resource_allocation_config(): with mock.patch('node_cli.core.resources.get_disk_size', return_value=NORMAL_DISK_SIZE): resource_allocation_config = compose_resource_allocation_config(DEFAULT_ENV_TYPE) - assert resource_allocation_config['schain']['cpu_shares']['test4'] == 102 - assert resource_allocation_config['schain']['cpu_shares']['test'] == 102 - assert resource_allocation_config['schain']['cpu_shares']['small'] == 6 - assert resource_allocation_config['schain']['cpu_shares']['medium'] == 102 - assert resource_allocation_config['schain']['cpu_shares']['large'] == 819 - - assert isinstance(resource_allocation_config['schain']['mem']['test4'], int) - assert isinstance(resource_allocation_config['schain']['mem']['test'], int) - assert isinstance(resource_allocation_config['schain']['mem']['small'], int) - assert isinstance(resource_allocation_config['schain']['mem']['medium'], int) - assert isinstance(resource_allocation_config['schain']['mem']['large'], int) - - assert resource_allocation_config['schain']['disk']['test4'] == 8879996928 - assert resource_allocation_config['schain']['disk']['test'] == 8879996928 - assert resource_allocation_config['schain']['disk']['small'] == 554999808 - assert resource_allocation_config['schain']['disk']['medium'] == 8879996928 - assert resource_allocation_config['schain']['disk']['large'] == 71039975424 + assert resource_allocation_config['skaled']['cpu_shares']['test4'] == 102 + assert resource_allocation_config['skaled']['cpu_shares']['test'] == 102 + assert resource_allocation_config['skaled']['cpu_shares']['small'] == 6 + assert resource_allocation_config['skaled']['cpu_shares']['medium'] == 102 + assert resource_allocation_config['skaled']['cpu_shares']['large'] == 819 + + assert isinstance(resource_allocation_config['skaled']['mem']['test4'], int) + assert isinstance(resource_allocation_config['skaled']['mem']['test'], int) + assert isinstance(resource_allocation_config['skaled']['mem']['small'], int) + assert isinstance(resource_allocation_config['skaled']['mem']['medium'], int) + assert isinstance(resource_allocation_config['skaled']['mem']['large'], int) + + assert resource_allocation_config['skaled']['disk']['test4'] == 8879996928 + assert resource_allocation_config['skaled']['disk']['test'] == 8879996928 + assert resource_allocation_config['skaled']['disk']['small'] == 554999808 + assert resource_allocation_config['skaled']['disk']['medium'] == 8879996928 + assert resource_allocation_config['skaled']['disk']['large'] == 71039975424 assert resource_allocation_config['ima']['cpu_shares'] == { 'large': 204, @@ -108,7 +108,7 @@ def test_generate_resource_allocation_config(): } assert isinstance(resource_allocation_config['ima']['mem'], dict) - assert resource_allocation_config['schain']['volume_limits'] == SCHAIN_VOLUME_PARTS + assert resource_allocation_config['skaled']['volume_limits'] == SCHAIN_VOLUME_PARTS def test_update_allocation_config(resource_alloc_config): @@ -194,7 +194,7 @@ def test_get_memory_alloc(params_by_env_type): def test_leveldb_limits(): with mock.patch('node_cli.core.resources.get_disk_size', return_value=NORMAL_DISK_SIZE): resource_allocation_config = compose_resource_allocation_config(DEFAULT_ENV_TYPE) - assert resource_allocation_config['schain']['leveldb_limits'] == { + assert resource_allocation_config['skaled']['leveldb_limits'] == { 'large': {'contract_storage': 12787195576, 'db_storage': 4262398525}, 'medium': {'contract_storage': 1598399446, 'db_storage': 532799815}, 'small': {'contract_storage': 99899965, 'db_storage': 33299988}, diff --git a/tests/routes_test.py b/tests/routes_test.py index 872c53e1..39490ab3 100644 --- a/tests/routes_test.py +++ b/tests/routes_test.py @@ -33,6 +33,20 @@ '/api/v1/wallet/send-eth', '/api/v1/fair-node/info', '/api/v1/fair-node/register', + '/api/v1/fair-node/set-domain-name', + '/api/v1/fair-node/change-ip', + '/api/v1/fair-node/exit', + '/api/v1/fair-chain/record', + '/api/v1/fair-chain/checks', + '/api/v1/fair-node-passive/setup', + '/api/v1/fair-staking/add-receiver', + '/api/v1/fair-staking/remove-receiver', + '/api/v1/fair-staking/set-fee-rate', + '/api/v1/fair-staking/request-fees', + '/api/v1/fair-staking/request-send-fees', + '/api/v1/fair-staking/claim-request', + '/api/v1/fair-staking/get-earned-fee-amount', + '/api/v1/fair-staking/get-exit-requests', ] diff --git a/tests/tools_meta_test.py b/tests/tools_meta_test.py index a2a6a2fc..c82e435c 100644 --- a/tests/tools_meta_test.py +++ b/tests/tools_meta_test.py @@ -17,21 +17,21 @@ def test_get_meta_info_v1(meta_file_v1): meta = CliMetaManager().get_meta_info() assert meta.version == TEST_META_V1['version'] assert meta.config_stream == TEST_META_V1['config_stream'] - assert meta.docker_lvmpy_stream == '1.0.0' + assert meta.docker_lvmpy_version == '1.0.0' def test_get_meta_info_v2(meta_file_v2): meta = CliMetaManager().get_meta_info() assert meta.version == TEST_META_V2['version'] assert meta.config_stream == TEST_META_V2['config_stream'] - assert meta.docker_lvmpy_stream == TEST_META_V2['docker_lvmpy_stream'] + assert meta.docker_lvmpy_version == TEST_META_V2['docker_lvmpy_version'] def test_get_meta_info_v3(meta_file_v3): meta = CliMetaManager().get_meta_info() assert meta.version == TEST_META_V3['version'] assert meta.config_stream == TEST_META_V3['config_stream'] - assert meta.docker_lvmpy_stream == TEST_META_V3['docker_lvmpy_stream'] + assert meta.docker_lvmpy_version == TEST_META_V3['docker_lvmpy_version'] assert meta.os_id == TEST_META_V3['os_id'] assert meta.os_version == TEST_META_V3['os_version'] @@ -45,7 +45,7 @@ def test_compose_default_meta(): meta = CliMetaManager().compose_default_meta() assert meta.version == '1.0.0' assert meta.config_stream == '1.1.0' - assert meta.docker_lvmpy_stream == '1.0.0' + assert meta.docker_lvmpy_version == '1.0.0' assert meta.os_id == 'ubuntu' assert meta.os_version == '18.04' @@ -58,7 +58,7 @@ def test_save_meta(meta_file_v2): assert saved_json == { 'version': '1.1.2', 'config_stream': '2.2.2', - 'docker_lvmpy_stream': '1.0.0', + 'docker_lvmpy_version': '1.0.0', 'os_id': 'ubuntu', 'os_version': '18.04', } @@ -69,14 +69,14 @@ def test_update_meta_from_v2_to_v3(meta_file_v2): CliMetaManager().update_meta( version='3.3.3', config_stream='1.1.1', - docker_lvmpy_stream='1.2.2', + docker_lvmpy_version='1.2.2', os_id='debian', os_version='11', ) meta = CliMetaManager().get_meta_info() assert meta.version == '3.3.3' assert meta.config_stream == '1.1.1' - assert meta.docker_lvmpy_stream == '1.2.2' + assert meta.docker_lvmpy_version == '1.2.2' assert meta.os_id == 'debian' assert meta.os_version == '11' assert meta != old_meta @@ -86,14 +86,14 @@ def test_update_meta_from_v1(meta_file_v1): CliMetaManager().update_meta( version='4.4.4', config_stream='beta', - docker_lvmpy_stream='1.3.3', + docker_lvmpy_version='1.3.3', os_id='debian', os_version='11', ) meta = CliMetaManager().get_meta_info() assert meta.version == '4.4.4' assert meta.config_stream == 'beta' - assert meta.docker_lvmpy_stream == '1.3.3' + assert meta.docker_lvmpy_version == '1.3.3' assert meta.os_id == 'debian' assert meta.os_version == '11' @@ -102,14 +102,14 @@ def test_update_meta_from_v3(meta_file_v3): CliMetaManager().update_meta( version='5.5.5', config_stream='stable', - docker_lvmpy_stream='1.2.3', + docker_lvmpy_version='1.2.3', os_id='ubuntu', os_version='20.04', ) meta = CliMetaManager().get_meta_info() assert meta.version == '5.5.5' assert meta.config_stream == 'stable' - assert meta.docker_lvmpy_stream == '1.2.3' + assert meta.docker_lvmpy_version == '1.2.3' assert meta.os_id == 'ubuntu' assert meta.os_version == '20.04' @@ -156,7 +156,7 @@ def test_fair_compose_default_meta(): assert meta.config_stream == '1.1.0' assert meta.os_id == 'ubuntu' assert meta.os_version == '18.04' - assert not hasattr(meta, 'docker_lvmpy_stream') + assert not hasattr(meta, 'docker_lvmpy_version') def test_fair_save_meta(meta_file_v2): @@ -172,7 +172,7 @@ def test_fair_save_meta(meta_file_v2): 'os_id': 'debian', 'os_version': '11', } - assert 'docker_lvmpy_stream' not in saved_json + assert 'docker_lvmpy_version' not in saved_json def test_fair_update_meta_from_v2_to_v3(meta_file_v2): @@ -237,7 +237,7 @@ def test_fair_get_meta_info_raw(meta_file_v3): assert raw_meta['config_stream'] == TEST_META_V3['config_stream'] assert raw_meta['os_id'] == TEST_META_V3['os_id'] assert raw_meta['os_version'] == TEST_META_V3['os_version'] - assert 'docker_lvmpy_stream' not in raw_meta + assert 'docker_lvmpy_version' not in raw_meta def test_fair_get_meta_info_raw_empty(): @@ -257,7 +257,7 @@ def test_fair_asdict(): 'os_version': '35', } assert meta_dict == expected - assert 'docker_lvmpy_stream' not in meta_dict + assert 'docker_lvmpy_version' not in meta_dict def test_fair_meta_compatibility_with_cli_meta_file(meta_file_v3): @@ -266,21 +266,21 @@ def test_fair_meta_compatibility_with_cli_meta_file(meta_file_v3): assert meta.config_stream == TEST_META_V3['config_stream'] assert meta.os_id == TEST_META_V3['os_id'] assert meta.os_version == TEST_META_V3['os_version'] - # Should not have docker_lvmpy_stream even though it's in the file - assert not hasattr(meta, 'docker_lvmpy_stream') + # Should not have docker_lvmpy_version even though it's in the file + assert not hasattr(meta, 'docker_lvmpy_version') def test_fair_save_meta_overwrites_cli_meta(meta_file_v3): with open(META_FILEPATH) as f: original_data = json.load(f) - assert 'docker_lvmpy_stream' in original_data + assert 'docker_lvmpy_version' in original_data fair_meta = FairCliMeta(version='2.0.0', config_stream='fair-new') FairCliMetaManager().save_meta(fair_meta) with open(META_FILEPATH) as f: saved_data = json.load(f) - assert 'docker_lvmpy_stream' not in saved_data + assert 'docker_lvmpy_version' not in saved_data assert saved_data['version'] == '2.0.0' assert saved_data['config_stream'] == 'fair-new' diff --git a/node_cli/core/node_config.py b/tests/utils/settings_test.py similarity index 71% rename from node_cli/core/node_config.py rename to tests/utils/settings_test.py index c7050918..e7b3f4be 100644 --- a/node_cli/core/node_config.py +++ b/tests/utils/settings_test.py @@ -2,7 +2,7 @@ # # This file is part of node-cli # -# Copyright (C) 2021 SKALE Labs +# Copyright (C) 2026 SKALE Labs # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -17,19 +17,3 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . - -class NodeConfig: - def __init__(self, config_filepath, env_filepath=None): - pass - - def load_env(self): - pass - - def validate_env(self): - pass - - def load_config(self): - pass - - def validate_config(self): - pass diff --git a/text.yml b/text.yml index c93a743e..6013f2d0 100644 --- a/text.yml +++ b/text.yml @@ -60,13 +60,13 @@ exit: wait_for_rotations: "Node is waiting to finish rotations" completed: "Node exiting is completed" -sync_node: +passive_node: init: - help: Initialize sync SKALE node - indexer: Run sync node in indexer mode (disable block rotation) + help: Initialize passive SKALE node + indexer: Run passive node in indexer mode (disable block rotation) archive: Enable historic state and disable block rotation snapshot_from: IP of the node to take snapshot from - snapshot: Start sync node from snapshot + snapshot: Start passive node from snapshot lvmpy: help: Lvmpy commands @@ -80,14 +80,25 @@ lvmpy: fair: node: repair: - help: Repair fair chain node - warning: Are you sure you want to repair fair chain node? In rare cases may cause data loss and require additional maintenance - snapshot_from: IP of the node to take snapshot from + help: Repair Fair chain node + warning: Are you sure you want to repair Fair chain node? In rare cases may cause data loss and require additional maintenance + snapshot: IP of the node to take snapshot from (put "any" to use any available node) repair_requested: Repair mode is requested not_inited: Node should be initialized to proceed with operation - registered: Node is registered in fair manager. + setup: + help: Setup passive Fair node + id: ID of the node in Fair manager + + registered: Node is registered in Fair manager. + setup_complete: Passive node setup complete. register: - help: Register node in fair manager - name: Name of the node in fair manager - ip: IP address of the node in fair manager + help: Register node in Fair manager + ip: IP address of the node in Fair manager + ip_changed: Node IP changed in Fair manager + change-ip: + help: Change the node IP in Fair manager + exited: Node removed from Fair manager + exit: + help: Remove node from Fair manager + prompt: Are you sure you want to remove the node from Fair manager?