diff --git a/.travis.yml b/.travis.yml index 74e34ee7a..78c9f07e7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,6 +21,8 @@ before_script: - VERSION="$(node_load_version)" - if [[ ! -z "$DOCKER_USERNAME" ]] ; then echo "${DOCKER_PASSWORD}" | docker login -u "${DOCKER_USERNAME}" --password-stdin; fi - log_env_variables + - if ! [ -x "$(command -v aws)" ]; then curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" ; unzip awscliv2.zip ; sudo ./aws/install ; fi + - aws s3 ls s3://symbol-bootstrap script: - npm pack - npm test @@ -37,14 +39,14 @@ jobs: name: github alpha pages script: /bin/bash travis/node-functions.sh node_push_github_pages if: branch = env(DEV_BRANCH) AND type = push - - name: alpha npm - script: npm pack && /bin/bash travis/node-functions.sh node_publish_alpha + - name: alpha npm - s3 + script: npm pack && /bin/bash travis/node-functions.sh node_publish_alpha && npm run dist && npm run publish if: (branch = env(DEV_BRANCH) AND type = push) OR (type = api AND commit_message = alpha) - stage: release - name: release npm - script: npm pack && /bin/bash travis/node-functions.sh node_publish_release + name: release npm - s3 + script: npm pack && /bin/bash travis/node-functions.sh node_publish_release && npm run dist && npm run publish if: branch = env(RELEASE_BRANCH) AND type = api AND commit_message = env(RELEASE_MESSAGE) - stage: post release name: tag and version upgrade - script: npm pack && /bin/bash travis/node-functions.sh node_post_release + script: npm pack && /bin/bash travis/node-functions.sh node_post_release && npm run dist && npm run publish if: branch = env(RELEASE_BRANCH) AND type = api AND commit_message = env(RELEASE_MESSAGE) diff --git a/README.md b/README.md index c09c777df..eab03351f 100644 --- a/README.md +++ b/README.md @@ -267,7 +267,7 @@ General users should install this tool like any other node module. * [`symbol-bootstrap autocomplete`](docs/autocomplete.md) - display autocomplete installation instructions * [`symbol-bootstrap clean`](docs/clean.md) - It removes the target folder deleting the generated configuration and data * [`symbol-bootstrap compose`](docs/compose.md) - It generates the `docker-compose.yml` file from the configured network. -* [`symbol-bootstrap config`](docs/config.md) - Command used to set up the configuration files and the nemesis block for the current network +* [`symbol-bootstrap config`](docs/config.md) - Command used to set up the configuration files and the nemesis block for the current network. * [`symbol-bootstrap decrypt`](docs/decrypt.md) - It decrypts a yml file using the provided password. The source file can be a custom preset file, a preset.yml file or an addresses.yml. * [`symbol-bootstrap encrypt`](docs/encrypt.md) - It encrypts a yml file using the provided password. The source files would be a custom preset file, a preset.yml file or an addresses.yml. * [`symbol-bootstrap enrolRewardProgram`](docs/enrolRewardProgram.md) - It enrols the nodes in the rewards program by announcing the enrol transaction to the registration address. You can also use this command to update the program registration when you change the node public key or server host. diff --git a/docs/clean.md b/docs/clean.md index 03fedf992..1bcae5ae3 100644 --- a/docs/clean.md +++ b/docs/clean.md @@ -3,6 +3,8 @@ It removes the target folder deleting the generated configuration and data +This command can be used in OFFLINE mode. + * [`symbol-bootstrap clean`](#symbol-bootstrap-clean) ## `symbol-bootstrap clean` @@ -17,6 +19,9 @@ OPTIONS -h, --help It shows the help of this command. -t, --target=target [default: target] The target folder where the symbol-bootstrap network is generated +DESCRIPTION + This command can be used in OFFLINE mode. + EXAMPLE $ symbol-bootstrap clean ``` diff --git a/docs/compose.md b/docs/compose.md index 468d7575f..7fcd42eb9 100644 --- a/docs/compose.md +++ b/docs/compose.md @@ -3,6 +3,8 @@ It generates the `docker-compose.yml` file from the configured network. +This command can be used in OFFLINE mode. + * [`symbol-bootstrap compose`](#symbol-bootstrap-compose) ## `symbol-bootstrap compose` @@ -29,6 +31,9 @@ OPTIONS --upgrade It regenerates the docker compose and utility files from the /docker folder +DESCRIPTION + This command can be used in OFFLINE mode. + EXAMPLE $ symbol-bootstrap compose ``` diff --git a/docs/config.md b/docs/config.md index c7a7dfc20..e1caf7520 100644 --- a/docs/config.md +++ b/docs/config.md @@ -1,13 +1,19 @@ `symbol-bootstrap config` ========================= -Command used to set up the configuration files and the nemesis block for the current network +Command used to set up the configuration files and the nemesis block for the current network. + +This command is by default an ONLINE tool, as it may use docker to run some operations like nemesis or certificate generation. +It's possible to run this command in OFFLINE mode, without using docker, by providing the --offline parameter. + +Note: OFFLINE requires Linux/Mac OS, and the openssl command installed. If you are creating a new network (bootstrap preset), +the nemesis seed needs to be provided with a `nemesisSeedFolder` preset property. Nemesis generation is an online feature that requires catapult tools and docker. * [`symbol-bootstrap config`](#symbol-bootstrap-config) ## `symbol-bootstrap config` -Command used to set up the configuration files and the nemesis block for the current network +Command used to set up the configuration files and the nemesis block for the current network. ``` USAGE @@ -33,14 +39,14 @@ OPTIONS --noPassword When provided, Bootstrap will not use a password, so private keys will be stored in plain text. Use with caution. + --offline If --offline is used, Bootstrap rejects any online operation when generating the + configuration. + --password=password A password used to encrypt and decrypt private keys in preset files like addresses.yml and preset.yml. Bootstrap prompts for a password by default, can be provided in the command line (--password=XXXX) or disabled in the command line (--noPassword). - --pullImages It pulls the utility images from DockerHub when running the configuration. It only - affects alpha/dev docker images. - --report It generates reStructuredText (.rst) reports describing the configuration of each node. @@ -49,8 +55,19 @@ OPTIONS local data. The original preset (-t), assembly (-a), and custom preset (-a) must be used. Backup the target folder before upgrading. +DESCRIPTION + This command is by default an ONLINE tool, as it may use docker to run some operations like nemesis or certificate + generation. + It's possible to run this command in OFFLINE mode, without using docker, by providing the --offline parameter. + + Note: OFFLINE requires Linux/Mac OS, and the openssl command installed. If you are creating a new network (bootstrap + preset), + the nemesis seed needs to be provided with a `nemesisSeedFolder` preset property. Nemesis generation is an online + feature that requires catapult tools and docker. + EXAMPLES $ symbol-bootstrap config -p bootstrap + $ symbol-bootstrap config -p testnet -a dual --customPreset my-encrypted-custom-preset.yml --offline $ symbol-bootstrap config -p testnet -a dual --password 1234 $ echo "$MY_ENV_VAR_PASSWORD" | symbol-bootstrap config -p testnet -a dual ``` diff --git a/docs/decrypt.md b/docs/decrypt.md index ce228c107..d27f21a85 100644 --- a/docs/decrypt.md +++ b/docs/decrypt.md @@ -5,6 +5,8 @@ It decrypts a yml file using the provided password. The source file can be a cus The main use case of this command is to verify private keys in encrypted files after encrypting a custom preset or running a bootstrap command with a provided --password. +This command can be used in OFFLINE mode. + * [`symbol-bootstrap decrypt`](#symbol-bootstrap-decrypt) ## `symbol-bootstrap decrypt` @@ -29,6 +31,8 @@ DESCRIPTION The main use case of this command is to verify private keys in encrypted files after encrypting a custom preset or running a bootstrap command with a provided --password. + This command can be used in OFFLINE mode. + EXAMPLES $ symbol-bootstrap start --password 1234 --preset testnet --assembly dual --customPreset decrypted-custom-preset.yml diff --git a/docs/encrypt.md b/docs/encrypt.md index 8a93e7f31..89ecd7ac4 100644 --- a/docs/encrypt.md +++ b/docs/encrypt.md @@ -5,6 +5,8 @@ It encrypts a yml file using the provided password. The source files would be a The main use case of this command is encrypting custom presets files. If your custom preset contains private keys, it's highly recommended to encrypt it and use provide --password when starting or configuring the node with Bootstrap. +This command can be used in OFFLINE mode. + * [`symbol-bootstrap encrypt`](#symbol-bootstrap-encrypt) ## `symbol-bootstrap encrypt` @@ -30,6 +32,8 @@ DESCRIPTION The main use case of this command is encrypting custom presets files. If your custom preset contains private keys, it's highly recommended to encrypt it and use provide --password when starting or configuring the node with Bootstrap. + This command can be used in OFFLINE mode. + EXAMPLES $ symbol-bootstrap encrypt --source plain-custom-preset.yml --destination encrypted-custom-preset.yml diff --git a/docs/enrolRewardProgram.md b/docs/enrolRewardProgram.md index 44da2ccec..c204d3340 100644 --- a/docs/enrolRewardProgram.md +++ b/docs/enrolRewardProgram.md @@ -5,6 +5,8 @@ It enrols the nodes in the rewards program by announcing the enrol transaction t Currently, the only program that can be enrolled post-launch is 'SuperNode'. +This is an ONLINE command as it creates the transaction depending on the main account type (simple or multisig), and it announces the transaction to the network. + * [`symbol-bootstrap enrolRewardProgram`](#symbol-bootstrap-enrolrewardprogram) ## `symbol-bootstrap enrolRewardProgram` @@ -39,6 +41,9 @@ OPTIONS DESCRIPTION Currently, the only program that can be enrolled post-launch is 'SuperNode'. + This is an ONLINE command as it creates the transaction depending on the main account type (simple or multisig), and + it announces the transaction to the network. + EXAMPLES $ symbol-bootstrap enrolRewardProgram $ symbol-bootstrap enrolRewardProgram --noPassword diff --git a/docs/healthCheck.md b/docs/healthCheck.md index 78ef0e582..1695d666b 100644 --- a/docs/healthCheck.md +++ b/docs/healthCheck.md @@ -10,6 +10,8 @@ This command checks: The health check process handles 'repeat' and custom 'openPort' services. +Although it's highly possible your node is connected to the internet, this is an OFFLINE command as it health checks the local nodes and services. + * [`symbol-bootstrap healthCheck`](#symbol-bootstrap-healthcheck) ## `symbol-bootstrap healthCheck` @@ -32,6 +34,9 @@ DESCRIPTION The health check process handles 'repeat' and custom 'openPort' services. + Although it's highly possible your node is connected to the internet, this is an OFFLINE command as it health checks + the local nodes and services. + EXAMPLE $ symbol-bootstrap healthCheck ``` diff --git a/docs/link.md b/docs/link.md index 83b2b5544..ecfc36cef 100644 --- a/docs/link.md +++ b/docs/link.md @@ -3,6 +3,8 @@ It announces VRF and Voting Link transactions to the network for each node with 'Peer' or 'Voting' roles. This command finalizes the node registration to an existing network. +This is an ONLINE command as it creates the transactions depending on the main account type (simple or multisig), the main account link status, and it announces the transactions to the network. + * [`symbol-bootstrap link`](#symbol-bootstrap-link) ## `symbol-bootstrap link` @@ -36,6 +38,10 @@ OPTIONS --useKnownRestGateways Use the best NEM node available when announcing. Otherwise the command will use the node provided by the --url parameter. +DESCRIPTION + This is an ONLINE command as it creates the transactions depending on the main account type (simple or multisig), the + main account link status, and it announces the transactions to the network. + EXAMPLES $ symbol-bootstrap link $ echo "$MY_ENV_VAR_PASSWORD" | symbol-bootstrap link --unlink --useKnownRestGateways diff --git a/docs/report.md b/docs/report.md index f7a931a9d..d6f49d34c 100644 --- a/docs/report.md +++ b/docs/report.md @@ -3,6 +3,8 @@ it generates reStructuredText (.rst) reports describing the configuration of each node. +This command can be used in OFFLINE mode. + * [`symbol-bootstrap report`](#symbol-bootstrap-report) ## `symbol-bootstrap report` @@ -17,6 +19,9 @@ OPTIONS -h, --help It shows the help of this command. -t, --target=target [default: target] The target folder where the symbol-bootstrap network is generated +DESCRIPTION + This command can be used in OFFLINE mode. + EXAMPLE $ symbol-bootstrap report ``` diff --git a/docs/resetData.md b/docs/resetData.md index 235ac5939..3285caea2 100644 --- a/docs/resetData.md +++ b/docs/resetData.md @@ -3,6 +3,8 @@ It removes the data keeping the generated configuration, certificates, keys and block 1. +This command can be used in OFFLINE mode. + * [`symbol-bootstrap resetData`](#symbol-bootstrap-resetdata) ## `symbol-bootstrap resetData` @@ -17,6 +19,9 @@ OPTIONS -h, --help It shows the help of this command. -t, --target=target [default: target] The target folder where the symbol-bootstrap network is generated +DESCRIPTION + This command can be used in OFFLINE mode. + EXAMPLE $ symbol-bootstrap resetData ``` diff --git a/docs/run.md b/docs/run.md index 81728af3b..e08183780 100644 --- a/docs/run.md +++ b/docs/run.md @@ -3,6 +3,8 @@ It boots the network via docker using the generated `docker-compose.yml` file and configuration. The config and compose methods/commands need to be called before this method. This is just a wrapper for the `docker-compose up` bash call. +This command can only be used in ONLINE mode. It uses docker, docker compose, and images must be pulled from the internet. It's expected for a mainnet/testnet node to be connected to the internet. + * [`symbol-bootstrap run`](#symbol-bootstrap-run) ## `symbol-bootstrap run` @@ -40,8 +42,11 @@ OPTIONS The health check process handles 'repeat' and custom 'openPort' services. + Although it's highly possible your node is connected to the internet, this is an OFFLINE command as it health checks + the local nodes and services. + --pullImages - It pulls the images from DockerHub when running the configuration. It only affects alpha/dev docker images. + It pulls the utility images from DockerHub when running the configuration. It only affects alpha/dev docker images. --resetData It reset the database and node data but keeps the generated configuration, keys, voting tree files and block 1 @@ -49,6 +54,10 @@ OPTIONS --timeout=timeout [default: 60000] If running in detached mode, how long before timing out (in milliseconds) +DESCRIPTION + This command can only be used in ONLINE mode. It uses docker, docker compose, and images must be pulled from the + internet. It's expected for a mainnet/testnet node to be connected to the internet. + EXAMPLE $ symbol-bootstrap run ``` diff --git a/docs/start.md b/docs/start.md index 3ed8075cc..8441eae72 100644 --- a/docs/start.md +++ b/docs/start.md @@ -3,6 +3,11 @@ Single command that aggregates config, compose and run in one line! +This command is typically run in ONLINE mode. It uses docker, docker compose, and images must be pulled from the internet. It's expected for a mainnet/testnet node to be connected to the internet. + +However, if you require your setup to be OFFLINE, you can run `config`, and `compose` in your offline machine, +zip and copy the target folder into the online machine, and execute `run` or `start` there. + * [`symbol-bootstrap start`](#symbol-bootstrap-start) ## `symbol-bootstrap start` @@ -56,9 +61,15 @@ OPTIONS The health check process handles 'repeat' and custom 'openPort' services. + Although it's highly possible your node is connected to the internet, this is an OFFLINE command as it health checks + the local nodes and services. + --noPassword When provided, Bootstrap will not use a password, so private keys will be stored in plain text. Use with caution. + --offline + If --offline is used, Bootstrap rejects any online operation when generating the configuration. + --password=password A password used to encrypt and decrypt private keys in preset files like addresses.yml and preset.yml. Bootstrap prompts for a password by default, can be provided in the command line (--password=XXXX) or disabled in the command @@ -81,6 +92,13 @@ OPTIONS keep your node up to date without dropping the local data. The original preset (-t), assembly (-a), and custom preset (-a) must be used. Backup the target folder before upgrading. +DESCRIPTION + This command is typically run in ONLINE mode. It uses docker, docker compose, and images must be pulled from the + internet. It's expected for a mainnet/testnet node to be connected to the internet. + + However, if you require your setup to be OFFLINE, you can run `config`, and `compose` in your offline machine, + zip and copy the target folder into the online machine, and execute `run` or `start` there. + EXAMPLES $ symbol-bootstrap start $ symbol-bootstrap start -p bootstrap diff --git a/docs/stop.md b/docs/stop.md index 6a1042202..e06cb7041 100644 --- a/docs/stop.md +++ b/docs/stop.md @@ -3,6 +3,8 @@ It stops the docker-compose network if running (symbol-bootstrap started with --detached). This is just a wrapper for the `docker-compose down` bash call. +This command can be run in OFFLINE mode, since it just stops the docker services. + * [`symbol-bootstrap stop`](#symbol-bootstrap-stop) ## `symbol-bootstrap stop` @@ -17,6 +19,9 @@ OPTIONS -h, --help It shows the help of this command. -t, --target=target [default: target] The target folder where the symbol-bootstrap network is generated +DESCRIPTION + This command can be run in OFFLINE mode, since it just stops the docker services. + EXAMPLE $ symbol-bootstrap stop ``` diff --git a/package.json b/package.json index 84988b8cf..ac15460e6 100644 --- a/package.json +++ b/package.json @@ -103,6 +103,8 @@ "coveralls-report": "cat ./coverage/lcov.info | coveralls", "version": "echo $npm_package_version", "install-cli": "npm pack && npm i -g", + "publish": "aws s3 cp --recursive ./dist/symbol-bootstrap-* s3://symbol-bootstrap/dist/v$npm_package_version --acl public-read", + "dist": "oclif-dev pack", "clean-network": "symbol-bootstrap clean -t target/bootstrap-test", "start-network": "symbol-bootstrap start -t target/bootstrap-test -u 'current'" }, diff --git a/src/commands/clean.ts b/src/commands/clean.ts index 7fec7dbde..a267a7927 100644 --- a/src/commands/clean.ts +++ b/src/commands/clean.ts @@ -19,7 +19,10 @@ import { BootstrapUtils } from '../service'; import { CommandUtils } from '../service/CommandUtils'; export default class Clean extends Command { - static description = 'It removes the target folder deleting the generated configuration and data'; + static description = `It removes the target folder deleting the generated configuration and data + +This command can be used in OFFLINE mode. + `; static examples = [`$ symbol-bootstrap clean`]; diff --git a/src/commands/compose.ts b/src/commands/compose.ts index 9b21ca1a5..016d83138 100644 --- a/src/commands/compose.ts +++ b/src/commands/compose.ts @@ -19,7 +19,10 @@ import { BootstrapService, BootstrapUtils, ComposeService } from '../service'; import { CommandUtils } from '../service/CommandUtils'; export default class Compose extends Command { - static description = 'It generates the `docker-compose.yml` file from the configured network.'; + static description = `It generates the \`docker-compose.yml\` file from the configured network. + +This command can be used in OFFLINE mode. +`; static examples = [`$ symbol-bootstrap compose`]; diff --git a/src/commands/config.ts b/src/commands/config.ts index 52a7bc42d..eaf659ba6 100644 --- a/src/commands/config.ts +++ b/src/commands/config.ts @@ -19,10 +19,19 @@ import { BootstrapService, BootstrapUtils, ConfigService, Preset } from '../serv import { CommandUtils } from '../service/CommandUtils'; export default class Config extends Command { - static description = 'Command used to set up the configuration files and the nemesis block for the current network'; + static description = `Command used to set up the configuration files and the nemesis block for the current network. + +This command is by default an ONLINE tool, as it may use docker to run some operations like nemesis or certificate generation. +It's possible to run this command in OFFLINE mode, without using docker, by providing the --offline parameter. + +Note: OFFLINE requires Linux/Mac OS, and the openssl command installed. If you are creating a new network (bootstrap preset), +the nemesis seed needs to be provided with a \`nemesisSeedFolder\` preset property. Nemesis generation is an online feature that requires catapult tools and docker. + + `; static examples = [ `$ symbol-bootstrap config -p bootstrap`, + `$ symbol-bootstrap config -p testnet -a dual --customPreset my-encrypted-custom-preset.yml --offline`, `$ symbol-bootstrap config -p testnet -a dual --password 1234`, `$ echo "$MY_ENV_VAR_PASSWORD" | symbol-bootstrap config -p testnet -a dual`, ]; @@ -64,10 +73,9 @@ export default class Config extends Command { default: ConfigService.defaultParams.report, }), - pullImages: flags.boolean({ - description: - 'It pulls the utility images from DockerHub when running the configuration. It only affects alpha/dev docker images.', - default: ConfigService.defaultParams.pullImages, + offline: flags.boolean({ + description: 'If --offline is used, Bootstrap rejects any online operation when generating the configuration.', + default: ConfigService.defaultParams.offline, }), user: flags.string({ diff --git a/src/commands/decrypt.ts b/src/commands/decrypt.ts index c2fcb51e9..db9f850fb 100644 --- a/src/commands/decrypt.ts +++ b/src/commands/decrypt.ts @@ -27,7 +27,10 @@ const logger: Logger = LoggerFactory.getLogger(LogType.System); export default class Decrypt extends Command { static description = `It decrypts a yml file using the provided password. The source file can be a custom preset file, a preset.yml file or an addresses.yml. -The main use case of this command is to verify private keys in encrypted files after encrypting a custom preset or running a bootstrap command with a provided --password.`; +The main use case of this command is to verify private keys in encrypted files after encrypting a custom preset or running a bootstrap command with a provided --password. + +This command can be used in OFFLINE mode. +`; static examples = [ ` diff --git a/src/commands/encrypt.ts b/src/commands/encrypt.ts index 0baec8d80..9cf1da51d 100644 --- a/src/commands/encrypt.ts +++ b/src/commands/encrypt.ts @@ -28,7 +28,10 @@ const logger: Logger = LoggerFactory.getLogger(LogType.System); export default class Encrypt extends Command { static description = `It encrypts a yml file using the provided password. The source files would be a custom preset file, a preset.yml file or an addresses.yml. -The main use case of this command is encrypting custom presets files. If your custom preset contains private keys, it's highly recommended to encrypt it and use provide --password when starting or configuring the node with Bootstrap.`; +The main use case of this command is encrypting custom presets files. If your custom preset contains private keys, it's highly recommended to encrypt it and use provide --password when starting or configuring the node with Bootstrap. + +This command can be used in OFFLINE mode. +`; static examples = [ ` diff --git a/src/commands/enrolRewardProgram.ts b/src/commands/enrolRewardProgram.ts index ea0d6ae1b..eabc32983 100644 --- a/src/commands/enrolRewardProgram.ts +++ b/src/commands/enrolRewardProgram.ts @@ -22,7 +22,10 @@ import { CommandUtils } from '../service/CommandUtils'; export default class EnrolRewardProgram extends Command { static description = `It enrols the nodes in the rewards program by announcing the enrol transaction to the registration address. You can also use this command to update the program registration when you change the node public key or server host. -Currently, the only program that can be enrolled post-launch is 'SuperNode'.`; +Currently, the only program that can be enrolled post-launch is 'SuperNode'. + +This is an ONLINE command as it creates the transaction depending on the main account type (simple or multisig), and it announces the transaction to the network. +`; static examples = [ `$ symbol-bootstrap enrolRewardProgram`, diff --git a/src/commands/healthCheck.ts b/src/commands/healthCheck.ts index dd3708be5..e0612bc0d 100644 --- a/src/commands/healthCheck.ts +++ b/src/commands/healthCheck.ts @@ -27,6 +27,8 @@ This command checks: - Whether the rest gateways' /node/health are OK. The health check process handles 'repeat' and custom 'openPort' services. + +Although it's highly possible your node is connected to the internet, this is an OFFLINE command as it health checks the local nodes and services. `; static examples = [`$ symbol-bootstrap healthCheck`]; diff --git a/src/commands/link.ts b/src/commands/link.ts index 4174b2580..43929324e 100644 --- a/src/commands/link.ts +++ b/src/commands/link.ts @@ -20,7 +20,10 @@ import { AnnounceService } from '../service/AnnounceService'; import { CommandUtils } from '../service/CommandUtils'; export default class Link extends Command { - static description = `It announces VRF and Voting Link transactions to the network for each node with 'Peer' or 'Voting' roles. This command finalizes the node registration to an existing network.`; + static description = `It announces VRF and Voting Link transactions to the network for each node with 'Peer' or 'Voting' roles. This command finalizes the node registration to an existing network. + +This is an ONLINE command as it creates the transactions depending on the main account type (simple or multisig), the main account link status, and it announces the transactions to the network. +`; static examples = [`$ symbol-bootstrap link`, `$ echo "$MY_ENV_VAR_PASSWORD" | symbol-bootstrap link --unlink --useKnownRestGateways`]; diff --git a/src/commands/report.ts b/src/commands/report.ts index b008c9e33..380873ee0 100644 --- a/src/commands/report.ts +++ b/src/commands/report.ts @@ -19,7 +19,10 @@ import { BootstrapService, BootstrapUtils } from '../service'; import { CommandUtils } from '../service/CommandUtils'; export default class Clean extends Command { - static description = 'it generates reStructuredText (.rst) reports describing the configuration of each node.'; + static description = `it generates reStructuredText (.rst) reports describing the configuration of each node. + +This command can be used in OFFLINE mode. +`; static examples = [`$ symbol-bootstrap report`]; diff --git a/src/commands/resetData.ts b/src/commands/resetData.ts index c981511a4..d69e9937d 100644 --- a/src/commands/resetData.ts +++ b/src/commands/resetData.ts @@ -19,7 +19,10 @@ import { BootstrapService, BootstrapUtils } from '../service'; import { CommandUtils } from '../service/CommandUtils'; export default class ResetData extends Command { - static description = 'It removes the data keeping the generated configuration, certificates, keys and block 1.'; + static description = `It removes the data keeping the generated configuration, certificates, keys and block 1. + +This command can be used in OFFLINE mode. +`; static examples = [`$ symbol-bootstrap resetData`]; diff --git a/src/commands/run.ts b/src/commands/run.ts index 0ef870fcc..e7060272c 100644 --- a/src/commands/run.ts +++ b/src/commands/run.ts @@ -20,8 +20,10 @@ import { CommandUtils } from '../service/CommandUtils'; import HealthCheck from './healthCheck'; export default class Run extends Command { - static description = - 'It boots the network via docker using the generated `docker-compose.yml` file and configuration. The config and compose methods/commands need to be called before this method. This is just a wrapper for the `docker-compose up` bash call.'; + static description = `It boots the network via docker using the generated \`docker-compose.yml\` file and configuration. The config and compose methods/commands need to be called before this method. This is just a wrapper for the \`docker-compose up\` bash call. + +This command can only be used in ONLINE mode. It uses docker, docker compose, and images must be pulled from the internet. It's expected for a mainnet/testnet node to be connected to the internet. +`; static examples = [`$ symbol-bootstrap run`]; @@ -43,7 +45,8 @@ export default class Run extends Command { }), pullImages: flags.boolean({ - description: 'It pulls the images from DockerHub when running the configuration. It only affects alpha/dev docker images.', + description: + 'It pulls the utility images from DockerHub when running the configuration. It only affects alpha/dev docker images.', default: RunService.defaultParams.pullImages, }), diff --git a/src/commands/start.ts b/src/commands/start.ts index b45824676..6ad0b0350 100644 --- a/src/commands/start.ts +++ b/src/commands/start.ts @@ -23,7 +23,13 @@ import Config from './config'; import Run from './run'; export default class Start extends Command { - static description = 'Single command that aggregates config, compose and run in one line!'; + static description = `Single command that aggregates config, compose and run in one line! + +This command is typically run in ONLINE mode. It uses docker, docker compose, and images must be pulled from the internet. It's expected for a mainnet/testnet node to be connected to the internet. + +However, if you require your setup to be OFFLINE, you can run \`config\`, and \`compose\` in your offline machine, +zip and copy the target folder into the online machine, and execute \`run\` or \`start\` there. +`; static examples = [ `$ symbol-bootstrap start`, diff --git a/src/commands/stop.ts b/src/commands/stop.ts index 28b910967..be4556c7e 100644 --- a/src/commands/stop.ts +++ b/src/commands/stop.ts @@ -19,8 +19,10 @@ import { BootstrapUtils, RunService } from '../service'; import { CommandUtils } from '../service/CommandUtils'; export default class Stop extends Command { - static description = - 'It stops the docker-compose network if running (symbol-bootstrap started with --detached). This is just a wrapper for the `docker-compose down` bash call.'; + static description = `It stops the docker-compose network if running (symbol-bootstrap started with --detached). This is just a wrapper for the \`docker-compose down\` bash call. + +This command can be run in OFFLINE mode, since it just stops the docker services. +`; static examples = [`$ symbol-bootstrap stop`]; static flags = { diff --git a/src/model/ConfigPreset.ts b/src/model/ConfigPreset.ts index 15d0342a0..d40d8e7f5 100644 --- a/src/model/ConfigPreset.ts +++ b/src/model/ConfigPreset.ts @@ -153,6 +153,18 @@ export interface FaucetPreset extends DockerServicePreset { name: string; } +export interface PeerInfo { + publicKey: string; + endpoint: { + host: string; + port: number; + }; + metadata: { + name: string; + roles: string; + }; +} + export interface ConfigPreset { preset: Preset; privateKeySecurityMode: string; @@ -191,7 +203,7 @@ export interface ConfigPreset { harvestingMosaicId: string; baseNamespace: string; databases?: DatabasePreset[]; - knownPeers?: Record; + knownPeers?: Record; knownRestGateways?: string[]; mongoComposeRunParam: string; mongoImage: string; diff --git a/src/service/AgentCertificateService.ts b/src/service/AgentCertificateService.ts index c1cf10d2d..d59d7fb38 100644 --- a/src/service/AgentCertificateService.ts +++ b/src/service/AgentCertificateService.ts @@ -18,11 +18,12 @@ import { join, resolve } from 'path'; import { LogType } from '../logger'; import Logger from '../logger/Logger'; import LoggerFactory from '../logger/LoggerFactory'; -import { BootstrapUtils } from './BootstrapUtils'; +import { BootstrapUtils, KnownError } from './BootstrapUtils'; export interface AgentCertificateParams { readonly target: string; readonly user: string; + readonly offline?: boolean; } const logger: Logger = LoggerFactory.getLogger(LogType.System); @@ -36,29 +37,45 @@ export class AgentCertificateService { await BootstrapUtils.mkdir(certFolder); const generatedContext = { name }; await BootstrapUtils.generateConfiguration(generatedContext, copyFrom, certFolder, []); - const command = this.createCertCommands('/data'); + const command = this.createCertCommands(); await BootstrapUtils.writeTextFile(join(certFolder, 'createAgentCertificate.sh'), command); const cmd = ['bash', 'createAgentCertificate.sh']; - const binds = [`${resolve(certFolder)}:/data:rw`]; - const userId = await BootstrapUtils.resolveDockerUserFromParam(this.params.user); - const { stdout, stderr } = await BootstrapUtils.runImageUsingExec({ - image: symbolServerToolsImage, - userId: userId, - workdir: '/data', - cmds: cmd, - binds: binds, - }); - if (stdout.indexOf('Certificate Created') < 0) { - logger.info(BootstrapUtils.secureString(stdout)); - logger.error(BootstrapUtils.secureString(stderr)); - throw new Error('Certificate creation failed. Check the logs!'); + + try { + logger.info('Creating Agent Certificate using exec and local openssl'); + const { stdout, stderr } = await BootstrapUtils.exec(cmd.join(' '), { cwd: certFolder }); + if (stdout.indexOf('Certificate Created') < 0) { + logger.info(BootstrapUtils.secureString(stdout)); + logger.error(BootstrapUtils.secureString(stderr)); + throw new Error('Certificate creation failed. Check the logs!'); + } + } catch (e) { + if (this.params.offline) { + throw new KnownError( + `Agent Certificates could not be created using local openssl. Do you have openssl installed? Remove --offline to fallback to docker. Error: ${e.message}`, + ); + } + logger.warn('Agent certificate could not be created using local openssl. Falling back to docker run...'); + const binds = [`${resolve(certFolder)}:/data:rw`]; + const userId = await BootstrapUtils.resolveDockerUserFromParam(this.params.user); + const { stdout, stderr } = await BootstrapUtils.runImageUsingExec({ + image: symbolServerToolsImage, + userId: userId, + workdir: '/data', + cmds: cmd, + binds: binds, + }); + if (stdout.indexOf('Certificate Created') < 0) { + logger.info(BootstrapUtils.secureString(stdout)); + logger.error(BootstrapUtils.secureString(stderr)); + throw new Error('Certificate creation failed. Check the logs!'); + } } logger.info(`Agent Certificate for node ${name} created`); } - private createCertCommands(target: string): string { + private createCertCommands(): string { return `set -e - cd ${target} # Creating a self signed certificate for the agent. This will be created by bootstrap when setting up a supernode openssl genrsa -out agent-key.pem 4096 openssl req -new -config agent.cnf -key agent-key.pem -out agent-csr.pem diff --git a/src/service/BootstrapUtils.ts b/src/service/BootstrapUtils.ts index 44688ae8a..8e8c65d26 100644 --- a/src/service/BootstrapUtils.ts +++ b/src/service/BootstrapUtils.ts @@ -53,6 +53,10 @@ export type Password = string | false | undefined; export class KnownError extends Error { public readonly known = true; } +export interface ExecOutput { + stdout: string; + stderr: string; +} /** * The operation to migrate the data. @@ -269,7 +273,7 @@ export class BootstrapUtils { workdir?: string; cmds: string[]; binds: string[]; - }): Promise<{ stdout: string; stderr: string }> { + }): Promise { const volumes = binds.map((b) => `-v ${b}`).join(' '); const userParam = userId ? `-u ${userId}` : ''; const workdirParam = workdir ? `--workdir=${workdir}` : ''; @@ -471,9 +475,9 @@ export class BootstrapUtils { return (await this.exec(runCommand)).stdout; } - public static async exec(runCommand: string): Promise<{ stdout: string; stderr: string }> { + public static async exec(runCommand: string, extraParams: any = undefined): Promise { logger.debug(`Exec command: ${runCommand}`); - const { stdout, stderr } = await exec(runCommand); + const { stdout, stderr } = await exec(runCommand, extraParams); return { stdout, stderr }; } diff --git a/src/service/CertificateService.ts b/src/service/CertificateService.ts index e8af19ec0..b9fc5cde3 100644 --- a/src/service/CertificateService.ts +++ b/src/service/CertificateService.ts @@ -20,13 +20,14 @@ import { LogType } from '../logger'; import Logger from '../logger/Logger'; import LoggerFactory from '../logger/LoggerFactory'; import { CertificatePair } from '../model'; -import { BootstrapUtils } from './BootstrapUtils'; +import { BootstrapUtils, ExecOutput, KnownError } from './BootstrapUtils'; import { CommandUtils } from './CommandUtils'; import { KeyName } from './ConfigService'; export interface CertificateParams { readonly target: string; readonly user: string; + readonly offline?: boolean; } export interface CertificateMetadata { @@ -87,7 +88,6 @@ export class CertificateService { providedCertificates: NodeCertificates, customCertFolder?: string, ): Promise { - const copyFrom = `${this.root}/config/cert`; const certFolder = customCertFolder || BootstrapUtils.getTargetNodesFolder(this.params.target, false, name, 'cert'); const metadataFile = join(certFolder, 'metadata.yml'); @@ -96,12 +96,34 @@ export class CertificateService { logger.info(`Certificates for node ${name} have been previously generated. Reusing...`); return; } + await this.exec(networkType, symbolServerToolsImage, name, providedCertificates, certFolder); + logger.info(`Certificates for node ${name} created`); + const metadata: CertificateMetadata = { + version: CertificateService.METADATA_VERSION, + transportPublicKey: providedCertificates.transport.publicKey, + mainPublicKey: providedCertificates.main.publicKey, + }; + await BootstrapUtils.writeYaml(metadataFile, metadata, undefined); + } + + private async exec( + networkType: NetworkType, + symbolServerToolsImage: string, + name: string, + providedCertificates: NodeCertificates, + certFolder: string, + ): Promise { + const copyFrom = `${this.root}/config/cert`; await BootstrapUtils.deleteFolder(certFolder); await BootstrapUtils.mkdir(certFolder); const newCertsFolder = join(certFolder, 'new_certs'); await BootstrapUtils.mkdir(newCertsFolder); const generatedContext = { name }; await BootstrapUtils.generateConfiguration(generatedContext, copyFrom, certFolder, []); + // TODO. Migrate this process to forge, sshpk or any node native implementation. + const command = this.createCertCommands(); + await BootstrapUtils.writeTextFile(join(certFolder, 'createNodeCertificates.sh'), command); + const cmd = ['bash', 'createNodeCertificates.sh']; const mainAccountPrivateKey = await CommandUtils.resolvePrivateKey(networkType, providedCertificates.main, KeyName.Main, name); const transportPrivateKey = await CommandUtils.resolvePrivateKey( @@ -110,22 +132,51 @@ export class CertificateService { KeyName.Transport, name, ); - BootstrapUtils.createDerFile(mainAccountPrivateKey, join(certFolder, 'ca.der')); - BootstrapUtils.createDerFile(transportPrivateKey, join(certFolder, 'node.der')); + const caDerFile = join(certFolder, 'ca.der'); + const nodeDerFile = join(certFolder, 'node.der'); + try { + BootstrapUtils.createDerFile(mainAccountPrivateKey, caDerFile); + BootstrapUtils.createDerFile(transportPrivateKey, nodeDerFile); + try { + logger.info('Creating Node Certificates using exec and local openssl'); + const execOutput = await BootstrapUtils.exec(cmd.join(' '), { + cwd: certFolder, + }); + await this.validateOutput(execOutput, providedCertificates, mainAccountPrivateKey, transportPrivateKey); + return execOutput; + } catch (e) { + if (this.params.offline) { + throw new KnownError( + `Node Certificates could not be created using local openssl. Do you have openssl installed? Remove --offline to fallback to docker. Error: ${e.message}`, + ); + } + logger.warn('Node Certificates could not be created using local openssl. Falling back to docker run...'); + } + const binds = [`${resolve(certFolder)}:/data:rw`]; + const userId = await BootstrapUtils.resolveDockerUserFromParam(this.params.user); + const execOutput = await BootstrapUtils.runImageUsingExec({ + image: symbolServerToolsImage, + userId: userId, + workdir: '/data', + cmds: cmd, + binds: binds, + }); + await this.validateOutput(execOutput, providedCertificates, mainAccountPrivateKey, transportPrivateKey); + return execOutput; + } finally { + //To be sure we are not keeping any private file + BootstrapUtils.deleteFile(join(certFolder, 'ca.key.pem')); + BootstrapUtils.deleteFile(caDerFile); + BootstrapUtils.deleteFile(nodeDerFile); + } + } - // TODO. Migrate this process to forge, sshpk or any node native implementation. - const command = this.createCertCommands('/data'); - await BootstrapUtils.writeTextFile(join(certFolder, 'createNodeCertificates.sh'), command); - const cmd = ['bash', 'createNodeCertificates.sh']; - const binds = [`${resolve(certFolder)}:/data:rw`]; - const userId = await BootstrapUtils.resolveDockerUserFromParam(this.params.user); - const { stdout, stderr } = await BootstrapUtils.runImageUsingExec({ - image: symbolServerToolsImage, - userId: userId, - workdir: '/data', - cmds: cmd, - binds: binds, - }); + private async validateOutput( + { stdout, stderr }: ExecOutput, + providedCertificates: NodeCertificates, + mainAccountPrivateKey: string, + transportPrivateKey: string, + ) { if (stdout.indexOf('Certificate Created') < 0) { logger.info(BootstrapUtils.secureString(stdout)); logger.error(BootstrapUtils.secureString(stderr)); @@ -136,21 +187,12 @@ export class CertificateService { if (certificates.length != 2) { throw new Error('Certificate creation failed. 2 certificates should have been created but got: ' + certificates.length); } - logger.info(`Certificate for node ${name} created`); const caCertificate = certificates[0]; const nodeCertificate = certificates[1]; - BootstrapUtils.validateIsTrue(caCertificate.privateKey === mainAccountPrivateKey, 'Invalid ca private key'); BootstrapUtils.validateIsTrue(caCertificate.publicKey === providedCertificates.main.publicKey, 'Invalid ca public key'); BootstrapUtils.validateIsTrue(nodeCertificate.privateKey === transportPrivateKey, 'Invalid Node private key'); BootstrapUtils.validateIsTrue(nodeCertificate.publicKey === providedCertificates.transport.publicKey, 'Invalid Node public key'); - - const metadata: CertificateMetadata = { - version: CertificateService.METADATA_VERSION, - transportPublicKey: providedCertificates.transport.publicKey, - mainPublicKey: providedCertificates.main.publicKey, - }; - await BootstrapUtils.writeYaml(metadataFile, metadata, undefined); } private async shouldGenerateCertificate(metadataFile: string, providedCertificates: NodeCertificates): Promise { @@ -166,9 +208,8 @@ export class CertificateService { } } - private createCertCommands(target: string): string { + private createCertCommands(): string { return `set -e -cd ${target} chmod 700 new_certs touch index.txt.attr touch index.txt diff --git a/src/service/ConfigService.ts b/src/service/ConfigService.ts index 03e0d1eed..face73cf3 100644 --- a/src/service/ConfigService.ts +++ b/src/service/ConfigService.ts @@ -32,7 +32,7 @@ import { import { LogType } from '../logger'; import Logger from '../logger/Logger'; import LoggerFactory from '../logger/LoggerFactory'; -import { Addresses, ConfigPreset, NodeAccount, NodePreset, NodeType } from '../model'; +import { Addresses, ConfigPreset, NodeAccount, NodePreset, NodeType, PeerInfo } from '../model'; import { AgentCertificateService } from './AgentCertificateService'; import { BootstrapUtils, KnownError } from './BootstrapUtils'; import { CertificateService } from './CertificateService'; @@ -70,7 +70,7 @@ export interface ConfigParams { target: string; password?: string; user: string; - pullImages?: boolean; + offline?: boolean; assembly?: string; customPreset?: string; customPresetObject?: any; @@ -89,8 +89,8 @@ export class ConfigService { report: false, preset: Preset.bootstrap, reset: false, + offline: false, upgrade: false, - pullImages: false, user: BootstrapUtils.CURRENT_USER, }; private readonly configLoader: ConfigLoader; @@ -138,7 +138,6 @@ export class ConfigService { const presetData: ConfigPreset = this.resolveCurrentPresetData(oldPresetData, password); const addresses = await this.configLoader.generateRandomConfiguration(oldAddresses, presetData); - if (this.params.pullImages) await BootstrapUtils.pullImage(presetData.symbolServerToolsImage); const privateKeySecurityMode = CryptoUtils.getPrivateKeySecurityMode(presetData.privateKeySecurityMode); await BootstrapUtils.mkdir(target); @@ -398,7 +397,7 @@ export class ConfigService { nodePresetDataFunction: (nodePresetData: NodePreset) => boolean, jsonFileName: string, ) { - const thisNetworkKnownPeers = (presetData.nodes || []) + const thisNetworkKnownPeers: PeerInfo[] = (presetData.nodes || []) .map((nodePresetData, index) => { if (!nodePresetDataFunction(nodePresetData)) { return undefined; @@ -416,7 +415,8 @@ export class ConfigService { }, }; }) - .filter((i) => i); + .filter((i) => i) + .map((i) => i as PeerInfo); const globalKnownPeers = presetData.knownPeers?.[type] || []; const data = { _info: `this file contains a list of ${type} peers`, diff --git a/src/service/NemgenService.ts b/src/service/NemgenService.ts index becaef8d5..619fb9fe4 100644 --- a/src/service/NemgenService.ts +++ b/src/service/NemgenService.ts @@ -16,11 +16,11 @@ import { promises } from 'fs'; import { join } from 'path'; +import { LogType } from '../logger'; import Logger from '../logger/Logger'; import LoggerFactory from '../logger/LoggerFactory'; -import { LogType } from '../logger/LogType'; import { ConfigPreset } from '../model'; -import { BootstrapUtils } from './BootstrapUtils'; +import { BootstrapUtils, KnownError } from './BootstrapUtils'; import { ConfigParams } from './ConfigService'; type NemgenParams = ConfigParams; @@ -34,7 +34,9 @@ export class NemgenService { const networkIdentifier = presetData.networkIdentifier; const symbolServerToolsImage = presetData.symbolServerToolsImage; const target = this.params.target; - + if (this.params.offline) { + throw new KnownError(`Nemesis generation is not an offline feature. It requires catapult.tools.nemgen via Docker images.`); + } if (!presetData.nodes || !presetData.nodes.length) { throw new Error('Nodes must be defined in preset when running nemgen'); }