From 550547211a8d6c5db3732121e6ba75d2a82035e2 Mon Sep 17 00:00:00 2001 From: fernando Date: Tue, 2 Mar 2021 17:06:20 -0300 Subject: [PATCH 1/6] Running openssl locally --- src/service/BootstrapUtils.ts | 10 +++-- src/service/CertificateService.ts | 64 ++++++++++++++++++++++--------- 2 files changed, 52 insertions(+), 22 deletions(-) diff --git a/src/service/BootstrapUtils.ts b/src/service/BootstrapUtils.ts index 44688ae8a..8e8c65d26 100644 --- a/src/service/BootstrapUtils.ts +++ b/src/service/BootstrapUtils.ts @@ -53,6 +53,10 @@ export type Password = string | false | undefined; export class KnownError extends Error { public readonly known = true; } +export interface ExecOutput { + stdout: string; + stderr: string; +} /** * The operation to migrate the data. @@ -269,7 +273,7 @@ export class BootstrapUtils { workdir?: string; cmds: string[]; binds: string[]; - }): Promise<{ stdout: string; stderr: string }> { + }): Promise { const volumes = binds.map((b) => `-v ${b}`).join(' '); const userParam = userId ? `-u ${userId}` : ''; const workdirParam = workdir ? `--workdir=${workdir}` : ''; @@ -471,9 +475,9 @@ export class BootstrapUtils { return (await this.exec(runCommand)).stdout; } - public static async exec(runCommand: string): Promise<{ stdout: string; stderr: string }> { + public static async exec(runCommand: string, extraParams: any = undefined): Promise { logger.debug(`Exec command: ${runCommand}`); - const { stdout, stderr } = await exec(runCommand); + const { stdout, stderr } = await exec(runCommand, extraParams); return { stdout, stderr }; } diff --git a/src/service/CertificateService.ts b/src/service/CertificateService.ts index e8af19ec0..4f41496ac 100644 --- a/src/service/CertificateService.ts +++ b/src/service/CertificateService.ts @@ -20,7 +20,7 @@ import { LogType } from '../logger'; import Logger from '../logger/Logger'; import LoggerFactory from '../logger/LoggerFactory'; import { CertificatePair } from '../model'; -import { BootstrapUtils } from './BootstrapUtils'; +import { BootstrapUtils, ExecOutput } from './BootstrapUtils'; import { CommandUtils } from './CommandUtils'; import { KeyName } from './ConfigService'; @@ -87,7 +87,6 @@ export class CertificateService { providedCertificates: NodeCertificates, customCertFolder?: string, ): Promise { - const copyFrom = `${this.root}/config/cert`; const certFolder = customCertFolder || BootstrapUtils.getTargetNodesFolder(this.params.target, false, name, 'cert'); const metadataFile = join(certFolder, 'metadata.yml'); @@ -96,12 +95,34 @@ export class CertificateService { logger.info(`Certificates for node ${name} have been previously generated. Reusing...`); return; } + await this.exec(networkType, symbolServerToolsImage, name, providedCertificates, certFolder); + logger.info(`Certificate for node ${name} created`); + const metadata: CertificateMetadata = { + version: CertificateService.METADATA_VERSION, + transportPublicKey: providedCertificates.transport.publicKey, + mainPublicKey: providedCertificates.main.publicKey, + }; + await BootstrapUtils.writeYaml(metadataFile, metadata, undefined); + } + + private async exec( + networkType: NetworkType, + symbolServerToolsImage: string, + name: string, + providedCertificates: NodeCertificates, + certFolder: string, + ): Promise { + const copyFrom = `${this.root}/config/cert`; await BootstrapUtils.deleteFolder(certFolder); await BootstrapUtils.mkdir(certFolder); const newCertsFolder = join(certFolder, 'new_certs'); await BootstrapUtils.mkdir(newCertsFolder); const generatedContext = { name }; await BootstrapUtils.generateConfiguration(generatedContext, copyFrom, certFolder, []); + // TODO. Migrate this process to forge, sshpk or any node native implementation. + const command = this.createCertCommands(); + await BootstrapUtils.writeTextFile(join(certFolder, 'createNodeCertificates.sh'), command); + const cmd = ['bash', 'createNodeCertificates.sh']; const mainAccountPrivateKey = await CommandUtils.resolvePrivateKey(networkType, providedCertificates.main, KeyName.Main, name); const transportPrivateKey = await CommandUtils.resolvePrivateKey( @@ -112,20 +133,35 @@ export class CertificateService { ); BootstrapUtils.createDerFile(mainAccountPrivateKey, join(certFolder, 'ca.der')); BootstrapUtils.createDerFile(transportPrivateKey, join(certFolder, 'node.der')); - - // TODO. Migrate this process to forge, sshpk or any node native implementation. - const command = this.createCertCommands('/data'); - await BootstrapUtils.writeTextFile(join(certFolder, 'createNodeCertificates.sh'), command); - const cmd = ['bash', 'createNodeCertificates.sh']; + try { + logger.info('Creating certificate using exec and local openssl'); + const execOutput = await BootstrapUtils.exec(cmd.join(' '), { + cwd: certFolder, + }); + await this.validateOutput(execOutput, providedCertificates, mainAccountPrivateKey, transportPrivateKey); + return execOutput; + } catch (e) { + logger.warn('Certificate could not be created using local openssl. Falling back to docker run...'); + } const binds = [`${resolve(certFolder)}:/data:rw`]; const userId = await BootstrapUtils.resolveDockerUserFromParam(this.params.user); - const { stdout, stderr } = await BootstrapUtils.runImageUsingExec({ + const execOutput = await BootstrapUtils.runImageUsingExec({ image: symbolServerToolsImage, userId: userId, workdir: '/data', cmds: cmd, binds: binds, }); + await this.validateOutput(execOutput, providedCertificates, mainAccountPrivateKey, transportPrivateKey); + return execOutput; + } + + private async validateOutput( + { stdout, stderr }: ExecOutput, + providedCertificates: NodeCertificates, + mainAccountPrivateKey: string, + transportPrivateKey: string, + ) { if (stdout.indexOf('Certificate Created') < 0) { logger.info(BootstrapUtils.secureString(stdout)); logger.error(BootstrapUtils.secureString(stderr)); @@ -136,21 +172,12 @@ export class CertificateService { if (certificates.length != 2) { throw new Error('Certificate creation failed. 2 certificates should have been created but got: ' + certificates.length); } - logger.info(`Certificate for node ${name} created`); const caCertificate = certificates[0]; const nodeCertificate = certificates[1]; - BootstrapUtils.validateIsTrue(caCertificate.privateKey === mainAccountPrivateKey, 'Invalid ca private key'); BootstrapUtils.validateIsTrue(caCertificate.publicKey === providedCertificates.main.publicKey, 'Invalid ca public key'); BootstrapUtils.validateIsTrue(nodeCertificate.privateKey === transportPrivateKey, 'Invalid Node private key'); BootstrapUtils.validateIsTrue(nodeCertificate.publicKey === providedCertificates.transport.publicKey, 'Invalid Node public key'); - - const metadata: CertificateMetadata = { - version: CertificateService.METADATA_VERSION, - transportPublicKey: providedCertificates.transport.publicKey, - mainPublicKey: providedCertificates.main.publicKey, - }; - await BootstrapUtils.writeYaml(metadataFile, metadata, undefined); } private async shouldGenerateCertificate(metadataFile: string, providedCertificates: NodeCertificates): Promise { @@ -166,9 +193,8 @@ export class CertificateService { } } - private createCertCommands(target: string): string { + private createCertCommands(): string { return `set -e -cd ${target} chmod 700 new_certs touch index.txt.attr touch index.txt From dfbf00f6b126c5c64f802308918ddbb79cc8f02b Mon Sep 17 00:00:00 2001 From: fernando Date: Tue, 2 Mar 2021 17:20:35 -0300 Subject: [PATCH 2/6] Migrated agent certificates to local openssl --- src/service/AgentCertificateService.ts | 43 ++++++++++++++++---------- src/service/CertificateService.ts | 6 ++-- 2 files changed, 30 insertions(+), 19 deletions(-) diff --git a/src/service/AgentCertificateService.ts b/src/service/AgentCertificateService.ts index c1cf10d2d..469783a11 100644 --- a/src/service/AgentCertificateService.ts +++ b/src/service/AgentCertificateService.ts @@ -36,29 +36,40 @@ export class AgentCertificateService { await BootstrapUtils.mkdir(certFolder); const generatedContext = { name }; await BootstrapUtils.generateConfiguration(generatedContext, copyFrom, certFolder, []); - const command = this.createCertCommands('/data'); + const command = this.createCertCommands(); await BootstrapUtils.writeTextFile(join(certFolder, 'createAgentCertificate.sh'), command); const cmd = ['bash', 'createAgentCertificate.sh']; - const binds = [`${resolve(certFolder)}:/data:rw`]; - const userId = await BootstrapUtils.resolveDockerUserFromParam(this.params.user); - const { stdout, stderr } = await BootstrapUtils.runImageUsingExec({ - image: symbolServerToolsImage, - userId: userId, - workdir: '/data', - cmds: cmd, - binds: binds, - }); - if (stdout.indexOf('Certificate Created') < 0) { - logger.info(BootstrapUtils.secureString(stdout)); - logger.error(BootstrapUtils.secureString(stderr)); - throw new Error('Certificate creation failed. Check the logs!'); + + try { + logger.info('Creating Agent Certificate using exec and local openssl'); + const { stdout, stderr } = await BootstrapUtils.exec(cmd.join(' '), { cwd: certFolder }); + if (stdout.indexOf('Certificate Created') < 0) { + logger.info(BootstrapUtils.secureString(stdout)); + logger.error(BootstrapUtils.secureString(stderr)); + throw new Error('Certificate creation failed. Check the logs!'); + } + } catch (e) { + logger.warn('Agent certificate could not be created using local openssl. Falling back to docker run...'); + const binds = [`${resolve(certFolder)}:/data:rw`]; + const userId = await BootstrapUtils.resolveDockerUserFromParam(this.params.user); + const { stdout, stderr } = await BootstrapUtils.runImageUsingExec({ + image: symbolServerToolsImage, + userId: userId, + workdir: '/data', + cmds: cmd, + binds: binds, + }); + if (stdout.indexOf('Certificate Created') < 0) { + logger.info(BootstrapUtils.secureString(stdout)); + logger.error(BootstrapUtils.secureString(stderr)); + throw new Error('Certificate creation failed. Check the logs!'); + } } logger.info(`Agent Certificate for node ${name} created`); } - private createCertCommands(target: string): string { + private createCertCommands(): string { return `set -e - cd ${target} # Creating a self signed certificate for the agent. This will be created by bootstrap when setting up a supernode openssl genrsa -out agent-key.pem 4096 openssl req -new -config agent.cnf -key agent-key.pem -out agent-csr.pem diff --git a/src/service/CertificateService.ts b/src/service/CertificateService.ts index 4f41496ac..05512895c 100644 --- a/src/service/CertificateService.ts +++ b/src/service/CertificateService.ts @@ -96,7 +96,7 @@ export class CertificateService { return; } await this.exec(networkType, symbolServerToolsImage, name, providedCertificates, certFolder); - logger.info(`Certificate for node ${name} created`); + logger.info(`Certificates for node ${name} created`); const metadata: CertificateMetadata = { version: CertificateService.METADATA_VERSION, transportPublicKey: providedCertificates.transport.publicKey, @@ -134,14 +134,14 @@ export class CertificateService { BootstrapUtils.createDerFile(mainAccountPrivateKey, join(certFolder, 'ca.der')); BootstrapUtils.createDerFile(transportPrivateKey, join(certFolder, 'node.der')); try { - logger.info('Creating certificate using exec and local openssl'); + logger.info('Creating Node Certificates using exec and local openssl'); const execOutput = await BootstrapUtils.exec(cmd.join(' '), { cwd: certFolder, }); await this.validateOutput(execOutput, providedCertificates, mainAccountPrivateKey, transportPrivateKey); return execOutput; } catch (e) { - logger.warn('Certificate could not be created using local openssl. Falling back to docker run...'); + logger.warn('Node Certificates could not be created using local openssl. Falling back to docker run...'); } const binds = [`${resolve(certFolder)}:/data:rw`]; const userId = await BootstrapUtils.resolveDockerUserFromParam(this.params.user); From 1e68dcabd18dac576a544ff571342d6029d2d4ff Mon Sep 17 00:00:00 2001 From: fernando Date: Wed, 3 Mar 2021 06:38:29 -0300 Subject: [PATCH 3/6] Added offline validation --- .travis.yml | 12 +++--- README.md | 2 +- docs/clean.md | 5 +++ docs/compose.md | 5 +++ docs/config.md | 24 +++++++++++- docs/decrypt.md | 4 ++ docs/encrypt.md | 4 ++ docs/enrolRewardProgram.md | 5 +++ docs/healthCheck.md | 5 +++ docs/link.md | 6 +++ docs/report.md | 5 +++ docs/resetData.md | 5 +++ docs/run.md | 9 +++++ docs/start.md | 18 +++++++++ docs/stop.md | 6 +++ package.json | 2 + src/commands/clean.ts | 5 ++- src/commands/compose.ts | 5 ++- src/commands/config.ts | 18 ++++++++- src/commands/decrypt.ts | 5 ++- src/commands/encrypt.ts | 5 ++- src/commands/enrolRewardProgram.ts | 5 ++- src/commands/healthCheck.ts | 2 + src/commands/link.ts | 5 ++- src/commands/report.ts | 5 ++- src/commands/resetData.ts | 5 ++- src/commands/run.ts | 6 ++- src/commands/start.ts | 8 +++- src/commands/stop.ts | 6 ++- src/service/AgentCertificateService.ts | 8 +++- src/service/CertificateService.ts | 53 +++++++++++++++++--------- src/service/ConfigService.ts | 2 + src/service/NemgenService.ts | 8 ++-- 33 files changed, 222 insertions(+), 46 deletions(-) diff --git a/.travis.yml b/.travis.yml index 74e34ee7a..78c9f07e7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,6 +21,8 @@ before_script: - VERSION="$(node_load_version)" - if [[ ! -z "$DOCKER_USERNAME" ]] ; then echo "${DOCKER_PASSWORD}" | docker login -u "${DOCKER_USERNAME}" --password-stdin; fi - log_env_variables + - if ! [ -x "$(command -v aws)" ]; then curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" ; unzip awscliv2.zip ; sudo ./aws/install ; fi + - aws s3 ls s3://symbol-bootstrap script: - npm pack - npm test @@ -37,14 +39,14 @@ jobs: name: github alpha pages script: /bin/bash travis/node-functions.sh node_push_github_pages if: branch = env(DEV_BRANCH) AND type = push - - name: alpha npm - script: npm pack && /bin/bash travis/node-functions.sh node_publish_alpha + - name: alpha npm - s3 + script: npm pack && /bin/bash travis/node-functions.sh node_publish_alpha && npm run dist && npm run publish if: (branch = env(DEV_BRANCH) AND type = push) OR (type = api AND commit_message = alpha) - stage: release - name: release npm - script: npm pack && /bin/bash travis/node-functions.sh node_publish_release + name: release npm - s3 + script: npm pack && /bin/bash travis/node-functions.sh node_publish_release && npm run dist && npm run publish if: branch = env(RELEASE_BRANCH) AND type = api AND commit_message = env(RELEASE_MESSAGE) - stage: post release name: tag and version upgrade - script: npm pack && /bin/bash travis/node-functions.sh node_post_release + script: npm pack && /bin/bash travis/node-functions.sh node_post_release && npm run dist && npm run publish if: branch = env(RELEASE_BRANCH) AND type = api AND commit_message = env(RELEASE_MESSAGE) diff --git a/README.md b/README.md index c09c777df..eab03351f 100644 --- a/README.md +++ b/README.md @@ -267,7 +267,7 @@ General users should install this tool like any other node module. * [`symbol-bootstrap autocomplete`](docs/autocomplete.md) - display autocomplete installation instructions * [`symbol-bootstrap clean`](docs/clean.md) - It removes the target folder deleting the generated configuration and data * [`symbol-bootstrap compose`](docs/compose.md) - It generates the `docker-compose.yml` file from the configured network. -* [`symbol-bootstrap config`](docs/config.md) - Command used to set up the configuration files and the nemesis block for the current network +* [`symbol-bootstrap config`](docs/config.md) - Command used to set up the configuration files and the nemesis block for the current network. * [`symbol-bootstrap decrypt`](docs/decrypt.md) - It decrypts a yml file using the provided password. The source file can be a custom preset file, a preset.yml file or an addresses.yml. * [`symbol-bootstrap encrypt`](docs/encrypt.md) - It encrypts a yml file using the provided password. The source files would be a custom preset file, a preset.yml file or an addresses.yml. * [`symbol-bootstrap enrolRewardProgram`](docs/enrolRewardProgram.md) - It enrols the nodes in the rewards program by announcing the enrol transaction to the registration address. You can also use this command to update the program registration when you change the node public key or server host. diff --git a/docs/clean.md b/docs/clean.md index 03fedf992..3756af27a 100644 --- a/docs/clean.md +++ b/docs/clean.md @@ -3,6 +3,8 @@ It removes the target folder deleting the generated configuration and data +This is an OFFLINE command. + * [`symbol-bootstrap clean`](#symbol-bootstrap-clean) ## `symbol-bootstrap clean` @@ -17,6 +19,9 @@ OPTIONS -h, --help It shows the help of this command. -t, --target=target [default: target] The target folder where the symbol-bootstrap network is generated +DESCRIPTION + This is an OFFLINE command. + EXAMPLE $ symbol-bootstrap clean ``` diff --git a/docs/compose.md b/docs/compose.md index 468d7575f..03ec29179 100644 --- a/docs/compose.md +++ b/docs/compose.md @@ -3,6 +3,8 @@ It generates the `docker-compose.yml` file from the configured network. +This is an OFFLINE command. + * [`symbol-bootstrap compose`](#symbol-bootstrap-compose) ## `symbol-bootstrap compose` @@ -29,6 +31,9 @@ OPTIONS --upgrade It regenerates the docker compose and utility files from the /docker folder +DESCRIPTION + This is an OFFLINE command. + EXAMPLE $ symbol-bootstrap compose ``` diff --git a/docs/config.md b/docs/config.md index c7a7dfc20..1b94cb4d1 100644 --- a/docs/config.md +++ b/docs/config.md @@ -1,13 +1,19 @@ `symbol-bootstrap config` ========================= -Command used to set up the configuration files and the nemesis block for the current network +Command used to set up the configuration files and the nemesis block for the current network. + +This command is by default an ONLINE tool, as it may use docker to run some operations like nemesis or certificate generation. +It's possible to run this command in OFFLINE mode, without using docker, by providing the --offline param. + +Note: OFFLINE requires Linux/Mac OS, and the openssl command installed. If you are creating a new network (bootstrap preset), +the nemesis seed needs to be provided with a `nemesisSeedFolder` preset property. Nemesis generation is an online feature that requires catapult tools and docker. * [`symbol-bootstrap config`](#symbol-bootstrap-config) ## `symbol-bootstrap config` -Command used to set up the configuration files and the nemesis block for the current network +Command used to set up the configuration files and the nemesis block for the current network. ``` USAGE @@ -33,6 +39,9 @@ OPTIONS --noPassword When provided, Bootstrap will not use a password, so private keys will be stored in plain text. Use with caution. + --offline If --offline is used, Bootstrap rejects any offline operation when generating the + configuration. + --password=password A password used to encrypt and decrypt private keys in preset files like addresses.yml and preset.yml. Bootstrap prompts for a password by default, can be provided in the command line (--password=XXXX) or disabled in the command line @@ -49,8 +58,19 @@ OPTIONS local data. The original preset (-t), assembly (-a), and custom preset (-a) must be used. Backup the target folder before upgrading. +DESCRIPTION + This command is by default an ONLINE tool, as it may use docker to run some operations like nemesis or certificate + generation. + It's possible to run this command in OFFLINE mode, without using docker, by providing the --offline param. + + Note: OFFLINE requires Linux/Mac OS, and the openssl command installed. If you are creating a new network (bootstrap + preset), + the nemesis seed needs to be provided with a `nemesisSeedFolder` preset property. Nemesis generation is an online + feature that requires catapult tools and docker. + EXAMPLES $ symbol-bootstrap config -p bootstrap + $ symbol-bootstrap config -p testnet -a dual --customPreset my-encrypted-custom-preset.yml --offline $ symbol-bootstrap config -p testnet -a dual --password 1234 $ echo "$MY_ENV_VAR_PASSWORD" | symbol-bootstrap config -p testnet -a dual ``` diff --git a/docs/decrypt.md b/docs/decrypt.md index ce228c107..84fc6c7d2 100644 --- a/docs/decrypt.md +++ b/docs/decrypt.md @@ -5,6 +5,8 @@ It decrypts a yml file using the provided password. The source file can be a cus The main use case of this command is to verify private keys in encrypted files after encrypting a custom preset or running a bootstrap command with a provided --password. +This is an OFFLINE command. + * [`symbol-bootstrap decrypt`](#symbol-bootstrap-decrypt) ## `symbol-bootstrap decrypt` @@ -29,6 +31,8 @@ DESCRIPTION The main use case of this command is to verify private keys in encrypted files after encrypting a custom preset or running a bootstrap command with a provided --password. + This is an OFFLINE command. + EXAMPLES $ symbol-bootstrap start --password 1234 --preset testnet --assembly dual --customPreset decrypted-custom-preset.yml diff --git a/docs/encrypt.md b/docs/encrypt.md index 8a93e7f31..993b0273b 100644 --- a/docs/encrypt.md +++ b/docs/encrypt.md @@ -5,6 +5,8 @@ It encrypts a yml file using the provided password. The source files would be a The main use case of this command is encrypting custom presets files. If your custom preset contains private keys, it's highly recommended to encrypt it and use provide --password when starting or configuring the node with Bootstrap. +This is an OFFLINE command. + * [`symbol-bootstrap encrypt`](#symbol-bootstrap-encrypt) ## `symbol-bootstrap encrypt` @@ -30,6 +32,8 @@ DESCRIPTION The main use case of this command is encrypting custom presets files. If your custom preset contains private keys, it's highly recommended to encrypt it and use provide --password when starting or configuring the node with Bootstrap. + This is an OFFLINE command. + EXAMPLES $ symbol-bootstrap encrypt --source plain-custom-preset.yml --destination encrypted-custom-preset.yml diff --git a/docs/enrolRewardProgram.md b/docs/enrolRewardProgram.md index 44da2ccec..c204d3340 100644 --- a/docs/enrolRewardProgram.md +++ b/docs/enrolRewardProgram.md @@ -5,6 +5,8 @@ It enrols the nodes in the rewards program by announcing the enrol transaction t Currently, the only program that can be enrolled post-launch is 'SuperNode'. +This is an ONLINE command as it creates the transaction depending on the main account type (simple or multisig), and it announces the transaction to the network. + * [`symbol-bootstrap enrolRewardProgram`](#symbol-bootstrap-enrolrewardprogram) ## `symbol-bootstrap enrolRewardProgram` @@ -39,6 +41,9 @@ OPTIONS DESCRIPTION Currently, the only program that can be enrolled post-launch is 'SuperNode'. + This is an ONLINE command as it creates the transaction depending on the main account type (simple or multisig), and + it announces the transaction to the network. + EXAMPLES $ symbol-bootstrap enrolRewardProgram $ symbol-bootstrap enrolRewardProgram --noPassword diff --git a/docs/healthCheck.md b/docs/healthCheck.md index 78ef0e582..1695d666b 100644 --- a/docs/healthCheck.md +++ b/docs/healthCheck.md @@ -10,6 +10,8 @@ This command checks: The health check process handles 'repeat' and custom 'openPort' services. +Although it's highly possible your node is connected to the internet, this is an OFFLINE command as it health checks the local nodes and services. + * [`symbol-bootstrap healthCheck`](#symbol-bootstrap-healthcheck) ## `symbol-bootstrap healthCheck` @@ -32,6 +34,9 @@ DESCRIPTION The health check process handles 'repeat' and custom 'openPort' services. + Although it's highly possible your node is connected to the internet, this is an OFFLINE command as it health checks + the local nodes and services. + EXAMPLE $ symbol-bootstrap healthCheck ``` diff --git a/docs/link.md b/docs/link.md index 83b2b5544..ecfc36cef 100644 --- a/docs/link.md +++ b/docs/link.md @@ -3,6 +3,8 @@ It announces VRF and Voting Link transactions to the network for each node with 'Peer' or 'Voting' roles. This command finalizes the node registration to an existing network. +This is an ONLINE command as it creates the transactions depending on the main account type (simple or multisig), the main account link status, and it announces the transactions to the network. + * [`symbol-bootstrap link`](#symbol-bootstrap-link) ## `symbol-bootstrap link` @@ -36,6 +38,10 @@ OPTIONS --useKnownRestGateways Use the best NEM node available when announcing. Otherwise the command will use the node provided by the --url parameter. +DESCRIPTION + This is an ONLINE command as it creates the transactions depending on the main account type (simple or multisig), the + main account link status, and it announces the transactions to the network. + EXAMPLES $ symbol-bootstrap link $ echo "$MY_ENV_VAR_PASSWORD" | symbol-bootstrap link --unlink --useKnownRestGateways diff --git a/docs/report.md b/docs/report.md index f7a931a9d..6adc05dee 100644 --- a/docs/report.md +++ b/docs/report.md @@ -3,6 +3,8 @@ it generates reStructuredText (.rst) reports describing the configuration of each node. +This is an OFFLINE command. + * [`symbol-bootstrap report`](#symbol-bootstrap-report) ## `symbol-bootstrap report` @@ -17,6 +19,9 @@ OPTIONS -h, --help It shows the help of this command. -t, --target=target [default: target] The target folder where the symbol-bootstrap network is generated +DESCRIPTION + This is an OFFLINE command. + EXAMPLE $ symbol-bootstrap report ``` diff --git a/docs/resetData.md b/docs/resetData.md index 235ac5939..db94a2f06 100644 --- a/docs/resetData.md +++ b/docs/resetData.md @@ -3,6 +3,8 @@ It removes the data keeping the generated configuration, certificates, keys and block 1. +This is an OFFLINE command. + * [`symbol-bootstrap resetData`](#symbol-bootstrap-resetdata) ## `symbol-bootstrap resetData` @@ -17,6 +19,9 @@ OPTIONS -h, --help It shows the help of this command. -t, --target=target [default: target] The target folder where the symbol-bootstrap network is generated +DESCRIPTION + This is an OFFLINE command. + EXAMPLE $ symbol-bootstrap resetData ``` diff --git a/docs/run.md b/docs/run.md index 81728af3b..83dda34f7 100644 --- a/docs/run.md +++ b/docs/run.md @@ -3,6 +3,8 @@ It boots the network via docker using the generated `docker-compose.yml` file and configuration. The config and compose methods/commands need to be called before this method. This is just a wrapper for the `docker-compose up` bash call. +This is an ONLINE command. It uses docker, docker compose, and images must be pulled from the internet. It's expected for a mainnet/testnet node to be connected to the internet. + * [`symbol-bootstrap run`](#symbol-bootstrap-run) ## `symbol-bootstrap run` @@ -40,6 +42,9 @@ OPTIONS The health check process handles 'repeat' and custom 'openPort' services. + Although it's highly possible your node is connected to the internet, this is an OFFLINE command as it health checks + the local nodes and services. + --pullImages It pulls the images from DockerHub when running the configuration. It only affects alpha/dev docker images. @@ -49,6 +54,10 @@ OPTIONS --timeout=timeout [default: 60000] If running in detached mode, how long before timing out (in milliseconds) +DESCRIPTION + This is an ONLINE command. It uses docker, docker compose, and images must be pulled from the internet. It's expected + for a mainnet/testnet node to be connected to the internet. + EXAMPLE $ symbol-bootstrap run ``` diff --git a/docs/start.md b/docs/start.md index 3ed8075cc..4dcc5a947 100644 --- a/docs/start.md +++ b/docs/start.md @@ -3,6 +3,11 @@ Single command that aggregates config, compose and run in one line! +This is an ONLINE command. It uses docker, docker compose, and images must be pulled from the internet. It's expected for a mainnet/testnet node to be connected to the internet. + +Note: If you require your setup to be OFFLINE, you can run `config`, and `compose` in your offline machine, +zip and copy the target folder into the online machine, and execute `run` or `start` there. + * [`symbol-bootstrap start`](#symbol-bootstrap-start) ## `symbol-bootstrap start` @@ -56,9 +61,15 @@ OPTIONS The health check process handles 'repeat' and custom 'openPort' services. + Although it's highly possible your node is connected to the internet, this is an OFFLINE command as it health checks + the local nodes and services. + --noPassword When provided, Bootstrap will not use a password, so private keys will be stored in plain text. Use with caution. + --offline + If --offline is used, Bootstrap rejects any offline operation when generating the configuration. + --password=password A password used to encrypt and decrypt private keys in preset files like addresses.yml and preset.yml. Bootstrap prompts for a password by default, can be provided in the command line (--password=XXXX) or disabled in the command @@ -81,6 +92,13 @@ OPTIONS keep your node up to date without dropping the local data. The original preset (-t), assembly (-a), and custom preset (-a) must be used. Backup the target folder before upgrading. +DESCRIPTION + This is an ONLINE command. It uses docker, docker compose, and images must be pulled from the internet. It's expected + for a mainnet/testnet node to be connected to the internet. + + Note: If you require your setup to be OFFLINE, you can run `config`, and `compose` in your offline machine, + zip and copy the target folder into the online machine, and execute `run` or `start` there. + EXAMPLES $ symbol-bootstrap start $ symbol-bootstrap start -p bootstrap diff --git a/docs/stop.md b/docs/stop.md index 6a1042202..38f2b94a2 100644 --- a/docs/stop.md +++ b/docs/stop.md @@ -3,6 +3,8 @@ It stops the docker-compose network if running (symbol-bootstrap started with --detached). This is just a wrapper for the `docker-compose down` bash call. +Although it's highly possible your node is connected to the internet, this is an OFFLINE command as it just stops the docker services. + * [`symbol-bootstrap stop`](#symbol-bootstrap-stop) ## `symbol-bootstrap stop` @@ -17,6 +19,10 @@ OPTIONS -h, --help It shows the help of this command. -t, --target=target [default: target] The target folder where the symbol-bootstrap network is generated +DESCRIPTION + Although it's highly possible your node is connected to the internet, this is an OFFLINE command as it just stops the + docker services. + EXAMPLE $ symbol-bootstrap stop ``` diff --git a/package.json b/package.json index 84988b8cf..ac15460e6 100644 --- a/package.json +++ b/package.json @@ -103,6 +103,8 @@ "coveralls-report": "cat ./coverage/lcov.info | coveralls", "version": "echo $npm_package_version", "install-cli": "npm pack && npm i -g", + "publish": "aws s3 cp --recursive ./dist/symbol-bootstrap-* s3://symbol-bootstrap/dist/v$npm_package_version --acl public-read", + "dist": "oclif-dev pack", "clean-network": "symbol-bootstrap clean -t target/bootstrap-test", "start-network": "symbol-bootstrap start -t target/bootstrap-test -u 'current'" }, diff --git a/src/commands/clean.ts b/src/commands/clean.ts index 7fec7dbde..7da3a392c 100644 --- a/src/commands/clean.ts +++ b/src/commands/clean.ts @@ -19,7 +19,10 @@ import { BootstrapUtils } from '../service'; import { CommandUtils } from '../service/CommandUtils'; export default class Clean extends Command { - static description = 'It removes the target folder deleting the generated configuration and data'; + static description = `It removes the target folder deleting the generated configuration and data + +This is an OFFLINE command. + `; static examples = [`$ symbol-bootstrap clean`]; diff --git a/src/commands/compose.ts b/src/commands/compose.ts index 9b21ca1a5..fb23244ce 100644 --- a/src/commands/compose.ts +++ b/src/commands/compose.ts @@ -19,7 +19,10 @@ import { BootstrapService, BootstrapUtils, ComposeService } from '../service'; import { CommandUtils } from '../service/CommandUtils'; export default class Compose extends Command { - static description = 'It generates the `docker-compose.yml` file from the configured network.'; + static description = `It generates the \`docker-compose.yml\` file from the configured network. + +This is an OFFLINE command. +`; static examples = [`$ symbol-bootstrap compose`]; diff --git a/src/commands/config.ts b/src/commands/config.ts index 52a7bc42d..1cb43f6dc 100644 --- a/src/commands/config.ts +++ b/src/commands/config.ts @@ -19,10 +19,19 @@ import { BootstrapService, BootstrapUtils, ConfigService, Preset } from '../serv import { CommandUtils } from '../service/CommandUtils'; export default class Config extends Command { - static description = 'Command used to set up the configuration files and the nemesis block for the current network'; + static description = `Command used to set up the configuration files and the nemesis block for the current network. + +This command is by default an ONLINE tool, as it may use docker to run some operations like nemesis or certificate generation. +It's possible to run this command in OFFLINE mode, without using docker, by providing the --offline param. + +Note: OFFLINE requires Linux/Mac OS, and the openssl command installed. If you are creating a new network (bootstrap preset), +the nemesis seed needs to be provided with a \`nemesisSeedFolder\` preset property. Nemesis generation is an online feature that requires catapult tools and docker. + + `; static examples = [ `$ symbol-bootstrap config -p bootstrap`, + `$ symbol-bootstrap config -p testnet -a dual --customPreset my-encrypted-custom-preset.yml --offline`, `$ symbol-bootstrap config -p testnet -a dual --password 1234`, `$ echo "$MY_ENV_VAR_PASSWORD" | symbol-bootstrap config -p testnet -a dual`, ]; @@ -64,10 +73,15 @@ export default class Config extends Command { default: ConfigService.defaultParams.report, }), + offline: flags.boolean({ + description: 'If --offline is used, Bootstrap rejects any offline operation when generating the configuration.', + default: ConfigService.defaultParams.pullImages, + }), + pullImages: flags.boolean({ description: 'It pulls the utility images from DockerHub when running the configuration. It only affects alpha/dev docker images.', - default: ConfigService.defaultParams.pullImages, + default: ConfigService.defaultParams.offline, }), user: flags.string({ diff --git a/src/commands/decrypt.ts b/src/commands/decrypt.ts index c2fcb51e9..d0efc8e91 100644 --- a/src/commands/decrypt.ts +++ b/src/commands/decrypt.ts @@ -27,7 +27,10 @@ const logger: Logger = LoggerFactory.getLogger(LogType.System); export default class Decrypt extends Command { static description = `It decrypts a yml file using the provided password. The source file can be a custom preset file, a preset.yml file or an addresses.yml. -The main use case of this command is to verify private keys in encrypted files after encrypting a custom preset or running a bootstrap command with a provided --password.`; +The main use case of this command is to verify private keys in encrypted files after encrypting a custom preset or running a bootstrap command with a provided --password. + +This is an OFFLINE command. +`; static examples = [ ` diff --git a/src/commands/encrypt.ts b/src/commands/encrypt.ts index 0baec8d80..1abad44fc 100644 --- a/src/commands/encrypt.ts +++ b/src/commands/encrypt.ts @@ -28,7 +28,10 @@ const logger: Logger = LoggerFactory.getLogger(LogType.System); export default class Encrypt extends Command { static description = `It encrypts a yml file using the provided password. The source files would be a custom preset file, a preset.yml file or an addresses.yml. -The main use case of this command is encrypting custom presets files. If your custom preset contains private keys, it's highly recommended to encrypt it and use provide --password when starting or configuring the node with Bootstrap.`; +The main use case of this command is encrypting custom presets files. If your custom preset contains private keys, it's highly recommended to encrypt it and use provide --password when starting or configuring the node with Bootstrap. + +This is an OFFLINE command. +`; static examples = [ ` diff --git a/src/commands/enrolRewardProgram.ts b/src/commands/enrolRewardProgram.ts index ea0d6ae1b..eabc32983 100644 --- a/src/commands/enrolRewardProgram.ts +++ b/src/commands/enrolRewardProgram.ts @@ -22,7 +22,10 @@ import { CommandUtils } from '../service/CommandUtils'; export default class EnrolRewardProgram extends Command { static description = `It enrols the nodes in the rewards program by announcing the enrol transaction to the registration address. You can also use this command to update the program registration when you change the node public key or server host. -Currently, the only program that can be enrolled post-launch is 'SuperNode'.`; +Currently, the only program that can be enrolled post-launch is 'SuperNode'. + +This is an ONLINE command as it creates the transaction depending on the main account type (simple or multisig), and it announces the transaction to the network. +`; static examples = [ `$ symbol-bootstrap enrolRewardProgram`, diff --git a/src/commands/healthCheck.ts b/src/commands/healthCheck.ts index dd3708be5..e0612bc0d 100644 --- a/src/commands/healthCheck.ts +++ b/src/commands/healthCheck.ts @@ -27,6 +27,8 @@ This command checks: - Whether the rest gateways' /node/health are OK. The health check process handles 'repeat' and custom 'openPort' services. + +Although it's highly possible your node is connected to the internet, this is an OFFLINE command as it health checks the local nodes and services. `; static examples = [`$ symbol-bootstrap healthCheck`]; diff --git a/src/commands/link.ts b/src/commands/link.ts index 4174b2580..43929324e 100644 --- a/src/commands/link.ts +++ b/src/commands/link.ts @@ -20,7 +20,10 @@ import { AnnounceService } from '../service/AnnounceService'; import { CommandUtils } from '../service/CommandUtils'; export default class Link extends Command { - static description = `It announces VRF and Voting Link transactions to the network for each node with 'Peer' or 'Voting' roles. This command finalizes the node registration to an existing network.`; + static description = `It announces VRF and Voting Link transactions to the network for each node with 'Peer' or 'Voting' roles. This command finalizes the node registration to an existing network. + +This is an ONLINE command as it creates the transactions depending on the main account type (simple or multisig), the main account link status, and it announces the transactions to the network. +`; static examples = [`$ symbol-bootstrap link`, `$ echo "$MY_ENV_VAR_PASSWORD" | symbol-bootstrap link --unlink --useKnownRestGateways`]; diff --git a/src/commands/report.ts b/src/commands/report.ts index b008c9e33..64b22fbed 100644 --- a/src/commands/report.ts +++ b/src/commands/report.ts @@ -19,7 +19,10 @@ import { BootstrapService, BootstrapUtils } from '../service'; import { CommandUtils } from '../service/CommandUtils'; export default class Clean extends Command { - static description = 'it generates reStructuredText (.rst) reports describing the configuration of each node.'; + static description = `it generates reStructuredText (.rst) reports describing the configuration of each node. + +This is an OFFLINE command. +`; static examples = [`$ symbol-bootstrap report`]; diff --git a/src/commands/resetData.ts b/src/commands/resetData.ts index c981511a4..2331635a3 100644 --- a/src/commands/resetData.ts +++ b/src/commands/resetData.ts @@ -19,7 +19,10 @@ import { BootstrapService, BootstrapUtils } from '../service'; import { CommandUtils } from '../service/CommandUtils'; export default class ResetData extends Command { - static description = 'It removes the data keeping the generated configuration, certificates, keys and block 1.'; + static description = `It removes the data keeping the generated configuration, certificates, keys and block 1. + +This is an OFFLINE command. +`; static examples = [`$ symbol-bootstrap resetData`]; diff --git a/src/commands/run.ts b/src/commands/run.ts index 0ef870fcc..dd8e6a4cf 100644 --- a/src/commands/run.ts +++ b/src/commands/run.ts @@ -20,8 +20,10 @@ import { CommandUtils } from '../service/CommandUtils'; import HealthCheck from './healthCheck'; export default class Run extends Command { - static description = - 'It boots the network via docker using the generated `docker-compose.yml` file and configuration. The config and compose methods/commands need to be called before this method. This is just a wrapper for the `docker-compose up` bash call.'; + static description = `It boots the network via docker using the generated \`docker-compose.yml\` file and configuration. The config and compose methods/commands need to be called before this method. This is just a wrapper for the \`docker-compose up\` bash call. + +This is an ONLINE command. It uses docker, docker compose, and images must be pulled from the internet. It's expected for a mainnet/testnet node to be connected to the internet. +`; static examples = [`$ symbol-bootstrap run`]; diff --git a/src/commands/start.ts b/src/commands/start.ts index b45824676..11c163a3d 100644 --- a/src/commands/start.ts +++ b/src/commands/start.ts @@ -23,7 +23,13 @@ import Config from './config'; import Run from './run'; export default class Start extends Command { - static description = 'Single command that aggregates config, compose and run in one line!'; + static description = `Single command that aggregates config, compose and run in one line! + +This is an ONLINE command. It uses docker, docker compose, and images must be pulled from the internet. It's expected for a mainnet/testnet node to be connected to the internet. + +Note: If you require your setup to be OFFLINE, you can run \`config\`, and \`compose\` in your offline machine, +zip and copy the target folder into the online machine, and execute \`run\` or \`start\` there. +`; static examples = [ `$ symbol-bootstrap start`, diff --git a/src/commands/stop.ts b/src/commands/stop.ts index 28b910967..33496b389 100644 --- a/src/commands/stop.ts +++ b/src/commands/stop.ts @@ -19,8 +19,10 @@ import { BootstrapUtils, RunService } from '../service'; import { CommandUtils } from '../service/CommandUtils'; export default class Stop extends Command { - static description = - 'It stops the docker-compose network if running (symbol-bootstrap started with --detached). This is just a wrapper for the `docker-compose down` bash call.'; + static description = `It stops the docker-compose network if running (symbol-bootstrap started with --detached). This is just a wrapper for the \`docker-compose down\` bash call. + +Although it's highly possible your node is connected to the internet, this is an OFFLINE command as it just stops the docker services. +`; static examples = [`$ symbol-bootstrap stop`]; static flags = { diff --git a/src/service/AgentCertificateService.ts b/src/service/AgentCertificateService.ts index 469783a11..d59d7fb38 100644 --- a/src/service/AgentCertificateService.ts +++ b/src/service/AgentCertificateService.ts @@ -18,11 +18,12 @@ import { join, resolve } from 'path'; import { LogType } from '../logger'; import Logger from '../logger/Logger'; import LoggerFactory from '../logger/LoggerFactory'; -import { BootstrapUtils } from './BootstrapUtils'; +import { BootstrapUtils, KnownError } from './BootstrapUtils'; export interface AgentCertificateParams { readonly target: string; readonly user: string; + readonly offline?: boolean; } const logger: Logger = LoggerFactory.getLogger(LogType.System); @@ -49,6 +50,11 @@ export class AgentCertificateService { throw new Error('Certificate creation failed. Check the logs!'); } } catch (e) { + if (this.params.offline) { + throw new KnownError( + `Agent Certificates could not be created using local openssl. Do you have openssl installed? Remove --offline to fallback to docker. Error: ${e.message}`, + ); + } logger.warn('Agent certificate could not be created using local openssl. Falling back to docker run...'); const binds = [`${resolve(certFolder)}:/data:rw`]; const userId = await BootstrapUtils.resolveDockerUserFromParam(this.params.user); diff --git a/src/service/CertificateService.ts b/src/service/CertificateService.ts index 05512895c..b9fc5cde3 100644 --- a/src/service/CertificateService.ts +++ b/src/service/CertificateService.ts @@ -20,13 +20,14 @@ import { LogType } from '../logger'; import Logger from '../logger/Logger'; import LoggerFactory from '../logger/LoggerFactory'; import { CertificatePair } from '../model'; -import { BootstrapUtils, ExecOutput } from './BootstrapUtils'; +import { BootstrapUtils, ExecOutput, KnownError } from './BootstrapUtils'; import { CommandUtils } from './CommandUtils'; import { KeyName } from './ConfigService'; export interface CertificateParams { readonly target: string; readonly user: string; + readonly offline?: boolean; } export interface CertificateMetadata { @@ -131,29 +132,43 @@ export class CertificateService { KeyName.Transport, name, ); - BootstrapUtils.createDerFile(mainAccountPrivateKey, join(certFolder, 'ca.der')); - BootstrapUtils.createDerFile(transportPrivateKey, join(certFolder, 'node.der')); + const caDerFile = join(certFolder, 'ca.der'); + const nodeDerFile = join(certFolder, 'node.der'); try { - logger.info('Creating Node Certificates using exec and local openssl'); - const execOutput = await BootstrapUtils.exec(cmd.join(' '), { - cwd: certFolder, + BootstrapUtils.createDerFile(mainAccountPrivateKey, caDerFile); + BootstrapUtils.createDerFile(transportPrivateKey, nodeDerFile); + try { + logger.info('Creating Node Certificates using exec and local openssl'); + const execOutput = await BootstrapUtils.exec(cmd.join(' '), { + cwd: certFolder, + }); + await this.validateOutput(execOutput, providedCertificates, mainAccountPrivateKey, transportPrivateKey); + return execOutput; + } catch (e) { + if (this.params.offline) { + throw new KnownError( + `Node Certificates could not be created using local openssl. Do you have openssl installed? Remove --offline to fallback to docker. Error: ${e.message}`, + ); + } + logger.warn('Node Certificates could not be created using local openssl. Falling back to docker run...'); + } + const binds = [`${resolve(certFolder)}:/data:rw`]; + const userId = await BootstrapUtils.resolveDockerUserFromParam(this.params.user); + const execOutput = await BootstrapUtils.runImageUsingExec({ + image: symbolServerToolsImage, + userId: userId, + workdir: '/data', + cmds: cmd, + binds: binds, }); await this.validateOutput(execOutput, providedCertificates, mainAccountPrivateKey, transportPrivateKey); return execOutput; - } catch (e) { - logger.warn('Node Certificates could not be created using local openssl. Falling back to docker run...'); + } finally { + //To be sure we are not keeping any private file + BootstrapUtils.deleteFile(join(certFolder, 'ca.key.pem')); + BootstrapUtils.deleteFile(caDerFile); + BootstrapUtils.deleteFile(nodeDerFile); } - const binds = [`${resolve(certFolder)}:/data:rw`]; - const userId = await BootstrapUtils.resolveDockerUserFromParam(this.params.user); - const execOutput = await BootstrapUtils.runImageUsingExec({ - image: symbolServerToolsImage, - userId: userId, - workdir: '/data', - cmds: cmd, - binds: binds, - }); - await this.validateOutput(execOutput, providedCertificates, mainAccountPrivateKey, transportPrivateKey); - return execOutput; } private async validateOutput( diff --git a/src/service/ConfigService.ts b/src/service/ConfigService.ts index ee20d7ac9..60a73aed6 100644 --- a/src/service/ConfigService.ts +++ b/src/service/ConfigService.ts @@ -70,6 +70,7 @@ export interface ConfigParams { target: string; password?: string; user: string; + offline?: boolean; pullImages?: boolean; assembly?: string; customPreset?: string; @@ -89,6 +90,7 @@ export class ConfigService { report: false, preset: Preset.bootstrap, reset: false, + offline: false, upgrade: false, pullImages: false, user: BootstrapUtils.CURRENT_USER, diff --git a/src/service/NemgenService.ts b/src/service/NemgenService.ts index 94e054685..822008522 100644 --- a/src/service/NemgenService.ts +++ b/src/service/NemgenService.ts @@ -16,11 +16,11 @@ import { promises } from 'fs'; import { join } from 'path'; +import { LogType } from '../logger'; import Logger from '../logger/Logger'; import LoggerFactory from '../logger/LoggerFactory'; -import { LogType } from '../logger/LogType'; import { ConfigPreset } from '../model'; -import { BootstrapUtils } from './BootstrapUtils'; +import { BootstrapUtils, KnownError } from './BootstrapUtils'; import { ConfigParams } from './ConfigService'; type NemgenParams = ConfigParams; @@ -34,7 +34,9 @@ export class NemgenService { const networkIdentifier = presetData.networkIdentifier; const symbolServerToolsImage = presetData.symbolServerToolsImage; const target = this.params.target; - + if (this.params.offline) { + throw new KnownError(`Nemesis generation is not an offline feature. It requires catapult.tools.nemgen via Docker images.`); + } if (!presetData.nodes || !presetData.nodes.length) { throw new Error('Nodes must be defined in preset when running nemgen'); } From b213bba04d9eb508827fe612601abc30232d6025 Mon Sep 17 00:00:00 2001 From: fernando Date: Wed, 3 Mar 2021 08:29:53 -0300 Subject: [PATCH 4/6] removed pullImages from config --- docs/config.md | 3 --- docs/run.md | 2 +- src/commands/config.ts | 6 ------ src/commands/run.ts | 3 ++- src/service/ConfigService.ts | 3 --- 5 files changed, 3 insertions(+), 14 deletions(-) diff --git a/docs/config.md b/docs/config.md index 1b94cb4d1..1e25d0bb9 100644 --- a/docs/config.md +++ b/docs/config.md @@ -47,9 +47,6 @@ OPTIONS provided in the command line (--password=XXXX) or disabled in the command line (--noPassword). - --pullImages It pulls the utility images from DockerHub when running the configuration. It only - affects alpha/dev docker images. - --report It generates reStructuredText (.rst) reports describing the configuration of each node. diff --git a/docs/run.md b/docs/run.md index 83dda34f7..bc24c7222 100644 --- a/docs/run.md +++ b/docs/run.md @@ -46,7 +46,7 @@ OPTIONS the local nodes and services. --pullImages - It pulls the images from DockerHub when running the configuration. It only affects alpha/dev docker images. + It pulls the utility images from DockerHub when running the configuration. It only affects alpha/dev docker images. --resetData It reset the database and node data but keeps the generated configuration, keys, voting tree files and block 1 diff --git a/src/commands/config.ts b/src/commands/config.ts index 1cb43f6dc..ac824609f 100644 --- a/src/commands/config.ts +++ b/src/commands/config.ts @@ -75,12 +75,6 @@ the nemesis seed needs to be provided with a \`nemesisSeedFolder\` preset proper offline: flags.boolean({ description: 'If --offline is used, Bootstrap rejects any offline operation when generating the configuration.', - default: ConfigService.defaultParams.pullImages, - }), - - pullImages: flags.boolean({ - description: - 'It pulls the utility images from DockerHub when running the configuration. It only affects alpha/dev docker images.', default: ConfigService.defaultParams.offline, }), diff --git a/src/commands/run.ts b/src/commands/run.ts index dd8e6a4cf..28a588425 100644 --- a/src/commands/run.ts +++ b/src/commands/run.ts @@ -45,7 +45,8 @@ This is an ONLINE command. It uses docker, docker compose, and images must be pu }), pullImages: flags.boolean({ - description: 'It pulls the images from DockerHub when running the configuration. It only affects alpha/dev docker images.', + description: + 'It pulls the utility images from DockerHub when running the configuration. It only affects alpha/dev docker images.', default: RunService.defaultParams.pullImages, }), diff --git a/src/service/ConfigService.ts b/src/service/ConfigService.ts index 60a73aed6..615908201 100644 --- a/src/service/ConfigService.ts +++ b/src/service/ConfigService.ts @@ -71,7 +71,6 @@ export interface ConfigParams { password?: string; user: string; offline?: boolean; - pullImages?: boolean; assembly?: string; customPreset?: string; customPresetObject?: any; @@ -92,7 +91,6 @@ export class ConfigService { reset: false, offline: false, upgrade: false, - pullImages: false, user: BootstrapUtils.CURRENT_USER, }; private readonly configLoader: ConfigLoader; @@ -140,7 +138,6 @@ export class ConfigService { const presetData: ConfigPreset = this.resolveCurrentPresetData(oldPresetData, password); const addresses = await this.configLoader.generateRandomConfiguration(oldAddresses, presetData); - if (this.params.pullImages) await BootstrapUtils.pullImage(presetData.symbolServerToolsImage); const privateKeySecurityMode = CryptoUtils.getPrivateKeySecurityMode(presetData.privateKeySecurityMode); await BootstrapUtils.mkdir(target); From c6d64f80d1fa2910c6db22867cd7720a22700654 Mon Sep 17 00:00:00 2001 From: fernando Date: Thu, 4 Mar 2021 06:44:41 -0300 Subject: [PATCH 5/6] Feedback fixes --- docs/clean.md | 4 ++-- docs/compose.md | 4 ++-- docs/config.md | 6 +++--- docs/decrypt.md | 4 ++-- docs/encrypt.md | 4 ++-- docs/report.md | 4 ++-- docs/resetData.md | 4 ++-- docs/run.md | 6 +++--- docs/start.md | 12 ++++++------ docs/stop.md | 5 ++--- src/commands/clean.ts | 2 +- src/commands/compose.ts | 2 +- src/commands/config.ts | 4 ++-- src/commands/decrypt.ts | 2 +- src/commands/encrypt.ts | 2 +- src/commands/report.ts | 2 +- src/commands/resetData.ts | 2 +- src/commands/run.ts | 2 +- src/commands/start.ts | 4 ++-- src/commands/stop.ts | 2 +- src/model/ConfigPreset.ts | 14 +++++++++++++- 21 files changed, 51 insertions(+), 40 deletions(-) diff --git a/docs/clean.md b/docs/clean.md index 3756af27a..1bcae5ae3 100644 --- a/docs/clean.md +++ b/docs/clean.md @@ -3,7 +3,7 @@ It removes the target folder deleting the generated configuration and data -This is an OFFLINE command. +This command can be used in OFFLINE mode. * [`symbol-bootstrap clean`](#symbol-bootstrap-clean) @@ -20,7 +20,7 @@ OPTIONS -t, --target=target [default: target] The target folder where the symbol-bootstrap network is generated DESCRIPTION - This is an OFFLINE command. + This command can be used in OFFLINE mode. EXAMPLE $ symbol-bootstrap clean diff --git a/docs/compose.md b/docs/compose.md index 03ec29179..7fcd42eb9 100644 --- a/docs/compose.md +++ b/docs/compose.md @@ -3,7 +3,7 @@ It generates the `docker-compose.yml` file from the configured network. -This is an OFFLINE command. +This command can be used in OFFLINE mode. * [`symbol-bootstrap compose`](#symbol-bootstrap-compose) @@ -32,7 +32,7 @@ OPTIONS --upgrade It regenerates the docker compose and utility files from the /docker folder DESCRIPTION - This is an OFFLINE command. + This command can be used in OFFLINE mode. EXAMPLE $ symbol-bootstrap compose diff --git a/docs/config.md b/docs/config.md index 1e25d0bb9..e1caf7520 100644 --- a/docs/config.md +++ b/docs/config.md @@ -4,7 +4,7 @@ Command used to set up the configuration files and the nemesis block for the current network. This command is by default an ONLINE tool, as it may use docker to run some operations like nemesis or certificate generation. -It's possible to run this command in OFFLINE mode, without using docker, by providing the --offline param. +It's possible to run this command in OFFLINE mode, without using docker, by providing the --offline parameter. Note: OFFLINE requires Linux/Mac OS, and the openssl command installed. If you are creating a new network (bootstrap preset), the nemesis seed needs to be provided with a `nemesisSeedFolder` preset property. Nemesis generation is an online feature that requires catapult tools and docker. @@ -39,7 +39,7 @@ OPTIONS --noPassword When provided, Bootstrap will not use a password, so private keys will be stored in plain text. Use with caution. - --offline If --offline is used, Bootstrap rejects any offline operation when generating the + --offline If --offline is used, Bootstrap rejects any online operation when generating the configuration. --password=password A password used to encrypt and decrypt private keys in preset files like @@ -58,7 +58,7 @@ OPTIONS DESCRIPTION This command is by default an ONLINE tool, as it may use docker to run some operations like nemesis or certificate generation. - It's possible to run this command in OFFLINE mode, without using docker, by providing the --offline param. + It's possible to run this command in OFFLINE mode, without using docker, by providing the --offline parameter. Note: OFFLINE requires Linux/Mac OS, and the openssl command installed. If you are creating a new network (bootstrap preset), diff --git a/docs/decrypt.md b/docs/decrypt.md index 84fc6c7d2..d27f21a85 100644 --- a/docs/decrypt.md +++ b/docs/decrypt.md @@ -5,7 +5,7 @@ It decrypts a yml file using the provided password. The source file can be a cus The main use case of this command is to verify private keys in encrypted files after encrypting a custom preset or running a bootstrap command with a provided --password. -This is an OFFLINE command. +This command can be used in OFFLINE mode. * [`symbol-bootstrap decrypt`](#symbol-bootstrap-decrypt) @@ -31,7 +31,7 @@ DESCRIPTION The main use case of this command is to verify private keys in encrypted files after encrypting a custom preset or running a bootstrap command with a provided --password. - This is an OFFLINE command. + This command can be used in OFFLINE mode. EXAMPLES diff --git a/docs/encrypt.md b/docs/encrypt.md index 993b0273b..89ecd7ac4 100644 --- a/docs/encrypt.md +++ b/docs/encrypt.md @@ -5,7 +5,7 @@ It encrypts a yml file using the provided password. The source files would be a The main use case of this command is encrypting custom presets files. If your custom preset contains private keys, it's highly recommended to encrypt it and use provide --password when starting or configuring the node with Bootstrap. -This is an OFFLINE command. +This command can be used in OFFLINE mode. * [`symbol-bootstrap encrypt`](#symbol-bootstrap-encrypt) @@ -32,7 +32,7 @@ DESCRIPTION The main use case of this command is encrypting custom presets files. If your custom preset contains private keys, it's highly recommended to encrypt it and use provide --password when starting or configuring the node with Bootstrap. - This is an OFFLINE command. + This command can be used in OFFLINE mode. EXAMPLES diff --git a/docs/report.md b/docs/report.md index 6adc05dee..d6f49d34c 100644 --- a/docs/report.md +++ b/docs/report.md @@ -3,7 +3,7 @@ it generates reStructuredText (.rst) reports describing the configuration of each node. -This is an OFFLINE command. +This command can be used in OFFLINE mode. * [`symbol-bootstrap report`](#symbol-bootstrap-report) @@ -20,7 +20,7 @@ OPTIONS -t, --target=target [default: target] The target folder where the symbol-bootstrap network is generated DESCRIPTION - This is an OFFLINE command. + This command can be used in OFFLINE mode. EXAMPLE $ symbol-bootstrap report diff --git a/docs/resetData.md b/docs/resetData.md index db94a2f06..3285caea2 100644 --- a/docs/resetData.md +++ b/docs/resetData.md @@ -3,7 +3,7 @@ It removes the data keeping the generated configuration, certificates, keys and block 1. -This is an OFFLINE command. +This command can be used in OFFLINE mode. * [`symbol-bootstrap resetData`](#symbol-bootstrap-resetdata) @@ -20,7 +20,7 @@ OPTIONS -t, --target=target [default: target] The target folder where the symbol-bootstrap network is generated DESCRIPTION - This is an OFFLINE command. + This command can be used in OFFLINE mode. EXAMPLE $ symbol-bootstrap resetData diff --git a/docs/run.md b/docs/run.md index bc24c7222..e08183780 100644 --- a/docs/run.md +++ b/docs/run.md @@ -3,7 +3,7 @@ It boots the network via docker using the generated `docker-compose.yml` file and configuration. The config and compose methods/commands need to be called before this method. This is just a wrapper for the `docker-compose up` bash call. -This is an ONLINE command. It uses docker, docker compose, and images must be pulled from the internet. It's expected for a mainnet/testnet node to be connected to the internet. +This command can only be used in ONLINE mode. It uses docker, docker compose, and images must be pulled from the internet. It's expected for a mainnet/testnet node to be connected to the internet. * [`symbol-bootstrap run`](#symbol-bootstrap-run) @@ -55,8 +55,8 @@ OPTIONS [default: 60000] If running in detached mode, how long before timing out (in milliseconds) DESCRIPTION - This is an ONLINE command. It uses docker, docker compose, and images must be pulled from the internet. It's expected - for a mainnet/testnet node to be connected to the internet. + This command can only be used in ONLINE mode. It uses docker, docker compose, and images must be pulled from the + internet. It's expected for a mainnet/testnet node to be connected to the internet. EXAMPLE $ symbol-bootstrap run diff --git a/docs/start.md b/docs/start.md index 4dcc5a947..8441eae72 100644 --- a/docs/start.md +++ b/docs/start.md @@ -3,9 +3,9 @@ Single command that aggregates config, compose and run in one line! -This is an ONLINE command. It uses docker, docker compose, and images must be pulled from the internet. It's expected for a mainnet/testnet node to be connected to the internet. +This command is typically run in ONLINE mode. It uses docker, docker compose, and images must be pulled from the internet. It's expected for a mainnet/testnet node to be connected to the internet. -Note: If you require your setup to be OFFLINE, you can run `config`, and `compose` in your offline machine, +However, if you require your setup to be OFFLINE, you can run `config`, and `compose` in your offline machine, zip and copy the target folder into the online machine, and execute `run` or `start` there. * [`symbol-bootstrap start`](#symbol-bootstrap-start) @@ -68,7 +68,7 @@ OPTIONS When provided, Bootstrap will not use a password, so private keys will be stored in plain text. Use with caution. --offline - If --offline is used, Bootstrap rejects any offline operation when generating the configuration. + If --offline is used, Bootstrap rejects any online operation when generating the configuration. --password=password A password used to encrypt and decrypt private keys in preset files like addresses.yml and preset.yml. Bootstrap @@ -93,10 +93,10 @@ OPTIONS preset (-a) must be used. Backup the target folder before upgrading. DESCRIPTION - This is an ONLINE command. It uses docker, docker compose, and images must be pulled from the internet. It's expected - for a mainnet/testnet node to be connected to the internet. + This command is typically run in ONLINE mode. It uses docker, docker compose, and images must be pulled from the + internet. It's expected for a mainnet/testnet node to be connected to the internet. - Note: If you require your setup to be OFFLINE, you can run `config`, and `compose` in your offline machine, + However, if you require your setup to be OFFLINE, you can run `config`, and `compose` in your offline machine, zip and copy the target folder into the online machine, and execute `run` or `start` there. EXAMPLES diff --git a/docs/stop.md b/docs/stop.md index 38f2b94a2..e06cb7041 100644 --- a/docs/stop.md +++ b/docs/stop.md @@ -3,7 +3,7 @@ It stops the docker-compose network if running (symbol-bootstrap started with --detached). This is just a wrapper for the `docker-compose down` bash call. -Although it's highly possible your node is connected to the internet, this is an OFFLINE command as it just stops the docker services. +This command can be run in OFFLINE mode, since it just stops the docker services. * [`symbol-bootstrap stop`](#symbol-bootstrap-stop) @@ -20,8 +20,7 @@ OPTIONS -t, --target=target [default: target] The target folder where the symbol-bootstrap network is generated DESCRIPTION - Although it's highly possible your node is connected to the internet, this is an OFFLINE command as it just stops the - docker services. + This command can be run in OFFLINE mode, since it just stops the docker services. EXAMPLE $ symbol-bootstrap stop diff --git a/src/commands/clean.ts b/src/commands/clean.ts index 7da3a392c..a267a7927 100644 --- a/src/commands/clean.ts +++ b/src/commands/clean.ts @@ -21,7 +21,7 @@ import { CommandUtils } from '../service/CommandUtils'; export default class Clean extends Command { static description = `It removes the target folder deleting the generated configuration and data -This is an OFFLINE command. +This command can be used in OFFLINE mode. `; static examples = [`$ symbol-bootstrap clean`]; diff --git a/src/commands/compose.ts b/src/commands/compose.ts index fb23244ce..016d83138 100644 --- a/src/commands/compose.ts +++ b/src/commands/compose.ts @@ -21,7 +21,7 @@ import { CommandUtils } from '../service/CommandUtils'; export default class Compose extends Command { static description = `It generates the \`docker-compose.yml\` file from the configured network. -This is an OFFLINE command. +This command can be used in OFFLINE mode. `; static examples = [`$ symbol-bootstrap compose`]; diff --git a/src/commands/config.ts b/src/commands/config.ts index ac824609f..eaf659ba6 100644 --- a/src/commands/config.ts +++ b/src/commands/config.ts @@ -22,7 +22,7 @@ export default class Config extends Command { static description = `Command used to set up the configuration files and the nemesis block for the current network. This command is by default an ONLINE tool, as it may use docker to run some operations like nemesis or certificate generation. -It's possible to run this command in OFFLINE mode, without using docker, by providing the --offline param. +It's possible to run this command in OFFLINE mode, without using docker, by providing the --offline parameter. Note: OFFLINE requires Linux/Mac OS, and the openssl command installed. If you are creating a new network (bootstrap preset), the nemesis seed needs to be provided with a \`nemesisSeedFolder\` preset property. Nemesis generation is an online feature that requires catapult tools and docker. @@ -74,7 +74,7 @@ the nemesis seed needs to be provided with a \`nemesisSeedFolder\` preset proper }), offline: flags.boolean({ - description: 'If --offline is used, Bootstrap rejects any offline operation when generating the configuration.', + description: 'If --offline is used, Bootstrap rejects any online operation when generating the configuration.', default: ConfigService.defaultParams.offline, }), diff --git a/src/commands/decrypt.ts b/src/commands/decrypt.ts index d0efc8e91..db9f850fb 100644 --- a/src/commands/decrypt.ts +++ b/src/commands/decrypt.ts @@ -29,7 +29,7 @@ export default class Decrypt extends Command { The main use case of this command is to verify private keys in encrypted files after encrypting a custom preset or running a bootstrap command with a provided --password. -This is an OFFLINE command. +This command can be used in OFFLINE mode. `; static examples = [ diff --git a/src/commands/encrypt.ts b/src/commands/encrypt.ts index 1abad44fc..9cf1da51d 100644 --- a/src/commands/encrypt.ts +++ b/src/commands/encrypt.ts @@ -30,7 +30,7 @@ export default class Encrypt extends Command { The main use case of this command is encrypting custom presets files. If your custom preset contains private keys, it's highly recommended to encrypt it and use provide --password when starting or configuring the node with Bootstrap. -This is an OFFLINE command. +This command can be used in OFFLINE mode. `; static examples = [ diff --git a/src/commands/report.ts b/src/commands/report.ts index 64b22fbed..380873ee0 100644 --- a/src/commands/report.ts +++ b/src/commands/report.ts @@ -21,7 +21,7 @@ import { CommandUtils } from '../service/CommandUtils'; export default class Clean extends Command { static description = `it generates reStructuredText (.rst) reports describing the configuration of each node. -This is an OFFLINE command. +This command can be used in OFFLINE mode. `; static examples = [`$ symbol-bootstrap report`]; diff --git a/src/commands/resetData.ts b/src/commands/resetData.ts index 2331635a3..d69e9937d 100644 --- a/src/commands/resetData.ts +++ b/src/commands/resetData.ts @@ -21,7 +21,7 @@ import { CommandUtils } from '../service/CommandUtils'; export default class ResetData extends Command { static description = `It removes the data keeping the generated configuration, certificates, keys and block 1. -This is an OFFLINE command. +This command can be used in OFFLINE mode. `; static examples = [`$ symbol-bootstrap resetData`]; diff --git a/src/commands/run.ts b/src/commands/run.ts index 28a588425..e7060272c 100644 --- a/src/commands/run.ts +++ b/src/commands/run.ts @@ -22,7 +22,7 @@ import HealthCheck from './healthCheck'; export default class Run extends Command { static description = `It boots the network via docker using the generated \`docker-compose.yml\` file and configuration. The config and compose methods/commands need to be called before this method. This is just a wrapper for the \`docker-compose up\` bash call. -This is an ONLINE command. It uses docker, docker compose, and images must be pulled from the internet. It's expected for a mainnet/testnet node to be connected to the internet. +This command can only be used in ONLINE mode. It uses docker, docker compose, and images must be pulled from the internet. It's expected for a mainnet/testnet node to be connected to the internet. `; static examples = [`$ symbol-bootstrap run`]; diff --git a/src/commands/start.ts b/src/commands/start.ts index 11c163a3d..6ad0b0350 100644 --- a/src/commands/start.ts +++ b/src/commands/start.ts @@ -25,9 +25,9 @@ import Run from './run'; export default class Start extends Command { static description = `Single command that aggregates config, compose and run in one line! -This is an ONLINE command. It uses docker, docker compose, and images must be pulled from the internet. It's expected for a mainnet/testnet node to be connected to the internet. +This command is typically run in ONLINE mode. It uses docker, docker compose, and images must be pulled from the internet. It's expected for a mainnet/testnet node to be connected to the internet. -Note: If you require your setup to be OFFLINE, you can run \`config\`, and \`compose\` in your offline machine, +However, if you require your setup to be OFFLINE, you can run \`config\`, and \`compose\` in your offline machine, zip and copy the target folder into the online machine, and execute \`run\` or \`start\` there. `; diff --git a/src/commands/stop.ts b/src/commands/stop.ts index 33496b389..be4556c7e 100644 --- a/src/commands/stop.ts +++ b/src/commands/stop.ts @@ -21,7 +21,7 @@ import { CommandUtils } from '../service/CommandUtils'; export default class Stop extends Command { static description = `It stops the docker-compose network if running (symbol-bootstrap started with --detached). This is just a wrapper for the \`docker-compose down\` bash call. -Although it's highly possible your node is connected to the internet, this is an OFFLINE command as it just stops the docker services. +This command can be run in OFFLINE mode, since it just stops the docker services. `; static examples = [`$ symbol-bootstrap stop`]; diff --git a/src/model/ConfigPreset.ts b/src/model/ConfigPreset.ts index 15d0342a0..9fc6fbba6 100644 --- a/src/model/ConfigPreset.ts +++ b/src/model/ConfigPreset.ts @@ -153,6 +153,18 @@ export interface FaucetPreset extends DockerServicePreset { name: string; } +export interface PeerInfo { + publicKey: string; + endpoint: { + host: string; + }; + port: number; + metadata: { + name: string; + roles: string; + }; +} + export interface ConfigPreset { preset: Preset; privateKeySecurityMode: string; @@ -191,7 +203,7 @@ export interface ConfigPreset { harvestingMosaicId: string; baseNamespace: string; databases?: DatabasePreset[]; - knownPeers?: Record; + knownPeers?: Record; knownRestGateways?: string[]; mongoComposeRunParam: string; mongoImage: string; From dbcdeb6aa5d34cca8aa8a2fa1f227733b52d9d87 Mon Sep 17 00:00:00 2001 From: fernando Date: Thu, 4 Mar 2021 06:55:11 -0300 Subject: [PATCH 6/6] improved types --- src/model/ConfigPreset.ts | 2 +- src/service/ConfigService.ts | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/model/ConfigPreset.ts b/src/model/ConfigPreset.ts index 9fc6fbba6..d40d8e7f5 100644 --- a/src/model/ConfigPreset.ts +++ b/src/model/ConfigPreset.ts @@ -157,8 +157,8 @@ export interface PeerInfo { publicKey: string; endpoint: { host: string; + port: number; }; - port: number; metadata: { name: string; roles: string; diff --git a/src/service/ConfigService.ts b/src/service/ConfigService.ts index f2ed43e86..face73cf3 100644 --- a/src/service/ConfigService.ts +++ b/src/service/ConfigService.ts @@ -32,7 +32,7 @@ import { import { LogType } from '../logger'; import Logger from '../logger/Logger'; import LoggerFactory from '../logger/LoggerFactory'; -import { Addresses, ConfigPreset, NodeAccount, NodePreset, NodeType } from '../model'; +import { Addresses, ConfigPreset, NodeAccount, NodePreset, NodeType, PeerInfo } from '../model'; import { AgentCertificateService } from './AgentCertificateService'; import { BootstrapUtils, KnownError } from './BootstrapUtils'; import { CertificateService } from './CertificateService'; @@ -397,7 +397,7 @@ export class ConfigService { nodePresetDataFunction: (nodePresetData: NodePreset) => boolean, jsonFileName: string, ) { - const thisNetworkKnownPeers = (presetData.nodes || []) + const thisNetworkKnownPeers: PeerInfo[] = (presetData.nodes || []) .map((nodePresetData, index) => { if (!nodePresetDataFunction(nodePresetData)) { return undefined; @@ -415,7 +415,8 @@ export class ConfigService { }, }; }) - .filter((i) => i); + .filter((i) => i) + .map((i) => i as PeerInfo); const globalKnownPeers = presetData.knownPeers?.[type] || []; const data = { _info: `this file contains a list of ${type} peers`,