From 4e1b5840ebc24fd4d22cf7c092a57976924054de Mon Sep 17 00:00:00 2001 From: fernando Date: Thu, 14 Jan 2021 00:26:46 -0300 Subject: [PATCH] Backup Sync Added --backup-sync to config command. This command downloads a known location via s3. First example back is https://symbol-bootstrap.s3-eu-west-1.amazonaws.com/testnet/testnet-partial-backup.zip --- .gitignore | 1 + CHANGELOG.md | 17 +- README.md | 1 + cmds/backup-full-dual.sh | 13 ++ cmds/backup-full-sync-testnet-backup.sh | 11 + cmds/backup-minimal-sync-testnet-backup.sh | 13 ++ cmds/backup-testnet-dual.sh | 3 + cmds/start-testnet-dual.sh | 2 +- cmds/start-testnet-voting.sh | 2 +- docs/backup.md | 36 ++++ docs/config.md | 6 + docs/start.md | 6 + package-lock.json | 229 ++++++++++++++++++++- package.json | 3 + presets/testnet/network.yml | 1 + src/commands/backup.ts | 42 ++++ src/commands/config.ts | 7 + src/model/ConfigPreset.ts | 2 + src/service/BackupSyncService.ts | 221 ++++++++++++++++++++ src/service/BootstrapService.ts | 13 ++ src/service/BootstrapUtils.ts | 4 +- src/service/ConfigService.ts | 40 +++- test/service/BackupSyncService.test.ts | 79 +++++++ test/testnet-custom-preset.yml | 2 + test/voting_preset.yml | 2 + 25 files changed, 733 insertions(+), 23 deletions(-) create mode 100644 cmds/backup-full-dual.sh create mode 100755 cmds/backup-full-sync-testnet-backup.sh create mode 100755 cmds/backup-minimal-sync-testnet-backup.sh create mode 100755 cmds/backup-testnet-dual.sh create mode 100644 docs/backup.md create mode 100644 src/commands/backup.ts create mode 100644 src/service/BackupSyncService.ts create mode 100644 test/service/BackupSyncService.test.ts create mode 100644 test/testnet-custom-preset.yml diff --git a/.gitignore b/.gitignore index 427f595ac..a4254d96b 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,4 @@ coverage /agent-linux.bin /privateSeeds/ /test/service/logs.log +/backup-sync/ diff --git a/CHANGELOG.md b/CHANGELOG.md index f27fbd125..6b773eaaf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,18 @@ All notable changes to this project will be documented in this file. The changelog format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). -## [0.4.2] - Feb-2-2020 +## [0.4.3] - NEXT + +**Milestone**: Hippopotamus(0.10.0.6) + +| Package | Version | Link | +| ---------------- | ------- | ------------------------------------------------------------------ | +| Symbol Bootstrap | v0.4.3 | [symbol-bootstrap](https://www.npmjs.com/package/symbol-bootstrap) | + +- Added `--backupSync` to `config` and `start` commands. It downloads a backup with the Mongo and RocksDb databases for faster synchronization. +- Added `backup` command. The command backups the Mongo and RocksDb data folder into a Zip file that can be used for `--backupSync` feature. + +## [0.4.2] - Feb-2-2021 **Milestone**: Hippopotamus(0.10.0.6) @@ -19,7 +30,7 @@ The changelog format is based on [Keep a Changelog](https://keepachangelog.com/e - Added Symbol Bootstrap version to generated configuration reports. - Renamed command from `supernode` for `enrolSupernode`. -## [0.4.1] - Jan-19-2020 +## [0.4.1] - Jan-19-2021 **Milestone**: Hippopotamus(0.10.0.5) @@ -34,7 +45,7 @@ The changelog format is based on [Keep a Changelog](https://keepachangelog.com/e - Added `CONTROLLER_PUBLIC_KEY` to Supernode's agent configuration - Upgraded Symbol Rest to version 2.3.1. -## [0.4.0] - Jan-14-2020 +## [0.4.0] - Jan-14-2021 **Milestone**: Hippopotamus(0.10.0.5) diff --git a/README.md b/README.md index baac8a479..7384dc927 100644 --- a/README.md +++ b/README.md @@ -264,6 +264,7 @@ General users should install this tool like any other node module. # Command Topics +* [`symbol-bootstrap backup`](docs/backup.md) - The command backs up the Mongo and RocksDb data folder into a Zip file that can then be used by the `--backupSync` feature. Bootstrap compose services must be stopped before calling this command. * [`symbol-bootstrap clean`](docs/clean.md) - It removes the target folder deleting the generated configuration and data * [`symbol-bootstrap compose`](docs/compose.md) - It generates the `docker-compose.yml` file from the configured network. * [`symbol-bootstrap config`](docs/config.md) - Command used to set up the configuration files and the nemesis block for the current network diff --git a/cmds/backup-full-dual.sh b/cmds/backup-full-dual.sh new file mode 100644 index 000000000..1f7173b9b --- /dev/null +++ b/cmds/backup-full-dual.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -e +rm -rf testnet-backup +mkdir -p testnet-backup +cp -rf target/databases/db testnet-backup/mongo +cp -rf target/nodes/api-node/data testnet-backup/data +rm -rf testnet-backup/data/spool + +cd testnet-backup +zip -r testnet-full-backup.zip * +cd .. + +# push testnet-backup/testnet-full-backup.zip into S3 diff --git a/cmds/backup-full-sync-testnet-backup.sh b/cmds/backup-full-sync-testnet-backup.sh new file mode 100755 index 000000000..be5714510 --- /dev/null +++ b/cmds/backup-full-sync-testnet-backup.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -e +rm -rf backup-sync/testnet-backup +mkdir -p backup-sync/testnet-backup +cp -rf target/testnet-dual/databases/db backup-sync/testnet-backup/mongo +cp -rf target/testnet-dual/nodes/api-node/data backup-sync/testnet-backup/data +rm -rf backup-sync/testnet-backup/data/spool + +cd backup-sync/testnet-backup +zip -r testnet-local-full-backup.zip * +cd ../.. diff --git a/cmds/backup-minimal-sync-testnet-backup.sh b/cmds/backup-minimal-sync-testnet-backup.sh new file mode 100755 index 000000000..f7d535d10 --- /dev/null +++ b/cmds/backup-minimal-sync-testnet-backup.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -e -x +rm -rf backup-sync/testnet-backup +mkdir -p backup-sync/testnet-backup/data +cp -rf target/testnet-dual/nodes/api-node/data/0* backup-sync/testnet-backup/data +cp -rf target/testnet-dual/nodes/api-node/data/proof.index.dat backup-sync/testnet-backup/data +cp -rf target/testnet-dual/nodes/api-node/data/index.dat backup-sync/testnet-backup/data +rm -rf backup-sync/testnet-backup/data/spool +touch backup-sync/testnet-backup/data/server.lock # force docker compose to run a recover + +cd backup-sync/testnet-backup +zip -r testnet-local-minimal-backup.zip * +cd ../.. diff --git a/cmds/backup-testnet-dual.sh b/cmds/backup-testnet-dual.sh new file mode 100755 index 000000000..16720dd6f --- /dev/null +++ b/cmds/backup-testnet-dual.sh @@ -0,0 +1,3 @@ +#!/bin/bash +set -e +symbol-bootstrap backup -t target/testnet-dual --destinationFile ./backup-sync/testnet-local-backup.zip $1 $2 $3 diff --git a/cmds/start-testnet-dual.sh b/cmds/start-testnet-dual.sh index c17256f62..41533b05f 100755 --- a/cmds/start-testnet-dual.sh +++ b/cmds/start-testnet-dual.sh @@ -1,3 +1,3 @@ #!/bin/bash set -e -symbol-bootstrap start -p testnet -a dual -t target/testnet-dual $1 $2 $3 +symbol-bootstrap start -p testnet -a dual -t target/testnet-dual -c test/testnet-custom-preset.yml $1 $2 $3 diff --git a/cmds/start-testnet-voting.sh b/cmds/start-testnet-voting.sh index 776a78d05..e08d46305 100755 --- a/cmds/start-testnet-voting.sh +++ b/cmds/start-testnet-voting.sh @@ -1,3 +1,3 @@ #!/bin/bash set -e -symbol-bootstrap start -p testnet -a dual -t target/testnet -c test/voting_preset.yml $1 +symbol-bootstrap start -p testnet -a dual -t target/testnet -c test/voting_preset.yml $1 $2 $3 diff --git a/docs/backup.md b/docs/backup.md new file mode 100644 index 000000000..c8e99ee94 --- /dev/null +++ b/docs/backup.md @@ -0,0 +1,36 @@ +`symbol-bootstrap backup` +========================= + +The command backs up the Mongo and RocksDb data folder into a Zip file that can then be used by the `--backupSync` feature. Bootstrap compose services must be stopped before calling this command. + +Note: This command is designed for NGL to be used when running public main or public test networks. It's not backing up any node specific information. + +* [`symbol-bootstrap backup`](#symbol-bootstrap-backup) + +## `symbol-bootstrap backup` + +The command backs up the Mongo and RocksDb data folder into a Zip file that can then be used by the `--backupSync` feature. Bootstrap compose services must be stopped before calling this command. + +``` +USAGE + $ symbol-bootstrap backup + +OPTIONS + -h, --help It shows the help of this command. + -t, --target=target [default: target] The target folder where the symbol-bootstrap network is generated + + --destinationFile=destinationFile The file location where the backup zip file will be created. Default destination is + target/backup.zip. + + --nodeName=nodeName The dual/api node name to be used to backup the data. If not provided, the first + configured api/dual node would be used. + +DESCRIPTION + Note: This command is designed for NGL to be used when running public main or public test networks. It's not backing + up any node specific information. + +EXAMPLE + $ symbol-bootstrap backup +``` + +_See code: [src/commands/backup.ts](https://github.com/nemtech/symbol-bootstrap/blob/v0.4.2/src/commands/backup.ts)_ diff --git a/docs/config.md b/docs/config.md index 1227f3e18..e23ea510a 100644 --- a/docs/config.md +++ b/docs/config.md @@ -30,6 +30,12 @@ OPTIONS -u, --user=user [default: current] User used to run docker images when creating configuration files like certificates or nemesis block. "current" means the current user. + --backupSync It downloads a backup with the Mongo and RocksDb databases for faster + synchronization. + + The location of the backup can be found and changed using the 'backupSyncLocation' + preset configuration. This configuration allows local files and remote URLs + --pullImages It pulls the utility images from DockerHub when running the configuration. It only affects alpha/dev docker images. diff --git a/docs/start.md b/docs/start.md index 02fc22a55..a83f188de 100644 --- a/docs/start.md +++ b/docs/start.md @@ -46,6 +46,12 @@ OPTIONS --args=args Add extra arguments to the docker-compose up command. Check out https://docs.docker.com/compose/reference/up. + --backupSync + It downloads a backup with the Mongo and RocksDb databases for faster synchronization. + + The location of the backup can be found and changed using the 'backupSyncLocation' preset configuration. This + configuration allows local files and remote URLs + --healthCheck It checks if the services created with docker compose are up and running. diff --git a/package-lock.json b/package-lock.json index cf1a01e1a..949850f10 100644 --- a/package-lock.json +++ b/package-lock.json @@ -494,6 +494,15 @@ "fancy-test": "^1.4.3" } }, + "@types/archiver": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/@types/archiver/-/archiver-5.1.0.tgz", + "integrity": "sha512-baFOhanb/hxmcOd1Uey2TfFg43kTSmM6py1Eo7Rjbv/ivcl7PXLhY0QgXGf50Hx/eskGCFqPfhs/7IZLb15C5g==", + "dev": true, + "requires": { + "@types/glob": "*" + } + }, "@types/chai": { "version": "4.2.12", "resolved": "https://registry.npmjs.org/@types/chai/-/chai-4.2.12.tgz", @@ -790,6 +799,80 @@ "default-require-extensions": "^2.0.0" } }, + "archiver": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/archiver/-/archiver-5.2.0.tgz", + "integrity": "sha512-QEAKlgQuAtUxKeZB9w5/ggKXh21bZS+dzzuQ0RPBC20qtDCbTyzqmisoeJP46MP39fg4B4IcyvR+yeyEBdblsQ==", + "requires": { + "archiver-utils": "^2.1.0", + "async": "^3.2.0", + "buffer-crc32": "^0.2.1", + "readable-stream": "^3.6.0", + "readdir-glob": "^1.0.0", + "tar-stream": "^2.1.4", + "zip-stream": "^4.0.4" + }, + "dependencies": { + "tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "requires": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + } + } + } + }, + "archiver-utils": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/archiver-utils/-/archiver-utils-2.1.0.tgz", + "integrity": "sha512-bEL/yUb/fNNiNTuUz979Z0Yg5L+LzLxGJz8x79lYmR54fmTIb6ob/hNQgkQnIUDWIFjZVQwl9Xs356I6BAMHfw==", + "requires": { + "glob": "^7.1.4", + "graceful-fs": "^4.2.0", + "lazystream": "^1.0.0", + "lodash.defaults": "^4.2.0", + "lodash.difference": "^4.5.0", + "lodash.flatten": "^4.4.0", + "lodash.isplainobject": "^4.0.6", + "lodash.union": "^4.6.0", + "normalize-path": "^3.0.0", + "readable-stream": "^2.0.0" + }, + "dependencies": { + "readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "requires": { + "safe-buffer": "~5.1.0" + } + } + } + }, "archy": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/archy/-/archy-1.0.0.tgz", @@ -876,8 +959,7 @@ "base64-js": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz", - "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g==", - "dev": true + "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g==" }, "bcrypt-pbkdf": { "version": "1.0.2", @@ -898,7 +980,6 @@ "version": "4.0.3", "resolved": "https://registry.npmjs.org/bl/-/bl-4.0.3.tgz", "integrity": "sha512-fs4G6/Hu4/EE+F75J8DuN/0IpQqNjAdC7aEQv7Qt8MHGUH7Ckv2MwTEEeN9QehD0pfIDkMI1bkHYkKy7xHyKIg==", - "dev": true, "requires": { "buffer": "^5.5.0", "inherits": "^2.0.4", @@ -937,12 +1018,16 @@ "version": "5.6.0", "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.6.0.tgz", "integrity": "sha512-/gDYp/UtU0eA1ys8bOs9J6a+E/KWIY+DZ+Q2WESNUA0jFRsJOc0SNUO6xJ5SGA1xueg3NL65W6s+NY5l9cunuw==", - "dev": true, "requires": { "base64-js": "^1.0.2", "ieee754": "^1.1.4" } }, + "buffer-crc32": { + "version": "0.2.13", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", + "integrity": "sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI=" + }, "buffer-from": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", @@ -1383,6 +1468,17 @@ "integrity": "sha1-3dgA2gxmEnOTzKWVDqloo6rxJTs=", "dev": true }, + "compress-commons": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/compress-commons/-/compress-commons-4.0.2.tgz", + "integrity": "sha512-qhd32a9xgzmpfoga1VQEiLEwdKZ6Plnpx5UCgIsf89FSolyJ7WnifY4Gtjgv5WR6hWAyRaHxC5MiEhU/38U70A==", + "requires": { + "buffer-crc32": "^0.2.13", + "crc32-stream": "^4.0.1", + "normalize-path": "^3.0.0", + "readable-stream": "^3.6.0" + } + }, "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -1454,6 +1550,24 @@ } } }, + "crc-32": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/crc-32/-/crc-32-1.2.0.tgz", + "integrity": "sha512-1uBwHxF+Y/4yF5G48fwnKq6QsIXheor3ZLPT80yGBV1oEUwpPojlEhQbWKVw1VwcTQyMGHK1/XMmTjmlsmTTGA==", + "requires": { + "exit-on-epipe": "~1.0.1", + "printj": "~1.1.0" + } + }, + "crc32-stream": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/crc32-stream/-/crc32-stream-4.0.1.tgz", + "integrity": "sha512-FN5V+weeO/8JaXsamelVYO1PHyeCsuL3HcG4cqsj0ceARcocxalaShCsohZMSAF+db7UYFwBy1rARK/0oFItUw==", + "requires": { + "crc-32": "^1.2.0", + "readable-stream": "^3.4.0" + } + }, "create-ts-index": { "version": "1.13.6", "resolved": "https://registry.npmjs.org/create-ts-index/-/create-ts-index-1.13.6.tgz", @@ -1676,7 +1790,6 @@ "version": "1.4.4", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", - "dev": true, "requires": { "once": "^1.4.0" } @@ -1930,6 +2043,11 @@ } } }, + "exit-on-epipe": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/exit-on-epipe/-/exit-on-epipe-1.0.1.tgz", + "integrity": "sha512-h2z5mrROTxce56S+pnvAV890uu7ls7f1kEvVGJbw1OlFH3/mlJ5bkXu0KRyW94v37zzHPiUd55iLn3DA7TjWpw==" + }, "extend": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", @@ -2226,8 +2344,7 @@ "fs-constants": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", - "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", - "dev": true + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" }, "fs-extra": { "version": "7.0.1", @@ -2512,8 +2629,7 @@ "ieee754": { "version": "1.1.13", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", - "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==", - "dev": true + "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" }, "ignore": { "version": "5.1.8", @@ -2903,6 +3019,43 @@ "resolved": "https://registry.npmjs.org/kuler/-/kuler-2.0.0.tgz", "integrity": "sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==" }, + "lazystream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/lazystream/-/lazystream-1.0.0.tgz", + "integrity": "sha1-9plf4PggOS9hOWvolGJAe7dxaOQ=", + "requires": { + "readable-stream": "^2.0.5" + }, + "dependencies": { + "readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "requires": { + "safe-buffer": "~5.1.0" + } + } + } + }, "lcov-parse": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/lcov-parse/-/lcov-parse-1.0.0.tgz", @@ -2976,12 +3129,32 @@ "resolved": "https://registry.npmjs.org/lodash._reinterpolate/-/lodash._reinterpolate-3.0.0.tgz", "integrity": "sha1-DM8tiRZq8Ds2Y8eWU4t1rG4RTZ0=" }, + "lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha1-0JF4cW/+pN3p5ft7N/bwgCJ0WAw=" + }, + "lodash.difference": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.difference/-/lodash.difference-4.5.0.tgz", + "integrity": "sha1-nMtOUF1Ia5FlE0V3KIWi3yf9AXw=" + }, + "lodash.flatten": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz", + "integrity": "sha1-8xwiIlqWMtK7+OSt2+8kCqdlph8=" + }, "lodash.flattendeep": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/lodash.flattendeep/-/lodash.flattendeep-4.4.0.tgz", "integrity": "sha1-+wMJF/hqMTTlvJvsDWngAT3f7bI=", "dev": true }, + "lodash.isplainobject": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", + "integrity": "sha1-fFJqUtibRcRcxpC4gWO+BJf1UMs=" + }, "lodash.template": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/lodash.template/-/lodash.template-4.5.0.tgz", @@ -2999,6 +3172,11 @@ "lodash._reinterpolate": "^3.0.0" } }, + "lodash.union": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.union/-/lodash.union-4.6.0.tgz", + "integrity": "sha1-SLtQiECfFvGCFmZkHETdGqrjzYg=" + }, "log-driver": { "version": "1.2.7", "resolved": "https://registry.npmjs.org/log-driver/-/log-driver-1.2.7.tgz", @@ -3333,6 +3511,11 @@ "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.10.0.tgz", "integrity": "sha512-PPmu8eEeG9saEUvI97fm4OYxXVB6bFvyNTyiUOBichBpFG8A1Ljw3bY62+5oOjDEMHRnd0Y7HQ+x7uzxOzC6JA==" }, + "node-stream-zip": { + "version": "1.12.0", + "resolved": "https://registry.npmjs.org/node-stream-zip/-/node-stream-zip-1.12.0.tgz", + "integrity": "sha512-HZ3XehqShTFj9gHauRJ3Bri9eiCTOII7/crtXzURtT14NdnOFs9Ia5E82W7z3izVBNx760tqwddxrBJVG52Y1Q==" + }, "normalize-package-data": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.5.0.tgz", @@ -3345,6 +3528,11 @@ "validate-npm-package-license": "^3.0.1" } }, + "normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==" + }, "npm-run-path": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-2.0.2.tgz", @@ -3659,6 +3847,11 @@ "integrity": "sha512-rFA1lnek1FYkMGthm4xBKME41qUKItTovuo24bCGZu/Vu1n3gW71UPLAkIdwewwkZCe29gRVweSOPXvAdckFuw==", "dev": true }, + "printj": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/printj/-/printj-1.1.2.tgz", + "integrity": "sha512-zA2SmoLaxZyArQTOPj5LXecR+RagfPSU5Kw1qP+jkWeNlrq+eJZyY2oS68SU1Z/7/myXM4lo9716laOFAVStCQ==" + }, "process-nextick-args": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", @@ -3855,6 +4048,14 @@ "util-deprecate": "^1.0.1" } }, + "readdir-glob": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/readdir-glob/-/readdir-glob-1.1.1.tgz", + "integrity": "sha512-91/k1EzZwDx6HbERR+zucygRFfiPl2zkIYZtv3Jjr6Mn7SkKcVct8aVO+sSRiGMc6fLf72du3d92/uY63YPdEA==", + "requires": { + "minimatch": "^3.0.4" + } + }, "rechoir": { "version": "0.6.2", "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.6.2.tgz", @@ -4984,6 +5185,16 @@ "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==", "dev": true + }, + "zip-stream": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/zip-stream/-/zip-stream-4.0.4.tgz", + "integrity": "sha512-a65wQ3h5gcQ/nQGWV1mSZCEzCML6EK/vyVPcrPNynySP1j3VBbQKh3nhC8CbORb+jfl2vXvh56Ul5odP1bAHqw==", + "requires": { + "archiver-utils": "^2.1.0", + "compress-commons": "^4.0.2", + "readable-stream": "^3.6.0" + } } } } diff --git a/package.json b/package.json index 7ad14d881..42921024b 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,7 @@ "@oclif/command": "^1.7.0", "@oclif/config": "^1.16.0", "@oclif/plugin-help": "^3.1.0", + "archiver": "^5.2.0", "figlet": "^1.2.4", "handlebars": "^4.7.6", "inquirer": "^7.3.3", @@ -18,6 +19,7 @@ "lodash": "^4.17.20", "memorystream": "^0.3.1", "node-forge": "^0.10.0", + "node-stream-zip": "^1.12.0", "rxjs": "^6.6.3", "shx": "^0.3.2", "sshpk": "^1.16.1", @@ -29,6 +31,7 @@ "devDependencies": { "@oclif/dev-cli": "^1.22.2", "@oclif/test": "^1.2.6", + "@types/archiver": "^5.1.0", "@types/chai": "^4.2.12", "@types/figlet": "^1.2.0", "@types/handlebars": "^4.1.0", diff --git a/presets/testnet/network.yml b/presets/testnet/network.yml index 5227a8014..64abb35a3 100644 --- a/presets/testnet/network.yml +++ b/presets/testnet/network.yml @@ -35,6 +35,7 @@ knownRestGateways: - 'http://api-01.us-east-1.testnet.symboldev.network:3000' - 'http://api-01.us-west-1.testnet.symboldev.network:3000' trustedHosts: +backupSyncLocation: https://symbol-bootstrap.s3-eu-west-1.amazonaws.com/testnet/backup.zip inflation: starting-at-height-2: 95998521 starting-at-height-200: 91882261 diff --git a/src/commands/backup.ts b/src/commands/backup.ts new file mode 100644 index 000000000..abf4204ab --- /dev/null +++ b/src/commands/backup.ts @@ -0,0 +1,42 @@ +/* + * Copyright 2021 NEM + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Command, flags } from '@oclif/command'; +import { BootstrapService, BootstrapUtils } from '../service'; + +export default class Backup extends Command { + static description = `The command backs up the Mongo and RocksDb data folder into a Zip file that can then be used by the \`--backupSync\` feature. Bootstrap compose services must be stopped before calling this command. + +Note: This command is designed for NGL to be used when running public main or public test networks. It's not backing up any node specific information.`; + + static examples = [`$ symbol-bootstrap backup`]; + + static flags = { + help: BootstrapUtils.helpFlag, + target: BootstrapUtils.targetFlag, + nodeName: flags.string({ + description: `The dual/api node name to be used to backup the data. If not provided, the first configured api/dual node would be used.`, + }), + destinationFile: flags.string({ + description: `The file location where the backup zip file will be created. Default destination is target/backup.zip.`, + }), + }; + + public async run(): Promise { + const { flags } = this.parse(Backup); + BootstrapUtils.showBanner(); + await new BootstrapService(this.config.root).backup(flags); + } +} diff --git a/src/commands/config.ts b/src/commands/config.ts index b59898026..6b4bf0431 100644 --- a/src/commands/config.ts +++ b/src/commands/config.ts @@ -58,6 +58,13 @@ export default class Config extends Command { default: ConfigService.defaultParams.report, }), + backupSync: flags.boolean({ + description: `It downloads a backup with the Mongo and RocksDb databases for faster synchronization. + +The location of the backup can be found and changed using the 'backupSyncLocation' preset configuration. This configuration allows local files and remote URLs`, + default: ConfigService.defaultParams.backupSync, + }), + pullImages: flags.boolean({ description: 'It pulls the utility images from DockerHub when running the configuration. It only affects alpha/dev docker images.', diff --git a/src/model/ConfigPreset.ts b/src/model/ConfigPreset.ts index 15fd2c7f2..cb2d3d033 100644 --- a/src/model/ConfigPreset.ts +++ b/src/model/ConfigPreset.ts @@ -181,4 +181,6 @@ export interface ConfigPreset { votingKeyEndEpoch: number; supernodeControllerPublicKey?: string; votingKeyLinkV2: number | undefined; + backupSyncLocation?: string; + backupSyncLocalCacheFileName?: string; } diff --git a/src/service/BackupSyncService.ts b/src/service/BackupSyncService.ts new file mode 100644 index 000000000..940e697b9 --- /dev/null +++ b/src/service/BackupSyncService.ts @@ -0,0 +1,221 @@ +/* + * Copyright 2021 NEM + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import * as archiver from 'archiver'; +import { EntryDataFunction } from 'archiver'; +import { createWriteStream } from 'fs'; +import * as StreamZip from 'node-stream-zip'; +import { join } from 'path'; +import { LogType } from '../logger'; +import Logger from '../logger/Logger'; +import LoggerFactory from '../logger/LoggerFactory'; +import { ConfigPreset, DatabasePreset, NodePreset } from '../model'; +import { BootstrapUtils, KnownError } from './BootstrapUtils'; +import { ConfigLoader } from './ConfigLoader'; + +export type BackupSyncParams = { + readonly target: string; + readonly nodeName?: string; + readonly destinationFile?: string; +}; + +const logger: Logger = LoggerFactory.getLogger(LogType.System); + +export class BackupSyncService { + // Parial restore + recovery doesn't work yet. + public fullRestore = true; + + public static defaultParams: BackupSyncParams = { + target: BootstrapUtils.defaultTargetFolder, + }; + constructor(private readonly root: string, protected readonly params: BackupSyncParams) {} + + public async run(presetData: ConfigPreset): Promise { + if (!presetData.backupSyncLocation) { + throw new Error(`Backup Sync cannot be executed. backupSyncLocation has not been defined.`); + } + await BootstrapUtils.mkdir(join(this.root, 'backup-sync')); + const downloadLocation = join( + this.root, + 'backup-sync', + presetData.backupSyncLocalCacheFileName || `backup-${presetData.nemesisGenerationHashSeed}.zip`, + ); + const fileLocation = (await BootstrapUtils.download(presetData.backupSyncLocation, downloadLocation)).fileLocation; + logger.info(`Restoring data from zip backup '${fileLocation}'`); + if (this.fullRestore) + await Promise.all( + (presetData.databases || []).map(async (db) => { + const destinationFolder = BootstrapUtils.getTargetDatabasesFolder(this.params.target, false, db.name); + await BootstrapUtils.mkdir(destinationFolder); + await this.unzip(fileLocation, 'mongo', destinationFolder); + }), + ); + await Promise.all( + (presetData.nodes || []).map(async (node) => { + const destinationFolder = BootstrapUtils.getTargetNodesFolder(this.params.target, false, node.name, 'data'); + if (!this.fullRestore) { + const seedFolder = presetData.nemesisSeedFolder || join(this.root, 'presets', presetData.preset, 'seed'); + await BootstrapUtils.generateConfiguration({}, seedFolder, destinationFolder); + await BootstrapUtils.writeTextFile(join(destinationFolder, 'server.lock'), ''); + } + + await BootstrapUtils.mkdir(destinationFolder); + await this.unzip(fileLocation, 'data', destinationFolder); + }), + ); + logger.info( + `Zip backup '${fileLocation}' has been restored. HINT: You can remove this file if you want to reclaim the disk space for future use.`, + ); + } + + private unzip(globalDestination: string, innerFolder: string, targetFolder: string): Promise { + const zip = new StreamZip({ + file: globalDestination, + storeEntries: true, + }); + logger.info(`Unzipping Backup Sync's '${innerFolder}' into '${targetFolder}'. This could take a while!`); + let totalFiles = 0; + let process = 0; + return new Promise((resolve, reject) => { + zip.on('entry', (entry) => { + if (!entry.isDirectory && totalFiles) { + process++; + const percentage = ((process * 100) / totalFiles).toFixed(2); + const message = `${percentage}% | ${process} files unzipped out of ${totalFiles}`; + BootstrapUtils.logSameLineMessage(message); + } + if (BootstrapUtils.stopProcess) { + zip.close(); + reject(new Error('Process cancelled!')); + } + }); + zip.on('ready', () => { + totalFiles = zip.entriesCount; + zip.extract(innerFolder, targetFolder, (err) => { + zip.close(); + if (err) { + reject(err); + } else { + logger.info(`Unzipped '${targetFolder}' created`); + resolve(); + } + }); + }); + }); + } + + private zip(destination: string, node: NodePreset, database: DatabasePreset): Promise { + // create a file to stream archive data to. + const output = createWriteStream(destination); + const archive = archiver('zip', { + zlib: { level: 9 }, // Sets the compression level. + }); + logger.info(`Creating zip file ${destination}. This could take a while!`); + return new Promise(async (resolve, reject) => { + // listen for all archive data to be written + // 'close' event is fired only when a file descriptor is involved + output.on('close', () => { + console.log(''); + logger.info( + `Zip file ${destination} size ${archive.pointer() / 1024} MB has been created. You can now share it for --backupSync.`, + ); + resolve(); + }); + + const mongoFolder = BootstrapUtils.getTargetDatabasesFolder(this.params.target, false, database.name); + const mongoTotalFiles = BootstrapUtils.getFilesRecursively(mongoFolder).length; + logger.info(`Adding '${mongoFolder}' to zip file ${destination}`); + const dataFolder = BootstrapUtils.getTargetNodesFolder(this.params.target, false, `${node.name}/`, 'data'); + const dataTotalFiles = BootstrapUtils.getFilesRecursively(dataFolder).length; + logger.info(`Adding '${dataFolder}' to zip file ${destination}`); + const totalFiles = mongoTotalFiles + dataTotalFiles; + + // This event is fired when the data source is drained no matter what was the data source. + // It is not part of this library but rather from the NodeJS Stream API. + // @see: https://nodejs.org/api/stream.html#stream_event_end + output.on('end', () => { + console.log(''); + logger.warn('Data has been drained'); + }); + + // good practice to catch warnings (ie stat failures and other non-blocking errors) + archive.on('warning', (err: any) => { + console.log(''); + if (err.code === 'ENOENT') { + // log warning + logger.warn(`There has been an warning creating ZIP file '${destination}' ${err.message || err}`); + } else { + // throw error + logger.error(`There has been an error creating ZIP file '${destination}' ${err.message || err}`); + reject(err); + } + }); + + let process = 0; + // good practice to catch this error explicitly + archive.on('error', function (err: any) { + logger.error(`There has been an error creating ZIP file '${destination}' ${err.message || err}`); + reject(err); + }); + + // pipe archive data to the file + archive.pipe(output); + + const filter: EntryDataFunction = (entry) => { + if (!entry.stats?.isDirectory()) { + process++; + const percentage = ((process * 100) / totalFiles).toFixed(2); + const message = `${percentage}% | ${process} files zipped out of ${totalFiles}`; + BootstrapUtils.logSameLineMessage(message); + } + const ignoreFiles = ['server.lock', 'broker.started', 'broker.lock']; + if (ignoreFiles.indexOf(entry.name) > -1) { + console.log(`\nExcluding file '${entry.name}'`); + return false; + } + const ignoreDirectories = ['spool']; + const ignoreEntryDirectory = ignoreDirectories.find((d) => entry.name.startsWith(d)); + if (ignoreEntryDirectory) { + if (entry.name === ignoreEntryDirectory) console.log(`\nExcluding directory '${entry.name}'`); + return false; + } + return entry; + }; + archive.directory(mongoFolder, 'mongo', filter); + archive.directory(dataFolder, 'data', filter); + await archive.finalize(); + }); + } + + public async createBackup(passedPresetData?: ConfigPreset): Promise { + const configLoader = new ConfigLoader(); + const presetData = passedPresetData ?? configLoader.loadExistingPresetData(this.params.target, false); + + const node = presetData.nodes?.find((node) => (!this.params.nodeName || node.name == this.params.nodeName) && node.api); + if (!node && this.params.nodeName) { + throw new KnownError(`Api/Dual node with name '${this.params.nodeName}' has not been configured in this instance!`); + } + if (!node) { + throw new KnownError(`Api/Dual has not been configured in this instance!`); + } + const database = presetData.databases?.find((db) => db.name == node.databaseHost || db.host == node.databaseHost); + if (!database) { + throw new KnownError(`Database with name/host '${node.databaseHost}' does not exist!`); + } + const destination = this.params.destinationFile || join(this.params.target, 'backup.zip'); + BootstrapUtils.deleteFile(destination); + await this.zip(destination, node, database); + } +} diff --git a/src/service/BootstrapService.ts b/src/service/BootstrapService.ts index 3152d1b0f..230e1815c 100644 --- a/src/service/BootstrapService.ts +++ b/src/service/BootstrapService.ts @@ -16,6 +16,7 @@ import { Addresses, ConfigPreset } from '../model'; import { DockerCompose } from '../model/DockerCompose'; +import { BackupSyncParams, BackupSyncService } from './BackupSyncService'; import { ComposeParams, ComposeService } from './ComposeService'; import { ConfigParams, ConfigResult, ConfigService } from './ConfigService'; import { LinkParams, LinkService } from './LinkService'; @@ -57,6 +58,18 @@ export class BootstrapService { return new ComposeService(this.root, config).run(passedPresetData, passedAddresses); } + /** + * It creates a zip backup with the node's data and mongo database folders ready for backupSync. + * + * Docker Compose must not be running when executing this operation. + * + * @param config the params of the compose command. + * @param passedPresetData the created preset if you know it, otherwise will load the latest one resolved from the target folder. + */ + public backup(config: BackupSyncParams = BackupSyncService.defaultParams, passedPresetData?: ConfigPreset): Promise { + return new BackupSyncService(this.root, config).createBackup(passedPresetData); + } + /** * It calls a running server announcing all the node transactions like VRF and Voting. * diff --git a/src/service/BootstrapUtils.ts b/src/service/BootstrapUtils.ts index e2dc5cdc9..00dda90ff 100644 --- a/src/service/BootstrapUtils.ts +++ b/src/service/BootstrapUtils.ts @@ -76,9 +76,9 @@ export class BootstrapUtils { public static readonly CURRENT_USER = 'current'; private static readonly pulledImages: string[] = []; - public static readonly VERSION = version; + public static stopProcess = false; - private static stopProcess = false; + public static readonly VERSION = version; public static helpFlag = flags.help({ char: 'h', description: 'It shows the help of this command.' }); diff --git a/src/service/ConfigService.ts b/src/service/ConfigService.ts index ca2e6d1c0..7113b88b0 100644 --- a/src/service/ConfigService.ts +++ b/src/service/ConfigService.ts @@ -34,6 +34,7 @@ import Logger from '../logger/Logger'; import LoggerFactory from '../logger/LoggerFactory'; import { Addresses, ConfigPreset, NodeAccount, NodePreset, NodeType } from '../model'; import { AgentCertificateService } from './AgentCertificateService'; +import { BackupSyncService } from './BackupSyncService'; import { BootstrapUtils, KnownError } from './BootstrapUtils'; import { CertificateService } from './CertificateService'; import { ConfigLoader } from './ConfigLoader'; @@ -57,6 +58,7 @@ export interface ConfigParams { target: string; password?: string; user: string; + backupSync?: boolean; pullImages?: boolean; assembly?: string; customPreset?: string; @@ -78,6 +80,7 @@ export class ConfigService { reset: false, upgrade: false, pullImages: false, + backupSync: false, user: BootstrapUtils.CURRENT_USER, }; private readonly configLoader: ConfigLoader; @@ -135,18 +138,13 @@ export class ConfigService { await BootstrapUtils.mkdir(target); this.cleanUpConfiguration(presetData); - await this.generateNodeCertificates(presetData, addresses); await this.generateAgentCertificates(presetData); await this.generateNodes(presetData, addresses); + await this.generateDataFolders(oldPresetData, oldAddresses, presetData, addresses); await this.generateGateways(presetData); await this.generateExplorers(presetData); await this.generateWallets(presetData); - if (!oldPresetData && !oldAddresses) { - await this.generateNemesis(presetData, addresses); - } else { - logger.info('Nemesis data cannot be generated or copied when upgrading...'); - } if (this.params.report) { await new ReportService(this.root, this.params).run(presetData); @@ -168,10 +166,32 @@ export class ConfigService { } } + private async generateDataFolders( + oldPresetData: ConfigPreset | undefined, + oldAddresses: Addresses | undefined, + presetData: ConfigPreset, + addresses: Addresses, + ) { + if (!oldPresetData && !oldAddresses) { + if (this.params.backupSync) { + const backupSyncService = new BackupSyncService(this.root, this.params); + await backupSyncService.run(presetData); + logger.info('Backup Sync has been executed...'); + } else { + await this.generateNemesis(presetData, addresses); + } + } else { + if (this.params.backupSync) { + logger.info('Backup Sync cannot be executed when upgrading...'); + } else { + logger.info('Nemesis data cannot be generated or copied when upgrading...'); + } + } + } + private async generateNemesis(presetData: ConfigPreset, addresses: Addresses) { const target = this.params.target; const nemesisSeedFolder = BootstrapUtils.getTargetNemesisFolder(target, false, 'seed'); - if (!presetData.nemesisSeedFolder && presetData.nemesis) { await this.generateNemesisConfig(presetData, addresses); } else { @@ -179,6 +199,12 @@ export class ConfigService { await BootstrapUtils.generateConfiguration({}, copyFrom, nemesisSeedFolder); } + return await this.copyNemesisSeed(presetData, addresses); + } + + private async copyNemesisSeed(presetData: ConfigPreset, addresses: Addresses) { + const target = this.params.target; + const nemesisSeedFolder = BootstrapUtils.getTargetNemesisFolder(target, false, 'seed'); BootstrapUtils.validateFolder(nemesisSeedFolder); await Promise.all( (addresses.nodes || []).map(async (account) => { diff --git a/test/service/BackupSyncService.test.ts b/test/service/BackupSyncService.test.ts new file mode 100644 index 000000000..19c72798b --- /dev/null +++ b/test/service/BackupSyncService.test.ts @@ -0,0 +1,79 @@ +/* + * Copyright 2021 NEM + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { expect } from '@oclif/test'; +import { existsSync } from 'fs'; +import 'mocha'; +import { BootstrapService, BootstrapUtils, ConfigLoader, ConfigService, Preset, StartParams } from '../../src/service'; +import { BackupSyncService } from '../../src/service/BackupSyncService'; + +describe('BackupSyncService', () => { + it('run', async () => { + const target = 'target/BackupSyncService.test'; + await BootstrapUtils.deleteFolder(target); + await BootstrapUtils.mkdir(target); + const preset = Preset.testnet; + const service = new BackupSyncService('.', { target: target }); + + const presetData = new ConfigLoader().createPresetData({ + root: '.', + preset: preset, + assembly: 'dual', + password: undefined, + customPresetObject: { + backupSyncLocation: 'https://symbol-bootstrap.s3-eu-west-1.amazonaws.com/testnet/testnet-unit-test.zip', + backupSyncLocalCacheFileName: 'testnet-unit-test.zip', + }, + }); + await service.run(presetData); + expect(existsSync(`${target}/nodes/api-node/data/00000/00002.dat`)).eq(true); + expect(existsSync(`${target}/databases/db/mongod.lock`)).eq(true); + }); + + it('run, stop, create backup', async () => { + const target = 'target/BackupSyncService.e2e'; + const backupSyncLocation = './backup-sync/testnet-unittest-backup.zip'; + BootstrapUtils.deleteFile(backupSyncLocation); + const config: StartParams = { + ...ConfigService.defaultParams, + preset: Preset.testnet, + reset: true, + detached: true, + healthCheck: true, + assembly: 'dual', + pullImages: true, + target, + customPresetObject: { + backupSyncLocation: backupSyncLocation, + backupSyncLocalCacheFileName: 'testnet-unittest-backup.zip', + }, + }; + + const service = new BootstrapService('.'); + + await service.start(config); + + await BootstrapUtils.sleep(5000); + + await service.stop(config); + + await service.backup({ + target: target, + destinationFile: backupSyncLocation, + }); + expect(existsSync(backupSyncLocation)).eq(true); + }); +}); diff --git a/test/testnet-custom-preset.yml b/test/testnet-custom-preset.yml new file mode 100644 index 000000000..9d201a38c --- /dev/null +++ b/test/testnet-custom-preset.yml @@ -0,0 +1,2 @@ +backupSyncLocation: ./backup-sync/testnet-local-backup.zip +backupSyncLocalCacheFileName: testnet-local-backup.zip diff --git a/test/voting_preset.yml b/test/voting_preset.yml index d631a1e6e..a827610a2 100644 --- a/test/voting_preset.yml +++ b/test/voting_preset.yml @@ -1,3 +1,5 @@ +backupSyncLocation: ./backup-sync/testnet-local-backup.zip +backupSyncLocalCacheFileName: testnet-local-backup.zip databases: - compose: mem_limit: ~