From b900ffddf769fe8ab02fdb0c76b1b7cbda7536dc Mon Sep 17 00:00:00 2001 From: Daniel Masegosa Date: Mon, 22 Jul 2019 23:54:26 +0200 Subject: [PATCH 1/5] First version of script to migrate data from local to cloud elasticsearch --- docker/docker-compose.yml | 4 ++-- migrate-elasticsearch-data.sh | 24 ++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 migrate-elasticsearch-data.sh diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 73dc113..538bbc1 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -5,7 +5,7 @@ services: # ELK ############# elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:6.1.4 + image: docker.elastic.co/elasticsearch/elasticsearch:7.2.0 container_name: elasticsearch environment: - node.name=needlehack-node @@ -30,7 +30,7 @@ services: retries: 3 kibana: - image: docker.elastic.co/kibana/kibana:6.1.4 + image: docker.elastic.co/kibana/kibana:7.2.0 container_name: kibana volumes: - ./kibana/config/kibana.yml:/usr/share/kibana/config/kibana.yml diff --git a/migrate-elasticsearch-data.sh b/migrate-elasticsearch-data.sh new file mode 100644 index 0000000..06f030d --- /dev/null +++ b/migrate-elasticsearch-data.sh @@ -0,0 +1,24 @@ +#!/bin/bash +localElastic=$(docker ps | grep 'docker.elastic.co/elasticsearch/elasticsearch' | awk '{print $1}') + +if [[ "$localElastic" != "" ]] +then + input="http://elasticsearch:9200/shakespeare" + echo "Moving data from: $input to $1" + + docker run --name=elasticsearch-dump --link=${localElastic}:elasticsearch taskrabbit/elasticsearch-dump --input=${input} \ + --output=$1 \ + --type=data + + echo "Stopped docker container:" $(docker stop elasticsearch-dump) + echo "Removed docker container:" $(docker rm elasticsearch-dump) + + echo "**********************************************************************************" + echo "******************** Imported data to cloud elasticsearch! :) ********************" + echo "**********************************************************************************" + +else + echo "**********************************************************************************" + echo "************************** Local elastic is not running **************************" + echo "**********************************************************************************" +fi \ No newline at end of file From 5fc6c7239a0a93bb2ac93494ccd95f283682c1c4 Mon Sep 17 00:00:00 2001 From: Daniel Masegosa Date: Sun, 28 Jul 2019 00:23:55 +0200 Subject: [PATCH 2/5] [NH-008] - Updated script to migrate elasticsearch data and added to README --- README.md | 13 +++++++++++++ migrate-elasticsearch-data.sh | 9 +++++---- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 4ee8ce6..369dd2c 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,19 @@ $ mvn clean install $ ./run.sh ``` +## Migrate elasticsearch data + +To migrate the collected data from our local elasticsearch to the cloud elasticsearch we can use the following command: +``` +$ ./migrate-elasticsearch-data.sh localIndex cloudElasticPath +``` +As first parameter we need to insert the local index to migrate and as second parameter the elastic cloud index path. + +Example: +``` +$ ./migrate-elasticsearch-data.sh sampleIndex https://elasticCloudHost/sampleIndex +``` + ## Built With * [Spring Framework](https://spring.io/) - The framework used diff --git a/migrate-elasticsearch-data.sh b/migrate-elasticsearch-data.sh index 06f030d..f709da6 100644 --- a/migrate-elasticsearch-data.sh +++ b/migrate-elasticsearch-data.sh @@ -3,11 +3,12 @@ localElastic=$(docker ps | grep 'docker.elastic.co/elasticsearch/elasticsearch' if [[ "$localElastic" != "" ]] then - input="http://elasticsearch:9200/shakespeare" - echo "Moving data from: $input to $1" + from="http://elasticsearch:9200/$1" + to=$2 + echo "Moving data from: $from to $to" - docker run --name=elasticsearch-dump --link=${localElastic}:elasticsearch taskrabbit/elasticsearch-dump --input=${input} \ - --output=$1 \ + docker run --name=elasticsearch-dump --link=${localElastic}:elasticsearch taskrabbit/elasticsearch-dump --input=${from} \ + --output=${to} \ --type=data echo "Stopped docker container:" $(docker stop elasticsearch-dump) From 5c1274353da41c8d3faa1d8af0d72a1a6bd2d7ba Mon Sep 17 00:00:00 2001 From: Daniel Masegosa Date: Sat, 3 Aug 2019 13:16:19 +0200 Subject: [PATCH 3/5] [NH-008] - Updated gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index d8b846b..2db39d3 100644 --- a/.gitignore +++ b/.gitignore @@ -75,6 +75,9 @@ local.properties # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 .idea/ +*.iws +*.iml +*.ipr # User-specific stuff .idea/**/workspace.xml From f551e22ce8d9d908aac69f0f42c8c426f7f68ea7 Mon Sep 17 00:00:00 2001 From: Daniel Masegosa Date: Sat, 3 Aug 2019 13:16:59 +0200 Subject: [PATCH 4/5] [NH-008] - Updated migrate data script to run docker command in the correct network --- migrate-elasticsearch-data.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/migrate-elasticsearch-data.sh b/migrate-elasticsearch-data.sh index f709da6..c426a60 100644 --- a/migrate-elasticsearch-data.sh +++ b/migrate-elasticsearch-data.sh @@ -7,7 +7,7 @@ then to=$2 echo "Moving data from: $from to $to" - docker run --name=elasticsearch-dump --link=${localElastic}:elasticsearch taskrabbit/elasticsearch-dump --input=${from} \ + docker run --name=elasticsearch-dump --network=docker_elk --link=${localElastic}:elasticsearch taskrabbit/elasticsearch-dump --input=${from} \ --output=${to} \ --type=data From def6f4c0ecdba5d77a274ecd01222b5733f6d652 Mon Sep 17 00:00:00 2001 From: Daniel Masegosa Date: Sat, 3 Aug 2019 13:20:02 +0200 Subject: [PATCH 5/5] [NH-008] - Updated README to detail how we can use our users to migrate the elasticsearch data --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 369dd2c..e853c0b 100644 --- a/README.md +++ b/README.md @@ -42,11 +42,11 @@ To migrate the collected data from our local elasticsearch to the cloud elastics ``` $ ./migrate-elasticsearch-data.sh localIndex cloudElasticPath ``` -As first parameter we need to insert the local index to migrate and as second parameter the elastic cloud index path. +As first parameter we need to insert the local index to migrate and as second parameter the elastic cloud index path adding the user and password. Example: ``` -$ ./migrate-elasticsearch-data.sh sampleIndex https://elasticCloudHost/sampleIndex +$ ./migrate-elasticsearch-data.sh sampleIndex https://user:password@elasticCloudHost/sampleIndex ``` ## Built With