From 5138b0e5e5d998fd5580ba374debce3267f779ef Mon Sep 17 00:00:00 2001 From: Calvin Vette Date: Tue, 14 Apr 2020 08:04:22 -0400 Subject: [PATCH 1/4] Added YUM repos for ES6 & ES7, install directions for YUM-based distros, including fallback for no available repo --- .idea/$CACHE_FILE$ | 6 + .idea/.gitignore | 2 + .idea/codeStyles/codeStyleConfig.xml | 5 + .idea/elastic.iml | 9 ++ .idea/misc.xml | 6 + .idea/modules.xml | 8 ++ .idea/vcs.xml | 6 + elasticsearch6.repo | 8 ++ elasticsearch7.repo | 8 ++ labs/01-install/index-yum.md | 208 +++++++++++++++++++++++++++ 10 files changed, 266 insertions(+) create mode 100644 .idea/$CACHE_FILE$ create mode 100644 .idea/.gitignore create mode 100644 .idea/codeStyles/codeStyleConfig.xml create mode 100644 .idea/elastic.iml create mode 100644 .idea/misc.xml create mode 100644 .idea/modules.xml create mode 100644 .idea/vcs.xml create mode 100644 elasticsearch6.repo create mode 100644 elasticsearch7.repo create mode 100644 labs/01-install/index-yum.md diff --git a/.idea/$CACHE_FILE$ b/.idea/$CACHE_FILE$ new file mode 100644 index 0000000..6cb8985 --- /dev/null +++ b/.idea/$CACHE_FILE$ @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..5c98b42 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,2 @@ +# Default ignored files +/workspace.xml \ No newline at end of file diff --git a/.idea/codeStyles/codeStyleConfig.xml b/.idea/codeStyles/codeStyleConfig.xml new file mode 100644 index 0000000..a55e7a1 --- /dev/null +++ b/.idea/codeStyles/codeStyleConfig.xml @@ -0,0 +1,5 @@ + + + + \ No newline at end of file diff --git a/.idea/elastic.iml b/.idea/elastic.iml new file mode 100644 index 0000000..d6ebd48 --- /dev/null +++ b/.idea/elastic.iml @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..28a804d --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,6 @@ + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..e3b0661 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..35eb1dd --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/elasticsearch6.repo b/elasticsearch6.repo new file mode 100644 index 0000000..e3ed383 --- /dev/null +++ b/elasticsearch6.repo @@ -0,0 +1,8 @@ +[elasticsearch] +name=Elasticsearch repository for 6.x packages +baseurl=https://artifacts.elastic.co/packages/6.x/yum +gpgcheck=1 +gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch +enabled=0 +autorefresh=1 +type=rpm-md diff --git a/elasticsearch7.repo b/elasticsearch7.repo new file mode 100644 index 0000000..a3b2391 --- /dev/null +++ b/elasticsearch7.repo @@ -0,0 +1,8 @@ +[elasticsearch] +name=Elasticsearch repository for 7.x packages +baseurl=https://artifacts.elastic.co/packages/7.x/yum +gpgcheck=1 +gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch +enabled=0 +autorefresh=1 +type=rpm-md diff --git a/labs/01-install/index-yum.md b/labs/01-install/index-yum.md new file mode 100644 index 0000000..81f28c3 --- /dev/null +++ b/labs/01-install/index-yum.md @@ -0,0 +1,208 @@ +# Elastic Stack Lab01 +In this lab we will be installing and setting up Elasticsearch on a CentOS/RHEL VM. + +These directions are based on the official documentation available at: + +https://www.elastic.co/guide/en/elasticsearch/reference/7.6/rpm.html#rpm-repo + +## Install Elasticsearch +Elasticsearch is based on Java, so we need to install a Java environment. + +Install pre-requisites +```bash +sudo yum update -y +``` + +Check for a Java 8 VM +```bash +[student@ip-172-30-0-35 ~]$ java -version +openjdk version "1.8.0_242" +OpenJDK Runtime Environment (build 1.8.0_242-b08) +OpenJDK 64-Bit Server VM (build 25.242-b08, mixed mode) +``` + +Install Java if not installed +```bash +sudo yum install java-1.8.0-openjdk.x86_64 java-1.8.0-openjdk-devel.x86_64 +``` + +Now we can install Elasticsearch itself. + +First let’s add the Elasticsearch GPG key to our VM +```bash +wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add - +``` + +Now we need to add the Elasticsearch repo to our VM. +```bash +cd ~/downloads +wget https://github.com/calvinvette/elastic/blob/master/elasticsearch6.repo +wget https://github.com/calvinvette/elastic/blob/master/elasticsearch7.repo +sudo cp downloads/elasticsearch* /etc/yum.repos.d/ +``` + +Finally let’s install Elasticsearch +```bash +sudo yum update -y +# For ES6, use: +sudo yum install --enablerepo=elasticsearch6 elasticsearch +# For ES7, use: +sudo yum install --enablerepo=elasticsearch7 elasticsearch +``` + +If the yum install fails (as of this writing, the repo is returning 404-Not Found), you can download the RPMs +directly and install them: + +```bash +# For ElasticSearch 7 +wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.6.2-x86_64.rpm +wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.6.2-x86_64.rpm.sha512 +shasum -a 512 -c elasticsearch-7.6.2-x86_64.rpm.sha512 +sudo rpm --install elasticsearch-7.6.2-x86_64.rpm +``` + +```bash +# For ElasticSearch 6 +wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.8.8.rpm +wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.8.8.rpm.sha512 +shasum -a 512 -c elasticsearch-6.8.8.rpm.sha512 +sudo rpm --install elasticsearch-6.8.8.rpm +``` + + +After this completes we need to allow external access to our Elasticsearch instance. + +Edit `elasticsearch.yml` +```bash +sudo vi /etc/elasticsearch/elasticsearch.yml +``` + +Change `http.host` (earlier ES6) or `network.host` (later ES6 and ES7) around line 55 to `0.0.0.0` + +![](index/0D7C537F-F1FA-4199-A63E-AA6EC3B74708%204.png) + + (in vi, use the arrow keys to move where you want to edit, then hit “i” to enter “insert mode” and make your edits. When done, hit `ESC` to exit “insert mode”, then type `:wq` to write your changes and quit vi.) + +Modify the init script to export a proper JAVA_HOME, around line 59: +```bash +sudo vi /etc/init.d/elasticsearch +``` + +Change this: + +``` +export JAVA_HOME +``` + +to this (or an appropriate JAVA_HOME): + +``` +export JAVA_HOME=/usr/lib/jvm/java +``` + +Enable the `Elasticsearch` service and start it up. +```bash +sudo chkconfig --add elasticsearch +sudo service elasticsearch start +``` + +If everything restarted without any errors Elasticsearch has been successfully installed! + +Let’s confirm it is working as expected by connecting to the API. +```bash +curl 127.0.0.1:9200 +``` + +```bash +$ curl 127.0.0.1:9200 +{ + "name" : "QDJyUOw", + "cluster_name" : "elasticsearch", + "cluster_uuid" : "FS96-DAATMKss1b-0xTL7g", + "version" : { + "number" : "6.8.8", + "build_flavor" : "default", + "build_type" : "rpm", + "build_hash" : "2f4c224", + "build_date" : "2020-03-18T23:22:18.622755Z", + "build_snapshot" : false, + "lucene_version" : "7.7.2", + "minimum_wire_compatibility_version" : "5.6.0", + "minimum_index_compatibility_version" : "5.0.0" + }, + "tagline" : "You Know, for Search" +} +``` + +You can also test it by loading http://VMIP:9200 in a browser, and if you see something like the following it’s working correctly. +![](index/05CDF398-09D6-4AE2-BA56-7A5BAA985A2D%208.png) + +## Loading data into Elasticsearch +Now that we have Elasticsearch installed it needs some data to aggregate and index. Let’s go ahead and load in the complete works of William Shakespeare + +Download and create the mapping +```bash +wget http://bit.ly/es-shakes-mapping -O shakes-mapping.json +curl -H 'Content-Type: application/json' -XPUT 127.0.0.1:9200/shakespeare --data-binary @shakes-mapping.json +``` + +Download the data +```bash +wget http://bit.ly/es-shakes-data -O shakespeare_6.0.json +``` + +Now we are going to load this data into Elasticsearch through it’s API +```bash +curl -H 'Content-Type: application/json' -X POST 'localhost:9200/shakespeare/doc/_bulk?pretty' --data-binary @shakespeare_6.0.json +``` + +And finally let’s go ahead and search the data we just inserted. +```bash +curl -H 'Content-Type: application/json' -XGET '127.0.0.1:9200/shakespeare/_search?pretty' -d ' +{ + "query" : { + "match_phrase" : { + "text_entry" : "to be or not to be" + } + } +} +' +``` + +We are searching all of the data we inserted for “to be or not to be” and our result is… Wow, pulled it out very quickly and we now know that it came from Hamlet. +```json +{ + "took" : 153, + "timed_out" : false, + "_shards" : { + "total" : 5, + "successful" : 5, + "skipped" : 0, + "failed" : 0 + }, + "hits" : { + "total" : 1, + "max_score" : 13.874454, + "hits" : [ + { + "_index" : "shakespeare", + "_type" : "doc", + "_id" : "34229", + "_score" : 13.874454, + "_source" : { + "type" : "line", + "line_id" : 34230, + "play_name" : "Hamlet", + "speech_number" : 19, + "line_number" : "3.1.64", + "speaker" : "HAMLET", + "text_entry" : "To be, or not to be: that is the question:" + } + } + ] + } +} + +``` + +## Lab Complete From 459bddbe34c8351cf7d2bc111e5fdbb7198c29c3 Mon Sep 17 00:00:00 2001 From: Calvin Vette Date: Tue, 14 Apr 2020 08:04:58 -0400 Subject: [PATCH 2/4] Added YUM repos for ES6 & ES7, install directions for YUM-based distros, including fallback for no available repo --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index e43b0f9..4befed3 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ .DS_Store +.idea From dd53dcb02e3f407b2f9095a2e544026b0d2c8903 Mon Sep 17 00:00:00 2001 From: Calvin Vette Date: Tue, 14 Apr 2020 08:06:22 -0400 Subject: [PATCH 3/4] Removed .idea folder --- .idea/$CACHE_FILE$ | 6 ------ .idea/.gitignore | 2 -- .idea/codeStyles/codeStyleConfig.xml | 5 ----- .idea/elastic.iml | 9 --------- .idea/misc.xml | 6 ------ .idea/modules.xml | 8 -------- .idea/vcs.xml | 6 ------ 7 files changed, 42 deletions(-) delete mode 100644 .idea/$CACHE_FILE$ delete mode 100644 .idea/.gitignore delete mode 100644 .idea/codeStyles/codeStyleConfig.xml delete mode 100644 .idea/elastic.iml delete mode 100644 .idea/misc.xml delete mode 100644 .idea/modules.xml delete mode 100644 .idea/vcs.xml diff --git a/.idea/$CACHE_FILE$ b/.idea/$CACHE_FILE$ deleted file mode 100644 index 6cb8985..0000000 --- a/.idea/$CACHE_FILE$ +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/.gitignore b/.idea/.gitignore deleted file mode 100644 index 5c98b42..0000000 --- a/.idea/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -# Default ignored files -/workspace.xml \ No newline at end of file diff --git a/.idea/codeStyles/codeStyleConfig.xml b/.idea/codeStyles/codeStyleConfig.xml deleted file mode 100644 index a55e7a1..0000000 --- a/.idea/codeStyles/codeStyleConfig.xml +++ /dev/null @@ -1,5 +0,0 @@ - - - - \ No newline at end of file diff --git a/.idea/elastic.iml b/.idea/elastic.iml deleted file mode 100644 index d6ebd48..0000000 --- a/.idea/elastic.iml +++ /dev/null @@ -1,9 +0,0 @@ - - - - - - - - - \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml deleted file mode 100644 index 28a804d..0000000 --- a/.idea/misc.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml deleted file mode 100644 index e3b0661..0000000 --- a/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index 35eb1dd..0000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file From 828342c6d6290842b989be94a93dfd294c77b1a9 Mon Sep 17 00:00:00 2001 From: Calvin Vette Date: Tue, 14 Apr 2020 09:51:22 -0400 Subject: [PATCH 4/4] Added prelim YUM support for Lab#13 - Logstash --- labs/01-install/index-yum.md | 7 +- labs/13-logstash/index-yum.md | 238 ++++++++++++++++++++++++++++++++++ 2 files changed, 239 insertions(+), 6 deletions(-) create mode 100644 labs/13-logstash/index-yum.md diff --git a/labs/01-install/index-yum.md b/labs/01-install/index-yum.md index 81f28c3..1c3d947 100644 --- a/labs/01-install/index-yum.md +++ b/labs/01-install/index-yum.md @@ -28,11 +28,6 @@ sudo yum install java-1.8.0-openjdk.x86_64 java-1.8.0-openjdk-devel.x86_64 Now we can install Elasticsearch itself. -First let’s add the Elasticsearch GPG key to our VM -```bash -wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add - -``` - Now we need to add the Elasticsearch repo to our VM. ```bash cd ~/downloads @@ -41,7 +36,7 @@ wget https://github.com/calvinvette/elastic/blob/master/elasticsearch7.repo sudo cp downloads/elasticsearch* /etc/yum.repos.d/ ``` -Finally let’s install Elasticsearch +Finally let’s install Elasticsearch (note that the repos are disabled by default): ```bash sudo yum update -y # For ES6, use: diff --git a/labs/13-logstash/index-yum.md b/labs/13-logstash/index-yum.md new file mode 100644 index 0000000..31fb5cb --- /dev/null +++ b/labs/13-logstash/index-yum.md @@ -0,0 +1,238 @@ +# Elastic Stack Lab13 +In this lab we are going to install and configure Logstash. + +Run the following to install Logstash. +``` +sudo apt-get update +sudo yum install -y logstash +``` + +If the Elastic repo is returning 404, use the RPMs directly: + +ES7 +``` +wget https://artifacts.elastic.co/downloads/logstash/logstash-7.6.2.rpm +rpm install logstash-7.6.2.rpm +``` + +``` +wget https://artifacts.elastic.co/downloads/logstash/logstash-6.8.8.rpm +rpm install logstash-6.8.8.rpm +``` + +After it's installed we have to configure it. + +Now create `/etc/logstash/conf.d/logstash.conf` in vi +And insert the following + +``` +input { + file { + path => "/home/student/access_log" + start_position => "beginning" + } +} + +filter { + if [path] =~ "access" { + mutate { replace => { "type" => "apache_access" } } + grok { + match => { "message" => "%{COMBINEDAPACHELOG}" } + } + } + date { + match => [ "timestamp" , "dd/MMM/yyyy:HH:mm:ss Z" ] + } +} + +output { + elasticsearch { + hosts => ["localhost:9200"] + } + stdout { codec => rubydebug } +} +``` + +We need to have logs that can be imported and manipulated by Logstash. Let's go ahead and download an Apache access log to use. +``` +wget http://bit.ly/logstash-apache -O access_log +``` + +Now that we have installed and configured Logstash let's go ahead and start it up. + +``` +cd /usr/share/logstash/ +sudo bin/logstash -f /etc/logstash/conf.d/logstash.conf +``` + +After starting Logstash wait a couple seconds and you should see it start to index all of the data from our `access_log`. All of this data is being imported into our Elasticsearch index. + +Logstash normally runs as a daemon monitoring for new data from the `input` systems and then it sends that data over to the `output` systems. + +Confirm the access log data was imported into Elasticsearch successfully. + +List all the indices +``` +curl -XGET 127.0.0.1:9200/_cat/indices?v +``` + +You should see quite a few new indices with data for each day in our log file. + +``` +health status index uuid pri rep docs.count docs.deleted store.size pri.store.size +yellow open logstash-2017.05.01 rUEL5_TpQQGIlnTR2rwrGw 5 1 15948 0 7mb 7mb +yellow open movies qHJxfatIRlOQzIVMHHXhIA 5 1 9125 0 1.4mb 1.4mb +yellow open logstash-2015.05.20 w_T-y-3OQNGwXhe0CzTbuQ 5 1 4750 0 22.7mb 22.7mb +yellow open ratings jWzRWMcNSbS3H-hz0CyPaQ 5 1 100004 0 14.4mb 14.4mb +``` + +Look at one of the indices +``` +curl -XGET '127.0.0.1:9200/logstash-2017.05.02/_search?pretty' +``` + +Notice that all of the data was imported and fields created and indexed successfully. + +## Logstash - MySQL + +Install MySQL first +``` +sudo apt-get update +sudo apt-get install -y mysql-server +``` + +Now we need to import some data into MySQL. + +Start by changing back to the student home directory +``` +cd ~ +``` + +Download the movie data +``` +wget http://files.grouplens.org/datasets/movielens/ml-100k.zip +unzip ml-100k.zip +``` + +Now look at the format of `ml-100k/u.item` and you'll see it's a pipe delimited file. + +Log into MySQL +``` +sudo mysql -uroot -ppassword +``` + +Create a new database +``` +CREATE DATABASE movielens; +``` + +Create a new table +``` +CREATE TABLE movielens.movies ( +-> movieID INT PRIMARY KEY NOT NULL, +-> title TEXT, +-> releaseDate DATE +-> ); +``` + + +Now load the data into the database +``` +LOAD DATA LOCAL INFILE 'ml-100k/u.item' INTO TABLE movielens.movies FIELDS TERMINATED BY '|' +-> (movieID, title, @var3) +-> set releaseDate = STR_TO_DATE(@var3, '%d-%M-%Y'); +``` + +Now if the above commands were all successful you should be able to run the following to confirm all the records were imported. +``` +SELECT * FROM movielens.movies WHERE title like 'Star%'; +``` + +You should get back something similar to +``` ++---------+------------------------------------------------+-------------+ +| movieID | title | releaseDate | ++---------+------------------------------------------------+-------------+ +| 50 | Star Wars (1977) | 1977-01-01 | +| 62 | Stargate (1994) | 1994-01-01 | +| 222 | Star Trek: First Contact (1996) | 1996-11-22 | +| 227 | Star Trek VI: The Undiscovered Country (1991) | 1991-01-01 | +| 228 | Star Trek: The Wrath of Khan (1982) | 1982-01-01 | +| 229 | Star Trek III: The Search for Spock (1984) | 1984-01-01 | +| 230 | Star Trek IV: The Voyage Home (1986) | 1986-01-01 | +| 271 | Starship Troopers (1997) | 1997-01-01 | +| 380 | Star Trek: Generations (1994) | 1994-01-01 | +| 449 | Star Trek: The Motion Picture (1979) | 1979-01-01 | +| 450 | Star Trek V: The Final Frontier (1989) | 1989-01-01 | +| 1068 | Star Maker, The (Uomo delle stelle, L') (1995) | 1996-03-01 | +| 1265 | Star Maps (1997) | 1997-01-01 | +| 1293 | Star Kid (1997) | 1998-01-16 | +| 1464 | Stars Fell on Henrietta, The (1995) | 1995-01-01 | ++---------+------------------------------------------------+-------------+ +15 rows in set (0.00 sec) +``` + +Now we need to create a new `jdbc` user to connect to MySQL + +Run the following in MySQL +``` +CREATE USER 'jdbc'@'%' IDENTIFIED BY 'password'; +GRANT ALL PRIVILEGES ON *.* TO 'jdbc'@'%'; +FLUSH PRIVILEGES; +``` + +Confirm you can log into mysql as the `jdbc` user + +``` +mysql -ujdbc -ppassword +``` + +Logstash is a Java application and to connect to MySQL is requires the `jdbc` driver. Installing this is fairly simple. + +Download the latest connector and unzip it. + +``` +wget https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-5.1.46.zip +unzip mysql-connector-java-5.1.46.zip +``` + +Now we need to configure Logstash to connect to MySQL. + +Create a new configuration file for Logstash `/etc/logstash/conf.d/mysql.conf` with the following data. +``` +input { + jdbc { + jdbc_connection_string => "jdbc:mysql://localhost:3306/movielens" + jdbc_user => "jdbc" + jdbc_password => "password" + jdbc_driver_library => "/home/student/mysql-connector-java-5.1.46/mysql-connector-java-5.1.46-bin.jar" + jdbc_driver_class => "com.mysql.jdbc.Driver" + statement => "SELECT * FROM movies" + } +} + +output { + stdout { codec => json_lines } + elasticsearch { + "hosts" => "localhost:9200" + "index" => "movielens-sql" + "document_type" => "data" + } +} +``` + +Now we need to restart Logstash and pass it the new mysql config file. + +``` +cd /usr/share/logstash/ +sudo bin/logstash -f /etc/logstash/conf.d/mysql.conf +``` + + +Now let's confirm the data was imported correctly. + +``` +curl -XGET '127.0.0.1:9200/movielens-sql/_search?q=title:Star&pretty' +``` + +# Lab Complete