From 568b7583be5902c54e36ff1a6c009a8d376f9a7d Mon Sep 17 00:00:00 2001 From: Ben Hall Date: Mon, 25 Feb 2019 11:15:50 +0000 Subject: [PATCH 1/5] Docker images need to be lowercase --- Centos/step1.md | 4 ++-- Centos/step3.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Centos/step1.md b/Centos/step1.md index 4eaf9a1..cdc5b0f 100644 --- a/Centos/step1.md +++ b/Centos/step1.md @@ -2,8 +2,8 @@ Let's build a CentOS 7 base image. Create a directory for base images and navigate to it -`mkdir -p ~/src/base_images.`{{execute}} +`mkdir -p ~/src/base_images`{{execute}} -`cd ~/src/base_images.`{{execute}} +`cd ~/src/base_images`{{execute}} You are in the base image directory. diff --git a/Centos/step3.md b/Centos/step3.md index 6bfa33e..a3a9604 100644 --- a/Centos/step3.md +++ b/Centos/step3.md @@ -1,9 +1,9 @@ To add your organisation name to the build, run the following command: Here we have used -Organization name as BlueData +Organization name as bluedata -`export BASE_IMG_ORGNAME='BlueData'`{{execute}} +`export BASE_IMG_ORGNAME='bluedata'`{{execute}} To identify your build, you add a build version to image. To do so run the following command: From e89b726c8c2e67e71efd5b3615694796937bb1d7 Mon Sep 17 00:00:00 2001 From: Ben Hall Date: Mon, 25 Feb 2019 11:21:13 +0000 Subject: [PATCH 2/5] Creating custom to save downloading large assets --- environments/bluedata-workbench/scripts/1_configure.sh | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 environments/bluedata-workbench/scripts/1_configure.sh diff --git a/environments/bluedata-workbench/scripts/1_configure.sh b/environments/bluedata-workbench/scripts/1_configure.sh new file mode 100644 index 0000000..b0a2f92 --- /dev/null +++ b/environments/bluedata-workbench/scripts/1_configure.sh @@ -0,0 +1,8 @@ +pip install --upgrade pip +pip install --upgrade bdworkbench +yum install wget -y +wget https://www.dropbox.com/s/xyv64z1xt9l6lw7/bdcatalog-centos7-bluedata-cdh514_centos7x-1.7.tar +tar xvf bdcatalog-centos7-bluedata-cdh514_centos7x-1.7.tar +chmod +x bdcatalog-centos7-bluedata-cdh514_centos7x-1.7.bin + + From bc6f9a9f66fffbfb93456f29c4b22309a0d91fbd Mon Sep 17 00:00:00 2001 From: Ben Hall Date: Mon, 25 Feb 2019 11:24:59 +0000 Subject: [PATCH 3/5] HTML markup makes code snippets easier to read --- CDH_upgrade_scenario/step7.md | 91 ++++++++++++++++++----------------- 1 file changed, 47 insertions(+), 44 deletions(-) diff --git a/CDH_upgrade_scenario/step7.md b/CDH_upgrade_scenario/step7.md index 68bee8d..4917bc9 100644 --- a/CDH_upgrade_scenario/step7.md +++ b/CDH_upgrade_scenario/step7.md @@ -16,17 +16,18 @@ To view the content of the file, execute the following
  • Identification

    • Below is the example snippet for identification: -
    -"distro_id": "bluedata/cdh515_centos7x"
    -"label": {
    - "name": ""name": "CDH 5.15.0 on 7x with Cloudera Manager",
    - "description": "CDH 5.15.0 with MRv1/YARN and HBase support. Includes Pig, Hive, Hue and Spark."
    - },
    -"version": "1.7",
    -"epic_compatible_versions": ["3.4"],
    -"categories": [ "Hadoop", "HBase" ],
    -
    +
    +"distro_id": "bluedata/cdh515_centos7x"
+"label": {
+  "name": ""name": "CDH 5.15.0 on 7x  with Cloudera Manager",
+  "description": "CDH 5.15.0 with MRv1/YARN and HBase support. Includes Pig, Hive, Hue and Spark."
+  },
+"version": "1.7",
+"epic_compatible_versions": ["3.4"],
+"categories": [ "Hadoop", "HBase" ],
+
    + distro_id is unique identifier for either a Catalog entry or a versioned set of Catalog entries.

    The label is a property contains the following parameters:
    @@ -48,17 +49,19 @@ Name and distro_id needs to be updated based on the version of the CDH image to
  • Components

    • Below is the example snippet for components: -
    -"image": {
    + +
    +"image": {
  "checksum": "65cd4ffd0cac3feec3e0719ae4f3dbcd",
-        "source_file": "bluedata-cdh5122_centos7x-centos7-2.0.tar.gz"
    
-},
    
-"setup_package": {
    
+        "source_file": "bluedata-cdh5122_centos7x-centos7-2.0.tar.gz"
+},
+"setup_package": {
   "config_api_version": 7,
        
     "checksum": "a7ea64f9b1de60ab9f87d76e2f571069",
-     
       "source_file": "cdh5-cm-setup.tgz"
    
-},
    
-
    
+     
       "source_file": "cdh5-cm-setup.tgz"
+},
+
    +
    image is a property that identifies the location for the image used to launch virtual nodes for this Catalog entry.

    setup_package is similar to the image property except for the configuration scripts package that runs inside the launched virtual node. @@ -67,22 +70,22 @@ Name and distro_id needs to be updated based on the version of the CDH image to
  • Services

    • Below is the example snippet for services: -
    -"services": [
    - {
    - "id": "hbase_master",
    - "exported_service": "hbase",
    - "label": {
    - "name": "HMaster"
    - },
    - "endpoint" : {
    - "url_scheme" : "http",
    - "port" : "60010",
    - "path" : "/",
    - "is_dashboard" : true
    - }
    - },
    - +
    +"services": [
+  {
+    "id": "hbase_master",
+    "exported_service": "hbase",
+    "label": {
+      "name": "HMaster"
+      },
+    "endpoint" : {
+      "url_scheme" : "http",
+      "port" : "60010",
+      "path" : "/",
+      "is_dashboard" : true
+      }
+    },
+

    In this example, services is a list of service objects.
    The defined services will be referenced by other elements of this JSON file to determine which services are active on which nodes within the cluster.
    @@ -90,16 +93,16 @@ Name and distro_id needs to be updated based on the version of the CDH image to
  • Node Roles

    • The example snippet for Node Roles: -
    -"node_roles": [
    - {
    - "id": "controller",
    - "cardinality": "1",
    - "anti_affinity_group_id": "CM",
    - "min_cores": "4",
    - "min_memory": "12288"
    - },
    -
    +
    +"node_roles": [
+  {
+    "id": "controller",
+    "cardinality": "1",
+    "anti_affinity_group_id": "CM",
+    "min_cores": "4",
+    "min_memory": "12288"
+  },
+
    In this example,node_roles is a list of objects describing roles that may be deployed for this Catalog entry. Each role is a particular configuration instantiated from the entry's virtual node image and configured by the setup scripts
  • Configuration
    • From 247794b01f1111191f399bad39280c68bc456342 Mon Sep 17 00:00:00 2001 From: Ben Hall Date: Mon, 25 Feb 2019 11:28:57 +0000 Subject: [PATCH 4/5] Other packages that could be pre-installed --- .../bluedata-workbench/scripts/1_configure.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/environments/bluedata-workbench/scripts/1_configure.sh b/environments/bluedata-workbench/scripts/1_configure.sh index b0a2f92..f9dd3d6 100644 --- a/environments/bluedata-workbench/scripts/1_configure.sh +++ b/environments/bluedata-workbench/scripts/1_configure.sh @@ -1,8 +1,20 @@ +yum install -y python-pip +pip install --upgrade setuptools +pip install --upgrade requests +pip install --upgrade argparse + pip install --upgrade pip pip install --upgrade bdworkbench +yum install -y epel-release + +yum install -y python-requests python-setuptools python-argparse python-pip + yum install wget -y wget https://www.dropbox.com/s/xyv64z1xt9l6lw7/bdcatalog-centos7-bluedata-cdh514_centos7x-1.7.tar tar xvf bdcatalog-centos7-bluedata-cdh514_centos7x-1.7.tar chmod +x bdcatalog-centos7-bluedata-cdh514_centos7x-1.7.bin +wget https://www.dropbox.com/s/wbnr83q26przbs6/appconfig.zip +yum install unzip -y +unzip appconfig.zip From db741abbe39a241c5095bccd4dd91c117a1f6108 Mon Sep 17 00:00:00 2001 From: Ben Hall Date: Mon, 6 May 2019 12:33:30 +0100 Subject: [PATCH 5/5] Add new bin files --- .../bluedata-workbench/{scripts => build}/1_configure.sh | 7 +++++++ 1 file changed, 7 insertions(+) rename environments/bluedata-workbench/{scripts => build}/1_configure.sh (59%) diff --git a/environments/bluedata-workbench/scripts/1_configure.sh b/environments/bluedata-workbench/build/1_configure.sh similarity index 59% rename from environments/bluedata-workbench/scripts/1_configure.sh rename to environments/bluedata-workbench/build/1_configure.sh index f9dd3d6..d3e5d37 100644 --- a/environments/bluedata-workbench/scripts/1_configure.sh +++ b/environments/bluedata-workbench/build/1_configure.sh @@ -14,6 +14,13 @@ wget https://www.dropbox.com/s/xyv64z1xt9l6lw7/bdcatalog-centos7-bluedata-cdh514 tar xvf bdcatalog-centos7-bluedata-cdh514_centos7x-1.7.tar chmod +x bdcatalog-centos7-bluedata-cdh514_centos7x-1.7.bin +wget https://www.dropbox.com/s/r4m3bba2qtueu5q/bdcatalog-centos7-bluedata-spark221-1.0.bin.tar +tar xvf bdcatalog-centos7-bluedata-spark221-1.0.bin.tar +chmod +x bdcatalog-centos7-bluedata-spark221-1.0.bin + +wget https://www.dropbox.com/s/2mh0pzsq1092xr8/bdcatalog-centos7-bluedata-hdp25-ambari-3.3.bin.tar.gz +tar xvf bdcatalog-centos7-bluedata-hdp25-ambari-3.3.bin.tar.gz +chmod +x bdcatalog-centos7-bluedata-hdp25-ambari-3.3.bin wget https://www.dropbox.com/s/wbnr83q26przbs6/appconfig.zip yum install unzip -y