Phase1

job_analyzer/a.yaml

@@ -0,0 +1,17 @@
+on: [push]
+
+jobs:
+  build:
+
+    runs-on: self-hosted
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up JDK 18
+        uses: actions/setup-java@v2
+        with:
+          java-version: '18'
+          distribution: 'adopt'
+          cache: maven
+      - name: Build with Maven Wrapper
+        run: ./mvnw spring-boot:run

job_analyzer/b.yaml

@@ -0,0 +1,19 @@
+
+jobs:
+  build:
+
+    runs-on: self-hosted
+
+    steps:
+      - run: touch file1
+      - uses: actions/checkout@v2
+      - run: touch file2
+      - name: Set up JDK 18
+        uses: actions/setup-java@v2
+        with:
+          java-version: '18'
+          distribution: 'adopt'
+          cache: maven
+      - run: touch file3
+      - name: Build with Maven Wrapper
+        run: ./mvnw spring-boot:run

job_analyzer/fork.py

@@ -0,0 +1,96 @@
+import json
+import requests
+import sys
+import time
+import os
+import base64
+
+
+# API endpoint
+GITHUB_API = 'https://api.github.com'
+
+github_auth_token = os.environ["G_AUTH_OP"]
+headers = {
+    'Authorization': 'token ' + github_auth_token
+}
+
+
+# Fork project on GitHub
+def fork_project(repository):
+    try:
+        url = GITHUB_API + '/repos/' + repository
+        response = requests.delete(url=url, headers=headers).json()
+        print(response)
+    except:
+        time.sleep(1)
+    time.sleep(1)
+    try:
+        url = GITHUB_API + '/repos/' + repository + '/forks'
+        response = requests.post(url=url, headers=headers).json()
+        print(response)
+    except:
+        time.sleep(1)
+
+
+def take_ci_file(repository):
+    file_path = ".github/workflows/RunPetClinic.yml"
+    try:
+        response = requests.get(
+            url="https://api.github.com/repos/" + repository + "/contents/" + file_path,
+            headers=headers).json()
+        if "content" in response:
+            content = base64.b64decode(response["content"]).decode("utf-8")
+            with open('a.yaml', 'w') as a:
+                a.write(content)
+            parse_ci_file()
+    except:
+        time.sleep(1)
+
+
+def parse_ci_file():
+    with open('a.yaml') as a, open('b.yaml', 'w') as b:
+        x = a.readline()
+        indent = 0
+        file_count = 1
+        while x != "":
+            x = a.readline()
+            if "- uses" in x or "- name" in x or "- run" in x:
+                indent = len(x) - len(x.lstrip())
+                change = ' ' * indent + "- run: touch file" + str(file_count) + "\n"
+                file_count = file_count + 1
+                b.write(change)
+                b.write(x)
+            else:
+                b.write(x)
+
+
+def main():
+    # PHASE-1: COLLECTION
+    """FORKING THE PROJECT (VIA GITHUB API)"""
+    """PARSING THE YAML FILE"""
+    """CHANGING THE YAML FILE"""
+    fork_project("bsknblc/spring-petclinic")
+    take_ci_file("optimizing-ci-builds/spring-petclinic")
+
+    # # PHASE-2: SETUP
+    # """SETTING UP RUNNER"""
+    # """SETTING UP THE ENVIRONMENT FOR THE RUNNER AND THE INOTIFYWAIT"""
+    # """RUNNING THE RUNNER AND THE INOTIFYWAIT"""
+    # token: str = "some another dummy token"
+    # runner_version: str = "2.294.0"
+    # tar_filename: str = f"linux-x64-{runner_version}.tar.gz"
+    # utils.setup_runner(tar_filename=tar_filename, runner_version=runner_version, token=token)
+    #
+    # # PHASE-3: EXECUTION
+    # """COMMITTING THE CHANGES IN THE YAML, TRIGGERING THE RUNNER AND INOTIFYWAIT"""
+    # monitored_dir_path: str = ""
+    # csv_filename: str = ""
+    # os.popen(f"inotifywait -mr {monitored_dir_path} --format %T,%w%f,%e --timefmt %T -o {csv_filename}.csv")
+    #
+    # # PHASE-4: ANALYSIS
+    # """ANALYZING THE CSV PRODUCED BY INOTIFYWAIT"""
+    # """PRINTING THE JOB (LINE NUMBER) FROM THE YAML FILE CAUSING UNNECESSARY USAGE"""
+
+
+if __name__ == "__main__":
+    main()

scripts/content_collector.py

@@ -12,7 +12,7 @@
 
 # Read the data from the csv file.
 repositories = []
-with open("../data/file_paths.csv", "r", newline="", encoding="utf8") as csv_file:
+with open("data/file_paths.csv", "r", newline="", encoding="utf8") as csv_file:
     csv_reader = csv.reader(csv_file, delimiter=',')
     next(csv_reader, None)
     for row in csv_reader:
@@ -39,12 +39,16 @@
 
     # Travis tools
     repository["Tyml_codecov"] = ""
+    repository["Tyml_coveralls"] = ""
+    repository["Tyml_codacy"] = ""
     repository["Tyml_jacoco"] = ""
     repository["Tyml_cobertura"] = ""
     repository["Tyml_javadoc"] = ""
 
     # Github Action Tools
     repository["Gyml_codecov"] = ""
+    repository["Gyml_coveralls"] = ""
+    repository["Gyml_codacy"] = ""
     repository["Gyml_jacoco"] = ""
     repository["Gyml_cobertura"] = ""
     repository["Gyml_javadoc"] = ""
@@ -138,8 +142,12 @@
                         headers=headers).json()
                     if "content" in response:
                         travis_content = base64.b64decode(response["content"]).decode("utf-8")
-                        if "https://codecov.io/bash" in travis_content:
+                        if "codecov" in travis_content:
                             repository["Tyml_codecov"] = repository["Tyml_codecov"] + file_path + ";"
+                        if "coveralls" in travis_content or "COVERALLS" in travis_content:
+                            repository["Tyml_coveralls"] = repository["Tyml_coveralls"] + file_path + ";"
+                        if "codacy" in travis_content or "CODACY" in travis_content:
+                            repository["Tyml_codacy"] = repository["Tyml_codacy"] + file_path + ";"
                         if "jacoco" in travis_content and "jacoco.skip=true" not in travis_content:
                             repository["Tyml_jacoco"] = repository["Tyml_jacoco"] + file_path + ";"
                         if "cobertura" in travis_content:
@@ -151,6 +159,8 @@
                         j = j + 1
                 except:
                     repository["Tyml_codecov"] = repository["Tyml_codecov"] + "Skipped;"
+                    repository["Tyml_coveralls"] = repository["Tyml_coveralls"] + "Skipped;"
+                    repository["Tyml_codacy"] = repository["Tyml_codacy"] + "Skipped;"
                     repository["Tyml_jacoco"] = repository["Tyml_jacoco"] + "Skipped;"
                     repository["Tyml_cobertura"] = repository["Tyml_cobertura"] + "Skipped;"
                     repository["Tyml_javadoc"] = repository["Tyml_javadoc"] + "Skipped;"
@@ -176,9 +186,13 @@
                         headers=headers).json()
                     if "content" in response:
                         yml_content = base64.b64decode(response["content"]).decode("utf-8")
-                        # checking if the *.yml contains "https://codecov.io/bash" keyword
-                        if "https://codecov.io/bash" in yml_content:
+                        # checking if the *.yml contains "codecov" keyword
+                        if "codecov" in yml_content:
                             repository["Gyml_codecov"] = repository["Gyml_codecov"] + file_path + ";"
+                        if "coveralls" in yml_content or "COVERALLS" in yml_content:
+                            repository["Gyml_coveralls"] = repository["Gyml_coveralls"] + file_path + ";"
+                        if "codacy" in yml_content or "CODACY" in yml_content:
+                            repository["Gyml_codacy"] = repository["Gyml_codacy"] + file_path + ";"
                         if "jacoco" in yml_content and "jacoco.skip=true" not in yml_content:
                             repository["Gyml_jacoco"] = repository["Gyml_jacoco"] + file_path + ";"
                         if "cobertura" in yml_content:
@@ -190,6 +204,8 @@
                         j = j + 1
                 except:
                     repository["Gyml_codecov"] = repository["Gyml_codecov"] + "Skipped;"
+                    repository["Gyml_coveralls"] = repository["Gyml_coveralls"] + "Skipped;"
+                    repository["Gyml_codacy"] = repository["Gyml_codacy"] + "Skipped;"
                     repository["Gyml_jacoco"] = repository["Gyml_jacoco"] + "Skipped;"
                     repository["Gyml_cobertura"] = repository["Gyml_cobertura"] + "Skipped;"
                     repository["Gyml_javadoc"] = repository["Gyml_javadoc"] + "Skipped;"
@@ -200,15 +216,17 @@
     print(i)
 
 # Save repositories to a csv file
-with open("../data/file_contents.csv", "w", newline="", encoding="utf-8") as csv_file:
+with open("data/file_contents.csv", "w", newline="", encoding="utf-8") as csv_file:
     csv_writer = csv.writer(csv_file)
     csv_writer.writerow(
         ["name", "link", "Default Branch", "SHA", "Stargazers Count", "Forks Count",
          "Maven", "Gradle", "Travis CI", "Github Actions",
          "Maven Jacoco", " Maven Cobertura", "Maven Javadoc",
          "Gradle Jacoco", "Gradle Cobertura", "Gradle Javadoc",
-         "Travis Codecov", "Travis Jacoco", " Travis Cobertura", "Travis Javadoc",
-         "GA Codecov", "GA Jacoco", " GA Cobertura", "GA Javadoc"
+         "Travis Codecov", "Travis Coveralls", "Travis Codacy",
+         "Travis Jacoco", " Travis Cobertura", "Travis Javadoc",
+         "GA Codecov", "GA Coveralls", "GA Codacy",
+         "GA Jacoco", " GA Cobertura", "GA Javadoc"
          ])
 
     for repository in repositories:
@@ -218,7 +236,7 @@
                              repository["Travis CI"], repository["Github Actions"],
                              repository["MJacoco"], repository["MCobertura"], repository["MJavadoc"],
                              repository["GJacoco"], repository["GCobertura"], repository["GJavadoc"],
-                             repository["Tyml_codecov"], repository["Tyml_jacoco"], repository["Tyml_cobertura"],
-                             repository["Tyml_javadoc"],
-                             repository["Gyml_codecov"], repository["Gyml_jacoco"], repository["Gyml_cobertura"],
-                             repository["Gyml_javadoc"]])
+                             repository["Tyml_codecov"], repository["Tyml_coveralls"], repository["Tyml_codacy"],
+                             repository["Tyml_jacoco"], repository["Tyml_cobertura"], repository["Tyml_javadoc"],
+                             repository["Gyml_codecov"], repository["Gyml_coveralls"], repository["Gyml_codacy"],
+                             repository["Gyml_jacoco"], repository["Gyml_cobertura"], repository["Gyml_javadoc"]])

scripts/file_collector.py

@@ -11,7 +11,7 @@
 
 # Read the repositories from the csv file.
 repositories = []
-with open("../data/repositories.csv", "r", newline="", encoding="utf-8") as csv_file:
+with open("data/repositories.csv", "r", newline="", encoding="utf-8") as csv_file:
     csv_reader = csv.reader(csv_file, delimiter=',')
     next(csv_reader, None)
     for row in csv_reader:

scripts/statistics.py

@@ -5,7 +5,7 @@
 import time
 
 repositories = []
-with open("../data/file_contents.csv", "r", newline="", encoding="utf8") as csv_file:
+with open("data/file_contents.csv", "r", newline="", encoding="utf8") as csv_file:
     csv_reader = csv.reader(csv_file, delimiter=',')
     next(csv_reader, None)
     for row in csv_reader:
@@ -299,7 +299,7 @@
             MGyml_jacoco_cobertura_javadoc += 1
 
 # Save statistics to a csv file
-with open("../data/statistics.csv", "w", newline="", encoding="utf-8") as csv_file:
+with open("data/statistics.csv", "w", newline="", encoding="utf-8") as csv_file:
     csv_writer = csv.writer(csv_file)
     csv_writer.writerow(
         ["Maven", "Gradle", "Travis CI", "Github Actions", "Travis or GA"])