Lit testing with coverage

.github/workflows/basic-ci.yml

@@ -57,9 +57,6 @@ jobs:
           - llvm-version: 14
             os: ubuntu-22.04
             preset: develop
-          - llvm-version: 18
-            os: ubuntu-24.04
-            preset: develop
           - llvm-version: 21
             os: ubuntu-24.04
             preset: develop
@@ -92,12 +89,18 @@ jobs:
           echo "CLANG_CMAKE_DIR=/usr/lib/llvm-${{ matrix.llvm-version }}/lib/cmake/clang" >> $GITHUB_ENV
           echo "EXTERNAL_LIT=/usr/lib/llvm-${{ matrix.llvm-version }}/build/utils/lit/lit.py" >> $GITHUB_ENV
 
-      - name: Build IRPrinter
-        run: |
-          cmake -B build -DCMAKE_BUILD_TYPE=Debug -DLLVM_DIR=${LLVM_CMAKE_DIR} -DClang_DIR=${CLANG_CMAKE_DIR}
-          cmake --build build --parallel
-
       - name: Build IRPrinter release
         run: |
-          cmake -B build_rel -DCMAKE_BUILD_TYPE=Release -DLLVM_DIR=${LLVM_CMAKE_DIR} -DClang_DIR=${CLANG_CMAKE_DIR}
+          cmake --preset release -DLLVM_DIR=${LLVM_CMAKE_DIR} -DClang_DIR=${CLANG_CMAKE_DIR} -DLLVM_EXTERNAL_LIT=${EXTERNAL_LIT}
           cmake --build build_rel --parallel --target install
+
+      - name: Test and Coverage
+        run: |
+          cmake --preset coverage -DLLVM_DIR=${LLVM_CMAKE_DIR} -DClang_DIR=${CLANG_CMAKE_DIR} -DLLVM_EXTERNAL_LIT=${EXTERNAL_LIT}
+          cmake --build build_cov --target coverage-irprinter
+
+      - name: Upload coverage
+        uses: actions/upload-artifact@v6
+        with:
+          name: coverage-report-llvm${{ matrix.llvm-version }}
+          path: build_cov/coverage_report/

CMakeLists.txt

@@ -1,6 +1,6 @@
-CMAKE_MINIMUM_REQUIRED(VERSION 3.20)
+CMAKE_MINIMUM_REQUIRED(VERSION 3.21)
 PROJECT(irprinter
-  VERSION 0.3
+  VERSION 0.4
 )
 
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
@@ -24,3 +24,6 @@ irprinter_add_format_target(format-sources
 )
 
 add_subdirectory(src)
+if(PROJECT_IS_TOP_LEVEL)
+  add_subdirectory(test)
+endif()

CMakePresets.json

@@ -0,0 +1,73 @@
+{
+  "version": 2,
+  "cmakeMinimumRequired": {
+    "major": 3,
+    "minor": 20,
+    "patch": 0
+  },
+  "configurePresets": [
+    {
+      "name": "clang-toolchain",
+      "hidden": true,
+      "generator": "Unix Makefiles",
+      "cacheVariables": {
+        "CMAKE_C_COMPILER": "clang",
+        "CMAKE_CXX_COMPILER": "clang++"
+      }
+    },
+    {
+      "name": "develop",
+      "displayName": "Develop (Debug)",
+      "description": "Default develop build options for Clang",
+      "binaryDir": "${sourceDir}/build",
+      "inherits": [
+        "clang-toolchain"
+      ],
+      "cacheVariables": {
+        "CMAKE_BUILD_TYPE": "Debug"
+      }
+    },
+    {
+      "name": "release",
+      "displayName": "Release",
+      "description": "Default release build options for Clang",
+      "binaryDir": "${sourceDir}/build_rel",
+      "inherits": [
+        "clang-toolchain"
+      ],
+      "cacheVariables": {
+        "CMAKE_BUILD_TYPE": "Release"
+      }
+    },
+    {
+      "name": "coverage",
+      "displayName": "Coverage (LLVM)",
+      "description": "Default coverage build options for Clang (LLVM-based)",
+      "binaryDir": "${sourceDir}/build_cov",
+      "inherits": [
+        "clang-toolchain"
+      ],
+      "cacheVariables": {
+        "CMAKE_BUILD_TYPE": "Debug",
+        "IRPRINTER_ENABLE_COVERAGE": "ON",
+        "CMAKE_C_FLAGS": "-fprofile-instr-generate -fcoverage-mapping",
+        "CMAKE_CXX_FLAGS": "-fprofile-instr-generate -fcoverage-mapping",
+        "CMAKE_EXE_LINKER_FLAGS": "-fprofile-instr-generate -fcoverage-mapping"
+      }
+    }
+  ],
+  "buildPresets": [
+    {
+      "name": "develop",
+      "configurePreset": "develop"
+    },
+    {
+      "name": "release",
+      "configurePreset": "release"
+    },
+    {
+      "name": "coverage",
+      "configurePreset": "coverage"
+    }
+  ]
+}

cmake/ToolchainOptions.cmake

@@ -27,9 +27,10 @@ include(clang-format)
 include(log-util)
 include(target-util)
 
-set(LOG_LEVEL 0 CACHE STRING "Granularity of the logger. 3 is most verbose, 0 is least.")
+set(IRPRINTER_LOG_LEVEL 0 CACHE STRING "Granularity of the logger. 3 is most verbose, 0 is least.")
 
 option(IRPRINTER_AUTO_RESOURCE_DIR "Try to automatically set the Clang resource directory" OFF)
+option(IRPRINTER_ENABLE_COVERAGE "Enable LLVM-based coverage" OFF)
 
 if(IRPRINTER_AUTO_RESOURCE_DIR AND NOT IRPRINTER_CLANG_RESOURCE_DIR)
   find_program(CLANG_EXECUTABLE NAMES clang-${LLVM_VERSION_MAJOR} clang)

cmake/modules/target-util.cmake

@@ -37,3 +37,54 @@ function(irprinter_target_project_compile_definitions target)
       )
   endif ()
 endfunction()
+
+function(irprinter_find_llvm_progs target names)
+  cmake_parse_arguments(ARG "ABORT_IF_MISSING;SHOW_VAR" "DEFAULT_EXE" "HINTS" ${ARGN})
+  set(TARGET_TMP ${target})
+
+  find_program(
+    ${target}
+    NAMES ${names}
+    PATHS ${LLVM_TOOLS_BINARY_DIR}
+    NO_DEFAULT_PATH
+  )
+  if(NOT ${target})
+    find_program(
+      ${target}
+      NAMES ${names}
+      HINTS ${ARG_HINTS}
+    )
+  endif()
+
+  if(NOT ${target})
+    set(target_missing_message "")
+    if(ARG_DEFAULT_EXE)
+      unset(${target} CACHE)
+      set(${target}
+          ${ARG_DEFAULT_EXE}
+          CACHE
+          STRING
+          "Default value for ${TARGET_TMP}."
+      )
+      set(target_missing_message "Using default: ${ARG_DEFAULT_EXE}")
+    endif()
+
+    set(message_status STATUS)
+    if(ARG_ABORT_IF_MISSING AND NOT ARG_DEFAULT_EXE)
+      set(message_status SEND_ERROR)
+    endif()
+    message(${message_status}
+      "Did find LLVM program " "${names}"
+      " in ${LLVM_TOOLS_BINARY_DIR}, in system path or hints " "\"${ARG_HINTS}\"" ". "
+      ${target_missing_message}
+    )
+  endif()
+
+  set(${TARGET_TMP} "${${TARGET_TMP}}" PARENT_SCOPE)
+
+  if(ARG_SHOW_VAR)
+    mark_as_advanced(CLEAR ${target})
+  else()
+    mark_as_advanced(${target})
+  endif()
+endfunction()

include/Util.h

@@ -133,7 +133,10 @@ inline llvm::SmallVector<const llvm::Function*, 4> regex_find(const llvm::Module
   if (use_mangle) {
     return detail::find(m, [&](const llvm::Function& f) { return r.match(f.getName()); });
   } else {
-    return detail::find(m, [&](const llvm::Function& f) { return r.match(try_demangle(f.getName())); });
+    return detail::find(m, [&](const llvm::Function& f) {
+      std::string name = try_demangle(f.getName());
+      return r.match(name);
+    });
   }
 }
 

src/main.cpp

@@ -121,4 +121,4 @@ int main(int argc, const char** argv) {
   }
 
   return 0;
-}
+}

src/printer/IRNodeFinder.cpp

@@ -21,13 +21,20 @@ using namespace llvm;
 namespace irprinter {
 
 namespace {
+std::string strip_parens(std::string name) {
+  if (name.size() > 2 && name.substr(name.size() - 2) == "()") {
+    name.erase(name.size() - 2);
+  }
+  return name;
+}
+
 template <typename F>
 void applyToMatchingFunction(llvm::raw_ostream& os, const llvm::Module* m, const std::string& regex, F&& func) {
   const auto fvec = util::regex_find(*m, regex);
   unsigned count{0};
   for (auto f : fvec) {
     auto fname = f->getName();
-    os << "Match " << ++count << " [" << util::try_demangle(fname) << "]:";
+    os << "Match " << ++count << " [" << strip_parens(util::try_demangle(fname)) << "]:";
     func(f);
     os << "\n";
   }
@@ -101,7 +108,7 @@ void IRNodeFinder::listFunction(const std::string& regex) const {
     oss.flush();
 
     if (f->isDeclaration()) {
-      os << "\n" << s.substr(1);
+      os << "\n" << s;
     } else {
       llvm::Regex r("((;|define)[^{]+){");
       SmallVector<StringRef, 2> match;
@@ -114,7 +121,7 @@ void IRNodeFinder::listFunction(const std::string& regex) const {
 }
 
 std::string IRNodeFinder::demangle(const std::string& name) {
-  return util::try_demangle(name);
+  return strip_parens(util::try_demangle(name));
 }
 
 } /* namespace irprinter */

test/CMakeLists.txt

@@ -0,0 +1,54 @@
+set(LLVM_IR_PRINTER_BINARY $<TARGET_FILE:llvm-ir-printer${EXE_SUFFIX}>)
+
+irprinter_find_llvm_progs(IRPRINTER_FILECHECK_EXEC "FileCheck-${LLVM_VERSION_MAJOR};FileCheck")
+
+if(LLVM_EXTERNAL_LIT)
+  cmake_path(GET LLVM_EXTERNAL_LIT PARENT_PATH LLVM_EXTERNAL_LIT_DIR)
+endif()
+irprinter_find_llvm_progs(IRPRINTER_LIT_EXEC
+  "llvm-lit;lit;lit.py"
+  HINTS ${LLVM_EXTERNAL_LIT_DIR} /usr/lib/llvm-${LLVM_VERSION_MAJOR} /usr/lib/llvm /usr/bin /usr/local/bin /opt/local/bin
+  ABORT_IF_MISSING
+)
+
+add_custom_target(check-irprinter
+  COMMAND ${IRPRINTER_LIT_EXEC} -vv ${CMAKE_CURRENT_BINARY_DIR}
+  DEPENDS llvm-ir-printer${EXE_SUFFIX}
+  COMMENT "Running irprinter regression tests"
+  USES_TERMINAL
+)
+
+if(IRPRINTER_ENABLE_COVERAGE)
+  set(COVERAGE_DIR ${CMAKE_BINARY_DIR}/coverage_report)
+  irprinter_find_llvm_progs(IRPRINTER_PROFDATA_EXEC "llvm-profdata-${LLVM_VERSION_MAJOR};llvm-profdata")
+  irprinter_find_llvm_progs(IRPRINTER_COV_EXEC "llvm-cov-${LLVM_VERSION_MAJOR};llvm-cov")
+
+  set(IGNORE_REGEX ".*(test|build|build_cov|external).*")
+
+  add_custom_target(coverage-irprinter
+    COMMAND ${CMAKE_COMMAND} -E make_directory ${COVERAGE_DIR}
+    COMMAND ${CMAKE_COMMAND} -E rm -f "${CMAKE_CURRENT_BINARY_DIR}/*.profraw" "${CMAKE_CURRENT_BINARY_DIR}/merged.profdata"
+    # Execute tests
+    COMMAND ${CMAKE_COMMAND} --build ${CMAKE_BINARY_DIR} --target check-irprinter
+    # Merge all profraw files
+    COMMAND ${CMAKE_COMMAND} -E env bash -c "${IRPRINTER_PROFDATA_EXEC} merge -sparse ${CMAKE_CURRENT_BINARY_DIR}/*.profraw -o ${CMAKE_CURRENT_BINARY_DIR}/merged.profdata"
+    # Generate HTML
+    COMMAND ${IRPRINTER_COV_EXEC} show "${LLVM_IR_PRINTER_BINARY}"
+            "-instr-profile=${CMAKE_CURRENT_BINARY_DIR}/merged.profdata"
+            -format=html
+            "-output-dir=${COVERAGE_DIR}"
+            -show-line-counts-or-regions
+            "-ignore-filename-regex=${IGNORE_REGEX}"
+    # Show summary
+    COMMAND ${IRPRINTER_COV_EXEC} report "${LLVM_IR_PRINTER_BINARY}"
+            "-instr-profile=${CMAKE_CURRENT_BINARY_DIR}/merged.profdata"
+            "-ignore-filename-regex=${IGNORE_REGEX}"
+    COMMENT "Generating coverage report"
+    DEPENDS check-irprinter
+    WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+    VERBATIM
+  )
+endif()
+
+configure_file(lit.site.cfg.py.in lit.site.cfg.py.tmp @ONLY)
+file(GENERATE OUTPUT lit.site.cfg.py INPUT ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg.py.tmp)

test/codes/decl.cpp

@@ -0,0 +1,9 @@
+// RUN: echo "l printf" | %llvm-ir-printer %s -- | %filecheck %s
+
+// CHECK: Match 1 [printf]:
+// CHECK-NEXT: declare i32 @printf
+extern "C" int printf(const char*, ...);
+int main() {
+  printf("test");
+  return 0;
+}

test/codes/demangle.cpp

@@ -0,0 +1,8 @@
+// RUN: echo "d _Z3foov" | %llvm-ir-printer %s -- | %filecheck %s
+
+// CHECK: Demangled name: foo
+void foo() {
+}
+int main() {
+  return 0;
+}

test/codes/dump.cpp

@@ -0,0 +1,7 @@
+// RUN: echo "dump" | %llvm-ir-printer %s -- | %filecheck %s
+
+// CHECK: source_filename = "{{.*}}dump.cpp"
+// CHECK: define {{.*}} @main
+int main() {
+  return 0;
+}

test/codes/errors.cpp

@@ -0,0 +1,11 @@
+// RUN: echo "10 5" | %llvm-ir-printer %s -- | %filecheck %s --check-prefix=LOC-ORDER
+// RUN: echo "5 invalid" | %llvm-ir-printer %s -- | %filecheck %s --check-prefix=LOC-PARSE
+// RUN: echo "l [" | %llvm-ir-printer %s -- | %filecheck %s --check-prefix=REGEX-ERR
+
+// LOC-ORDER: Error: end location (5) is less than start location (10)
+// LOC-PARSE: Invalid end location: invalid
+// REGEX-ERR: Invalid regex ({{.*}}): "["
+
+int main() {
+  return 0;
+}

test/codes/flags.cpp

@@ -0,0 +1,8 @@
+// RUN: echo -e "f -Wall\nf -O3\np main" | %llvm-ir-printer %s -- | %filecheck %s
+
+// CHECK: Set flag to -Wall
+// CHECK: Set flag to -O3
+// CHECK: define {{.*}} @main
+int main() {
+  return 0;
+}

test/codes/generate.cpp

@@ -0,0 +1,6 @@
+// RUN: echo "g" | %llvm-ir-printer %s -- | %filecheck %s
+// Without "-f -g" no effect, but no explicit error either.
+// CHECK-NOT: Error
+int main() {
+  return 0;
+}

test/codes/list.cpp

@@ -0,0 +1,9 @@
+// RUN: echo "l foo" | %llvm-ir-printer %s -- | %filecheck %s
+
+// CHECK: _Z3foov
+// CHECK-NOT: main
+void foo() {
+}
+int main() {
+  return 0;
+}

test/codes/list_all.cpp

@@ -0,0 +1,9 @@
+// RUN: echo "l" | %llvm-ir-printer %s -- | %filecheck %s
+
+// CHECK: Match 1 [foo]
+// CHECK: Match 2 [main]
+void foo() {
+}
+int main() {
+  return 0;
+}