From 8141d239f6be9e80f3c00519912991eda5a6c366 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Thu, 11 Jun 2026 17:59:42 -0400 Subject: [PATCH 01/21] Include pre-build differenty --- Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 6506208d58..d0e1df3de9 100644 --- a/Makefile +++ b/Makefile @@ -1092,6 +1092,8 @@ test/%.d: ; rm $${TESTDIR}/empty.proto $${TESTDIR}/empty.pb.h $${TESTDIR}/empty.pb.cc; \ rmdir $${TESTDIR}; \ fi; + echo "" > .pre-build + echo "Did pre-build" # A note about Protobuf: # We have a lot of logic here to make sure that the protoc we have henerates headers with exactly the same @@ -1102,7 +1104,7 @@ test/%.d: ; # run .pre-build before we make anything at all. --include .pre-build +include .pre-build # for rebuilding just vg clean-vg: From c40e43a7f4abcb0b2d90a2726a2be4f9afb88d6f Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Thu, 11 Jun 2026 18:19:16 -0400 Subject: [PATCH 02/21] De-phony-ify pre-build --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index d0e1df3de9..df8813d479 100644 --- a/Makefile +++ b/Makefile @@ -519,7 +519,7 @@ DEPS += $(INC_DIR)/BooPHF.h DEPS += $(INC_DIR)/mio/mmap.hpp DEPS += $(INC_DIR)/atomic_queue.h -.PHONY: clean clean-tests get-deps deps lint test set-path objs static static-docker docs man .pre-build version +.PHONY: clean clean-tests get-deps deps lint test set-path objs static static-docker docs man version # Aggregate all libvg deps, and exe deps other than libvg LIBVG_DEPS = $(OBJ) $(ALGORITHMS_OBJ) $(IO_OBJ) $(DEP_OBJ) $(DEPS) @@ -1092,7 +1092,7 @@ test/%.d: ; rm $${TESTDIR}/empty.proto $${TESTDIR}/empty.pb.h $${TESTDIR}/empty.pb.cc; \ rmdir $${TESTDIR}; \ fi; - echo "" > .pre-build + echo '$$(info Ran pre-build previously)' > .pre-build echo "Did pre-build" # A note about Protobuf: From a7e3569faa14fc6815428302baafbdd5afffa181 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Thu, 11 Jun 2026 18:55:27 -0400 Subject: [PATCH 03/21] Move pre-build target to a script and shell calls to support GNU Make ~4.4+ --- .gitignore | 2 +- Makefile | 79 +++++++++++++++++++--------------------------------- pre-build.sh | 31 +++++++++++++++++++++ 3 files changed, 61 insertions(+), 51 deletions(-) create mode 100755 pre-build.sh diff --git a/.gitignore b/.gitignore index 7cb3dfee5f..25412b5335 100644 --- a/.gitignore +++ b/.gitignore @@ -8,7 +8,7 @@ share/ include/ obj/ cpp/ -.pre-build +pre-build.log src/vg_git_version.hpp src/vg_environment_version.hpp *.o diff --git a/Makefile b/Makefile index df8813d479..2cc7e2c921 100644 --- a/Makefile +++ b/Makefile @@ -1055,56 +1055,35 @@ test/%.d: ; ## VG source code compilation ends here #################################### - -# Make directories before quitting target due to missing protoc. -# If we run the rest of the build without these, lib and include can become files. -# TODO: quitting if no protoc doesn't reliably stop the build. -.pre-build: - @if [ ! -d $(BIN_DIR) ]; then mkdir -p $(BIN_DIR); fi - @if [ ! -d $(UNITTEST_BIN_DIR) ]; then mkdir -p $(UNITTEST_BIN_DIR); fi - @if [ ! -d $(LIB_DIR) ]; then mkdir -p $(LIB_DIR); fi - @if [ ! -d $(OBJ_DIR) ]; then mkdir -p $(OBJ_DIR); fi - @if [ ! -d $(SHARED_OBJ_DIR) ]; then mkdir -p $(SHARED_OBJ_DIR); fi - @if [ ! -d $(ALGORITHMS_OBJ_DIR) ]; then mkdir -p $(ALGORITHMS_OBJ_DIR); fi - @if [ ! -d $(ALGORITHMS_SHARED_OBJ_DIR) ]; then mkdir -p $(ALGORITHMS_SHARED_OBJ_DIR); fi - @if [ ! -d $(CONFIG_OBJ_DIR) ]; then mkdir -p $(CONFIG_OBJ_DIR); fi - @if [ ! -d $(IO_OBJ_DIR) ]; then mkdir -p $(IO_OBJ_DIR); fi - @if [ ! -d $(IO_SHARED_OBJ_DIR) ]; then mkdir -p $(IO_SHARED_OBJ_DIR); fi - @if [ ! -d $(SUBCOMMAND_OBJ_DIR) ]; then mkdir -p $(SUBCOMMAND_OBJ_DIR); fi - @if [ ! -d $(UNITTEST_OBJ_DIR) ]; then mkdir -p $(UNITTEST_OBJ_DIR); fi - @if [ ! -d $(UNITTEST_SUPPORT_OBJ_DIR) ]; then mkdir -p $(UNITTEST_SUPPORT_OBJ_DIR); fi - @if [ ! -d $(INC_DIR) ]; then mkdir -p $(INC_DIR); fi - @protoc --version >/dev/null 2>/dev/null || (echo "Error: protobuf compiler (protoc) not available!" ; exit 1) - @if [ -e $(INC_DIR)/vg/vg.pb.h ] ; then \ - HEADER_VER=$$(cat $(INC_DIR)/vg/vg.pb.h | grep GOOGLE_PROTOBUF_VERSION | sed 's/[^0-9]*\([0-9]*\)[^0-9]*/\1/' | head -n1); \ - WORKDIR=$$(pwd); \ - TESTDIR=$$(mktemp -d); \ - echo 'syntax = "proto3";' > $${TESTDIR}/empty.proto; \ - protoc $${TESTDIR}/empty.proto --proto_path=$${TESTDIR} --cpp_out=$${TESTDIR}; \ - PROTOC_VER=$$(cat $${TESTDIR}/empty.pb.h | grep GOOGLE_PROTOBUF_VERSION | sed 's/[^0-9]*\([0-9]*\)[^0-9]*/\1/' | head -n1); \ - if [ "$${HEADER_VER}" != "$${PROTOC_VER}" ] ; then \ - echo "Protobuf version has changed!"; \ - echo "Headers are for $${HEADER_VER} but we make headers for $${PROTOC_VER}"; \ - echo "Need to rebuild libvgio"; \ - rm -f $(LIB_DIR)/libvgio.a; \ - rm -f $(INC_DIR)/vg/vg.pb.h; \ - fi; \ - rm $${TESTDIR}/empty.proto $${TESTDIR}/empty.pb.h $${TESTDIR}/empty.pb.cc; \ - rmdir $${TESTDIR}; \ - fi; - echo '$$(info Ran pre-build previously)' > .pre-build - echo "Did pre-build" - -# A note about Protobuf: -# We have a lot of logic here to make sure that the protoc we have henerates headers with exactly the same -# version requirements as the headers we already have. -# If not, we regenerate them. -# Doesn't handle Protobuf 3.12.3 weirdness; just make clean if you change flavors of Protobuf 3.12.3. - - - -# run .pre-build before we make anything at all. -include .pre-build +# Make all the directories we need. +# If these don't exist, they can become files (somehow) when we run the rest of +# the build. +# We used to run a target to do this but since GNU Make 4.4 we can't easily run +# a target at startup by `-include`-ing a .PHONY target so we need to do it at +# top-level with shell calls. See +$(shell bash -c "if [ ! -d $(BIN_DIR) ]; then mkdir -p $(BIN_DIR); fi") +$(shell bash -c "if [ ! -d $(UNITTEST_BIN_DIR) ]; then mkdir -p $(UNITTEST_BIN_DIR); fi") +$(shell bash -c "if [ ! -d $(LIB_DIR) ]; then mkdir -p $(LIB_DIR); fi") +$(shell bash -c "if [ ! -d $(OBJ_DIR) ]; then mkdir -p $(OBJ_DIR); fi") +$(shell bash -c "if [ ! -d $(SHARED_OBJ_DIR) ]; then mkdir -p $(SHARED_OBJ_DIR); fi") +$(shell bash -c "if [ ! -d $(ALGORITHMS_OBJ_DIR) ]; then mkdir -p $(ALGORITHMS_OBJ_DIR); fi") +$(shell bash -c "if [ ! -d $(ALGORITHMS_SHARED_OBJ_DIR) ]; then mkdir -p $(ALGORITHMS_SHARED_OBJ_DIR); fi") +$(shell bash -c "if [ ! -d $(CONFIG_OBJ_DIR) ]; then mkdir -p $(CONFIG_OBJ_DIR); fi") +$(shell bash -c "if [ ! -d $(IO_OBJ_DIR) ]; then mkdir -p $(IO_OBJ_DIR); fi") +$(shell bash -c "if [ ! -d $(IO_SHARED_OBJ_DIR) ]; then mkdir -p $(IO_SHARED_OBJ_DIR); fi") +$(shell bash -c "if [ ! -d $(SUBCOMMAND_OBJ_DIR) ]; then mkdir -p $(SUBCOMMAND_OBJ_DIR); fi") +$(shell bash -c "if [ ! -d $(UNITTEST_OBJ_DIR) ]; then mkdir -p $(UNITTEST_OBJ_DIR); fi") +$(shell bash -c "if [ ! -d $(UNITTEST_SUPPORT_OBJ_DIR) ]; then mkdir -p $(UNITTEST_SUPPORT_OBJ_DIR); fi") +$(shell bash -c "if [ ! -d $(INC_DIR) ]; then mkdir -p $(INC_DIR); fi") + +# run pre-build.sh before we make anything at all. +# Make really doesn't want to depend on this succeeding, but we need it to. +# See . +PREBUILD_RESULT=$(shell ./pre-build.sh >pre-build.log 2>&1 ; echo $$?) +$(info $(shell cat pre-build.log)) +ifneq ($(PREBUILD_RESULT), 0) + $(error Pre-build script failed) +endif # for rebuilding just vg clean-vg: diff --git a/pre-build.sh b/pre-build.sh new file mode 100755 index 0000000000..662a899f0d --- /dev/null +++ b/pre-build.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# pre-build.sh: setup commands to run before building anything. +# Invoked by the Makefile automatically. +# If this script fails, the build will fail. +set -e + +# TODO: quitting if no protoc doesn't reliably stop the build. +protooc --version >/dev/null 2>/dev/null || (echo "Error: protobuf compiler (protoc) not available!" ; exit 1) +if [ -e include/vg/vg.pb.h ] ; then \ + HEADER_VER=$(cat include/vg/vg.pb.h | grep GOOGLE_PROTOBUF_VERSION | sed 's/[^0-9]*\([0-9]*\)[^0-9]*/\1/' | head -n1); \ + WORKDIR=$(pwd); \ + TESTDIR=$(mktemp -d); \ + echo 'syntax = "proto3";' > ${TESTDIR}/empty.proto; \ + protoc ${TESTDIR}/empty.proto --proto_path=${TESTDIR} --cpp_out=${TESTDIR}; \ + PROTOC_VER=$(cat ${TESTDIR}/empty.pb.h | grep GOOGLE_PROTOBUF_VERSION | sed 's/[^0-9]*\([0-9]*\)[^0-9]*/\1/' | head -n1); \ + if [ "$${HEADER_VER}" != "$${PROTOC_VER}" ] ; then \ + echo "Protobuf version has changed!"; \ + echo "Headers are for $${HEADER_VER} but we make headers for $${PROTOC_VER}"; \ + echo "Need to rebuild libvgio"; \ + rm -f lib/libvgio.a; \ + rm -f include/vg/vg.pb.h; \ + fi; \ + rm ${TESTDIR}/empty.proto ${TESTDIR}/empty.pb.h ${TESTDIR}/empty.pb.cc; \ + rmdir ${TESTDIR}; \ +fi; + +# A note about Protobuf: +# We have a lot of logic here to make sure that the protoc we have generates headers with exactly the same +# version requirements as the headers we already have. +# If not, we regenerate them. +# Doesn't handle Protobuf 3.12.3 weirdness; just make clean if you change flavors of Protobuf 3.12.3. From 399b720823e1e7764a38d453148eec21747cffe6 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Thu, 11 Jun 2026 18:59:52 -0400 Subject: [PATCH 04/21] Un-double more dollar signs --- pre-build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pre-build.sh b/pre-build.sh index 662a899f0d..bdb89b726d 100755 --- a/pre-build.sh +++ b/pre-build.sh @@ -13,9 +13,9 @@ if [ -e include/vg/vg.pb.h ] ; then \ echo 'syntax = "proto3";' > ${TESTDIR}/empty.proto; \ protoc ${TESTDIR}/empty.proto --proto_path=${TESTDIR} --cpp_out=${TESTDIR}; \ PROTOC_VER=$(cat ${TESTDIR}/empty.pb.h | grep GOOGLE_PROTOBUF_VERSION | sed 's/[^0-9]*\([0-9]*\)[^0-9]*/\1/' | head -n1); \ - if [ "$${HEADER_VER}" != "$${PROTOC_VER}" ] ; then \ + if [ "${HEADER_VER}" != "${PROTOC_VER}" ] ; then \ echo "Protobuf version has changed!"; \ - echo "Headers are for $${HEADER_VER} but we make headers for $${PROTOC_VER}"; \ + echo "Headers are for ${HEADER_VER} but we make headers for ${PROTOC_VER}"; \ echo "Need to rebuild libvgio"; \ rm -f lib/libvgio.a; \ rm -f include/vg/vg.pb.h; \ From c4c968043871e7f29bf496b653aa81471fd882ca Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Thu, 11 Jun 2026 19:03:54 -0400 Subject: [PATCH 05/21] Run script right away --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 2cc7e2c921..44464d3060 100644 --- a/Makefile +++ b/Makefile @@ -1079,7 +1079,8 @@ $(shell bash -c "if [ ! -d $(INC_DIR) ]; then mkdir -p $(INC_DIR); fi") # run pre-build.sh before we make anything at all. # Make really doesn't want to depend on this succeeding, but we need it to. # See . -PREBUILD_RESULT=$(shell ./pre-build.sh >pre-build.log 2>&1 ; echo $$?) +# Make sure to run right now. +PREBUILD_RESULT:=$(shell ./pre-build.sh >pre-build.log 2>&1 ; echo $$?) $(info $(shell cat pre-build.log)) ifneq ($(PREBUILD_RESULT), 0) $(error Pre-build script failed) From eea62dd6d71547879c20669983d16ee4afb9bc98 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Thu, 11 Jun 2026 19:24:53 -0400 Subject: [PATCH 06/21] Un-break protobuf compiler name --- pre-build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pre-build.sh b/pre-build.sh index bdb89b726d..416330b81c 100755 --- a/pre-build.sh +++ b/pre-build.sh @@ -5,8 +5,8 @@ set -e # TODO: quitting if no protoc doesn't reliably stop the build. -protooc --version >/dev/null 2>/dev/null || (echo "Error: protobuf compiler (protoc) not available!" ; exit 1) -if [ -e include/vg/vg.pb.h ] ; then \ +protoc --version >/dev/null 2>/dev/null || (echo "Error: protobuf compiler (protoc) not available!" ; exit 1) +if [ -e include/vg/vg.pb.h ] ; then HEADER_VER=$(cat include/vg/vg.pb.h | grep GOOGLE_PROTOBUF_VERSION | sed 's/[^0-9]*\([0-9]*\)[^0-9]*/\1/' | head -n1); \ WORKDIR=$(pwd); \ TESTDIR=$(mktemp -d); \ From bc4cecb539ac5ee41e2eff8b7551ef4c99637364 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Thu, 11 Jun 2026 19:37:10 -0400 Subject: [PATCH 07/21] Move protoc install check to before libvgio build so we can make get-deps without it --- Makefile | 4 ++-- pre-build.sh | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 44464d3060..6ba8a35c3e 100644 --- a/Makefile +++ b/Makefile @@ -722,6 +722,7 @@ $(LIB_DIR)/cleaned_old_catch: +touch $(LIB_DIR)/cleaned_old_catch $(LIB_DIR)/libvgio.a: $(LIB_DIR)/libhts.a $(LIB_DIR)/libhandlegraph.a $(LIB_DIR)/pkgconfig/htslib.pc $(LIB_DIR)/cleaned_old_protobuf_v003 $(LIBVGIO_DIR)/CMakeLists.txt $(LIBVGIO_DIR)/src/*.cpp $(LIBVGIO_DIR)/include/vg/io/*.hpp $(LIBVGIO_DIR)/deps/vg.proto + @protoc --version >/dev/null 2>/dev/null || (echo "Error: protobuf compiler (protoc) not available!" ; exit 1) +rm -f $(CWD)/$(INC_DIR)/vg.pb.h $(CWD)/$(INC_DIR)/vg/vg.pb.h +rm -Rf $(CWD)/$(INC_DIR)/vg/io/ +export CXXFLAGS="$(CPPFLAGS) $(CXXFLAGS)" && export LDFLAGS="$(LD_LIB_DIR_FLAGS) $(LDFLAGS)" && cd $(LIBVGIO_DIR) && rm -Rf CMakeCache.txt CMakeFiles *.cmake install_manifest.txt *.pb.cc *.pb.h *.a && rm -rf build-vg && mkdir build-vg && cd build-vg && PKG_CONFIG_PATH=$(CWD)/$(LIB_DIR)/pkgconfig:$(PKG_CONFIG_PATH) cmake -DCMAKE_C_COMPILER="$(CC)" -DCMAKE_CXX_COMPILER="$(CXX)" -DCMAKE_CXX_STANDARD=$(CXX_STANDARD) -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_PREFIX_PATH="/usr;$(OMP_PREFIXES)" -DCMAKE_INSTALL_PREFIX=$(CWD) -DCMAKE_INSTALL_LIBDIR=lib -DUSE_INSTALLED_LIBHANDLEGRAPH_ONLY=ON .. $(FILTER) && $(MAKE) clean && VERBOSE=1 $(MAKE) $(FILTER) && $(MAKE) install @@ -1081,9 +1082,8 @@ $(shell bash -c "if [ ! -d $(INC_DIR) ]; then mkdir -p $(INC_DIR); fi") # See . # Make sure to run right now. PREBUILD_RESULT:=$(shell ./pre-build.sh >pre-build.log 2>&1 ; echo $$?) -$(info $(shell cat pre-build.log)) ifneq ($(PREBUILD_RESULT), 0) - $(error Pre-build script failed) + $(error Pre-build script failed; see pre-build.log) endif # for rebuilding just vg diff --git a/pre-build.sh b/pre-build.sh index 416330b81c..9f3c474856 100755 --- a/pre-build.sh +++ b/pre-build.sh @@ -4,9 +4,10 @@ # If this script fails, the build will fail. set -e -# TODO: quitting if no protoc doesn't reliably stop the build. -protoc --version >/dev/null 2>/dev/null || (echo "Error: protobuf compiler (protoc) not available!" ; exit 1) -if [ -e include/vg/vg.pb.h ] ; then +# We don't want to put dependency checks here because we run before `make +# get-deps` can do any dep-getting. + +if which protoc >/dev/null 2>&1 && [ -e include/vg/vg.pb.h ] ; then HEADER_VER=$(cat include/vg/vg.pb.h | grep GOOGLE_PROTOBUF_VERSION | sed 's/[^0-9]*\([0-9]*\)[^0-9]*/\1/' | head -n1); \ WORKDIR=$(pwd); \ TESTDIR=$(mktemp -d); \ From b87ebaa9b22418c15a1777149d2d3781c80af6d7 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Thu, 11 Jun 2026 19:53:08 -0400 Subject: [PATCH 08/21] Update elfutils --- deps/elfutils | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/elfutils b/deps/elfutils index 983e86fd89..302252356d 160000 --- a/deps/elfutils +++ b/deps/elfutils @@ -1 +1 @@ -Subproject commit 983e86fd89e8bf02f2d27ba5dce5bf078af4ceda +Subproject commit 302252356da5475670ac5b10dadd091c59689425 From 7376dcfc87bf2d73a2020a6ac34a7a4cf7457b24 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Thu, 11 Jun 2026 19:54:04 -0400 Subject: [PATCH 09/21] Bump CI Docker image base so CI can test the 26.04 build for me --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b068542a67..34e660d8b0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ # Use Google's non-rate-limited mirror of Docker Hub to get our base image. # This helps automated Quay builds because Quay hasn't built a caching system # and exposes pull rate limits to users. -FROM mirror.gcr.io/library/ubuntu:22.04 AS base +FROM mirror.gcr.io/library/ubuntu:26.04 AS base MAINTAINER vgteam RUN echo base > /stage.txt From e79a5c1e05c87b7560bb1126bd2e76ea67f0c999 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Fri, 12 Jun 2026 13:29:21 -0400 Subject: [PATCH 10/21] Deal with missing src --- Makefile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 6ba8a35c3e..51d0d38754 100644 --- a/Makefile +++ b/Makefile @@ -935,6 +935,9 @@ $(LIB_DIR)/libxg.a: $(XG_DIR)/src/*.hpp $(XG_DIR)/src/*.cpp $(INC_DIR)/mmmultima +$(CXX) $(INCLUDE_FLAGS) $(CXXFLAGS) $(CPPFLAGS) -fPIC -DNO_GFAKLUGE -c -o $(XG_DIR)/xg.o $(XG_DIR)/src/xg.cpp $(FILTER) +ar rs $@ $(XG_DIR)/xg.o +# All this version header generation stuff needs to work even if SRC_DIR hasn't been copied into the container yet. +$(shell mkdir -p $(SRC_DIR) + # Auto-git-versioning # Can be overridden from the environment to supply a version if none is on disk. @@ -958,7 +961,7 @@ ifeq ($(shell if [ -d .git ]; then echo present; else echo absent; fi),present) else # Just use the version file we have, if any. $(info Do not check Git) - $(shell if [ ! -e $(SRC_DIR)/vg_git_version.hpp] ; then echo "#define VG_GIT_VERSION \"$(VG_GIT_VERSION)\"" > $(SRC_DIR)/vg_git_version.hpp ; fi) + $(shell if [ ! -e $(SRC_DIR)/vg_git_version.hpp ] ; then echo "#define VG_GIT_VERSION \"$(VG_GIT_VERSION)\"" > $(SRC_DIR)/vg_git_version.hpp ; fi) endif @@ -969,7 +972,7 @@ version: # Build an environment version file. # If it's not the same as the old one, replace the old one. # If it is the same, do nothing and don't rebuild dependent targets. -# Clean old path +# Clean old path. $(shell rm -f $(INC_DIR)/vg_environment_version.hpp) $(shell echo "#define VG_COMPILER_VERSION \"$(shell $(CXX) --version 2>/dev/null | head -n 1)\"" > $(SRC_DIR)/vg_environment_version.hpp.tmp) $(shell echo "#define VG_OS \"$(shell uname)\"" >> $(SRC_DIR)/vg_environment_version.hpp.tmp) From ea167b9d6e54ac2b4f449b73b20d1c38ae14a81f Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Fri, 12 Jun 2026 13:30:28 -0400 Subject: [PATCH 11/21] Dump script log --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 51d0d38754..3ca016fdec 100644 --- a/Makefile +++ b/Makefile @@ -1086,7 +1086,8 @@ $(shell bash -c "if [ ! -d $(INC_DIR) ]; then mkdir -p $(INC_DIR); fi") # Make sure to run right now. PREBUILD_RESULT:=$(shell ./pre-build.sh >pre-build.log 2>&1 ; echo $$?) ifneq ($(PREBUILD_RESULT), 0) - $(error Pre-build script failed; see pre-build.log) + $(info $(shell cat pre-build.log)) + $(error Pre-build script failed) endif # for rebuilding just vg From 56ddc760694b56b0ac389fc4f8ece0711eaf5106 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Fri, 12 Jun 2026 13:59:11 -0400 Subject: [PATCH 12/21] Make Makefile actually parse --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 3ca016fdec..e9ad4f806d 100644 --- a/Makefile +++ b/Makefile @@ -936,7 +936,7 @@ $(LIB_DIR)/libxg.a: $(XG_DIR)/src/*.hpp $(XG_DIR)/src/*.cpp $(INC_DIR)/mmmultima +ar rs $@ $(XG_DIR)/xg.o # All this version header generation stuff needs to work even if SRC_DIR hasn't been copied into the container yet. -$(shell mkdir -p $(SRC_DIR) +$(shell mkdir -p $(SRC_DIR)) # Auto-git-versioning From 764af4b8391e2149fa5346d7b3d737e73038924f Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Fri, 12 Jun 2026 15:25:54 -0400 Subject: [PATCH 13/21] Upgrade Python in run container --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 34e660d8b0..eaee1aa8da 100644 --- a/Dockerfile +++ b/Dockerfile @@ -124,7 +124,7 @@ RUN ls -lah /vg && \ fontconfig-config \ awscli \ binutils \ - libpython2.7 \ + python3 \ libperl-dev \ libelf1 \ libdw1 \ From 83bad80ae7e55d8d6c4f80f0c8926153ae201192 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Fri, 12 Jun 2026 15:33:24 -0400 Subject: [PATCH 14/21] Fix extra tab --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e9ad4f806d..9de561fd02 100644 --- a/Makefile +++ b/Makefile @@ -1086,7 +1086,7 @@ $(shell bash -c "if [ ! -d $(INC_DIR) ]; then mkdir -p $(INC_DIR); fi") # Make sure to run right now. PREBUILD_RESULT:=$(shell ./pre-build.sh >pre-build.log 2>&1 ; echo $$?) ifneq ($(PREBUILD_RESULT), 0) - $(info $(shell cat pre-build.log)) + $(info $(shell cat pre-build.log)) $(error Pre-build script failed) endif From 8d54f8cb9da16251d4395f08c4ee1938ac308184 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Fri, 12 Jun 2026 15:35:34 -0400 Subject: [PATCH 15/21] Actually send pre-build script --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index eaee1aa8da..ba0f7f3e40 100644 --- a/Dockerfile +++ b/Dockerfile @@ -60,6 +60,7 @@ RUN if [ -z "${TARGETARCH}" ] || [ "${TARGETARCH}" = "amd64" ] ; then sed -i s/m # Clear any CMake caches in case we are building from someone's checkout RUN find . -name CMakeCache.txt | xargs rm -f # Build the dependencies +COPY pre-build.sh /vg/pre-build.sh COPY Makefile /vg/Makefile RUN CXXFLAGS="$(if [ -z "${TARGETARCH}" ] || [ "${TARGETARCH}" = "amd64" ] ; then echo " -march=nehalem "; fi)" CFLAGS="$(if [ -z "${TARGETARCH}" ] || [ "${TARGETARCH}" = "amd64" ] ; then echo " -march=nehalem "; fi)" make -j $((THREADS < $(nproc) ? THREADS : $(nproc))) deps From c122af39bb2040f84e22ee144faef06d69c7f113 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Fri, 12 Jun 2026 15:46:39 -0400 Subject: [PATCH 16/21] Remove dependency .pre-build targets --- Makefile | 28 ++++++++++++++-------------- deps/libbdsg | 2 +- deps/structures | 2 +- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/Makefile b/Makefile index 9de561fd02..dea1b30a19 100644 --- a/Makefile +++ b/Makefile @@ -1065,20 +1065,20 @@ test/%.d: ; # We used to run a target to do this but since GNU Make 4.4 we can't easily run # a target at startup by `-include`-ing a .PHONY target so we need to do it at # top-level with shell calls. See -$(shell bash -c "if [ ! -d $(BIN_DIR) ]; then mkdir -p $(BIN_DIR); fi") -$(shell bash -c "if [ ! -d $(UNITTEST_BIN_DIR) ]; then mkdir -p $(UNITTEST_BIN_DIR); fi") -$(shell bash -c "if [ ! -d $(LIB_DIR) ]; then mkdir -p $(LIB_DIR); fi") -$(shell bash -c "if [ ! -d $(OBJ_DIR) ]; then mkdir -p $(OBJ_DIR); fi") -$(shell bash -c "if [ ! -d $(SHARED_OBJ_DIR) ]; then mkdir -p $(SHARED_OBJ_DIR); fi") -$(shell bash -c "if [ ! -d $(ALGORITHMS_OBJ_DIR) ]; then mkdir -p $(ALGORITHMS_OBJ_DIR); fi") -$(shell bash -c "if [ ! -d $(ALGORITHMS_SHARED_OBJ_DIR) ]; then mkdir -p $(ALGORITHMS_SHARED_OBJ_DIR); fi") -$(shell bash -c "if [ ! -d $(CONFIG_OBJ_DIR) ]; then mkdir -p $(CONFIG_OBJ_DIR); fi") -$(shell bash -c "if [ ! -d $(IO_OBJ_DIR) ]; then mkdir -p $(IO_OBJ_DIR); fi") -$(shell bash -c "if [ ! -d $(IO_SHARED_OBJ_DIR) ]; then mkdir -p $(IO_SHARED_OBJ_DIR); fi") -$(shell bash -c "if [ ! -d $(SUBCOMMAND_OBJ_DIR) ]; then mkdir -p $(SUBCOMMAND_OBJ_DIR); fi") -$(shell bash -c "if [ ! -d $(UNITTEST_OBJ_DIR) ]; then mkdir -p $(UNITTEST_OBJ_DIR); fi") -$(shell bash -c "if [ ! -d $(UNITTEST_SUPPORT_OBJ_DIR) ]; then mkdir -p $(UNITTEST_SUPPORT_OBJ_DIR); fi") -$(shell bash -c "if [ ! -d $(INC_DIR) ]; then mkdir -p $(INC_DIR); fi") +$(shell if [ ! -d $(BIN_DIR) ]; then mkdir -p $(BIN_DIR); fi) +$(shell if [ ! -d $(UNITTEST_BIN_DIR) ]; then mkdir -p $(UNITTEST_BIN_DIR); fi) +$(shell if [ ! -d $(LIB_DIR) ]; then mkdir -p $(LIB_DIR); fi) +$(shell if [ ! -d $(OBJ_DIR) ]; then mkdir -p $(OBJ_DIR); fi) +$(shell if [ ! -d $(SHARED_OBJ_DIR) ]; then mkdir -p $(SHARED_OBJ_DIR); fi) +$(shell if [ ! -d $(ALGORITHMS_OBJ_DIR) ]; then mkdir -p $(ALGORITHMS_OBJ_DIR); fi) +$(shell if [ ! -d $(ALGORITHMS_SHARED_OBJ_DIR) ]; then mkdir -p $(ALGORITHMS_SHARED_OBJ_DIR); fi) +$(shell if [ ! -d $(CONFIG_OBJ_DIR) ]; then mkdir -p $(CONFIG_OBJ_DIR); fi) +$(shell if [ ! -d $(IO_OBJ_DIR) ]; then mkdir -p $(IO_OBJ_DIR); fi) +$(shell if [ ! -d $(IO_SHARED_OBJ_DIR) ]; then mkdir -p $(IO_SHARED_OBJ_DIR); fi) +$(shell if [ ! -d $(SUBCOMMAND_OBJ_DIR) ]; then mkdir -p $(SUBCOMMAND_OBJ_DIR); fi) +$(shell if [ ! -d $(UNITTEST_OBJ_DIR) ]; then mkdir -p $(UNITTEST_OBJ_DIR); fi) +$(shell if [ ! -d $(UNITTEST_SUPPORT_OBJ_DIR) ]; then mkdir -p $(UNITTEST_SUPPORT_OBJ_DIR); fi) +$(shell if [ ! -d $(INC_DIR) ]; then mkdir -p $(INC_DIR); fi) # run pre-build.sh before we make anything at all. # Make really doesn't want to depend on this succeeding, but we need it to. diff --git a/deps/libbdsg b/deps/libbdsg index 73e74c223e..5429995457 160000 --- a/deps/libbdsg +++ b/deps/libbdsg @@ -1 +1 @@ -Subproject commit 73e74c223ed16c6b0db2c378f285819c5a3499e0 +Subproject commit 54299954575a980c8506624e52e1593333a2a6d1 diff --git a/deps/structures b/deps/structures index d9e184da87..d283203c3b 160000 --- a/deps/structures +++ b/deps/structures @@ -1 +1 @@ -Subproject commit d9e184da87492535c5e69d60706ee02a7eb2e82e +Subproject commit d283203c3b7021dc83352318429be5aad1995510 From bebd618f52b121e155388b73325e27f8fb8b0315 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Fri, 12 Jun 2026 16:21:59 -0400 Subject: [PATCH 17/21] Build more deps at deps stage and be super sure libvcfh.a has a symbol table --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index dea1b30a19..d738cc1d90 100644 --- a/Makefile +++ b/Makefile @@ -569,7 +569,7 @@ get-deps: sudo DEBIAN_FRONTEND=$(DEBIAN_FRONTEND) apt-get install -qq -y --no-upgrade $(shell cat Dockerfile | sed -n '/^###DEPS_BEGIN###/,$${p;/^###DEPS_END###/q}' | grep -v '^ *#' | grep -v "^RUN" | tr '\n' ' ' | tr -d '\\') # And we have submodule deps to build -deps: $(DEPS) +deps: $(DEPS) $(LINK_DEPS) $(PRE_LINK_DEPS) lint: $(SRC_DIR)/*.cpp $(SRC_DIR)/*.hpp $(ALGORITHMS_SRC_DIR)/*.cpp $(ALGORITHMS_SRC_DIR)/*.hpp $(SUBCOMMAND_SRC_DIR)/*.cpp $(SUBCOMMAND_SRC_DIR)/*.hpp $(UNITTEST_SRC_DIR)/*.cpp $(UNITTEST_SRC_DIR)/*.hpp $(UNITTEST_SUPPORT_SRC_DIR)/*.cpp scripts/check_options.py 1>&2 @@ -828,7 +828,7 @@ $(INC_DIR)/sparsepp/spp.h: $(wildcard $(SPARSEPP_DIR)/sparsepp/*.h) #$(INC_DIR)/Variant.h $(LIB_DIR)/libvcfh.a: $(DEP_DIR)/libVCFH/*.cpp $(DEP_DIR)/libVCFH/*.hpp - +cd $(DEP_DIR)/libVCFH && $(MAKE) $(FILTER) && cp libvcfh.a $(CWD)/$(LIB_DIR)/ && cp vcfheader.hpp $(CWD)/$(INC_DIR)/ + +cd $(DEP_DIR)/libVCFH && $(MAKE) $(FILTER) && ranlib libvcfh.a && cp libvcfh.a $(CWD)/$(LIB_DIR)/ && cp vcfheader.hpp $(CWD)/$(INC_DIR)/ $(LIB_DIR)/libsonlib.a: $(CWD)/$(DEP_DIR)/sonLib/C/inc/*.h $(CWD)/$(DEP_DIR)/sonLib/C/impl/*.c +cd $(DEP_DIR)/sonLib && $(MAKE) clean && kyotoTycoonLib="" CFLAGS="-fPIC $(CFLAGS)" CXXFLAGS="-fPIC $(CXXFLAGS)" $(MAKE) $(FILTER) && cp lib/sonLib.a $(CWD)/$(LIB_DIR)/libsonlib.a && mkdir -p $(CWD)/$(INC_DIR)/sonLib && cp lib/*.h $(CWD)/$(INC_DIR)/sonLib From 82d5729d4453da84a0f697e73f1e23e34cdc0e4b Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Fri, 12 Jun 2026 16:28:14 -0400 Subject: [PATCH 18/21] Update sparsehash to reduce compiler warnings --- deps/sparsehash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/sparsehash b/deps/sparsehash index a61a6ba7ad..1dffea3d91 160000 --- a/deps/sparsehash +++ b/deps/sparsehash @@ -1 +1 @@ -Subproject commit a61a6ba7adbc4e3a7545843a72c530bf35604dae +Subproject commit 1dffea3d917445d70d33d0c7492919fc4408fe5c From bf256368842448c771fea51b42cd08974d445568 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Fri, 12 Jun 2026 17:03:22 -0400 Subject: [PATCH 19/21] Remove libVCFH which seems unused and breaks the Docker build --- .gitmodules | 3 --- Makefile | 9 +-------- deps/libVCFH | 1 - 3 files changed, 1 insertion(+), 12 deletions(-) delete mode 160000 deps/libVCFH diff --git a/.gitmodules b/.gitmodules index 1bb80d2346..c15f27aa26 100644 --- a/.gitmodules +++ b/.gitmodules @@ -22,9 +22,6 @@ [submodule "sdsl-lite"] path = deps/sdsl-lite url = https://github.com/vgteam/sdsl-lite.git -[submodule "deps/libVCFH"] - path = deps/libVCFH - url = https://github.com/edawson/libVCFH.git [submodule "deps/sparsehash"] path = deps/sparsehash url = https://github.com/sparsehash/sparsehash.git diff --git a/Makefile b/Makefile index d738cc1d90..9838fde9fd 100644 --- a/Makefile +++ b/Makefile @@ -74,7 +74,7 @@ INCLUDE_FLAGS :=-I$(CWD)/$(INC_DIR) -I. -I$(CWD)/$(SRC_DIR) -I$(CWD)/$(UNITTEST_ # These need to come before library search paths from LDFLAGS or we won't # prefer linking vg-installed dependencies over system ones. LD_LIB_DIR_FLAGS := -L$(CWD)/$(LIB_DIR) -LD_LIB_FLAGS := -lvcflib -lwfa2 -ltabixpp -lgssw -lssw -lsublinearLS -lpthread -lncurses -lgcsa2 -lgbwtgraph -lgbwt -lkff -ldivsufsort -ldivsufsort64 -lvcfh -lraptor2 -lpinchesandcacti -l3edgeconnected -lsonlib -lfml -lstructures -lbdsg -lxg -lsdsl -lzstd -lhandlegraph -lcrypto +LD_LIB_FLAGS := -lvcflib -lwfa2 -ltabixpp -lgssw -lssw -lsublinearLS -lpthread -lncurses -lgcsa2 -lgbwtgraph -lgbwt -lkff -ldivsufsort -ldivsufsort64 -lraptor2 -lpinchesandcacti -l3edgeconnected -lsonlib -lfml -lstructures -lbdsg -lxg -lsdsl -lzstd -lhandlegraph -lcrypto # We omit Boost Program Options for now; we find it in a platform-dependent way. # By default it has no suffix BOOST_SUFFIX="" @@ -396,7 +396,6 @@ LIB_DEPS += $(LIB_DIR)/libhts.a LIB_DEPS += $(LIB_DIR)/libtabixpp.a LIB_DEPS += $(LIB_DIR)/libvcflib.a LIB_DEPS += $(LIB_DIR)/libgssw.a -LIB_DEPS += $(LIB_DIR)/libvcfh.a LIB_DEPS += $(LIB_DIR)/libsonlib.a LIB_DEPS += $(LIB_DIR)/libpinchesandcacti.a LIB_DEPS += $(LIB_DIR)/libraptor2.a @@ -826,10 +825,6 @@ $(INC_DIR)/sparsehash/sparse_hash_map: $(wildcard $(SPARSEHASH_DIR)/**/*.cc) $(w $(INC_DIR)/sparsepp/spp.h: $(wildcard $(SPARSEPP_DIR)/sparsepp/*.h) +cp -r $(SPARSEPP_DIR)/sparsepp $(INC_DIR)/ -#$(INC_DIR)/Variant.h -$(LIB_DIR)/libvcfh.a: $(DEP_DIR)/libVCFH/*.cpp $(DEP_DIR)/libVCFH/*.hpp - +cd $(DEP_DIR)/libVCFH && $(MAKE) $(FILTER) && ranlib libvcfh.a && cp libvcfh.a $(CWD)/$(LIB_DIR)/ && cp vcfheader.hpp $(CWD)/$(INC_DIR)/ - $(LIB_DIR)/libsonlib.a: $(CWD)/$(DEP_DIR)/sonLib/C/inc/*.h $(CWD)/$(DEP_DIR)/sonLib/C/impl/*.c +cd $(DEP_DIR)/sonLib && $(MAKE) clean && kyotoTycoonLib="" CFLAGS="-fPIC $(CFLAGS)" CXXFLAGS="-fPIC $(CXXFLAGS)" $(MAKE) $(FILTER) && cp lib/sonLib.a $(CWD)/$(LIB_DIR)/libsonlib.a && mkdir -p $(CWD)/$(INC_DIR)/sonLib && cp lib/*.h $(CWD)/$(INC_DIR)/sonLib @@ -1132,7 +1127,6 @@ clean: clean-vcflib cd $(DEP_DIR) && cd ssw && cd src && $(MAKE) clean cd $(DEP_DIR) && cd progress_bar && $(MAKE) clean cd $(DEP_DIR) && cd sdsl-lite && ./uninstall.sh || true - cd $(DEP_DIR) && cd libVCFH && $(MAKE) clean cd $(DEP_DIR) && cd vcflib && $(MAKE) clean cd $(DEP_DIR) && cd sha1 && $(MAKE) clean cd $(DEP_DIR) && cd structures && $(MAKE) clean @@ -1147,5 +1141,4 @@ clean: clean-vcflib clean-vcflib: cd $(DEP_DIR) && cd vcflib && $(MAKE) clean - rm -f $(LIB_DIR)/libvcfh.a cd $(INC_DIR) && rm -f BedReader.h convert.h join.h mt19937ar.h split.h Variant.h vec128int.h veclib_types.h diff --git a/deps/libVCFH b/deps/libVCFH deleted file mode 160000 index 615a06b5de..0000000000 --- a/deps/libVCFH +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 615a06b5de21f59294d7d806e9f505162ae8c2c1 From 3fbce5e7687709154f0162c28c5254be0ef35156 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Fri, 12 Jun 2026 17:06:40 -0400 Subject: [PATCH 20/21] Hide particularly volumenous new warning from the Docker build --- Dockerfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index ba0f7f3e40..7e7bf87650 100644 --- a/Dockerfile +++ b/Dockerfile @@ -62,17 +62,18 @@ RUN find . -name CMakeCache.txt | xargs rm -f # Build the dependencies COPY pre-build.sh /vg/pre-build.sh COPY Makefile /vg/Makefile -RUN CXXFLAGS="$(if [ -z "${TARGETARCH}" ] || [ "${TARGETARCH}" = "amd64" ] ; then echo " -march=nehalem "; fi)" CFLAGS="$(if [ -z "${TARGETARCH}" ] || [ "${TARGETARCH}" = "amd64" ] ; then echo " -march=nehalem "; fi)" make -j $((THREADS < $(nproc) ? THREADS : $(nproc))) deps +# Turn off user-defined conversion warning until https://github.com/greg7mdp/sparsepp/issues/98 can be fixed +RUN CXXFLAGS="$(if [ -z "${TARGETARCH}" ] || [ "${TARGETARCH}" = "amd64" ] ; then echo " -march=nehalem "; fi) -Wno-cast-user-defined" CFLAGS="$(if [ -z "${TARGETARCH}" ] || [ "${TARGETARCH}" = "amd64" ] ; then echo " -march=nehalem "; fi)" make -j $((THREADS < $(nproc) ? THREADS : $(nproc))) deps # Bring in the sources, which we need in order to build. COPY src /vg/src # Build all the object files for vg, but don't link. # Also pass the arch here -RUN CXXFLAGS="$(if [ -z "${TARGETARCH}" ] || [ "${TARGETARCH}" = "amd64" ] ; then echo " -march=nehalem "; fi)" make -j $((THREADS < $(nproc) ? THREADS : $(nproc))) objs +RUN CXXFLAGS="$(if [ -z "${TARGETARCH}" ] || [ "${TARGETARCH}" = "amd64" ] ; then echo " -march=nehalem "; fi) -Wno-cast-user-defined" make -j $((THREADS < $(nproc) ? THREADS : $(nproc))) objs # Do the final build and link, knowing the version. Trim down the resulting binary but make sure to include enough debug info for profiling. -RUN CXXFLAGS="$(if [ -z "${TARGETARCH}" ] || [ "${TARGETARCH}" = "amd64" ] ; then echo " -march=nehalem "; fi)" make -j $((THREADS < $(nproc) ? THREADS : $(nproc))) static && strip -d bin/vg +RUN CXXFLAGS="$(if [ -z "${TARGETARCH}" ] || [ "${TARGETARCH}" = "amd64" ] ; then echo " -march=nehalem "; fi) -Wno-cast-user-defined" make -j $((THREADS < $(nproc) ? THREADS : $(nproc))) static && strip -d bin/vg # Ship the scripts COPY scripts /vg/scripts From 9e47cae18c6d75a520f0d0a8b90e4e4a61731bc2 Mon Sep 17 00:00:00 2001 From: Adam Novak Date: Fri, 12 Jun 2026 17:32:38 -0400 Subject: [PATCH 21/21] Give up on letting the Docker build test 26.04 --- Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 7e7bf87650..e1e3e8402c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,10 @@ # Use Google's non-rate-limited mirror of Docker Hub to get our base image. # This helps automated Quay builds because Quay hasn't built a caching system # and exposes pull rate limits to users. -FROM mirror.gcr.io/library/ubuntu:26.04 AS base +# We need to use Ubuntu 22.04 because 24.04 and later suffer from +# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=1104888 and don't package a +# libcairo.a at all. +FROM mirror.gcr.io/library/ubuntu:22.04 AS base MAINTAINER vgteam RUN echo base > /stage.txt