diff --git a/cli/CMakeLists.txt b/cli/CMakeLists.txt index ce26788d0..91aa5ee2c 100644 --- a/cli/CMakeLists.txt +++ b/cli/CMakeLists.txt @@ -36,7 +36,9 @@ add_dependencies(zli commands utils logger - xgboost_external ) +if(TARGET xgboost_external) + add_dependencies(zli xgboost_external) +endif() apply_openzl_compile_options_to_target(zli) diff --git a/custom_parsers/CMakeLists.txt b/custom_parsers/CMakeLists.txt index aa35fdc40..b7b7db7d5 100644 --- a/custom_parsers/CMakeLists.txt +++ b/custom_parsers/CMakeLists.txt @@ -48,7 +48,10 @@ target_link_libraries(custom_parsers parquet_graph shared_components ) -add_dependencies(custom_parsers openzl openzl_cpp xgboost_external) +add_dependencies(custom_parsers openzl openzl_cpp) +if(TARGET xgboost_external) + add_dependencies(custom_parsers xgboost_external) +endif() apply_openzl_compile_options_to_target(custom_parsers csv_parser parquet_graph diff --git a/tools/ml_selector/CMakeLists.txt b/tools/ml_selector/CMakeLists.txt index e6b1d3152..c46c0f81a 100644 --- a/tools/ml_selector/CMakeLists.txt +++ b/tools/ml_selector/CMakeLists.txt @@ -1,92 +1,110 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. if(OPENZL_BUILD_ML_SELECTOR) - include(ExternalProject) - include(GNUInstallDirs) - - # Build xgboost as an external project - set(XGBOOST_INSTALL_DIR "${CMAKE_CURRENT_BINARY_DIR}/xgboost-install") - set(XGBOOST_LIB_DIR "${XGBOOST_INSTALL_DIR}/lib") + # Prefer a pre-installed xgboost (e.g. from conda/pixi) when one is + # available, and only fall back to building it from source otherwise. The + # CONDA_PREFIX hint lets `pixi run` environments be discovered without any + # extra -DCMAKE_PREFIX_PATH flag. + find_package(xgboost CONFIG QUIET HINTS "$ENV{CONDA_PREFIX}") + + if(xgboost_FOUND) + message(STATUS "Using pre-installed xgboost: ${xgboost_DIR}") + # The imported xgboost::xgboost target carries its own include dirs + # (covering both xgboost/ and dmlc/ headers) and statically bundles dmlc, + # so no separate dmlc link target or include path is required. + set(OPENZL_XGBOOST_LIBS xgboost::xgboost) + else() + message(STATUS + "Pre-installed xgboost not found; building from source via ExternalProject") + include(ExternalProject) + include(GNUInstallDirs) + + # Build xgboost as an external project + set(XGBOOST_INSTALL_DIR "${CMAKE_CURRENT_BINARY_DIR}/xgboost-install") + set(XGBOOST_LIB_DIR "${XGBOOST_INSTALL_DIR}/lib") + + # Create install directories at configure time + file(MAKE_DIRECTORY ${XGBOOST_INSTALL_DIR}/include) + file(MAKE_DIRECTORY ${XGBOOST_LIB_DIR}) + + # On Windows/MSVC, static libraries don't have a "lib" prefix + if(MSVC) + set(XGBOOST_LIB_NAME "xgboost${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(DMLC_LIB_NAME "dmlc${CMAKE_STATIC_LIBRARY_SUFFIX}") + else() + set(XGBOOST_LIB_NAME "libxgboost${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(DMLC_LIB_NAME "libdmlc${CMAKE_STATIC_LIBRARY_SUFFIX}") + endif() - # Create install directories at configure time - file(MAKE_DIRECTORY ${XGBOOST_INSTALL_DIR}/include) - file(MAKE_DIRECTORY ${XGBOOST_LIB_DIR}) + # Enable C++ exceptions for ClangCL. + if(MSVC) + set(XGBOOST_CXX_FLAGS + "-DDMLC_LOG_STACK_TRACE=0 /EHsc") + else() + set(XGBOOST_CXX_FLAGS + "-DDMLC_LOG_STACK_TRACE=0") + endif() - # On Windows/MSVC, static libraries don't have a "lib" prefix - if(MSVC) - set(XGBOOST_LIB_NAME "xgboost${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(DMLC_LIB_NAME "dmlc${CMAKE_STATIC_LIBRARY_SUFFIX}") - else() - set(XGBOOST_LIB_NAME "libxgboost${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(DMLC_LIB_NAME "libdmlc${CMAKE_STATIC_LIBRARY_SUFFIX}") - endif() + # Add XGBoost as external project + # Disable DMLC stack trace to avoid execinfo.h + ExternalProject_Add(xgboost_external + GIT_REPOSITORY https://github.com/dmlc/xgboost.git + GIT_TAG ccb511768e13d1670c10be07dea89d0edca138f3 # v3.1.0 + GIT_SUBMODULES "dmlc-core" + GIT_SHALLOW TRUE + CMAKE_ARGS + -DCMAKE_INSTALL_PREFIX=${XGBOOST_INSTALL_DIR} + -DCMAKE_INSTALL_LIBDIR=lib + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DUSE_OPENMP=OFF + -DUSE_CUDA=OFF + -DUSE_NCCL=OFF + -DCMAKE_POSITION_INDEPENDENT_CODE=ON + -DBUILD_STATIC_LIB=ON + ${XGBOOST_EXTRA_CMAKE_ARGS} + CMAKE_CACHE_ARGS + "-DCMAKE_C_FLAGS:STRING=-DDMLC_LOG_STACK_TRACE=0" + "-DCMAKE_CXX_FLAGS:STRING=${XGBOOST_CXX_FLAGS}" + "-DBUILD_TESTING:BOOL=OFF" + "-DCMAKE_POLICY_DEFAULT_CMP0091:STRING=NEW" + "-DCMAKE_MSVC_RUNTIME_LIBRARY:STRING=${CMAKE_MSVC_RUNTIME_LIBRARY}" + BUILD_BYPRODUCTS + ${XGBOOST_LIB_DIR}/${XGBOOST_LIB_NAME} + ${XGBOOST_LIB_DIR}/${DMLC_LIB_NAME} + LOG_DOWNLOAD ON + LOG_CONFIGURE ON + LOG_BUILD ON + LOG_INSTALL ON + ) - # Enable C++ exceptions for ClangCL. - if(MSVC) - set(XGBOOST_CXX_FLAGS - "-DDMLC_LOG_STACK_TRACE=0 /EHsc") - else() - set(XGBOOST_CXX_FLAGS - "-DDMLC_LOG_STACK_TRACE=0") - endif() + # Create imported targets + add_library(xgboost STATIC IMPORTED GLOBAL) + set_target_properties(xgboost PROPERTIES + IMPORTED_LOCATION + ${XGBOOST_LIB_DIR}/${XGBOOST_LIB_NAME} + INTERFACE_INCLUDE_DIRECTORIES ${XGBOOST_INSTALL_DIR}/include + INTERFACE_SYSTEM_INCLUDE_DIRECTORIES ${XGBOOST_INSTALL_DIR}/include + INTERFACE_COMPILE_DEFINITIONS "DMLC_LOG_STACK_TRACE=0" + ) + add_dependencies(xgboost xgboost_external) - # Add XGBoost as external project - # Disable DMLC stack trace to avoid execinfo.h - ExternalProject_Add(xgboost_external - GIT_REPOSITORY https://github.com/dmlc/xgboost.git - GIT_TAG ccb511768e13d1670c10be07dea89d0edca138f3 # v3.1.0 - GIT_SUBMODULES "dmlc-core" - GIT_SHALLOW TRUE - CMAKE_ARGS - -DCMAKE_INSTALL_PREFIX=${XGBOOST_INSTALL_DIR} - -DCMAKE_INSTALL_LIBDIR=lib - -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} - -DUSE_OPENMP=OFF - -DUSE_CUDA=OFF - -DUSE_NCCL=OFF - -DCMAKE_POSITION_INDEPENDENT_CODE=ON - -DBUILD_STATIC_LIB=ON - ${XGBOOST_EXTRA_CMAKE_ARGS} - CMAKE_CACHE_ARGS - "-DCMAKE_C_FLAGS:STRING=-DDMLC_LOG_STACK_TRACE=0" - "-DCMAKE_CXX_FLAGS:STRING=${XGBOOST_CXX_FLAGS}" - "-DBUILD_TESTING:BOOL=OFF" - "-DCMAKE_POLICY_DEFAULT_CMP0091:STRING=NEW" - "-DCMAKE_MSVC_RUNTIME_LIBRARY:STRING=${CMAKE_MSVC_RUNTIME_LIBRARY}" - BUILD_BYPRODUCTS - ${XGBOOST_LIB_DIR}/${XGBOOST_LIB_NAME} - ${XGBOOST_LIB_DIR}/${DMLC_LIB_NAME} - LOG_DOWNLOAD ON - LOG_CONFIGURE ON - LOG_BUILD ON - LOG_INSTALL ON - ) + if(WIN32 OR MINGW) + set_target_properties(xgboost PROPERTIES + INTERFACE_LINK_LIBRARIES "ws2_32") + endif() - # Create imported targets - add_library(xgboost STATIC IMPORTED GLOBAL) - set_target_properties(xgboost PROPERTIES - IMPORTED_LOCATION - ${XGBOOST_LIB_DIR}/${XGBOOST_LIB_NAME} - INTERFACE_INCLUDE_DIRECTORIES ${XGBOOST_INSTALL_DIR}/include - INTERFACE_SYSTEM_INCLUDE_DIRECTORIES ${XGBOOST_INSTALL_DIR}/include - INTERFACE_COMPILE_DEFINITIONS "DMLC_LOG_STACK_TRACE=0" - ) - add_dependencies(xgboost xgboost_external) + add_library(dmlc STATIC IMPORTED GLOBAL) + set_target_properties(dmlc PROPERTIES + IMPORTED_LOCATION + ${XGBOOST_LIB_DIR}/${DMLC_LIB_NAME} + INTERFACE_INCLUDE_DIRECTORIES ${XGBOOST_INSTALL_DIR}/include + INTERFACE_SYSTEM_INCLUDE_DIRECTORIES ${XGBOOST_INSTALL_DIR}/include + INTERFACE_COMPILE_DEFINITIONS "DMLC_LOG_STACK_TRACE=0" + ) + add_dependencies(dmlc xgboost_external) - if(WIN32 OR MINGW) - set_target_properties(xgboost PROPERTIES - INTERFACE_LINK_LIBRARIES "ws2_32") + set(OPENZL_XGBOOST_LIBS xgboost dmlc) endif() - add_library(dmlc STATIC IMPORTED GLOBAL) - set_target_properties(dmlc PROPERTIES - IMPORTED_LOCATION - ${XGBOOST_LIB_DIR}/${DMLC_LIB_NAME} - INTERFACE_INCLUDE_DIRECTORIES ${XGBOOST_INSTALL_DIR}/include - INTERFACE_SYSTEM_INCLUDE_DIRECTORIES ${XGBOOST_INSTALL_DIR}/include - INTERFACE_COMPILE_DEFINITIONS "DMLC_LOG_STACK_TRACE=0" - ) - add_dependencies(dmlc xgboost_external) - add_library(ml_selector ml_selector_trainer.cpp ml_selector_trainer.h @@ -94,10 +112,6 @@ if(OPENZL_BUILD_ML_SELECTOR) ml_features.h ) - target_include_directories(ml_selector - SYSTEM PUBLIC - ${XGBOOST_INSTALL_DIR}/include - ) target_include_directories(ml_selector PUBLIC ${PROJECT_SOURCE_DIR} @@ -113,15 +127,18 @@ if(OPENZL_BUILD_ML_SELECTOR) PUBLIC openzl openzl_cpp - xgboost - dmlc + ${OPENZL_XGBOOST_LIBS} ) - add_dependencies(ml_selector xgboost_external) + if(TARGET xgboost_external) + add_dependencies(ml_selector xgboost_external) + endif() if (OPENZL_BUILD_TESTS) add_executable(test_ml_selector tests/test_mlSelectorGraph.cpp) - add_dependencies(test_ml_selector xgboost_external) + if(TARGET xgboost_external) + add_dependencies(test_ml_selector xgboost_external) + endif() target_link_libraries(test_ml_selector PRIVATE @@ -139,7 +156,9 @@ if(OPENZL_BUILD_ML_SELECTOR) if (OPENZL_ALLOW_INTROSPECTION) add_executable(test_ml_selector_trainer tests/test_mlSelectorTrainer.cpp) - add_dependencies(test_ml_selector_trainer xgboost_external) + if(TARGET xgboost_external) + add_dependencies(test_ml_selector_trainer xgboost_external) + endif() target_link_libraries(test_ml_selector_trainer PRIVATE diff --git a/tools/parquet/CMakeLists.txt b/tools/parquet/CMakeLists.txt index 68675ef54..20af4e17b 100644 --- a/tools/parquet/CMakeLists.txt +++ b/tools/parquet/CMakeLists.txt @@ -1,80 +1,99 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. if (OPENZL_BUILD_PARQUET_TOOLS) - set(ARROW_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/arrow_ep") + # Prefer a pre-installed Arrow/Parquet (e.g. from conda/pixi) when available, + # and only fall back to building it from source otherwise. The CONDA_PREFIX + # hint lets `pixi run` environments be discovered without an extra + # -DCMAKE_PREFIX_PATH flag. + find_package(Arrow CONFIG QUIET HINTS "$ENV{CONDA_PREFIX}") + find_package(Parquet CONFIG QUIET HINTS "$ENV{CONDA_PREFIX}") - # Point arrow to the zstd build directory - list(APPEND CMAKE_PREFIX_PATH "${zstd_BINARY_DIR}") + if(Arrow_FOUND AND Parquet_FOUND) + message(STATUS "Using pre-installed Arrow/Parquet: ${Arrow_DIR}") + # Parquet::parquet_shared transitively links Arrow::arrow_shared and carries + # its own include dirs; no ExternalProject or imported targets are required. + set(OPENZL_PARQUET_LIBS Parquet::parquet_shared Arrow::arrow_shared) + else() + message(STATUS + "Pre-installed Arrow/Parquet not found; building from source via ExternalProject") + set(ARROW_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/arrow_ep") - set(ARROW_CMAKE_ARGS - -DARROW_PARQUET=ON - -DARROW_DEPENDENCY_SOURCE=AUTO - -DARROW_WITH_THRIFT=ON - -DARROW_WITH_LZ4=OFF # fix - -DARROW_WITH_SNAPPY=ON - -DARROW_WITH_ZLIB=ON - -DARROW_WITH_ZSTD=ON - -DARROW_MIMALLOC=OFF - -DCMAKE_INSTALL_PREFIX=${ARROW_PREFIX}/install - -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} - -DCMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH} - -DCMAKE_POLICY_VERSION_MINIMUM=3.5 - -DARROW_CXXFLAGS=-Wno-documentation) - set(ARROW_LIBDIR ${ARROW_PREFIX}/install/${CMAKE_INSTALL_LIBDIR}) + # Point arrow to the zstd build directory + list(APPEND CMAKE_PREFIX_PATH "${zstd_BINARY_DIR}") - set(ARROW_DEPS thrift snappy) - set(thrift_ROOT ${ARROW_PREFIX}/src/arrow_ep-build/thrift_ep-install) - set(snappy_ROOT ${ARROW_PREFIX}/src/arrow_ep-build/snappy_ep/src/snappy_ep-install) + set(ARROW_CMAKE_ARGS + -DARROW_PARQUET=ON + -DARROW_DEPENDENCY_SOURCE=AUTO + -DARROW_WITH_THRIFT=ON + -DARROW_WITH_LZ4=OFF # fix + -DARROW_WITH_SNAPPY=ON + -DARROW_WITH_ZLIB=ON + -DARROW_WITH_ZSTD=ON + -DARROW_MIMALLOC=OFF + -DCMAKE_INSTALL_PREFIX=${ARROW_PREFIX}/install + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + -DCMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH} + -DCMAKE_POLICY_VERSION_MINIMUM=3.5 + -DARROW_CXXFLAGS=-Wno-documentation) + set(ARROW_LIBDIR ${ARROW_PREFIX}/install/${CMAKE_INSTALL_LIBDIR}) - foreach (dep ${ARROW_DEPS}) - add_library(${dep} STATIC IMPORTED GLOBAL) - set(${dep}_LIB ${${dep}_ROOT}/lib/lib${dep}.a) - list(APPEND ARROW_DEPS_LIBS ${${dep}_LIB}) - file(MAKE_DIRECTORY ${${dep}_ROOT}/include) - set(${dep}_INCLUDE_DIR ${${dep}_ROOT}/include) - set_property(TARGET ${dep} PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${${dep}_INCLUDE_DIR}) - set_property(TARGET ${dep} PROPERTY IMPORTED_LOCATION ${${dep}_LIB}) - endforeach() + set(ARROW_DEPS thrift snappy) + set(thrift_ROOT ${ARROW_PREFIX}/src/arrow_ep-build/thrift_ep-install) + set(snappy_ROOT ${ARROW_PREFIX}/src/arrow_ep-build/snappy_ep/src/snappy_ep-install) - set(ZL_ARROW_BUILD_VERSION 20.0.0) - set(ZL_ARROW_BUILD_SHA256_CHECKSUM 67e31a4f46528634b8c3cbb0dc60ac8f85859d906b400d83d0b6f732b0c5b0e3) - set(ZL_ARROW_SOURCE_URL - "https://github.com/apache/arrow/archive/refs/tags/apache-arrow-${ZL_ARROW_BUILD_VERSION}.tar.gz" - ) - include(ExternalProject) - ExternalProject_Add( - arrow_ep - PREFIX ${ARROW_PREFIX} - URL ${ZL_ARROW_SOURCE_URL} - URL_HASH SHA256=${ZL_ARROW_BUILD_SHA256_CHECKSUM} - SOURCE_SUBDIR cpp - CMAKE_ARGS ${ARROW_CMAKE_ARGS} - BUILD_BYPRODUCTS ${ARROW_LIBDIR}/libarrow.a - ${ARROW_LIBDIR}/libparquet.a - ${ARROW_DEPS_LIBS} - ) + foreach (dep ${ARROW_DEPS}) + add_library(${dep} STATIC IMPORTED GLOBAL) + set(${dep}_LIB ${${dep}_ROOT}/lib/lib${dep}.a) + list(APPEND ARROW_DEPS_LIBS ${${dep}_LIB}) + file(MAKE_DIRECTORY ${${dep}_ROOT}/include) + set(${dep}_INCLUDE_DIR ${${dep}_ROOT}/include) + set_property(TARGET ${dep} PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${${dep}_INCLUDE_DIR}) + set_property(TARGET ${dep} PROPERTY IMPORTED_LOCATION ${${dep}_LIB}) + endforeach() - add_library(arrow STATIC IMPORTED GLOBAL) - add_library(parquet STATIC IMPORTED GLOBAL) - add_dependencies(arrow arrow_ep) - add_dependencies(parquet arrow) + set(ZL_ARROW_BUILD_VERSION 20.0.0) + set(ZL_ARROW_BUILD_SHA256_CHECKSUM 67e31a4f46528634b8c3cbb0dc60ac8f85859d906b400d83d0b6f732b0c5b0e3) + set(ZL_ARROW_SOURCE_URL + "https://github.com/apache/arrow/archive/refs/tags/apache-arrow-${ZL_ARROW_BUILD_VERSION}.tar.gz" + ) + include(ExternalProject) + ExternalProject_Add( + arrow_ep + PREFIX ${ARROW_PREFIX} + URL ${ZL_ARROW_SOURCE_URL} + URL_HASH SHA256=${ZL_ARROW_BUILD_SHA256_CHECKSUM} + SOURCE_SUBDIR cpp + CMAKE_ARGS ${ARROW_CMAKE_ARGS} + BUILD_BYPRODUCTS ${ARROW_LIBDIR}/libarrow.a + ${ARROW_LIBDIR}/libparquet.a + ${ARROW_DEPS_LIBS} + ) - file(MAKE_DIRECTORY ${ARROW_PREFIX}/install/include) - set_target_properties( - arrow parquet PROPERTIES INTERFACE_INCLUDE_DIRECTORIES - ${ARROW_PREFIX}/install/include) - set_target_properties(arrow PROPERTIES IMPORTED_LOCATION - ${ARROW_LIBDIR}/libarrow.a) - set_target_properties(parquet PROPERTIES IMPORTED_LOCATION - ${ARROW_LIBDIR}/libparquet.a) + add_library(arrow STATIC IMPORTED GLOBAL) + add_library(parquet STATIC IMPORTED GLOBAL) + add_dependencies(arrow arrow_ep) + add_dependencies(parquet arrow) - set_property( - TARGET - arrow - PROPERTY - INTERFACE_LINK_LIBRARIES - libzstd z ${ARROW_DEPS}) - set_property(TARGET parquet PROPERTY INTERFACE_LINK_LIBRARIES arrow) + file(MAKE_DIRECTORY ${ARROW_PREFIX}/install/include) + set_target_properties( + arrow parquet PROPERTIES INTERFACE_INCLUDE_DIRECTORIES + ${ARROW_PREFIX}/install/include) + set_target_properties(arrow PROPERTIES IMPORTED_LOCATION + ${ARROW_LIBDIR}/libarrow.a) + set_target_properties(parquet PROPERTIES IMPORTED_LOCATION + ${ARROW_LIBDIR}/libparquet.a) + + set_property( + TARGET + arrow + PROPERTY + INTERFACE_LINK_LIBRARIES + libzstd z ${ARROW_DEPS}) + set_property(TARGET parquet PROPERTY INTERFACE_LINK_LIBRARIES arrow) + + # The parquet target already pulls arrow transitively. + set(OPENZL_PARQUET_LIBS parquet) + endif() file( GLOB_RECURSE make_canonical_parquet_sources @@ -84,8 +103,11 @@ if (OPENZL_BUILD_PARQUET_TOOLS) target_link_libraries( make_canonical_parquet - PRIVATE arg openzl parquet tools_io) - add_dependencies(make_canonical_parquet arg openzl parquet tools_io) + PRIVATE arg openzl ${OPENZL_PARQUET_LIBS} tools_io) + add_dependencies(make_canonical_parquet arg openzl tools_io) + if(TARGET arrow_ep) + add_dependencies(make_canonical_parquet arrow_ep) + endif() apply_openzl_compile_options_to_target(make_canonical_parquet) endif() diff --git a/tools/training/CMakeLists.txt b/tools/training/CMakeLists.txt index c93b090ad..d6777316e 100644 --- a/tools/training/CMakeLists.txt +++ b/tools/training/CMakeLists.txt @@ -37,8 +37,10 @@ if (OPENZL_BUILD_TRAINING_TOOLS) openzl_cpp tools_io logger - ml_selector - xgboost_external) + ml_selector) + if(TARGET xgboost_external) + add_dependencies(tools_training xgboost_external) + endif() if (OPENZL_BUILD_TESTS AND OPENZL_ALLOW_INTROSPECTION) file( @@ -52,7 +54,9 @@ if (OPENZL_BUILD_TRAINING_TOOLS) CONFIGURE_DEPENDS "${CMAKE_CURRENT_LIST_DIR}/tests/*.h") add_executable(test_training ${test_training_srcs} ${test_training_headers}) - add_dependencies(test_training xgboost_external) + if(TARGET xgboost_external) + add_dependencies(test_training xgboost_external) + endif() target_link_libraries( test_training PRIVATE