From a0b7ff9ddefac48e3846e04e290f76cbc00e4d30 Mon Sep 17 00:00:00 2001 From: Sero Date: Thu, 25 Dec 2025 11:49:41 +0400 Subject: [PATCH] Switched to vcpkg provided Sentencepiece, kept submodule as a fallback. --- CMakeLists.txt | 13 +++- src/3rd_party/CMakeLists.txt | 113 +++++++++++++++++---------------- src/CMakeLists.txt | 20 ++++-- src/data/sentencepiece_vocab.h | 10 ++- 4 files changed, 92 insertions(+), 64 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b970fd854..1530a4e78 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -411,9 +411,18 @@ endif() # Downloading SentencePiece if requested and set to compile with it. # Requires all the dependencies imposed by SentencePiece if(USE_SENTENCEPIECE) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DUSE_SENTENCEPIECE -D_USE_INTERNAL_STRING_VIEW") + # Prefer external sentencepiece (e.g., provided by vcpkg) if available. + # If found, set a cache var so subdirectories can detect it. + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DUSE_SENTENCEPIECE") LIST(APPEND CUDA_NVCC_FLAGS -DUSE_SENTENCEPIECE; ) - set(EXT_LIBS ${EXT_LIBS} sentencepiece sentencepiece_train) + if(SENTENCEPIECE_TARGET) + set(EXT_LIBS ${EXT_LIBS} ${SENTENCEPIECE_TARGET}) + set(USE_EXTERNAL_SENTENCEPIECE ON CACHE BOOL "Use external sentencepiece via find_package" FORCE) + else() + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_USE_INTERNAL_STRING_VIEW") + set(EXT_LIBS ${EXT_LIBS} sentencepiece sentencepiece_train) + MESSAGE(WARNING "Using internal sentencepiece") + endif() endif() if(USE_ONNX) diff --git a/src/3rd_party/CMakeLists.txt b/src/3rd_party/CMakeLists.txt index e3eca527c..89fb02f78 100644 --- a/src/3rd_party/CMakeLists.txt +++ b/src/3rd_party/CMakeLists.txt @@ -90,69 +90,74 @@ if(USE_FBGEMM) endif(USE_FBGEMM) if(USE_SENTENCEPIECE) - if(USE_STATIC_LIBS) - set(_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES}) - if(WIN32) - list(INSERT CMAKE_FIND_LIBRARY_SUFFIXES 0 .lib .a) - else() - set(CMAKE_FIND_LIBRARY_SUFFIXES .a) + # If external sentencepiece is available, skip building the vendored one. + if(USE_EXTERNAL_SENTENCEPIECE) + message(STATUS "Using external sentencepiece (skipping vendored subdirectory)") + else() + if(USE_STATIC_LIBS) + set(_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES}) + if(WIN32) + list(INSERT CMAKE_FIND_LIBRARY_SUFFIXES 0 .lib .a) + else() + set(CMAKE_FIND_LIBRARY_SUFFIXES .a) + endif() endif() - endif() - # regardless of -DUSE_STATIC_LIBS setting always build sentencepiece statically - set(SPM_ENABLE_SHARED OFF CACHE BOOL "Builds shared libaries in addition to static libraries." FORCE) - set(SPM_ENABLE_TCMALLOC ON CACHE BOOL "Enable TCMalloc if available.") - - if(USE_STATIC_LIBS) - set(SPM_TCMALLOC_STATIC ON CACHE BOOL "Link static library of TCMALLOC." FORCE) - else(USE_STATIC_LIBS) - set(SPM_TCMALLOC_STATIC OFF CACHE BOOL "Link static library of TCMALLOC.") - endif(USE_STATIC_LIBS) - - add_subdirectory(./sentencepiece) - include_directories(./sentencepiece) - if (NOT USE_WASM_COMPATIBLE_SOURCE) - set_target_properties(spm_encode spm_decode spm_train spm_normalize spm_export_vocab - PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}") - endif() + # regardless of -DUSE_STATIC_LIBS setting always build sentencepiece statically + set(SPM_ENABLE_SHARED OFF CACHE BOOL "Builds shared libaries in addition to static libraries." FORCE) + set(SPM_ENABLE_TCMALLOC ON CACHE BOOL "Enable TCMalloc if available.") - list(APPEND SENTENCEPIECE_TARGETS sentencepiece-static sentencepiece_train-static) - if (NOT USE_WASM_COMPATIBLE_SOURCE) - list(APPEND SENTENCEPIECE_TARGETS spm_decode spm_encode spm_export_vocab spm_normalize spm_train) - endif() + if(USE_STATIC_LIBS) + set(SPM_TCMALLOC_STATIC ON CACHE BOOL "Link static library of TCMALLOC." FORCE) + else(USE_STATIC_LIBS) + set(SPM_TCMALLOC_STATIC OFF CACHE BOOL "Link static library of TCMALLOC.") + endif(USE_STATIC_LIBS) + + add_subdirectory(./sentencepiece) + include_directories(./sentencepiece) + if (NOT USE_WASM_COMPATIBLE_SOURCE) + set_target_properties(spm_encode spm_decode spm_train spm_normalize spm_export_vocab + PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}") + endif() - if (CMAKE_CXX_COMPILER_ID MATCHES "GNU" AND (CMAKE_CXX_COMPILER_VERSION VERSION_EQUAL 8.0 OR CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 8.0)) - foreach(t IN LISTS SENTENCEPIECE_TARGETS) - set_property(TARGET ${t} APPEND_STRING PROPERTY COMPILE_FLAGS " -Wno-class-memaccess") - endforeach(t) - endif() + list(APPEND SENTENCEPIECE_TARGETS sentencepiece-static sentencepiece_train-static) + if (NOT USE_WASM_COMPATIBLE_SOURCE) + list(APPEND SENTENCEPIECE_TARGETS spm_decode spm_encode spm_export_vocab spm_normalize spm_train) + endif() - if (CMAKE_CXX_COMPILER_ID MATCHES "Clang") - foreach(t IN LISTS SENTENCEPIECE_TARGETS) - set_property(TARGET ${t} APPEND_STRING PROPERTY COMPILE_FLAGS " -Wno-tautological-compare -Wno-unused") - if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 9.0) - set_property(TARGET ${t} APPEND_STRING PROPERTY COMPILE_FLAGS " -Wno-range-loop-construct") - endif() - # get_property(SENTENCEPIECE_COMPILE_FLAGS TARGET ${t} PROPERTY COMPILE_FLAGS) - # message("-- SENTENCPIECE: compile flags for target ${t}: ${SENTENCEPIECE_COMPILE_FLAGS}") - endforeach(t) - endif() + if (CMAKE_CXX_COMPILER_ID MATCHES "GNU" AND (CMAKE_CXX_COMPILER_VERSION VERSION_EQUAL 8.0 OR CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 8.0)) + foreach(t IN LISTS SENTENCEPIECE_TARGETS) + set_property(TARGET ${t} APPEND_STRING PROPERTY COMPILE_FLAGS " -Wno-class-memaccess") + endforeach(t) + endif() - if(USE_STATIC_LIBS) - set(CMAKE_FIND_LIBRARY_SUFFIXES ${_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES}) - endif() + if (CMAKE_CXX_COMPILER_ID MATCHES "Clang") + foreach(t IN LISTS SENTENCEPIECE_TARGETS) + set_property(TARGET ${t} APPEND_STRING PROPERTY COMPILE_FLAGS " -Wno-tautological-compare -Wno-unused") + if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 9.0) + set_property(TARGET ${t} APPEND_STRING PROPERTY COMPILE_FLAGS " -Wno-range-loop-construct") + endif() + # get_property(SENTENCEPIECE_COMPILE_FLAGS TARGET ${t} PROPERTY COMPILE_FLAGS) + # message("-- SENTENCPIECE: compile flags for target ${t}: ${SENTENCEPIECE_COMPILE_FLAGS}") + endforeach(t) + endif() - if(GENERATE_MARIAN_INSTALL_TARGETS) if(USE_STATIC_LIBS) - install(TARGETS sentencepiece-static sentencepiece_train-static - EXPORT marian-targets - DESTINATION sentencepiece) - else() - install(TARGETS sentencepiece sentencepiece_train - EXPORT marian-targets - DESTINATION sentencepiece) + set(CMAKE_FIND_LIBRARY_SUFFIXES ${_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES}) endif() - endif(GENERATE_MARIAN_INSTALL_TARGETS) + + if(GENERATE_MARIAN_INSTALL_TARGETS) + if(USE_STATIC_LIBS) + install(TARGETS sentencepiece-static sentencepiece_train-static + EXPORT marian-targets + DESTINATION sentencepiece) + else() + install(TARGETS sentencepiece sentencepiece_train + EXPORT marian-targets + DESTINATION sentencepiece) + endif() + endif(GENERATE_MARIAN_INSTALL_TARGETS) + endif() endif(USE_SENTENCEPIECE) include_directories(./CLI) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 7847455fd..5db9c3203 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -3,18 +3,23 @@ add_subdirectory(3rd_party) include_directories(.) include_directories(3rd_party) include_directories(3rd_party/SQLiteCpp/include) -include_directories(3rd_party/sentencepiece) -# Use external protobuf instead of builtin when SPM_PROTOBUF_PROVIDER=package -if(NOT SPM_PROTOBUF_PROVIDER STREQUAL "package") - include_directories(3rd_party/sentencepiece/third_party/protobuf-lite) +# Use vendored sentencepiece includes only when not using external package +if(NOT USE_EXTERNAL_SENTENCEPIECE) + include_directories(3rd_party/sentencepiece) + # Use external protobuf instead of builtin when SPM_PROTOBUF_PROVIDER=package + if(NOT SPM_PROTOBUF_PROVIDER STREQUAL "package") + include_directories(3rd_party/sentencepiece/third_party/protobuf-lite) + endif() endif() include_directories(3rd_party/fbgemm/include) include_directories(3rd_party/intgemm) include_directories(3rd_party/ruy) include_directories(${CMAKE_CURRENT_BINARY_DIR}/3rd_party/intgemm) include_directories(${CMAKE_CURRENT_BINARY_DIR}/3rd_party) -# Add sentencepiece build directory for external protobuf generated files -include_directories(${CMAKE_CURRENT_BINARY_DIR}/3rd_party/sentencepiece/src) +# Add sentencepiece build directory for external protobuf generated files (vendored case only) +if(NOT USE_EXTERNAL_SENTENCEPIECE) + include_directories(${CMAKE_CURRENT_BINARY_DIR}/3rd_party/sentencepiece/src) +endif() # Add external protobuf include directory when using package provider if(SPM_PROTOBUF_PROVIDER STREQUAL "package") find_package(Protobuf REQUIRED) @@ -144,6 +149,9 @@ endif() add_library(marian STATIC ${MARIAN_SOURCES}) target_compile_options(marian PRIVATE ${ALL_WARNINGS}) +if(USE_EXTERNAL_SENTENCEPIECE) + target_compile_definitions(marian PUBLIC USE_EXTERNAL_SENTENCEPIECE) +endif() # Generate git_revision.h to reflect current git revision information # [https://stackoverflow.com/questions/1435953/how-can-i-pass-git-sha1-to-compiler-as-definition-using-cmake] diff --git a/src/data/sentencepiece_vocab.h b/src/data/sentencepiece_vocab.h index 2e7051885..802ba9dc0 100644 --- a/src/data/sentencepiece_vocab.h +++ b/src/data/sentencepiece_vocab.h @@ -6,8 +6,14 @@ #pragma warning(disable : 4100) -#include "sentencepiece/src/sentencepiece_processor.h" -#include "sentencepiece/src/sentencepiece_trainer.h" +#ifdef USE_EXTERNAL_SENTENCEPIECE +# include +# include +#else +# include "sentencepiece/src/sentencepiece_processor.h" +# include "sentencepiece/src/sentencepiece_trainer.h" +#endif + /* https://github.com/google/googletest/issues/1063#issuecomment-332518392 */ #if __GNUC__ >= 5