Skip to content

Switched to vcpkg provided Sentencepiece, kept submodule as a fallback. #2

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Sero1000 wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Switched to vcpkg provided Sentencepiece, kept submodule as a fallback. #2

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 11 additions & 2 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -411,9 +411,18 @@
# Downloading SentencePiece if requested and set to compile with it.
# Requires all the dependencies imposed by SentencePiece
if(USE_SENTENCEPIECE)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DUSE_SENTENCEPIECE -D_USE_INTERNAL_STRING_VIEW")
# Prefer external sentencepiece (e.g., provided by vcpkg) if available.
# If found, set a cache var so subdirectories can detect it.
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DUSE_SENTENCEPIECE")
LIST(APPEND CUDA_NVCC_FLAGS -DUSE_SENTENCEPIECE; )
set(EXT_LIBS ${EXT_LIBS} sentencepiece sentencepiece_train)
if(SENTENCEPIECE_TARGET)
set(EXT_LIBS ${EXT_LIBS} ${SENTENCEPIECE_TARGET})
set(USE_EXTERNAL_SENTENCEPIECE ON CACHE BOOL "Use external sentencepiece via find_package" FORCE)
else()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_USE_INTERNAL_STRING_VIEW")
set(EXT_LIBS ${EXT_LIBS} sentencepiece sentencepiece_train)
MESSAGE(WARNING "Using internal sentencepiece")

Check warning on line 424 in CMakeLists.txt

View workflow job for this annotation

GitHub Actions / Windows CPU-only 

Using internal sentencepiece
endif()
endif()

if(USE_ONNX)
Expand Down Expand Up @@ -588,7 +597,7 @@
endif(CUDA_FOUND)

else(COMPILE_CUDA)
message(WARNING "COMPILE_CUDA=off : Building only CPU version")

Check warning on line 600 in CMakeLists.txt

View workflow job for this annotation

GitHub Actions / Windows CPU-only 

COMPILE_CUDA=off : Building only CPU version
endif(COMPILE_CUDA)

# TODO: make compatible with older CUDA versions
Expand Down
113 changes: 59 additions & 54 deletions src/3rd_party/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,69 +90,74 @@ if(USE_FBGEMM)
endif(USE_FBGEMM)

if(USE_SENTENCEPIECE)
if(USE_STATIC_LIBS)
set(_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES})
if(WIN32)
list(INSERT CMAKE_FIND_LIBRARY_SUFFIXES 0 .lib .a)
else()
set(CMAKE_FIND_LIBRARY_SUFFIXES .a)
# If external sentencepiece is available, skip building the vendored one.
if(USE_EXTERNAL_SENTENCEPIECE)
message(STATUS "Using external sentencepiece (skipping vendored subdirectory)")
else()
if(USE_STATIC_LIBS)
set(_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES})
if(WIN32)
list(INSERT CMAKE_FIND_LIBRARY_SUFFIXES 0 .lib .a)
else()
set(CMAKE_FIND_LIBRARY_SUFFIXES .a)
endif()
endif()
endif()

# regardless of -DUSE_STATIC_LIBS setting always build sentencepiece statically
set(SPM_ENABLE_SHARED OFF CACHE BOOL "Builds shared libaries in addition to static libraries." FORCE)
set(SPM_ENABLE_TCMALLOC ON CACHE BOOL "Enable TCMalloc if available.")

if(USE_STATIC_LIBS)
set(SPM_TCMALLOC_STATIC ON CACHE BOOL "Link static library of TCMALLOC." FORCE)
else(USE_STATIC_LIBS)
set(SPM_TCMALLOC_STATIC OFF CACHE BOOL "Link static library of TCMALLOC.")
endif(USE_STATIC_LIBS)

add_subdirectory(./sentencepiece)
include_directories(./sentencepiece)
if (NOT USE_WASM_COMPATIBLE_SOURCE)
set_target_properties(spm_encode spm_decode spm_train spm_normalize spm_export_vocab
PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}")
endif()
# regardless of -DUSE_STATIC_LIBS setting always build sentencepiece statically
set(SPM_ENABLE_SHARED OFF CACHE BOOL "Builds shared libaries in addition to static libraries." FORCE)
set(SPM_ENABLE_TCMALLOC ON CACHE BOOL "Enable TCMalloc if available.")

list(APPEND SENTENCEPIECE_TARGETS sentencepiece-static sentencepiece_train-static)
if (NOT USE_WASM_COMPATIBLE_SOURCE)
list(APPEND SENTENCEPIECE_TARGETS spm_decode spm_encode spm_export_vocab spm_normalize spm_train)
endif()
if(USE_STATIC_LIBS)
set(SPM_TCMALLOC_STATIC ON CACHE BOOL "Link static library of TCMALLOC." FORCE)
else(USE_STATIC_LIBS)
set(SPM_TCMALLOC_STATIC OFF CACHE BOOL "Link static library of TCMALLOC.")
endif(USE_STATIC_LIBS)

add_subdirectory(./sentencepiece)
include_directories(./sentencepiece)
if (NOT USE_WASM_COMPATIBLE_SOURCE)
set_target_properties(spm_encode spm_decode spm_train spm_normalize spm_export_vocab
PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}")
endif()

if (CMAKE_CXX_COMPILER_ID MATCHES "GNU" AND (CMAKE_CXX_COMPILER_VERSION VERSION_EQUAL 8.0 OR CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 8.0))
foreach(t IN LISTS SENTENCEPIECE_TARGETS)
set_property(TARGET ${t} APPEND_STRING PROPERTY COMPILE_FLAGS " -Wno-class-memaccess")
endforeach(t)
endif()
list(APPEND SENTENCEPIECE_TARGETS sentencepiece-static sentencepiece_train-static)
if (NOT USE_WASM_COMPATIBLE_SOURCE)
list(APPEND SENTENCEPIECE_TARGETS spm_decode spm_encode spm_export_vocab spm_normalize spm_train)
endif()

if (CMAKE_CXX_COMPILER_ID MATCHES "Clang")
foreach(t IN LISTS SENTENCEPIECE_TARGETS)
set_property(TARGET ${t} APPEND_STRING PROPERTY COMPILE_FLAGS " -Wno-tautological-compare -Wno-unused")
if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 9.0)
set_property(TARGET ${t} APPEND_STRING PROPERTY COMPILE_FLAGS " -Wno-range-loop-construct")
endif()
# get_property(SENTENCEPIECE_COMPILE_FLAGS TARGET ${t} PROPERTY COMPILE_FLAGS)
# message("-- SENTENCPIECE: compile flags for target ${t}: ${SENTENCEPIECE_COMPILE_FLAGS}")
endforeach(t)
endif()
if (CMAKE_CXX_COMPILER_ID MATCHES "GNU" AND (CMAKE_CXX_COMPILER_VERSION VERSION_EQUAL 8.0 OR CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 8.0))
foreach(t IN LISTS SENTENCEPIECE_TARGETS)
set_property(TARGET ${t} APPEND_STRING PROPERTY COMPILE_FLAGS " -Wno-class-memaccess")
endforeach(t)
endif()

if(USE_STATIC_LIBS)
set(CMAKE_FIND_LIBRARY_SUFFIXES ${_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES})
endif()
if (CMAKE_CXX_COMPILER_ID MATCHES "Clang")
foreach(t IN LISTS SENTENCEPIECE_TARGETS)
set_property(TARGET ${t} APPEND_STRING PROPERTY COMPILE_FLAGS " -Wno-tautological-compare -Wno-unused")
if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 9.0)
set_property(TARGET ${t} APPEND_STRING PROPERTY COMPILE_FLAGS " -Wno-range-loop-construct")
endif()
# get_property(SENTENCEPIECE_COMPILE_FLAGS TARGET ${t} PROPERTY COMPILE_FLAGS)
# message("-- SENTENCPIECE: compile flags for target ${t}: ${SENTENCEPIECE_COMPILE_FLAGS}")
endforeach(t)
endif()

if(GENERATE_MARIAN_INSTALL_TARGETS)
if(USE_STATIC_LIBS)
install(TARGETS sentencepiece-static sentencepiece_train-static
EXPORT marian-targets
DESTINATION sentencepiece)
else()
install(TARGETS sentencepiece sentencepiece_train
EXPORT marian-targets
DESTINATION sentencepiece)
set(CMAKE_FIND_LIBRARY_SUFFIXES ${_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES})
endif()
endif(GENERATE_MARIAN_INSTALL_TARGETS)

if(GENERATE_MARIAN_INSTALL_TARGETS)
if(USE_STATIC_LIBS)
install(TARGETS sentencepiece-static sentencepiece_train-static
EXPORT marian-targets
DESTINATION sentencepiece)
else()
install(TARGETS sentencepiece sentencepiece_train
EXPORT marian-targets
DESTINATION sentencepiece)
endif()
endif(GENERATE_MARIAN_INSTALL_TARGETS)
endif()
endif(USE_SENTENCEPIECE)

include_directories(./CLI)
Expand Down
20 changes: 14 additions & 6 deletions src/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,18 +3,23 @@ add_subdirectory(3rd_party)
include_directories(.)
include_directories(3rd_party)
include_directories(3rd_party/SQLiteCpp/include)
include_directories(3rd_party/sentencepiece)
# Use external protobuf instead of builtin when SPM_PROTOBUF_PROVIDER=package
if(NOT SPM_PROTOBUF_PROVIDER STREQUAL "package")
include_directories(3rd_party/sentencepiece/third_party/protobuf-lite)
# Use vendored sentencepiece includes only when not using external package
if(NOT USE_EXTERNAL_SENTENCEPIECE)
include_directories(3rd_party/sentencepiece)
# Use external protobuf instead of builtin when SPM_PROTOBUF_PROVIDER=package
if(NOT SPM_PROTOBUF_PROVIDER STREQUAL "package")
include_directories(3rd_party/sentencepiece/third_party/protobuf-lite)
endif()
endif()
include_directories(3rd_party/fbgemm/include)
include_directories(3rd_party/intgemm)
include_directories(3rd_party/ruy)
include_directories(${CMAKE_CURRENT_BINARY_DIR}/3rd_party/intgemm)
include_directories(${CMAKE_CURRENT_BINARY_DIR}/3rd_party)
# Add sentencepiece build directory for external protobuf generated files
include_directories(${CMAKE_CURRENT_BINARY_DIR}/3rd_party/sentencepiece/src)
# Add sentencepiece build directory for external protobuf generated files (vendored case only)
if(NOT USE_EXTERNAL_SENTENCEPIECE)
include_directories(${CMAKE_CURRENT_BINARY_DIR}/3rd_party/sentencepiece/src)
endif()
# Add external protobuf include directory when using package provider
if(SPM_PROTOBUF_PROVIDER STREQUAL "package")
find_package(Protobuf REQUIRED)
Expand Down Expand Up @@ -144,6 +149,9 @@ endif()
add_library(marian STATIC ${MARIAN_SOURCES})

target_compile_options(marian PRIVATE ${ALL_WARNINGS})
if(USE_EXTERNAL_SENTENCEPIECE)
target_compile_definitions(marian PUBLIC USE_EXTERNAL_SENTENCEPIECE)
endif()

# Generate git_revision.h to reflect current git revision information
# [https://stackoverflow.com/questions/1435953/how-can-i-pass-git-sha1-to-compiler-as-definition-using-cmake]
Expand Down
10 changes: 8 additions & 2 deletions src/data/sentencepiece_vocab.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,14 @@

#pragma warning(disable : 4100)

#include "sentencepiece/src/sentencepiece_processor.h"
#include "sentencepiece/src/sentencepiece_trainer.h"
#ifdef USE_EXTERNAL_SENTENCEPIECE
# include <sentencepiece_processor.h>
# include <sentencepiece_trainer.h>
#else
# include "sentencepiece/src/sentencepiece_processor.h"
# include "sentencepiece/src/sentencepiece_trainer.h"
#endif


/* https://github.com/google/googletest/issues/1063#issuecomment-332518392 */
#if __GNUC__ >= 5
Expand Down
Loading