diff --git a/pyproject.toml b/pyproject.toml index fed528d4a..37259982b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,3 @@ [build-system] -requires = ["setuptools"] +requires = ["setuptools", "pkgconfig"] build-backend = "setuptools.build_meta" diff --git a/python/README.md b/python/README.md index 4b6f63f1d..5ea135ba5 100644 --- a/python/README.md +++ b/python/README.md @@ -17,6 +17,16 @@ following command from this directory: $ make install +If you already have native Brotli installed on your system and want to use this one instead of the vendored sources, you +should set the `USE_SYSTEM_BROTLI=1` environment variable when building the wheel, like this: + + $ USE_SYSTEM_BROTLI=1 pip install brotli --no-binary brotli + +Brotli is found via the `pkg-config` utility. Moreover, you must build all 3 `brotlicommon`, `brotlienc`, and `brotlidec` +components. If you're installing brotli from the package manager, you need the development package, like this on Fedora: + + $ dnf install brotli brotli-devel + ### Development You may run the following commands from this directory: diff --git a/setup.py b/setup.py index 6cd325d8c..c7b5df551 100644 --- a/setup.py +++ b/setup.py @@ -20,112 +20,125 @@ from distutils import log + CURR_DIR = os.path.abspath(os.path.dirname(os.path.realpath(__file__))) +def bool_from_environ(key: str): + value = os.environ.get(key) + if not value: + return False + if value == "1": + return True + if value == "0": + return False + raise ValueError(f"Environment variable {key} has invalid value {value}. Please set it to 1, 0 or an empty string") + + def read_define(path, macro): - """ Return macro value from the given file. """ - with open(path, 'r') as f: - for line in f: - m = re.match(rf'#define\s{macro}\s+(.+)', line) - if m: - return m.group(1) - return '' + """ Return macro value from the given file. """ + with open(path, 'r') as f: + for line in f: + m = re.match(rf'#define\s{macro}\s+(.+)', line) + if m: + return m.group(1) + + return '' def get_version(): - """ Return library version string from 'common/version.h' file. """ - version_file_path = os.path.join(CURR_DIR, 'c', 'common', 'version.h') - major = read_define(version_file_path, 'BROTLI_VERSION_MAJOR') - minor = read_define(version_file_path, 'BROTLI_VERSION_MINOR') - patch = read_define(version_file_path, 'BROTLI_VERSION_PATCH') - if not major or not minor or not patch: - return '' - return f'{major}.{minor}.{patch}' + """ Return library version string from 'common/version.h' file. """ + version_file_path = os.path.join(CURR_DIR, 'c', 'common', 'version.h') + major = read_define(version_file_path, 'BROTLI_VERSION_MAJOR') + minor = read_define(version_file_path, 'BROTLI_VERSION_MINOR') + patch = read_define(version_file_path, 'BROTLI_VERSION_PATCH') + if not major or not minor or not patch: + return '' + return f'{major}.{minor}.{patch}' def get_test_suite(): - test_loader = unittest.TestLoader() - test_suite = test_loader.discover('python', pattern='*_test.py') - return test_suite + test_loader = unittest.TestLoader() + test_suite = test_loader.discover('python', pattern='*_test.py') + return test_suite class BuildExt(build_ext): - def get_source_files(self): - filenames = build_ext.get_source_files(self) - for ext in self.extensions: - filenames.extend(ext.depends) - return filenames - - def build_extension(self, ext): - if ext.sources is None or not isinstance(ext.sources, (list, tuple)): - raise errors.DistutilsSetupError( - "in 'ext_modules' option (extension '%s'), " - "'sources' must be present and must be " - "a list of source filenames" % ext.name) - - ext_path = self.get_ext_fullpath(ext.name) - depends = ext.sources + ext.depends - if not (self.force or dep_util.newer_group(depends, ext_path, 'newer')): - log.debug("skipping '%s' extension (up-to-date)", ext.name) - return - else: - log.info("building '%s' extension", ext.name) - - c_sources = [] - for source in ext.sources: - if source.endswith('.c'): - c_sources.append(source) - extra_args = ext.extra_compile_args or [] - - objects = [] - - macros = ext.define_macros[:] - if platform.system() == 'Darwin': - macros.append(('OS_MACOSX', '1')) - elif self.compiler.compiler_type == 'mingw32': - # On Windows Python 2.7, pyconfig.h defines "hypot" as "_hypot", - # This clashes with GCC's cmath, and causes compilation errors when - # building under MinGW: http://bugs.python.org/issue11566 - macros.append(('_hypot', 'hypot')) - for undef in ext.undef_macros: - macros.append((undef,)) - - objs = self.compiler.compile( - c_sources, - output_dir=self.build_temp, - macros=macros, - include_dirs=ext.include_dirs, - debug=self.debug, - extra_postargs=extra_args, - depends=ext.depends) - objects.extend(objs) - - self._built_objects = objects[:] - if ext.extra_objects: - objects.extend(ext.extra_objects) - extra_args = ext.extra_link_args or [] - # when using GCC on Windows, we statically link libgcc and libstdc++, - # so that we don't need to package extra DLLs - if self.compiler.compiler_type == 'mingw32': - extra_args.extend(['-static-libgcc', '-static-libstdc++']) - - ext_path = self.get_ext_fullpath(ext.name) - # Detect target language, if not provided - language = ext.language or self.compiler.detect_language(c_sources) - - self.compiler.link_shared_object( - objects, - ext_path, - libraries=self.get_libraries(ext), - library_dirs=ext.library_dirs, - runtime_library_dirs=ext.runtime_library_dirs, - extra_postargs=extra_args, - export_symbols=self.get_export_symbols(ext), - debug=self.debug, - build_temp=self.build_temp, - target_lang=language) + def get_source_files(self): + filenames = build_ext.get_source_files(self) + for ext in self.extensions: + filenames.extend(ext.depends) + return filenames + + def build_extension(self, ext): + if ext.sources is None or not isinstance(ext.sources, (list, tuple)): + raise errors.DistutilsSetupError( + "in 'ext_modules' option (extension '%s'), " + "'sources' must be present and must be " + "a list of source filenames" % ext.name) + + ext_path = self.get_ext_fullpath(ext.name) + depends = ext.sources + ext.depends + if not (self.force or dep_util.newer_group(depends, ext_path, 'newer')): + log.debug("skipping '%s' extension (up-to-date)", ext.name) + return + else: + log.info("building '%s' extension", ext.name) + + c_sources = [] + for source in ext.sources: + if source.endswith('.c'): + c_sources.append(source) + extra_args = ext.extra_compile_args or [] + + objects = [] + + macros = ext.define_macros[:] + if platform.system() == 'Darwin': + macros.append(('OS_MACOSX', '1')) + elif self.compiler.compiler_type == 'mingw32': + # On Windows Python 2.7, pyconfig.h defines "hypot" as "_hypot", + # This clashes with GCC's cmath, and causes compilation errors when + # building under MinGW: http://bugs.python.org/issue11566 + macros.append(('_hypot', 'hypot')) + for undef in ext.undef_macros: + macros.append((undef,)) + + objs = self.compiler.compile( + c_sources, + output_dir=self.build_temp, + macros=macros, + include_dirs=ext.include_dirs, + debug=self.debug, + extra_postargs=extra_args, + depends=ext.depends) + objects.extend(objs) + + self._built_objects = objects[:] + if ext.extra_objects: + objects.extend(ext.extra_objects) + extra_args = ext.extra_link_args or [] + # when using GCC on Windows, we statically link libgcc and libstdc++, + # so that we don't need to package extra DLLs + if self.compiler.compiler_type == 'mingw32': + extra_args.extend(['-static-libgcc', '-static-libstdc++']) + + ext_path = self.get_ext_fullpath(ext.name) + # Detect target language, if not provided + language = ext.language or self.compiler.detect_language(c_sources) + + self.compiler.link_shared_object( + objects, + ext_path, + libraries=self.get_libraries(ext), + library_dirs=ext.library_dirs, + runtime_library_dirs=ext.runtime_library_dirs, + extra_postargs=extra_args, + export_symbols=self.get_export_symbols(ext), + debug=self.debug, + build_temp=self.build_temp, + target_lang=language) NAME = 'Brotli' @@ -172,103 +185,137 @@ def build_extension(self, ext): PY_MODULES = ['brotli'] -EXT_MODULES = [ - Extension( +USE_SYSTEM_BROTLI = bool_from_environ('USE_SYSTEM_BROTLI') + +if USE_SYSTEM_BROTLI: + import pkgconfig + + REQUIRED_BROTLI_SYSTEM_LIBRARIES = ["libbrotlicommon", "libbrotlienc", "libbrotlidec"] + + define_macros = [] + include_dirs = [] + libraries = [] + library_dirs = [] + + for required_system_library in REQUIRED_BROTLI_SYSTEM_LIBRARIES: + package_configuration = pkgconfig.parse(required_system_library) + + define_macros += package_configuration["define_macros"] + include_dirs += package_configuration["include_dirs"] + libraries += package_configuration["libraries"] + library_dirs += package_configuration["library_dirs"] + + brotli_extension = Extension( '_brotli', sources=[ - 'python/_brotli.c', - 'c/common/constants.c', - 'c/common/context.c', - 'c/common/dictionary.c', - 'c/common/platform.c', - 'c/common/shared_dictionary.c', - 'c/common/transform.c', - 'c/dec/bit_reader.c', - 'c/dec/decode.c', - 'c/dec/huffman.c', - 'c/dec/state.c', - 'c/enc/backward_references.c', - 'c/enc/backward_references_hq.c', - 'c/enc/bit_cost.c', - 'c/enc/block_splitter.c', - 'c/enc/brotli_bit_stream.c', - 'c/enc/cluster.c', - 'c/enc/command.c', - 'c/enc/compound_dictionary.c', - 'c/enc/compress_fragment.c', - 'c/enc/compress_fragment_two_pass.c', - 'c/enc/dictionary_hash.c', - 'c/enc/encode.c', - 'c/enc/encoder_dict.c', - 'c/enc/entropy_encode.c', - 'c/enc/fast_log.c', - 'c/enc/histogram.c', - 'c/enc/literal_cost.c', - 'c/enc/memory.c', - 'c/enc/metablock.c', - 'c/enc/static_dict.c', - 'c/enc/utf8_util.c', - ], - depends=[ - 'c/common/constants.h', - 'c/common/context.h', - 'c/common/dictionary.h', - 'c/common/platform.h', - 'c/common/shared_dictionary_internal.h', - 'c/common/transform.h', - 'c/common/version.h', - 'c/dec/bit_reader.h', - 'c/dec/huffman.h', - 'c/dec/prefix.h', - 'c/dec/state.h', - 'c/enc/backward_references.h', - 'c/enc/backward_references_hq.h', - 'c/enc/backward_references_inc.h', - 'c/enc/bit_cost.h', - 'c/enc/bit_cost_inc.h', - 'c/enc/block_encoder_inc.h', - 'c/enc/block_splitter.h', - 'c/enc/block_splitter_inc.h', - 'c/enc/brotli_bit_stream.h', - 'c/enc/cluster.h', - 'c/enc/cluster_inc.h', - 'c/enc/command.h', - 'c/enc/compound_dictionary.h', - 'c/enc/compress_fragment.h', - 'c/enc/compress_fragment_two_pass.h', - 'c/enc/dictionary_hash.h', - 'c/enc/encoder_dict.h', - 'c/enc/entropy_encode.h', - 'c/enc/entropy_encode_static.h', - 'c/enc/fast_log.h', - 'c/enc/find_match_length.h', - 'c/enc/hash.h', - 'c/enc/hash_composite_inc.h', - 'c/enc/hash_forgetful_chain_inc.h', - 'c/enc/hash_longest_match64_inc.h', - 'c/enc/hash_longest_match_inc.h', - 'c/enc/hash_longest_match_quickly_inc.h', - 'c/enc/hash_rolling_inc.h', - 'c/enc/hash_to_binary_tree_inc.h', - 'c/enc/histogram.h', - 'c/enc/histogram_inc.h', - 'c/enc/literal_cost.h', - 'c/enc/memory.h', - 'c/enc/metablock.h', - 'c/enc/metablock_inc.h', - 'c/enc/params.h', - 'c/enc/prefix.h', - 'c/enc/quality.h', - 'c/enc/ringbuffer.h', - 'c/enc/static_dict.h', - 'c/enc/static_dict_lut.h', - 'c/enc/utf8_util.h', - 'c/enc/write_bits.h', + 'python/_brotli.c' ], - include_dirs=[ - 'c/include', - ]), -] + include_dirs=include_dirs, + define_macros=define_macros, + libraries=libraries, + library_dirs=library_dirs + ) + + + EXT_MODULES = [brotli_extension] +else: + EXT_MODULES = [ + Extension( + '_brotli', + sources=[ + 'python/_brotli.c', + 'c/common/constants.c', + 'c/common/context.c', + 'c/common/dictionary.c', + 'c/common/platform.c', + 'c/common/shared_dictionary.c', + 'c/common/transform.c', + 'c/dec/bit_reader.c', + 'c/dec/decode.c', + 'c/dec/huffman.c', + 'c/dec/state.c', + 'c/enc/backward_references.c', + 'c/enc/backward_references_hq.c', + 'c/enc/bit_cost.c', + 'c/enc/block_splitter.c', + 'c/enc/brotli_bit_stream.c', + 'c/enc/cluster.c', + 'c/enc/command.c', + 'c/enc/compound_dictionary.c', + 'c/enc/compress_fragment.c', + 'c/enc/compress_fragment_two_pass.c', + 'c/enc/dictionary_hash.c', + 'c/enc/encode.c', + 'c/enc/encoder_dict.c', + 'c/enc/entropy_encode.c', + 'c/enc/fast_log.c', + 'c/enc/histogram.c', + 'c/enc/literal_cost.c', + 'c/enc/memory.c', + 'c/enc/metablock.c', + 'c/enc/static_dict.c', + 'c/enc/utf8_util.c', + ], + depends=[ + 'c/common/constants.h', + 'c/common/context.h', + 'c/common/dictionary.h', + 'c/common/platform.h', + 'c/common/shared_dictionary_internal.h', + 'c/common/transform.h', + 'c/common/version.h', + 'c/dec/bit_reader.h', + 'c/dec/huffman.h', + 'c/dec/prefix.h', + 'c/dec/state.h', + 'c/enc/backward_references.h', + 'c/enc/backward_references_hq.h', + 'c/enc/backward_references_inc.h', + 'c/enc/bit_cost.h', + 'c/enc/bit_cost_inc.h', + 'c/enc/block_encoder_inc.h', + 'c/enc/block_splitter.h', + 'c/enc/block_splitter_inc.h', + 'c/enc/brotli_bit_stream.h', + 'c/enc/cluster.h', + 'c/enc/cluster_inc.h', + 'c/enc/command.h', + 'c/enc/compound_dictionary.h', + 'c/enc/compress_fragment.h', + 'c/enc/compress_fragment_two_pass.h', + 'c/enc/dictionary_hash.h', + 'c/enc/encoder_dict.h', + 'c/enc/entropy_encode.h', + 'c/enc/entropy_encode_static.h', + 'c/enc/fast_log.h', + 'c/enc/find_match_length.h', + 'c/enc/hash.h', + 'c/enc/hash_composite_inc.h', + 'c/enc/hash_forgetful_chain_inc.h', + 'c/enc/hash_longest_match64_inc.h', + 'c/enc/hash_longest_match_inc.h', + 'c/enc/hash_longest_match_quickly_inc.h', + 'c/enc/hash_rolling_inc.h', + 'c/enc/hash_to_binary_tree_inc.h', + 'c/enc/histogram.h', + 'c/enc/histogram_inc.h', + 'c/enc/literal_cost.h', + 'c/enc/memory.h', + 'c/enc/metablock.h', + 'c/enc/metablock_inc.h', + 'c/enc/params.h', + 'c/enc/prefix.h', + 'c/enc/quality.h', + 'c/enc/ringbuffer.h', + 'c/enc/static_dict.h', + 'c/enc/static_dict_lut.h', + 'c/enc/utf8_util.h', + 'c/enc/write_bits.h', + ], + include_dirs=[ + 'c/include', + ]), + ] TEST_SUITE = 'setup.get_test_suite'