diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index f383277..223cd4d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -6,35 +6,37 @@ name: Python package on: push: branches: [ master ] + tags: + - 'v*' pull_request: branches: [ master ] jobs: - build: - + test: runs-on: ${{ matrix.os }} strategy: matrix: include: - - os: ubuntu-latest - python-version: 3.7 - os: ubuntu-latest python-version: 3.8 - os: ubuntu-latest python-version: 3.9 - os: ubuntu-latest python-version: '3.10' + - os: ubuntu-latest + python-version: '3.11' - os: macos-latest python-version: 3.9 - os: windows-latest python-version: 3.9 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: + fetch-depth: 0 submodules: true - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies @@ -48,3 +50,78 @@ jobs: run: | pip install pytest pytest + + build-for-publish: + name: Build distribution 📦 + runs-on: ubuntu-latest + if: "startsWith(github.ref, 'refs/tags')" + needs: test + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + submodules: true + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install dependencies + run: | + python -m pip install --upgrade build + + - name: Build a source tarball and wheel + run: python -m build . + + - name: Store the distribution packages + uses: actions/upload-artifact@v4 + with: + name: python-package-distributions + # NOTE: for now upload sdist only to prevent errors like unsupported platform tag 'linux_x86_64'. + path: dist/*.tar.gz + + publish-to-pypi: + name: >- + Publish Python 🐍 distribution 📦 to PyPI + if: startsWith(github.ref, 'refs/tags/') # only publish to PyPI on tag pushes + needs: + - build-for-publish + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/p/pyopenjtalk + permissions: + id-token: write # IMPORTANT: mandatory for trusted publishing + steps: + - name: Download all the dists + uses: actions/download-artifact@v4 + with: + name: python-package-distributions + path: dist/ + - name: Publish distribution 📦 to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + verbose: true + + # NOTE: for testing purposes only + # publish-to-testpypi: + # name: Publish Python 🐍 distribution 📦 to TestPyPI + # needs: + # - build-for-publish + # runs-on: ubuntu-latest + # environment: + # name: testpypi + # url: https://test.pypi.org/p/pyopenjtalk + # permissions: + # id-token: write # IMPORTANT: mandatory for trusted publishing + # steps: + # - name: Download all the dists + # uses: actions/download-artifact@v4 + # with: + # name: python-package-distributions + # path: dist/ + # - name: Publish distribution 📦 to TestPyPI + # uses: pypa/gh-action-pypi-publish@release/v1 + # with: + # repository-url: https://test.pypi.org/legacy/ + # verbose: true diff --git a/MANIFEST.in b/MANIFEST.in index d7063f2..9bc9ebe 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,7 @@ -include README.md LICENSE.md -recursive-include pyopenjtalk *.cpp *.pxd *.md *.pyx -recursive-include lib *.cpp *.c *.h README LICENSE COPYING CMakeLists.txt *.in +include pyopenjtalk/version.py +exclude .gitignore .gitmodules .travis.yml tox.ini +prune .github/* +prune docs/** +recursive-include lib *.cpp *.c *.h README COPYING CMakeLists.txt *.in exclude lib/open_jtalk/src/mecab/src/config.h +prune lib/open_jtalk/src/build/** diff --git a/README.md b/README.md index b6b0f70..6df5ee5 100644 --- a/README.md +++ b/README.md @@ -105,52 +105,56 @@ In [3]: pyopenjtalk.g2p("こんにちは", kana=True) Out[3]: 'コンニチワ' ``` -### About `run_marine` option +### Create/Apply user dictionary -After v0.3.0, the `run_marine` option has been available for estimating the Japanese accent with the DNN-based method (see [marine](https://github.com/6gsn/marine)). If you want to use the feature, please install pyopenjtalk as below; +1. Create a CSV file (e.g. `user.csv`) and write custom words like below: -```shell -pip install pyopenjtalk[marine] +```csv +ＧＮＵ,,,1,名詞,一般,*,*,*,*,ＧＮＵ,グヌー,グヌー,2/3,* ``` -And then, you can use the option as the following examples; +2. Call `mecab_dict_index` to compile the CSV file. ```python In [1]: import pyopenjtalk -In [2]: x, sr = pyopenjtalk.tts("おめでとうございます", run_marine=True) # for TTS +In [2]: pyopenjtalk.mecab_dict_index("user.csv", "user.dic") +reading user.csv ... 1 +emitting double-array: 100% |###########################################| -In [3]: label = pyopenjtalk.extract_fullcontext("こんにちは", run_marine=True) # for text processing frontend only +done! ``` -### Create/Apply user dictionary +3. Call `update_global_jtalk_with_user_dict` to apply the user dictionary. -1. Create a CSV file (e.g. `user.csv`) and write custom words like below: +```python +In [3]: pyopenjtalk.g2p("GNU") +Out[3]: 'j i i e n u y u u' -```csv -ＧＮＵ,,,1,名詞,一般,*,*,*,*,ＧＮＵ,グヌー,グヌー,2/3,* +In [4]: pyopenjtalk.update_global_jtalk_with_user_dict("user.dic") + +In [5]: pyopenjtalk.g2p("GNU") +Out[5]: 'g u n u u' ``` -2. Call `create_user_dict` to compile the CSV file. +### About `run_marine` option -``` ->>> import pyopenjtalk ->>> pyopenjtalk.create_user_dict("user.csv", "user.dic") -reading user.csv ... 1 -emitting double-array: 100% |###########################################| +After v0.3.0, the `run_marine` option has been available for estimating the Japanese accent with the DNN-based method (see [marine](https://github.com/6gsn/marine)). If you want to use the feature, please install pyopenjtalk as below; -done! +```shell +pip install pyopenjtalk[marine] ``` -3. Call `set_user_dict` to apply the user dictionary. +And then, you can use the option as the following examples; +```python +In [1]: import pyopenjtalk + +In [2]: x, sr = pyopenjtalk.tts("おめでとうございます", run_marine=True) # for TTS + +In [3]: label = pyopenjtalk.extract_fullcontext("こんにちは", run_marine=True) # for text processing frontend only ``` ->>> pyopenjtalk.g2p("GNU") -'j i i e n u y u u' ->>> pyopenjtalk.set_user_dict("user.dic") ->>> pyopenjtalk.g2p("GNU") -'g u n u u' -``` + ## LICENSE diff --git a/lib/open_jtalk b/lib/open_jtalk index b9b1bf6..d2a2cbf 160000 --- a/lib/open_jtalk +++ b/lib/open_jtalk @@ -1 +1 @@ -Subproject commit b9b1bf6a0cba6bc9550b4521913b20334a218dfc +Subproject commit d2a2cbfcbd69abc74993ed8bab3fabc0111c9727 diff --git a/pyopenjtalk/__init__.py b/pyopenjtalk/__init__.py index 1716ff9..3b3ac5c 100644 --- a/pyopenjtalk/__init__.py +++ b/pyopenjtalk/__init__.py @@ -1,16 +1,21 @@ +from __future__ import annotations + +import atexit import os +import sys +import tarfile +import tempfile +from collections.abc import Callable, Generator +from contextlib import ExitStack, contextmanager from os.path import exists +from threading import Lock +from typing import TypeVar +from urllib.request import urlopen -import pkg_resources -import six -from tqdm.auto import tqdm - -if six.PY2: - from urllib import urlretrieve +if sys.version_info >= (3, 9): + from importlib.resources import as_file, files else: - from urllib.request import urlretrieve - -import tarfile + from importlib_resources import as_file, files try: from .version import __version__ # NOQA @@ -18,60 +23,51 @@ raise ImportError("BUG: version.py doesn't exist. Please file a bug report.") from .htsengine import HTSEngine -from .openjtalk import CreateUserDict, OpenJTalk +from .openjtalk import OpenJTalk +from .openjtalk import mecab_dict_index as _mecab_dict_index from .utils import merge_njd_marine_features +_file_manager = ExitStack() +atexit.register(_file_manager.close) + +_pyopenjtalk_ref = files(__name__) +_dic_dir_name = "open_jtalk_dic_utf_8-1.11" + # Dictionary directory # defaults to the package directory where the dictionary will be automatically downloaded OPEN_JTALK_DICT_DIR = os.environ.get( "OPEN_JTALK_DICT_DIR", - pkg_resources.resource_filename(__name__, "open_jtalk_dic_utf_8-1.11"), + str(_file_manager.enter_context(as_file(_pyopenjtalk_ref / _dic_dir_name))), ).encode("utf-8") _dict_download_url = "https://github.com/r9y9/open_jtalk/releases/download/v1.11.1" _DICT_URL = f"{_dict_download_url}/open_jtalk_dic_utf_8-1.11.tar.gz" # Default mei_normal.voice for HMM-based TTS -DEFAULT_HTS_VOICE = pkg_resources.resource_filename( - __name__, "htsvoice/mei_normal.htsvoice" +DEFAULT_HTS_VOICE = str( + _file_manager.enter_context( + as_file(_pyopenjtalk_ref / "htsvoice/mei_normal.htsvoice") + ) ).encode("utf-8") -# Global instance of OpenJTalk -_global_jtalk = None -# Global instance of HTSEngine -# mei_normal.voice is used as default -_global_htsengine = None -# Global instance of Marine -_global_marine = None - - -# https://github.com/tqdm/tqdm#hooks-and-callbacks -class _TqdmUpTo(tqdm): # type: ignore - def update_to(self, b=1, bsize=1, tsize=None): - if tsize is not None: - self.total = tsize - return self.update(b * bsize - self.n) - def _extract_dic(): + from tqdm.auto import tqdm + global OPEN_JTALK_DICT_DIR - filename = pkg_resources.resource_filename(__name__, "dic.tar.gz") - print('Downloading: "{}"'.format(_DICT_URL)) - with _TqdmUpTo( - unit="B", - unit_scale=True, - unit_divisor=1024, - miniters=1, - desc="dic.tar.gz", - ) as t: # all optional kwargs - urlretrieve(_DICT_URL, filename, reporthook=t.update_to) - t.total = t.n - print("Extracting tar file {}".format(filename)) - with tarfile.open(filename, mode="r|gz") as f: - f.extractall(path=pkg_resources.resource_filename(__name__, "")) - OPEN_JTALK_DICT_DIR = pkg_resources.resource_filename( - __name__, "open_jtalk_dic_utf_8-1.11" - ).encode("utf-8") - os.remove(filename) + pyopenjtalk_dir = _file_manager.enter_context(as_file(_pyopenjtalk_ref)) + with tempfile.TemporaryFile() as t: + print('Downloading: "{}"'.format(_DICT_URL)) + with urlopen(_DICT_URL) as response: + with tqdm.wrapattr( + t, "write", total=getattr(response, "length", None) + ) as tar: + for chunk in response: + tar.write(chunk) + t.seek(0) + print("Extracting tar file") + with tarfile.open(mode="r|gz", fileobj=t) as f: + f.extractall(path=pyopenjtalk_dir) + OPEN_JTALK_DICT_DIR = str(pyopenjtalk_dir / _dic_dir_name).encode("utf-8") def _lazy_init(): @@ -79,6 +75,49 @@ def _lazy_init(): _extract_dic() +_T = TypeVar("_T") + + +def _global_instance_manager( + instance_factory: Callable[[], _T] | None = None, instance: _T | None = None +) -> Callable[[], Generator[_T, None, None]]: + assert instance_factory is not None or instance is not None + _instance = instance + mutex = Lock() + + @contextmanager + def manager() -> Generator[_T, None, None]: + nonlocal _instance + with mutex: + if _instance is None: + _instance = instance_factory() + yield _instance + + return manager + + +def _jtalk_factory() -> OpenJTalk: + _lazy_init() + return OpenJTalk(dn_mecab=OPEN_JTALK_DICT_DIR) + + +def _marine_factory(): + try: + from marine.predict import Predictor + except ImportError: + raise ImportError("Please install marine by `pip install pyopenjtalk[marine]`") + return Predictor() + + +# Global instance of OpenJTalk +_global_jtalk = _global_instance_manager(_jtalk_factory) +# Global instance of HTSEngine +# mei_normal.voice is used as default +_global_htsengine = _global_instance_manager(lambda: HTSEngine(DEFAULT_HTS_VOICE)) +# Global instance of Marine +_global_marine = _global_instance_manager(_marine_factory) + + def g2p(*args, **kwargs): """Grapheme-to-phoeneme (G2P) conversion @@ -94,11 +133,8 @@ def g2p(*args, **kwargs): Returns: str or list: G2P result in 1) str if join is True 2) list if join is False. """ - global _global_jtalk - if _global_jtalk is None: - _lazy_init() - _global_jtalk = OpenJTalk(dn_mecab=OPEN_JTALK_DICT_DIR) - return _global_jtalk.g2p(*args, **kwargs) + with _global_jtalk() as jtalk: + return jtalk.g2p(*args, **kwargs) def estimate_accent(njd_features): @@ -112,21 +148,13 @@ def estimate_accent(njd_features): Returns: list: features for NJDNode with estimation results by marine. """ - global _global_marine - if _global_marine is None: - try: - from marine.predict import Predictor - except BaseException: - raise ImportError( - "Please install marine by `pip install pyopenjtalk[marine]`" - ) - _global_marine = Predictor() - from marine.utils.openjtalk_util import convert_njd_feature_to_marine_feature + with _global_marine() as marine: + from marine.utils.openjtalk_util import convert_njd_feature_to_marine_feature - marine_feature = convert_njd_feature_to_marine_feature(njd_features) - marine_results = _global_marine.predict( - [marine_feature], require_open_jtalk_format=True - ) + marine_feature = convert_njd_feature_to_marine_feature(njd_features) + marine_results = marine.predict( + [marine_feature], require_open_jtalk_format=True + ) njd_features = merge_njd_marine_features(njd_features, marine_results) return njd_features @@ -165,13 +193,11 @@ def synthesize(labels, speed=1.0, half_tone=0.0): if isinstance(labels, tuple) and len(labels) == 2: labels = labels[1] - global _global_htsengine - if _global_htsengine is None: - _global_htsengine = HTSEngine(DEFAULT_HTS_VOICE) - sr = _global_htsengine.get_sampling_frequency() - _global_htsengine.set_speed(speed) - _global_htsengine.add_half_tone(half_tone) - return _global_htsengine.synthesize(labels), sr + with _global_htsengine() as htsengine: + sr = htsengine.get_sampling_frequency() + htsengine.set_speed(speed) + htsengine.add_half_tone(half_tone) + return htsengine.synthesize(labels), sr def tts(text, speed=1.0, half_tone=0.0, run_marine=False): @@ -203,11 +229,8 @@ def run_frontend(text): Returns: list: features for NJDNode. """ - global _global_jtalk - if _global_jtalk is None: - _lazy_init() - _global_jtalk = OpenJTalk(dn_mecab=OPEN_JTALK_DICT_DIR) - return _global_jtalk.run_frontend(text) + with _global_jtalk() as jtalk: + return jtalk.run_frontend(text) def make_label(njd_features): @@ -219,47 +242,55 @@ def make_label(njd_features): Returns: list: full-context labels. """ - global _global_jtalk - if _global_jtalk is None: - _lazy_init() - _global_jtalk = OpenJTalk(dn_mecab=OPEN_JTALK_DICT_DIR) - return _global_jtalk.make_label(njd_features) + with _global_jtalk() as jtalk: + return jtalk.make_label(njd_features) -def create_user_dict(path, out_path): +def mecab_dict_index(path, out_path, dn_mecab=None): """Create user dictionary Args: path (str): path to user csv out_path (str): path to output dictionary + dn_mecab (optional. str): path to mecab dictionary """ - global _global_jtalk - if _global_jtalk is None: - _lazy_init() if not exists(path): - raise ValueError("no such file or directory: %s" % path) - CreateUserDict(OPEN_JTALK_DICT_DIR, path.encode("utf-8"), out_path.encode("utf-8")) + raise FileNotFoundError("no such file or directory: %s" % path) + if dn_mecab is None: + with _global_jtalk(): # call _lazy_init() + pass + dn_mecab = OPEN_JTALK_DICT_DIR + r = _mecab_dict_index(dn_mecab, path.encode("utf-8"), out_path.encode("utf-8")) + # NOTE: mecab load returns 1 if success, but mecab_dict_index return the opposite + # yeah it's confusing... + if r != 0: + raise RuntimeError("Failed to create user dictionary") -def set_user_dict(path): - """Apply user dictionary + +def update_global_jtalk_with_user_dict(path): + """Update global openjtalk instance with the user dictionary + + Note that this will change the global state of the openjtalk module. Args: path (str): path to user dictionary """ global _global_jtalk - if _global_jtalk is None: - _lazy_init() - if not exists(path): - raise ValueError("no such file or directory: %s" % path) - _global_jtalk = OpenJTalk( - dn_mecab=OPEN_JTALK_DICT_DIR, user_mecab=path.encode("utf-8") - ) + with _global_jtalk(): + if not exists(path): + raise FileNotFoundError("no such file or directory: %s" % path) + _global_jtalk = _global_instance_manager( + instance=OpenJTalk( + dn_mecab=OPEN_JTALK_DICT_DIR, userdic=path.encode("utf-8") + ) + ) def unset_user_dict(): """Stop applying user dictionary""" global _global_jtalk - if _global_jtalk is None: - _lazy_init() - _global_jtalk = OpenJTalk(dn_mecab=OPEN_JTALK_DICT_DIR) + with _global_jtalk(): + _global_jtalk = _global_instance_manager( + instance=OpenJTalk(dn_mecab=OPEN_JTALK_DICT_DIR) + ) diff --git a/pyopenjtalk/htsengine.pyx b/pyopenjtalk/htsengine.pyx index 9936a6e..cddd16b 100644 --- a/pyopenjtalk/htsengine.pyx +++ b/pyopenjtalk/htsengine.pyx @@ -1,6 +1,10 @@ # coding: utf-8 # cython: boundscheck=True, wraparound=True # cython: c_string_type=unicode, c_string_encoding=ascii +# cython: language_level=3 + +from contextlib import contextmanager +from threading import RLock import numpy as np @@ -19,13 +23,25 @@ from .htsengine cimport ( HTS_Engine_get_generated_speech, HTS_Engine_get_nsamples ) -cdef class HTSEngine(object): +def _generate_lock_manager(): + lock = RLock() + + @contextmanager + def f(): + with lock: + yield + + return f + + +cdef class HTSEngine: """HTSEngine Args: voice (bytes): File path of htsvoice. """ cdef HTS_Engine* engine + _lock_manager = _generate_lock_manager() def __cinit__(self, bytes voice=b"htsvoice/mei_normal.htsvoice"): self.engine = new HTS_Engine() @@ -33,24 +49,29 @@ cdef class HTSEngine(object): HTS_Engine_initialize(self.engine) if self.load(voice) != 1: - self.clear() - raise RuntimeError("Failed to initalize HTS_Engine") + self.clear() + raise RuntimeError("Failed to initalize HTS_Engine") + @_lock_manager() def load(self, bytes voice): cdef char* voices = voice cdef char ret - ret = HTS_Engine_load(self.engine, &voices, 1) + with nogil: + ret = HTS_Engine_load(self.engine, &voices, 1) return ret + @_lock_manager() def get_sampling_frequency(self): """Get sampling frequency """ return HTS_Engine_get_sampling_frequency(self.engine) + @_lock_manager() def get_fperiod(self): """Get frame period""" return HTS_Engine_get_fperiod(self.engine) + @_lock_manager() def set_speed(self, speed=1.0): """Set speed @@ -59,6 +80,7 @@ cdef class HTSEngine(object): """ HTS_Engine_set_speed(self.engine, speed) + @_lock_manager() def add_half_tone(self, half_tone=0.0): """Additional half tone in log-f0 @@ -67,6 +89,7 @@ cdef class HTSEngine(object): """ HTS_Engine_add_half_tone(self.engine, half_tone) + @_lock_manager() def synthesize(self, list labels): """Synthesize waveform from list of full-context labels @@ -81,34 +104,43 @@ cdef class HTSEngine(object): self.refresh() return x + @_lock_manager() def synthesize_from_strings(self, list labels): """Synthesize from strings""" cdef size_t num_lines = len(labels) cdef char **lines = malloc((num_lines + 1) * sizeof(char*)) - for n in range(len(labels)): + for n in range(num_lines): lines[n] = labels[n] - cdef char ret = HTS_Engine_synthesize_from_strings(self.engine, lines, num_lines) - free(lines) + cdef char ret + with nogil: + ret = HTS_Engine_synthesize_from_strings(self.engine, lines, num_lines) + free(lines) if ret != 1: raise RuntimeError("Failed to run synthesize_from_strings") + @_lock_manager() def get_generated_speech(self): """Get generated speech""" cdef size_t nsamples = HTS_Engine_get_nsamples(self.engine) - cdef np.ndarray speech = np.zeros([nsamples], dtype=np.float64) + cdef np.ndarray speech = np.empty([nsamples], dtype=np.float64) + cdef double[:] speech_view = speech cdef size_t index - for index in range(nsamples): - speech[index] = HTS_Engine_get_generated_speech(self.engine, index) + with (nogil, cython.boundscheck(False)): + for index in range(nsamples): + speech_view[index] = HTS_Engine_get_generated_speech(self.engine, index) return speech + @_lock_manager() def get_fullcontext_label_format(self): """Get full-context label format""" return (HTS_Engine_get_fullcontext_label_format(self.engine)).decode("utf-8") + @_lock_manager() def refresh(self): - HTS_Engine_refresh(self.engine) + HTS_Engine_refresh(self.engine) + @_lock_manager() def clear(self): HTS_Engine_clear(self.engine) diff --git a/pyopenjtalk/htsengine/__init__.pxd b/pyopenjtalk/htsengine/__init__.pxd index c24d959..e919505 100644 --- a/pyopenjtalk/htsengine/__init__.pxd +++ b/pyopenjtalk/htsengine/__init__.pxd @@ -1,4 +1,5 @@ # distutils: language = c++ +# cython: language_level=3 cdef extern from "HTS_engine.h": @@ -7,16 +8,16 @@ cdef extern from "HTS_engine.h": ctypedef _HTS_Engine HTS_Engine void HTS_Engine_initialize(HTS_Engine * engine) - char HTS_Engine_load(HTS_Engine * engine, char **voices, size_t num_voices) + char HTS_Engine_load(HTS_Engine * engine, char **voices, size_t num_voices) nogil size_t HTS_Engine_get_sampling_frequency(HTS_Engine * engine) size_t HTS_Engine_get_fperiod(HTS_Engine * engine) void HTS_Engine_refresh(HTS_Engine * engine) void HTS_Engine_clear(HTS_Engine * engine) const char *HTS_Engine_get_fullcontext_label_format(HTS_Engine * engine) - char HTS_Engine_synthesize_from_strings(HTS_Engine * engine, char **lines, size_t num_lines) + char HTS_Engine_synthesize_from_strings(HTS_Engine * engine, char **lines, size_t num_lines) nogil char HTS_Engine_synthesize_from_fn(HTS_Engine * engine, const char *fn) - double HTS_Engine_get_generated_speech(HTS_Engine * engine, size_t index) + double HTS_Engine_get_generated_speech(HTS_Engine * engine, size_t index) nogil size_t HTS_Engine_get_nsamples(HTS_Engine * engine) void HTS_Engine_set_speed(HTS_Engine * engine, double f) - void HTS_Engine_add_half_tone(HTS_Engine * engine, double f) \ No newline at end of file + void HTS_Engine_add_half_tone(HTS_Engine * engine, double f) diff --git a/pyopenjtalk/openjtalk.pyx b/pyopenjtalk/openjtalk.pyx index 9e0978a..5c460a0 100644 --- a/pyopenjtalk/openjtalk.pyx +++ b/pyopenjtalk/openjtalk.pyx @@ -1,58 +1,27 @@ # coding: utf-8 # cython: boundscheck=True, wraparound=True # cython: c_string_type=unicode, c_string_encoding=ascii +# cython: language_level=3 -import errno -import numpy as np +from contextlib import contextmanager +from threading import Lock -cimport numpy as np -np.import_array() - -cimport cython +from libc cimport errno from libc.stdlib cimport calloc +from libc.string cimport strlen from .openjtalk.mecab cimport Mecab, Mecab_initialize, Mecab_load, Mecab_analysis from .openjtalk.mecab cimport Mecab_get_feature, Mecab_get_size, Mecab_refresh, Mecab_clear -from .openjtalk.mecab cimport mecab_dict_index, createModel, Model, Tagger, Lattice +from .openjtalk.mecab cimport createModel, Model, Tagger, Lattice +from .openjtalk.mecab cimport mecab_dict_index as _mecab_dict_index from .openjtalk.njd cimport NJD, NJD_initialize, NJD_refresh, NJD_print, NJD_clear from .openjtalk cimport njd as _njd from .openjtalk.jpcommon cimport JPCommon, JPCommon_initialize,JPCommon_make_label from .openjtalk.jpcommon cimport JPCommon_get_label_size, JPCommon_get_label_feature from .openjtalk.jpcommon cimport JPCommon_refresh, JPCommon_clear -from .openjtalk cimport njd2jpcommon from .openjtalk.text2mecab cimport text2mecab from .openjtalk.mecab2njd cimport mecab2njd from .openjtalk.njd2jpcommon cimport njd2jpcommon -from libc.string cimport strlen - -cdef inline int Mecab_load_ex(Mecab *m, char* dicdir, char* userdic): - if userdic == NULL or strlen(userdic) == 0: - return Mecab_load(m, dicdir) - - if m == NULL or dicdir == NULL or strlen(dicdir) == 0: - return 0 - - Mecab_clear(m) - - cdef (char*)[5] argv = ["mecab", "-d", dicdir, "-u", userdic] - cdef Model *model = createModel(5, argv) - - if model == NULL: - return 0 - m.model = model - - cdef Tagger *tagger = model.createTagger() - if tagger == NULL: - Mecab_clear(m) - return 0 - m.tagger = tagger - - cdef Lattice *lattice = model.createLattice() - if lattice == NULL: - Mecab_clear(m) - return 0 - m.lattice = lattice - return 1 cdef njd_node_get_string(_njd.NJDNode* node): return ((_njd.NJDNode_get_string(node))).decode("utf-8") @@ -84,48 +53,48 @@ cdef njd_node_get_read(_njd.NJDNode* node): cdef njd_node_get_pron(_njd.NJDNode* node): return ((_njd.NJDNode_get_pron(node))).decode("utf-8") -cdef njd_node_get_acc(_njd.NJDNode* node): +cdef int njd_node_get_acc(_njd.NJDNode* node) noexcept: return _njd.NJDNode_get_acc(node) -cdef njd_node_get_mora_size(_njd.NJDNode* node): +cdef int njd_node_get_mora_size(_njd.NJDNode* node) noexcept: return _njd.NJDNode_get_mora_size(node) cdef njd_node_get_chain_rule(_njd.NJDNode* node): return ((_njd.NJDNode_get_chain_rule(node))).decode("utf-8") -cdef njd_node_get_chain_flag(_njd.NJDNode* node): - return _njd.NJDNode_get_chain_flag(node) +cdef int njd_node_get_chain_flag(_njd.NJDNode* node) noexcept: + return _njd.NJDNode_get_chain_flag(node) cdef node2feature(_njd.NJDNode* node): - return { - "string": njd_node_get_string(node), - "pos": njd_node_get_pos(node), - "pos_group1": njd_node_get_pos_group1(node), - "pos_group2": njd_node_get_pos_group2(node), - "pos_group3": njd_node_get_pos_group3(node), - "ctype": njd_node_get_ctype(node), - "cform": njd_node_get_cform(node), - "orig": njd_node_get_orig(node), - "read": njd_node_get_read(node), - "pron": njd_node_get_pron(node), - "acc": njd_node_get_acc(node), - "mora_size": njd_node_get_mora_size(node), - "chain_rule": njd_node_get_chain_rule(node), - "chain_flag": njd_node_get_chain_flag(node), - } + return { + "string": njd_node_get_string(node), + "pos": njd_node_get_pos(node), + "pos_group1": njd_node_get_pos_group1(node), + "pos_group2": njd_node_get_pos_group2(node), + "pos_group3": njd_node_get_pos_group3(node), + "ctype": njd_node_get_ctype(node), + "cform": njd_node_get_cform(node), + "orig": njd_node_get_orig(node), + "read": njd_node_get_read(node), + "pron": njd_node_get_pron(node), + "acc": njd_node_get_acc(node), + "mora_size": njd_node_get_mora_size(node), + "chain_rule": njd_node_get_chain_rule(node), + "chain_flag": njd_node_get_chain_flag(node), + } cdef njd2feature(_njd.NJD* njd): cdef _njd.NJDNode* node = njd.head features = [] while node is not NULL: - features.append(node2feature(node)) - node = node.next + features.append(node2feature(node)) + node = node.next return features -cdef feature2njd(_njd.NJD* njd, features): +cdef void feature2njd(_njd.NJD* njd, features): cdef _njd.NJDNode* node for feature_node in features: @@ -148,67 +117,115 @@ cdef feature2njd(_njd.NJD* njd, features): _njd.NJDNode_set_chain_flag(node, feature_node["chain_flag"]) _njd.NJD_push_node(njd, node) +# based on Mecab_load in impl. from mecab.cpp +cdef inline int Mecab_load_with_userdic(Mecab *m, char* dicdir, char* userdic) noexcept nogil: + if userdic == NULL or strlen(userdic) == 0: + return Mecab_load(m, dicdir) + + if m == NULL or dicdir == NULL or strlen(dicdir) == 0: + return 0 + + Mecab_clear(m) + + cdef (char*)[5] argv = ["mecab", "-d", dicdir, "-u", userdic] + cdef Model *model = createModel(5, argv) + + if model == NULL: + return 0 + m.model = model + + cdef Tagger *tagger = model.createTagger() + if tagger == NULL: + Mecab_clear(m) + return 0 + m.tagger = tagger -cdef class OpenJTalk(object): + cdef Lattice *lattice = model.createLattice() + if lattice == NULL: + Mecab_clear(m) + return 0 + m.lattice = lattice + + return 1 + +def _generate_lock_manager(): + lock = Lock() + + @contextmanager + def f(): + with lock: + yield + + return f + + +cdef class OpenJTalk: """OpenJTalk Args: dn_mecab (bytes): Dictionaly path for MeCab. - user_mecab (bytes): Dictionary path for MeCab userdic. + userdic (bytes): Dictionary path for MeCab userdic. This option is ignored when empty bytestring is given. Default is empty. """ cdef Mecab* mecab cdef NJD* njd cdef JPCommon* jpcommon + _lock_manager = _generate_lock_manager() + + def __cinit__(self, bytes dn_mecab=b"/usr/local/dic", bytes userdic=b""): + cdef char* _dn_mecab = dn_mecab + cdef char* _userdic = userdic - def __cinit__(self, bytes dn_mecab=b"/usr/local/dic", bytes user_mecab=b""): self.mecab = new Mecab() self.njd = new NJD() self.jpcommon = new JPCommon() - Mecab_initialize(self.mecab) - NJD_initialize(self.njd) - JPCommon_initialize(self.jpcommon) + with nogil: + Mecab_initialize(self.mecab) + NJD_initialize(self.njd) + JPCommon_initialize(self.jpcommon) - r = self._load(dn_mecab, user_mecab) - if r != 1: - self._clear() - raise RuntimeError("Failed to initalize Mecab") + r = self._load(_dn_mecab, _userdic) + if r != 1: + self._clear() + raise RuntimeError("Failed to initalize Mecab") + cdef void _clear(self) noexcept nogil: + Mecab_clear(self.mecab) + NJD_clear(self.njd) + JPCommon_clear(self.jpcommon) - def _clear(self): - Mecab_clear(self.mecab) - NJD_clear(self.njd) - JPCommon_clear(self.jpcommon) - - def _load(self, bytes dn_mecab, bytes user_mecab): - return Mecab_load_ex(self.mecab, dn_mecab, user_mecab) - + cdef int _load(self, char* dn_mecab, char* userdic) noexcept nogil: + return Mecab_load_with_userdic(self.mecab, dn_mecab, userdic) + @_lock_manager() def run_frontend(self, text): """Run OpenJTalk's text processing frontend """ cdef char buff[8192] + if isinstance(text, str): text = text.encode("utf-8") - - cdef int result = text2mecab(buff, 8192, text) - if result != 0: - if result == errno.ERANGE: - raise RuntimeError("Text is too long") - if result == errno.EINVAL: - raise RuntimeError("Invalid input for text2mecab") - raise RuntimeError("Unknown error: " + str(result)) - - Mecab_analysis(self.mecab, buff) - mecab2njd(self.njd, Mecab_get_feature(self.mecab), Mecab_get_size(self.mecab)) - _njd.njd_set_pronunciation(self.njd) - _njd.njd_set_digit(self.njd) - _njd.njd_set_accent_phrase(self.njd) - _njd.njd_set_accent_type(self.njd) - _njd.njd_set_unvoiced_vowel(self.njd) - _njd.njd_set_long_vowel(self.njd) + cdef const char* _text = text + cdef int result + with nogil: + result = text2mecab(buff, 8192, _text) + if result != 0: + if result == errno.ERANGE: + raise RuntimeError("Text is too long") + if result == errno.EINVAL: + raise RuntimeError("Invalid input for text2mecab") + raise RuntimeError("Unknown error: " + str(result)) + + Mecab_analysis(self.mecab, buff) + mecab2njd(self.njd, Mecab_get_feature(self.mecab), Mecab_get_size(self.mecab)) + _njd.njd_set_pronunciation(self.njd) + _njd.njd_set_digit(self.njd) + _njd.njd_set_accent_phrase(self.njd) + _njd.njd_set_accent_type(self.njd) + _njd.njd_set_unvoiced_vowel(self.njd) + _njd.njd_set_long_vowel(self.njd) features = njd2feature(self.njd) # Note that this will release memory for njd feature @@ -217,23 +234,24 @@ cdef class OpenJTalk(object): return features + @_lock_manager() def make_label(self, features): """Make full-context label """ feature2njd(self.njd, features) - njd2jpcommon(self.jpcommon, self.njd) + with nogil: + njd2jpcommon(self.jpcommon, self.njd) - JPCommon_make_label(self.jpcommon) + JPCommon_make_label(self.jpcommon) - cdef int label_size = JPCommon_get_label_size(self.jpcommon) - cdef char** label_feature - label_feature = JPCommon_get_label_feature(self.jpcommon) + label_size = JPCommon_get_label_size(self.jpcommon) + label_feature = JPCommon_get_label_feature(self.jpcommon) labels = [] for i in range(label_size): - # This will create a copy of c string - # http://cython.readthedocs.io/en/latest/src/tutorial/strings.html - labels.append(label_feature[i]) + # This will create a copy of c string + # http://cython.readthedocs.io/en/latest/src/tutorial/strings.html + labels.append(label_feature[i]) # Note that this will release memory for label feature JPCommon_refresh(self.jpcommon) @@ -274,7 +292,7 @@ cdef class OpenJTalk(object): del self.njd del self.jpcommon -def CreateUserDict(bytes dn_mecab, bytes path, bytes out_path): +def mecab_dict_index(bytes dn_mecab, bytes path, bytes out_path): cdef (char*)[10] argv = [ "mecab-dict-index", "-d", @@ -287,4 +305,7 @@ def CreateUserDict(bytes dn_mecab, bytes path, bytes out_path): "utf-8", path ] - mecab_dict_index(10, argv) \ No newline at end of file + cdef int ret + with nogil: + ret = _mecab_dict_index(10, argv) + return ret diff --git a/pyopenjtalk/openjtalk/__init__.pxd b/pyopenjtalk/openjtalk/__init__.pxd index e69de29..f08f4aa 100644 --- a/pyopenjtalk/openjtalk/__init__.pxd +++ b/pyopenjtalk/openjtalk/__init__.pxd @@ -0,0 +1 @@ +# cython: language_level=3 diff --git a/pyopenjtalk/openjtalk/jpcommon.pxd b/pyopenjtalk/openjtalk/jpcommon.pxd index 8e86bea..d2b121a 100644 --- a/pyopenjtalk/openjtalk/jpcommon.pxd +++ b/pyopenjtalk/openjtalk/jpcommon.pxd @@ -1,29 +1,30 @@ # distutils: language = c++ +# cython: language_level=3 from libc.stdio cimport FILE cdef extern from "jpcommon.h": cdef cppclass JPCommonNode: - char *pron - char *pos - char *ctype - char *cform - int acc - int chain_flag - void *prev - void *next + char *pron + char *pos + char *ctype + char *cform + int acc + int chain_flag + void *prev + void *next cdef cppclass JPCommon: - JPCommonNode *head - JPCommonNode *tail - void *label + JPCommonNode *head + JPCommonNode *tail + void *label - void JPCommon_initialize(JPCommon * jpcommon) + void JPCommon_initialize(JPCommon * jpcommon) nogil void JPCommon_push(JPCommon * jpcommon, JPCommonNode * node) - void JPCommon_make_label(JPCommon * jpcommon) - int JPCommon_get_label_size(JPCommon * jpcommon) - char **JPCommon_get_label_feature(JPCommon * jpcommon) + void JPCommon_make_label(JPCommon * jpcommon) nogil + int JPCommon_get_label_size(JPCommon * jpcommon) nogil + char **JPCommon_get_label_feature(JPCommon * jpcommon) nogil void JPCommon_print(JPCommon * jpcommon) void JPCommon_fprint(JPCommon * jpcommon, FILE * fp) void JPCommon_refresh(JPCommon * jpcommon) - void JPCommon_clear(JPCommon * jpcommon) + void JPCommon_clear(JPCommon * jpcommon) nogil diff --git a/pyopenjtalk/openjtalk/mecab.pxd b/pyopenjtalk/openjtalk/mecab.pxd index 1538e05..746ac18 100644 --- a/pyopenjtalk/openjtalk/mecab.pxd +++ b/pyopenjtalk/openjtalk/mecab.pxd @@ -1,4 +1,5 @@ # distutils: language = c++ +# cython: language_level=3 cdef extern from "mecab.h": cdef cppclass Mecab: @@ -8,15 +9,15 @@ cdef extern from "mecab.h": void *tagger void *lattice - cdef int Mecab_initialize(Mecab *m) - cdef int Mecab_load(Mecab *m, const char *dicdir) - cdef int Mecab_analysis(Mecab *m, const char *str) + cdef int Mecab_initialize(Mecab *m) nogil + cdef int Mecab_load(Mecab *m, const char *dicdir) nogil + cdef int Mecab_analysis(Mecab *m, const char *str) nogil cdef int Mecab_print(Mecab *m) - int Mecab_get_size(Mecab *m) - char **Mecab_get_feature(Mecab *m) - cdef int Mecab_refresh(Mecab *m) - cdef int Mecab_clear(Mecab *m) - cdef int mecab_dict_index(int argc, char **argv) + int Mecab_get_size(Mecab *m) nogil + char **Mecab_get_feature(Mecab *m) nogil + cdef int Mecab_refresh(Mecab *m) nogil + cdef int Mecab_clear(Mecab *m) nogil + cdef int mecab_dict_index(int argc, char **argv) nogil cdef extern from "mecab.h" namespace "MeCab": cdef cppclass Tagger: @@ -24,6 +25,6 @@ cdef extern from "mecab.h" namespace "MeCab": cdef cppclass Lattice: pass cdef cppclass Model: - Tagger *createTagger() - Lattice *createLattice() - cdef Model *createModel(int argc, char **argv) + Tagger *createTagger() nogil + Lattice *createLattice() nogil + cdef Model *createModel(int argc, char **argv) nogil diff --git a/pyopenjtalk/openjtalk/mecab2njd.pxd b/pyopenjtalk/openjtalk/mecab2njd.pxd index be57ccc..33b6277 100644 --- a/pyopenjtalk/openjtalk/mecab2njd.pxd +++ b/pyopenjtalk/openjtalk/mecab2njd.pxd @@ -1,6 +1,7 @@ # distutils: language = c++ +# cython: language_level=3 from .njd cimport NJD cdef extern from "mecab2njd.h": - void mecab2njd(NJD * njd, char **feature, int size); + void mecab2njd(NJD * njd, char **feature, int size) nogil diff --git a/pyopenjtalk/openjtalk/njd.pxd b/pyopenjtalk/openjtalk/njd.pxd index 38d3887..5f7a3ba 100644 --- a/pyopenjtalk/openjtalk/njd.pxd +++ b/pyopenjtalk/openjtalk/njd.pxd @@ -1,4 +1,5 @@ # distutils: language = c++ +# cython: language_level=3 from libc.stdio cimport FILE @@ -66,7 +67,7 @@ cdef extern from "njd.h": NJDNode *head NJDNode *tail - void NJD_initialize(NJD * njd) + void NJD_initialize(NJD * njd) nogil void NJD_load(NJD * njd, const char *str) void NJD_load_from_fp(NJD * njd, FILE * fp) int NJD_get_size(NJD * njd) @@ -76,22 +77,22 @@ cdef extern from "njd.h": void NJD_fprint(NJD * njd, FILE * fp) void NJD_sprint(NJD * njd, char *buff, const char *split_code) void NJD_refresh(NJD * njd) - void NJD_clear(NJD * wl) + void NJD_clear(NJD * wl) nogil cdef extern from "njd_set_accent_phrase.h": - void njd_set_accent_phrase(NJD * njd) + void njd_set_accent_phrase(NJD * njd) nogil cdef extern from "njd_set_accent_type.h": - void njd_set_accent_type(NJD * njd) + void njd_set_accent_type(NJD * njd) nogil cdef extern from "njd_set_digit.h": - void njd_set_digit(NJD * njd) + void njd_set_digit(NJD * njd) nogil cdef extern from "njd_set_long_vowel.h": - void njd_set_long_vowel(NJD * njd) + void njd_set_long_vowel(NJD * njd) nogil cdef extern from "njd_set_pronunciation.h": - void njd_set_pronunciation(NJD * njd) + void njd_set_pronunciation(NJD * njd) nogil cdef extern from "njd_set_unvoiced_vowel.h": - void njd_set_unvoiced_vowel(NJD * njd) + void njd_set_unvoiced_vowel(NJD * njd) nogil diff --git a/pyopenjtalk/openjtalk/njd2jpcommon.pxd b/pyopenjtalk/openjtalk/njd2jpcommon.pxd index 8309288..7069815 100644 --- a/pyopenjtalk/openjtalk/njd2jpcommon.pxd +++ b/pyopenjtalk/openjtalk/njd2jpcommon.pxd @@ -1,7 +1,8 @@ # distutils: language = c++ +# cython: language_level=3 from .jpcommon cimport JPCommon from .njd cimport NJD cdef extern from "njd2jpcommon.h": - void njd2jpcommon(JPCommon * jpcommon, NJD * njd) + void njd2jpcommon(JPCommon * jpcommon, NJD * njd) nogil diff --git a/pyopenjtalk/openjtalk/text2mecab.pxd b/pyopenjtalk/openjtalk/text2mecab.pxd index 3d44553..64e7252 100644 --- a/pyopenjtalk/openjtalk/text2mecab.pxd +++ b/pyopenjtalk/openjtalk/text2mecab.pxd @@ -1,4 +1,5 @@ # distutils: language = c++ +# cython: language_level=3 cdef extern from "text2mecab.h": - int text2mecab(char *output, size_t sizeOfOutput, const char *input) + int text2mecab(char *output, size_t sizeOfOutput, const char *input) nogil diff --git a/pyproject.toml b/pyproject.toml index 9accebb..9de1588 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,13 +1,85 @@ [build-system] requires = [ - "setuptools=0.28.0,<3.0.0", - "cmake<4.0.0", + "setuptools>=64", + "setuptools_scm>=8", + "cython>=0.29.16", + "cmake", "numpy>=1.25.0; python_version>='3.9'", "oldest-supported-numpy; python_version<'3.9'", ] build-backend = "setuptools.build_meta" +[project] +name = "pyopenjtalk" +dynamic = ["version"] +description = "A python wrapper for OpenJTalk" +readme = "README.md" +requires-python = ">=3.8" +license.file = "LICENSE.md" +authors = [{ name = "Ryuichi Yamamoto", email = "zryuichi@gmail.com" }] +keywords = ["OpenJTalk", "Research"] +classifiers = [ + "Operating System :: POSIX", + "Operating System :: Unix", + "Operating System :: MacOS", + "Operating System :: Microsoft :: Windows", + "Programming Language :: Cython", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "License :: OSI Approved :: MIT License", + "Topic :: Scientific/Engineering", + "Topic :: Software Development", + "Intended Audience :: Science/Research", + "Intended Audience :: Developers", +] +urls.Homepage = "https://github.com/r9y9/pyopenjtalk" +dependencies = [ + "importlib_resources; python_version<'3.9'", + "numpy>=1.20.0", + "tqdm", +] + +[project.optional-dependencies] +docs = [ + "sphinx_rtd_theme", + "nbsphinx>=0.8.6", + "Jinja2>=3.0.1", + "pandoc", + "ipython", + "jupyter", +] +dev = [ + "cython>=0.29.16", + "pysen", + "types-setuptools", + "mypy<=0.910", + "black>=19.19b0,<=20.8", + "click<8.1.0", + "flake8>=3.7,<4", + "flake8-bugbear", + "isort>=4.3,<5.2.0", + "types-decorator", + "importlib-metadata<5.0", +] +test = ["pytest", "scipy"] +marine = ["marine>=0.0.5"] + +[tool.setuptools.packages.find] +include = ["pyopenjtalk*"] + +[tool.setuptools.exclude-package-data] +"*" = ["*.pyx", "*.pxd"] + +[tool.setuptools_scm] +version_file = "pyopenjtalk/version.py" +version_file_template = ''' +__version__ = "{version}" +''' +# local_scheme = "no-local-version" + [tool.pysen] version = "0.10.2" @@ -18,6 +90,6 @@ enable_isort = true enable_mypy = false # TODO: enable mypy in the future mypy_preset = "strict" line_length = 88 -py_version = "py37" +py_version = "py38" [[tool.pysen.lint.mypy_targets]] paths = ["."] diff --git a/release.sh b/release.sh deleted file mode 100755 index 3dabccc..0000000 --- a/release.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -# Script for Pypi release -# 0. Make sure you are on git tag -# 1. Run the script -# 2. Upload sdist - -set -e - -script_dir=$(cd $(dirname ${BASH_SOURCE:-$0}); pwd) -cd $script_dir - -TAG=$(git describe --exact-match --tags HEAD) - -VERSION=${TAG/v/} - -PYOPENJTALK_BUILD_VERSION=$VERSION python setup.py develop sdist -echo "*** Ready to release! pyopenjtalk $TAG ***" -echo "Please make sure that release verion is correct." -cat pyopenjtalk/version.py -echo "Please run the following command manually:" -echo twine upload dist/pyopenjtalk-${VERSION}.tar.gz --repository-url https://upload.pypi.org/legacy/ diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index af57bc0..0000000 --- a/setup.cfg +++ /dev/null @@ -1 +0,0 @@ -[bdist_wheel] diff --git a/setup.py b/setup.py index 9563649..0fb8fad 100644 --- a/setup.py +++ b/setup.py @@ -1,74 +1,55 @@ import os import subprocess import sys -from distutils.errors import DistutilsExecError -from distutils.spawn import spawn -from distutils.version import LooseVersion from glob import glob from itertools import chain from os.path import exists, join -from subprocess import run import numpy as np -import setuptools.command.build_py -import setuptools.command.develop -from setuptools import Extension, find_packages, setup +import setuptools.command.build_ext +from setuptools import Extension, setup platform_is_windows = sys.platform == "win32" -version = "0.3.2" - -min_cython_ver = "0.21.0" -try: - import Cython - - ver = Cython.__version__ - _CYTHON_INSTALLED = ver >= LooseVersion(min_cython_ver) -except ImportError: - _CYTHON_INSTALLED = False - - msvc_extra_compile_args_config = [ "/source-charset:utf-8", "/execution-charset:utf-8", ] -try: - if not _CYTHON_INSTALLED: - raise ImportError("No supported version of Cython installed.") - from Cython.Distutils import build_ext - cython = True -except ImportError: - cython = False +def msvc_extra_compile_args(compile_args): + cas = set(compile_args) + xs = filter(lambda x: x not in cas, msvc_extra_compile_args_config) + return list(chain(compile_args, xs)) + -if cython: - ext = ".pyx" +msvc_define_macros_config = [ + ("_CRT_NONSTDC_NO_WARNINGS", None), + ("_CRT_SECURE_NO_WARNINGS", None), +] - def msvc_extra_compile_args(compile_args): - cas = set(compile_args) - xs = filter(lambda x: x not in cas, msvc_extra_compile_args_config) - return list(chain(compile_args, xs)) - class custom_build_ext(build_ext): - def build_extensions(self): - compiler_type_is_msvc = self.compiler.compiler_type == "msvc" - for entry in self.extensions: - if compiler_type_is_msvc: - entry.extra_compile_args = msvc_extra_compile_args( - entry.extra_compile_args - if hasattr(entry, "extra_compile_args") - else [] - ) +def msvc_define_macros(macros): + mns = set([i[0] for i in macros]) + xs = filter(lambda x: x[0] not in mns, msvc_define_macros_config) + return list(chain(macros, xs)) - build_ext.build_extensions(self) - cmdclass = {"build_ext": custom_build_ext} -else: - ext = ".cpp" - cmdclass = {} - if not os.path.exists(join("pyopenjtalk", "openjtalk" + ext)): - raise RuntimeError("Cython is required to generate C++ code") +class custom_build_ext(setuptools.command.build_ext.build_ext): + def build_extensions(self): + compiler_type_is_msvc = self.compiler.compiler_type == "msvc" + for entry in self.extensions: + if compiler_type_is_msvc: + entry.extra_compile_args = msvc_extra_compile_args( + entry.extra_compile_args + if hasattr(entry, "extra_compile_args") + else [] + ) + entry.define_macros = msvc_define_macros( + entry.define_macros if hasattr(entry, "define_macros") else [] + ) + + setuptools.command.build_ext.build_ext.build_extensions(self) def check_cmake_in_path(): @@ -107,71 +88,6 @@ def check_cmake_in_path(): path." ) - -# Workaround for `distutils.spawn` problem on Windows python < 3.9 -# See details: [bpo-39763: distutils.spawn now uses subprocess (GH-18743)] -# (https://github.com/python/cpython/commit/1ec63b62035e73111e204a0e03b83503e1c58f2e) -def test_quoted_arg_change(): - child_script = """ -import os -import sys -if len(sys.argv) > 5: - try: - os.makedirs(sys.argv[1], exist_ok=True) - with open(sys.argv[2], mode=sys.argv[3], encoding=sys.argv[4]) as fd: - fd.write(sys.argv[5]) - except OSError: - pass -""" - - try: - # write - package_build_dir = "build" - file_name = join(package_build_dir, "quoted_arg_output") - output_mode = "w" - file_encoding = "utf8" - arg_value = '"ARG"' - - spawn( - [ - sys.executable, - "-c", - child_script, - package_build_dir, - file_name, - output_mode, - file_encoding, - arg_value, - ] - ) - - # read - with open(file_name, mode="r", encoding=file_encoding) as fd: - return fd.readline() != arg_value - except (DistutilsExecError, TypeError): - return False - - -def escape_string_macro_arg(s): - return s.replace("\\", "\\\\").replace('"', '\\"') - - -def escape_macro_element(x): - (k, arg) = x - return (k, escape_string_macro_arg(arg)) if type(arg) == str else x - - -def escape_macros(macros): - return list(map(escape_macro_element, macros)) - - -custom_define_macros = ( - escape_macros - if platform_is_windows and test_quoted_arg_change() - else (lambda macros: macros) -) - - # open_jtalk sources src_top = join("lib", "open_jtalk", "src") @@ -185,9 +101,7 @@ def escape_macros(macros): os.makedirs(build_dir, exist_ok=True) os.chdir(build_dir) - # NOTE: The wrapped OpenJTalk does not depend on HTS_Engine, - # but since HTSEngine is included in CMake's dependencies, it refers to a dummy path. - r = run(["cmake", "..", "-DHTS_ENGINE_INCLUDE_DIR=.", "-DHTS_ENGINE_LIB=dummy"]) + r = subprocess.run(["cmake", ".."]) r.check_returncode() os.chdir(cwd) @@ -215,21 +129,19 @@ def escape_macros(macros): ext_modules = [ Extension( name="pyopenjtalk.openjtalk", - sources=[join("pyopenjtalk", "openjtalk" + ext)] + all_src, - include_dirs=[np.get_include()] + include_dirs, + sources=[join("pyopenjtalk", "openjtalk.pyx")] + all_src, + include_dirs=include_dirs, extra_compile_args=[], extra_link_args=[], language="c++", - define_macros=custom_define_macros( - [ - ("HAVE_CONFIG_H", None), - ("DIC_VERSION", 102), - ("MECAB_DEFAULT_RC", '"dummy"'), - ("PACKAGE", '"open_jtalk"'), - ("VERSION", '"1.10"'), - ("CHARSET_UTF_8", None), - ] - ), + define_macros=[ + ("HAVE_CONFIG_H", None), + ("DIC_VERSION", "102"), + ("MECAB_DEFAULT_RC", '"dummy"'), + ("PACKAGE", '"open_jtalk"'), + ("VERSION", '"1.10"'), + ("CHARSET_UTF_8", None), + ], ) ] @@ -239,127 +151,17 @@ def escape_macros(macros): ext_modules += [ Extension( name="pyopenjtalk.htsengine", - sources=[join("pyopenjtalk", "htsengine" + ext)] + all_htsengine_src, + sources=[join("pyopenjtalk", "htsengine.pyx")] + all_htsengine_src, include_dirs=[np.get_include(), join(htsengine_src_top, "include")], extra_compile_args=[], extra_link_args=[], libraries=["winmm"] if platform_is_windows else [], language="c++", - define_macros=custom_define_macros( - [ - ("AUDIO_PLAY_NONE", None), - ] - ), + define_macros=[ + ("AUDIO_PLAY_NONE", None), + ("NPY_NO_DEPRECATED_API", "NPY_1_7_API_VERSION"), + ], ) ] -# Adapted from https://github.com/pytorch/pytorch -cwd = os.path.dirname(os.path.abspath(__file__)) -if os.getenv("PYOPENJTALK_BUILD_VERSION"): - version = os.getenv("PYOPENJTALK_BUILD_VERSION") -else: - try: - sha = ( - subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=cwd) - .decode("ascii") - .strip() - ) - version += "+" + sha[:7] - except subprocess.CalledProcessError: - pass - except IOError: # FileNotFoundError for python 3 - pass - - -class build_py(setuptools.command.build_py.build_py): - def run(self): - self.create_version_file() - setuptools.command.build_py.build_py.run(self) - - @staticmethod - def create_version_file(): - global version, cwd - print("-- Building version " + version) - version_path = os.path.join(cwd, "pyopenjtalk", "version.py") - with open(version_path, "w") as f: - f.write("__version__ = '{}'\n".format(version)) - - -class develop(setuptools.command.develop.develop): - def run(self): - build_py.create_version_file() - setuptools.command.develop.develop.run(self) - - -cmdclass["build_py"] = build_py -cmdclass["develop"] = develop - - -with open("README.md", "r", encoding="utf8") as fd: - long_description = fd.read() - -setup( - name="pyopenjtalk", - version=version, - description="A python wrapper for OpenJTalk", - long_description=long_description, - long_description_content_type="text/markdown", - author="Ryuichi Yamamoto", - author_email="zryuichi@gmail.com", - url="https://github.com/r9y9/pyopenjtalk", - license="MIT", - packages=find_packages(), - package_data={"": ["htsvoice/*"]}, - ext_modules=ext_modules, - cmdclass=cmdclass, - install_requires=[ - "numpy >= 1.20.0", - "six", - "tqdm", - ], - tests_require=["nose", "coverage"], - extras_require={ - "docs": [ - "sphinx_rtd_theme", - "nbsphinx>=0.8.6", - "Jinja2>=3.0.1", - "pandoc", - "ipython", - "jupyter", - ], - "dev": [ - "cython >= " + min_cython_ver + ",<3.0.0", - "pysen", - "types-setuptools", - "mypy<=0.910", - "black>=19.19b0,<=20.8", - "click<8.1.0", - "flake8>=3.7,<4", - "flake8-bugbear", - "isort>=4.3,<5.2.0", - "types-decorator", - "importlib-metadata<5.0", - ], - "test": ["pytest", "scipy"], - "marine": ["marine>=0.0.5"], - }, - classifiers=[ - "Operating System :: POSIX", - "Operating System :: Unix", - "Operating System :: MacOS", - "Operating System :: Microsoft :: Windows", - "Programming Language :: Cython", - "Programming Language :: Python", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "License :: OSI Approved :: MIT License", - "Topic :: Scientific/Engineering", - "Topic :: Software Development", - "Intended Audience :: Science/Research", - "Intended Audience :: Developers", - ], - keywords=["OpenJTalk", "Research"], -) +setup(ext_modules=ext_modules, cmdclass={"build_ext": custom_build_ext}) diff --git a/tests/test_data/.gitignore b/tests/test_data/.gitignore new file mode 100644 index 0000000..d6b7ef3 --- /dev/null +++ b/tests/test_data/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/tests/test_openjtalk.py b/tests/test_openjtalk.py index 0f43363..5a27466 100644 --- a/tests/test_openjtalk.py +++ b/tests/test_openjtalk.py @@ -1,3 +1,6 @@ +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + import pyopenjtalk @@ -66,7 +69,10 @@ def test_g2p_kana(): for text, pron in [ ("今日もこんにちは", "キョーモコンニチワ"), ("いやあん", "イヤーン"), - ("パソコンのとりあえず知っておきたい使い方", "パソコンノトリアエズシッテオキタイツカイカタ"), + ( + "パソコンのとりあえず知っておきたい使い方", + "パソコンノトリアエズシッテオキタイツカイカタ", + ), ]: p = pyopenjtalk.g2p(text, kana=True) assert p == pron @@ -80,3 +86,53 @@ def test_g2p_phone(): ]: p = pyopenjtalk.g2p(text, kana=False) assert p == pron + + +def test_userdic(): + for text, expected in [ + ("nnmn", "n a n a m i N"), + ("GNU", "g u n u u"), + ]: + p = pyopenjtalk.g2p(text) + assert p != expected + + user_csv = str(Path(__file__).parent / "test_data" / "user.csv") + user_dic = str(Path(__file__).parent / "test_data" / "user.dic") + + with open(user_csv, "w", encoding="utf-8") as f: + f.write("ｎｎｍｎ,,,1,名詞,一般,*,*,*,*,ｎｎｍｎ,ナナミン,ナナミン,1/4,*\n") + f.write("ＧＮＵ,,,1,名詞,一般,*,*,*,*,ＧＮＵ,グヌー,グヌー,2/3,*\n") + + pyopenjtalk.mecab_dict_index(f.name, user_dic) + pyopenjtalk.update_global_jtalk_with_user_dict(user_dic) + + for text, expected in [ + ("nnmn", "n a n a m i N"), + ("GNU", "g u n u u"), + ]: + p = pyopenjtalk.g2p(text) + assert p == expected + + +def test_multithreading(): + ojt = pyopenjtalk.openjtalk.OpenJTalk(pyopenjtalk.OPEN_JTALK_DICT_DIR) + texts = [ + "今日もいい天気ですね", + "こんにちは", + "マルチスレッドプログラミング", + "テストです", + "Pythonはプログラミング言語です", + "日本語テキストを音声合成します", + ] * 4 + + # Test consistency between single and multi-threaded runs + # make sure no corruptions happen in OJT internal + results_s = [ojt.run_frontend(text) for text in texts] + results_m = [] + with ThreadPoolExecutor() as e: + results_m = [i for i in e.map(ojt.run_frontend, texts)] + for s, m in zip(results_s, results_m): + assert len(s) == len(m) + for s_, m_ in zip(s, m): + # full context must exactly match + assert s_ == m_