diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8b8dd31..e215249 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -21,16 +21,16 @@ jobs: fail-fast: true matrix: os: [ubuntu-latest] - python_version: ["3.7", "3.11"] + python_version: ["3.8", "3.11"] runs-on: ${{ matrix.os }} steps: - name: Check out repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Configure Python version - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python_version }} architecture: x64 diff --git a/requirements.txt b/requirements.txt index 335ff57..55e453b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ spacy>=3.0.0,<4.0.0 -stanza>=1.2.0,<1.7.0 +stanza>=1.2.0, <2.0.0 # Development dependencies pytest>=5.2.0 diff --git a/setup.py b/setup.py index 67441d5..d9db2fd 100644 --- a/setup.py +++ b/setup.py @@ -32,8 +32,8 @@ def setup_package(): version=about["__version__"], license=about["__license__"], packages=find_packages(), - install_requires=["spacy>=3.0.0,<4.0.0", "stanza>=1.2.0,<1.7.0"], - python_requires=">=3.6", + install_requires=["spacy>=3.0.0,<4.0.0", "stanza>=1.2.0, <2.0.0"], + python_requires=">=3.8", entry_points={ "spacy_tokenizers": [ "spacy_stanza.PipelineAsTokenizer.v1 = spacy_stanza:tokenizer.create_tokenizer", @@ -43,8 +43,6 @@ def setup_package(): "Development Status :: 4 - Beta", "Intended Audience :: Developers", "Topic :: Scientific/Engineering :: Artificial Intelligence", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", diff --git a/spacy_stanza/__init__.py b/spacy_stanza/__init__.py index 6eebb3e..22301f2 100644 --- a/spacy_stanza/__init__.py +++ b/spacy_stanza/__init__.py @@ -10,7 +10,7 @@ def load_pipeline( lang: str = "", dir: Optional[str] = None, package: str = "default", - processors: Union[dict, str] = {}, + processors: Union[dict, str] = None, logging_level: Optional[Union[int, str]] = None, verbose: Optional[bool] = None, use_gpu: bool = True, @@ -32,6 +32,8 @@ def load_pipeline( **kwargs: Options for the individual stanza processors. RETURNS (Language): The nlp object. """ + if processors is None: + processors = {} # Create an empty config skeleton config = {"nlp": {"tokenizer": {"kwargs": {}}}} if lang == "": diff --git a/spacy_stanza/tokenizer.py b/spacy_stanza/tokenizer.py index 6026308..54d9fe2 100644 --- a/spacy_stanza/tokenizer.py +++ b/spacy_stanza/tokenizer.py @@ -15,12 +15,17 @@ def create_tokenizer( lang: str = "", dir: Optional[str] = None, package: str = "default", - processors: Union[dict, str] = {}, + processors: Union[dict, str] = None, logging_level: Optional[Union[int, str]] = None, verbose: Optional[bool] = None, use_gpu: bool = True, - kwargs: dict = {}, + kwargs: dict = None, ): + if processors is None: + processors = {} + if kwargs is None: + kwargs = {} + def tokenizer_factory( nlp, lang=lang, @@ -82,16 +87,14 @@ def __call__(self, text): snlp_doc = self.snlp(text) text = snlp_doc.text - snlp_tokens, snlp_heads = self.get_tokens_with_heads(snlp_doc) - words = [] - spaces = [] + snlp_tokens, snlp_heads, snlp_sent_starts = self.get_tokens_with_heads(snlp_doc) pos = [] tags = [] morphs = [] deps = [] heads = [] lemmas = [] - offset = 0 + sent_starts = [] token_texts = [t.text for t in snlp_tokens] is_aligned = True try: @@ -117,6 +120,7 @@ def __call__(self, text): morphs.append("") deps.append("") lemmas.append(word) + sent_starts.append(False) # increment any heads left of this position that point beyond # this position to the right (already present in heads) @@ -141,7 +145,7 @@ def __call__(self, text): else: token = snlp_tokens[i + offset] assert word == token.text - + sent_starts.append(snlp_sent_starts[i + offset]) pos.append(token.upos or "") tags.append(token.xpos or token.upos or "") morphs.append(token.feats or "") @@ -158,6 +162,7 @@ def __call__(self, text): morphs=morphs, lemmas=lemmas, deps=deps, + sent_starts=sent_starts, heads=[head + i for i, head in enumerate(heads)], ) ents = [] @@ -199,8 +204,11 @@ def get_tokens_with_heads(self, snlp_doc): """ tokens = [] heads = [] + sent_starts = [] offset = 0 + token_idx = 0 for sentence in snlp_doc.sentences: + first = True for token in sentence.tokens: for word in token.words: # Here, we're calculating the absolute token index in the doc, @@ -212,8 +220,31 @@ def get_tokens_with_heads(self, snlp_doc): head = 0 heads.append(head) tokens.append(word) + if first: + sent_starts.append(True) + first = False + else: + sent_starts.append(False) + token_idx += 1 offset += sum(len(token.words) for token in sentence.tokens) - return tokens, heads + return tokens, heads, sent_starts + + @staticmethod + def get_sentences(snlp_doc): + """Extract the sentences from the Stanza Doc. + + snlp_doc (stanza.Document): The processed Stanza doc. + RETURNS (list): The sentences. + """ + sentences = [] + offset = 0 + for sentence in snlp_doc.sentences: + words = [] + for token in sentence.tokens: + words.extend([word.text for word in token.words]) + sentences.append("".join(words)) + offset += len(words) + return sentences def get_words_and_spaces(self, words, text): if "".join("".join(words).split()) != "".join(text.split()): @@ -242,7 +273,7 @@ def get_words_and_spaces(self, words, text): if text_pos < len(text): text_words.append(text[text_pos:]) text_spaces.append(False) - return (text_words, text_spaces) + return text_words, text_spaces def token_vector(self, token): """Get Stanza's pretrained word embedding for given token.