deep-learning/text_preprocessing.py at main · Will1618sd/deep-learning · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
# import re  # not in image
import string # not in image

import os
os.environ["TRANSFORMERS_NO_FLASH_ATTN"] = "1"

from transformers import BertModel, BertConfig, BertTokenizer

def preprocess_text(text):
    text = text.strip("\n")

    # Tokenize
    tokens = word_tokenize(text.lower())

    # Remove stopwords
    stop_words = set(stopwords.words('english'))
    tokens_no_stopwords = [word for word in tokens if (word not in stop_words) and (len(word) >= 3)]
    text = tokens_no_stopwords

    # Substitution rules ?
    # text = re.sub(r"e-mail", "email", text)

    # Lemmatize (idk if useful)
    lemmatizer = WordNetLemmatizer()
    tokens_lemmatized = []
    for word in text:
        lem = lemmatizer.lemmatize(word)
        tokens_lemmatized.append(lem)
    text = tokens_lemmatized

    # Stemming (idk if useful)
    stemmer = PorterStemmer()
    tokens_stemmed = []
    for word in text:
        stem = stemmer.stem(word)
        tokens_stemmed.append(stem)
    text = tokens_stemmed

    # configuration = BertConfig()
    # model = BertModel(configuration)

    # txt = BertTokenizer(text)

    print("Preprocessing okay")

    return text