tweets/text_processing.py at main · dbadeev/tweets · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
import text_cleaninig
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer
import pandas as pd
from nltk.stem import WordNetLemmatizer


def word_exists(df, column):
    """
    This function takes a DataFrame 'df' and a column name 'column' from the DataFrame.
    It uses CountVectorizer to create a document-term matrix (DTM) from the specified column.
    The DTM is then transformed into a DataFrame with the same index as the original DataFrame.
    For each column in the DTM, the function applies a lambda function to replace 0 values with 1.
    This function does not return a DataFrame with an additional 'type' column.

    Parameters:
    df (pandas.DataFrame): The input DataFrame containing the text data.
    column (str): The name of the column in the DataFrame from which the text data will be extracted.

    Returns:
    pandas.DataFrame: A DataFrame containing the document-term matrix with 0 values replaced by 1.
    """
    cv = CountVectorizer()
    cv_matrix = cv.fit_transform(df[column])
    # create document term matrix
    df_dtm = pd.DataFrame(cv_matrix.toarray(), index=df['tweets'],
                                                  columns=cv.get_feature_names_out())
    columns = df_dtm.columns
    for col in columns:
        df_dtm[col] = df_dtm[col].apply(lambda item: item if item == 0 else 1)
#         df_dtm[col] = pd.to_numeric(df_dtm[col],downcast='integer')
#     df = df.set_index('tweets')
#     df_dtm = pd.concat([df_dtm, df['type']], axis=1)

    return df_dtm


def word_count(df, column):
    """
    This function takes a DataFrame 'df' and a column name 'column' from the DataFrame.
    It uses CountVectorizer to create a document-term matrix (DTM) from the specified column.
    The DTM is then transformed into a DataFrame with the same index as the original DataFrame.
    For each column in the DTM, the function applies a lambda function to replace 0 values with 1.
    This function does not return a DataFrame with an additional 'type' column.

    Parameters:
    df (pandas.DataFrame): The input DataFrame containing the text data.
    column (str): The name of the column in the DataFrame from which the text data will be extracted.

    Returns:
    pandas.DataFrame: A DataFrame containing the document-term matrix with 0 values replaced by 1.
    """
    cv = CountVectorizer()
    cv_matrix = cv.fit_transform(df[column])
    # create document term matrix
    df_dtm = pd.DataFrame(cv_matrix.toarray(), index=df['tweets'],
                                                  columns=cv.get_feature_names_out())
    columns = df_dtm.columns
    for col in columns:
        df_dtm[col] = df_dtm[col].apply(lambda item: item if item == 0 else 1)
#     df_dtm[col] = pd.to_numeric(df_dtm[col],downcast='integer')
#     df = df.set_index('tweets')
#     df_dtm = pd.concat([df_dtm, df['type']], axis=1)

    return df_dtm


def tfidf(df, column):
    """
    This function takes a DataFrame 'df' and a column name 'column' from the DataFrame.
    It uses TfidfVectorizer to create a document-term matrix (DTM) from the specified column.
    The DTM is then transformed into a DataFrame with the same index as the original DataFrame.
    The function does not return a DataFrame with an additional 'type' column.

    Parameters:
    df (pandas.DataFrame): The input DataFrame containing the text data.
    column (str): The name of the column in the DataFrame from which the text data will be extracted.

    Returns:
    pandas.DataFrame: A DataFrame containing the document-term matrix.
    """
    vectorizer = TfidfVectorizer()
    X = vectorizer.fit_transform(df[column])
    df_dtm = pd.DataFrame(data=X.todense(), index=df['tweets'],
                                                  columns=vectorizer.get_feature_names_out())
#     columns = df_dtm.columns
#     for col in columns:
#         df_dtm[col] = pd.to_numeric(df_dtm[col],downcast='float')
#     df = df.set_index('tweets')

#     df_dtm = pd.concat([df_dtm, df['type']], axis=1)
    return df_dtm


def stem_text(text, stemmer, misspelling=False):
    """
    This function takes a text string and a stemmer object as input.
    It cleans the input text using the `text_cleaninig.clean` function,
    splits the cleaned text into words, and then applies the stemming operation
    to each word using the provided stemmer. The stemmed words are then joined back into a single string.

    Parameters:
    text (str): The input text string to be stemmed.
    stemmer (object): A stemmer object, such as `nltk.stem.SnowballStemmer` or `nltk.stem.WordNetLemmatizer`.
    misspelling (bool, optional): A flag indicating whether to apply misspelling corrections to the input text.
        Defaults to `False`.

    Returns:
    str: A string containing the stemmed version of the input text.
    """
    text = text_cleaninig.clean(text, misspelling=misspelling)
    text = text.split()
    stemmed_words = [stemmer.stem(word) for word in text]
    text = " ".join(stemmed_words)

    return text


def lem_text(text, misspelling=False):
    """
    This function takes a text string and a boolean flag indicating whether to apply misspelling corrections.
    It cleans the input text using the `text_cleaninig.clean` function,
    splits the cleaned text into words, and then applies lemmatization to each word using the WordNetLemmatizer.
    The lemmatized words are then joined back into a single string.

    Parameters:
    text (str): The input text string to be lemmatized.
    misspelling (bool, optional): A flag indicating whether to apply misspelling corrections to the input text.
        Defaults to `False`.

    Returns:
    str: A string containing the lemmatized version of the input text.
    """
    wordnet_lemmatizer = WordNetLemmatizer()
    text = text_cleaninig.clean(text, misspelling=misspelling)
    text = text.split()
    lemmed_words = [wordnet_lemmatizer.lemmatize(word) for word in text]
    text = " ".join(lemmed_words)

    return text