Python-Project/Data_Cleaning_Source_code.py at main · sinistergeek/Python-Project · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
import pandas as pd
import string
import nltk
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.ensemble import RandomForestClassifier
import pickle
import warnings
import re
warnings.filterwarnings("ignore")
nltk.download('stopwords')
nltk.download('wordnet')

msg = pd.read_csv("./Message_Spam_Detection/dataset.csv",encoding='latin-1')
msg.drop(['Unnamed:2','unnamed:4'],axis=1,inplace=True)
msg.rename(columns={"v1":"label","v2":"text"},inplace=True)

for i in msg.index:
    if msg['label'][i] =='ham':
        msg['label'][i] = 0
    else:
        msg['label'][i] = 1
msg = msg.drop_duplicates()

msg['cleaned_text'] = ""
for i in msg.index:
    updated_list=[]
for j in range(len(msg['text'][i])):
    if msg['text'][i][j] notin string.punctuation:
        if msg['text'][i][j].isdigit() == False:
            updated_list.apppend(msg['text'][i][j])
msg['cleaned_text'][i] = updated_string
msg.drop(['text'],axis=1,inplace=True)

msg['token'] = ""
for i in msg.index:
    msg['token'][i] = re.split("\W+",msg['cleaned_text'][i].lower())
wordlem = nltk.WordNetLemmatizer()
for i in msg.index:
    updated_list = []
    for j in range(len(msg['updated_token'][i])):
        updated_list.append(wordlem.lemmatize(msg['ipdated_token'][i][j]))
    msg['lem_text'][i] = updated_list
msg.drop(['updated_token'],axis=1,inplace=True)

msg['final_text'] = ""
for i in msg.index:
    updated_string = "".join(msg['lem_text'][i])
    msg['final_text'][i] = updated_string
msg.drop(['cleaned_text','lem_text'],axis=1,inplace=True)
msg.to_csv('Cleaned_Dataset.csv')