-
Notifications
You must be signed in to change notification settings - Fork 0
/
covid_twitter_EDA.py
168 lines (75 loc) · 3.37 KB
/
covid_twitter_EDA.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
import pandas as pd
import numpy as np
import spacy
from spacy import displacy
from spacy.util import minibatch, compounding
from sklearn import preprocessing
import explacy
import urllib.request
import scattertext as st
import matplotlib.pyplot as plt
df_train = pd.read_csv('./Corona_NLP_train.csv')
spacy.load('en_core_web_md')
df_train.columns
df_train.head()
print(df_train.shape)
#EDA- Nan values
var_null_train = [var for var in df_train.columns if df_train[var].isnull().sum() > 0]
print(var_null_train)
df_train[var_null_train].isnull().mean()
ax = df_train.Sentiment.value_counts().plot(kind='bar')
fig = ax.get_figure()
train_df = df_train[['OriginalTweet', 'Sentiment']]
train_df.shape
train_df.isnull().sum()
label_encoder = preprocessing.LabelEncoder()
train_df['Sentiment']= label_encoder.fit_transform(train_df['Sentiment'])
train_df['Sentiment'].unique()
train_df.head()
# ## tokenization of tweets using spacy's 'en_core_web_md' model
spacy_tok = spacy.load('en_core_web_md') #English Language model for tokenization!
sample_review = train_df.OriginalTweet[55]
sample_review
spacy_parsed_review = spacy_tok(sample_review)
spacy_parsed_review
# ## Using explacy to see the tokenization and POS tag
#!wget https://raw.githubusercontent.com/tylerneylon/explacy/master/explacy.py
url = 'https://raw.githubusercontent.com/tylerneylon/explacy/master/explacy.py'
filename = 'explacy.py'
urllib.request.urlretrieve(url, filename)
explacy.print_parse_info(spacy_tok, 'Covid-19 has taken many lives all over India and other nations')
explacy.print_parse_info(spacy_tok,train_df.OriginalTweet[1])
# ## Visualizing Lemma, POS , deposition , shape of an example
tokenized_text = pd.DataFrame()
for i, token in enumerate(spacy_parsed_review):
tokenized_text.loc[i, 'text'] = token.text
tokenized_text.loc[i, 'lemma'] = token.lemma_,
tokenized_text.loc[i, 'pos'] = token.pos_
tokenized_text.loc[i, 'tag'] = token.tag_
tokenized_text.loc[i, 'dep'] = token.dep_
tokenized_text.loc[i, 'shape'] = token.shape_
tokenized_text.loc[i, 'is_alpha'] = token.is_alpha
tokenized_text.loc[i, 'is_stop'] = token.is_stop
tokenized_text.loc[i, 'is_punctuation'] = token.is_punct
tokenized_text[:20]
#spacy.displacy.render(spacy_parsed_review, style='ent', jupyter=True)
spacy.explain('npadvmod') # to explain POS tag
sentence_spans = list(spacy_parsed_review.sents)
sentence_spans
displacy.render(spacy_parsed_review, style='dep', jupyter=True,options={'distance': 140})
options = {'compact': True, 'bg': 'white','distance': 140,
'color': 'blue', 'font': 'Trebuchet MS'}
displacy.render(spacy_parsed_review, jupyter=True, style='dep', options=options)
noun_chunks_df = pd.DataFrame()
for i, chunk in enumerate(spacy_parsed_review.noun_chunks):
noun_chunks_df.loc[i, 'text'] = chunk.text
noun_chunks_df.loc[i, 'root'] = chunk.root,
noun_chunks_df.loc[i, 'root.text'] = chunk.root.text,
noun_chunks_df.loc[i, 'root.dep_'] = chunk.root.dep_
noun_chunks_df.loc[i, 'root.head.text'] = chunk.root.head.text
noun_chunks_df[:20]
nlp = spacy.load('en_core_web_md',disable=["tagger","ner"])
train_df['spacy_parsed'] = train_df.OriginalTweet.apply(nlp)
corpus = st.CorpusFromParsedDocuments(train_df,
category_col='Sentiment',
parsed_col='parsed').build()