28. Natural Language Processing with Python

corpus = ["To be, or not to be, that is the question:",
          "Whether 'tis nobler in the mind to suffer",
          "The slings and arrows of outrageous fortune,"]

from sklearn.feature_extraction import text

vectorizer = text.CountVectorizer()
print(vectorizer)

vectorizer.fit(corpus)

print("Vocabulary: ", vectorizer.vocabulary_)

print(vectorizer.get_feature_names_out())

print(list(vectorizer.vocabulary_.keys()))

token_count_matrix = vectorizer.transform(corpus)
print(token_count_matrix)

dense_tcm = token_count_matrix.toarray()
dense_tcm

feature_names = vectorizer.get_feature_names_out()
for el in vectorizer.vocabulary_:
    print(el, end=(", "))

import pandas as pd

pd.DataFrame(data=dense_tcm, 
             index=['corpus_0', 'corpus_1', 'corpus_2'],
             columns=vectorizer.get_feature_names_out())

word = "be"
i = 1
j = vectorizer.vocabulary_[word]
print("number of times '" + word + "' occurs in:")
for i in range(len(corpus)):
     print("    '" + corpus[i] + "': " + str(dense_tcm[i][j]))

txt = "That is the question and it is nobler in the mind."
vectorizer.transform([txt]).toarray()

print(vectorizer.get_feature_names_out())

print(vectorizer.vocabulary_)

from sklearn.feature_extraction import text
corpus = ["It does not matter what you are doing, just do it!",
          "Would you work if you won the lottery?",
          "You like Python, he likes Python, we like Python, everybody loves Python!"
          "You said: 'I wish I were a Python programmer'",
          "You can stay here, if you want to. I would, if I were you."
         ]

vectorizer = text.CountVectorizer()
vectorizer.fit(corpus)

token_count_matrix = vectorizer.transform(corpus)
print(token_count_matrix)

tf_idf = text.TfidfTransformer()
tf_idf.fit(token_count_matrix)

tf_idf.idf_

tf_idf.idf_[vectorizer.vocabulary_['python']]

da = vectorizer.transform(corpus).toarray()
i = 0

# check how often the word 'would' occurs in the the i'th sentence:
#vectorizer.vocabulary_['would']
word_ind = vectorizer.vocabulary_['would']
da[i][word_ind]
da[:,word_ind]

word_weight_list = list(zip(vectorizer.get_feature_names_out(), tf_idf.idf_))

word_weight_list.sort(key=lambda x:x[1])  # sort list by the weights (2nd component)
for word, idf_weight in word_weight_list:
    print(f"{word:15s}: {idf_weight:4.3f}")

from numpy import log
from sklearn.feature_extraction import text

corpus = ["It does not matter what you are doing, just do it!",
          "Would you work if you won the lottery?",
          "You like Python, he likes Python, we like Python, everybody loves Python!"
          "You said: 'I wish I were a Python programmer'",
          "You can stay here, if you want to. I would, if I were you."
         ]

n = len(corpus)


# the following variables are used globally (as free variables) in the functions :-(
vectorizer = text.CountVectorizer()
vectorizer.fit(corpus)
da = vectorizer.transform(corpus).toarray() 

def tf(t, d, mode="raw"):
    """ The Term Frequency 'tf' calculates how often a term 't' 
        occurs in a document 'd'.  ('d': document index)
        If t_in_d =  Number of times a term t appears in a document d
        and no_terms_d = Total number of terms in the document, 
        tf(t, d) = t_in_d / no_terms_d
        
    """

    if t in vectorizer.vocabulary_:
        word_ind = vectorizer.vocabulary_[t]
        t_occurences = da[d, word_ind]    # 'd' is the document index
    else:
        t_occurences = 0
    if mode == "raw":
        result =  t_occurences
    elif mode == "length":
        all_terms = (da[d] > 0).sum()  # calculate number of different terms in d
        result = t_occurences / all_terms
    elif mode == "log":
        result = log(1 + t_occurences)
    elif mode == "augfreq":
        result = 0.5 + 0.5 * t_occurences / da[d].max()
    
    return result

print("   raw    length log   augmented freq")
for term in ['matter', 'python', 'would']:
    for docu_index in range(len(corpus)):
        d = corpus[docu_index]
        print(f"\n'{term}' in '{d}''")
        for mode in ['raw', 'length', 'log', 'augfreq']:
            x = tf(term, docu_index, mode=mode)
            print(f"{x:7.2f}", end="")

def df(t):
    """ df(t) is the document frequency of t; the document frequency is 
        the number of documents  in the document set that contain the term t. """
    
    word_ind = vectorizer.vocabulary_[t]

    tf_in_docus = da[:, word_ind] # vector with the freqencies of word_ind in all docus
    existence_in_docus = tf_in_docus > 0 # binary vector, existence of word in docus
    return existence_in_docus.sum()
    
#df("would", vectorizer) 

def idf(t, smooth_idf=True):
    """ idf """
    if smooth_idf:
        return log((1 + n) / (1 + df(t)) ) + 1
    else:
        return log(n / df(t) ) + 1


    
    
def tf_idf(t, d):
    return idf(t) * tf(t, d)
    

res_idf = []
for word  in vectorizer.get_feature_names_out():
    tf_docus = []
    res_idf.append([word, idf(word)])

res_idf.sort(key=lambda x:x[1])
for item  in res_idf:
    print(item)

corpus

for word, word_index in vectorizer.vocabulary_.items():
    print(f"\n{word:12s}: ", end="")
    for d_index in range(len(corpus)):
        print(f"{d_index:1d} {tf_idf(word, d_index):3.2f}, ",  end="" )

from sklearn.feature_extraction import text

words = "Cold wind blows over the cornfields".split()
corpus = []
for i in range(1, len(words)+1):
    corpus.append(" ".join(words[:i]))
    
print(corpus)

vectorizer = text.CountVectorizer()

vectorizer = vectorizer.fit(corpus)
vectorized_text = vectorizer.transform(corpus)

tf_idf = text.TfidfTransformer()
tf_idf.fit(vectorized_text)

tf_idf.idf_

word_weight_list = list(zip(vectorizer.get_feature_names_out(), tf_idf.idf_))
word_weight_list.sort(key=lambda x:x[1])  # sort list by the weights (2nd component)
for word, idf_weight in word_weight_list:
    print(f"{word:15s}: {idf_weight:4.3f}")

TfidF = text.TfidfTransformer(smooth_idf=True, use_idf=True)
tfidf = TfidF.fit_transform(vectorized_text)


word_weight_list = list(zip(vectorizer.get_feature_names_out(), tf_idf.idf_))
word_weight_list.sort(key=lambda x:x[1])  # sort list by the weights (2nd component)
for word, idf_weight in word_weight_list:
    print(f"{word:15s}: {idf_weight:4.3f}")

from sklearn.datasets import fetch_20newsgroups
from sklearn.feature_extraction.text import CountVectorizer

import numpy as np

# Create our vectorizer
vectorizer = CountVectorizer()

# Let's fetch all the possible text data
newsgroups_data = fetch_20newsgroups()

print(newsgroups_data.data[0])

print(newsgroups_data.data[200])

vectorizer.fit(newsgroups_data.data)

counter = 0
n = 10
for word, index in vectorizer.vocabulary_.items():
    print(word, index)
    counter += 1
    if counter > n:
        break

a = vectorizer.transform([newsgroups_data.data[0]]).toarray()[0]
print(a)

len(vectorizer.vocabulary_)

newsgroups_data_cleaned = fetch_20newsgroups(remove=('headers', 'footers', 'quotes'))

print(newsgroups_data_cleaned.data[0])

print(newsgroups_data.data[0])

vectorizer_cleaned = vectorizer.fit(newsgroups_data_cleaned.data)
len(vectorizer_cleaned.vocabulary_)

newsgroups_train = fetch_20newsgroups(subset='train',
                                      remove=('headers', 'footers', 'quotes'))
newsgroups_test = fetch_20newsgroups(subset='test',
                                     remove=('headers', 'footers', 'quotes'))

from sklearn.datasets import fetch_20newsgroups
from sklearn.feature_extraction.text import CountVectorizer

from sklearn.naive_bayes import MultinomialNB
from sklearn import metrics

vectorizer = CountVectorizer()

train_data = vectorizer.fit_transform(newsgroups_train.data)

# creating a classifier
classifier = MultinomialNB(alpha=.01)
classifier.fit(train_data, newsgroups_train.target)

test_data = vectorizer.transform(newsgroups_test.data)

predictions = classifier.predict(test_data)
accuracy_score = metrics.accuracy_score(newsgroups_test.target, 
                                        predictions)
f1_score = metrics.f1_score(newsgroups_test.target, 
                            predictions, 
                            average='macro')

print("Accuracy score: ", accuracy_score)
print("F1 score: ", f1_score)

from sklearn.feature_extraction.text import CountVectorizer

corpus = ["A horse, a horse, my kingdom for a horse!",
          "Horse sense is the thing a horse has which keeps it from betting on people."
          "I’ve often said there is nothing better for the inside of the man, than the outside of the horse.",
          "A man on a horse is spiritually, as well as physically, bigger then a man on foot.",
          "No heaven can heaven be, if my horse isn’t there to welcome me."]

cv = CountVectorizer(stop_words=["my", "for","the", "has", "than", "if", 
                                 "from", "on", "of", "it", "there", "ve",
                                 "as", "no", "be", "which", "isn", "to", 
                                 "me", "is", "can", "then"])
count_vector = cv.fit_transform(corpus)
count_vector.shape

cv.vocabulary_

from sklearn.feature_extraction import text
n = 25
print(str(n) + " arbitrary words from ENGLISH_STOP_WORDS:")
counter = 0
for word in text.ENGLISH_STOP_WORDS:
    if counter == n - 1:
        print(word)
        break
    print(word, end=", ")
    counter += 1

from sklearn.datasets import fetch_20newsgroups
from sklearn.feature_extraction.text import CountVectorizer

from sklearn.naive_bayes import MultinomialNB
from sklearn import metrics

vectorizer = CountVectorizer(stop_words=list(text.ENGLISH_STOP_WORDS))

vectors = vectorizer.fit_transform(newsgroups_train.data)

# creating a classifier
classifier = MultinomialNB(alpha=.01)
classifier.fit(vectors, newsgroups_train.target)

vectors_test = vectorizer.transform(newsgroups_test.data)

predictions = classifier.predict(vectors_test)
accuracy_score = metrics.accuracy_score(newsgroups_test.target, 
                                        predictions)
f1_score = metrics.f1_score(newsgroups_test.target, 
                            predictions, 
                            average='macro')

print("accuracy score: ", accuracy_score)
print("F1-score: ", f1_score)

corpus = ["""People say you cannot live without love, 
             but I think oxygen is more important""",
          "Sometimes, when you close your eyes, you cannot see."
          "A horse, a horse, my kingdom for a horse!",
          """Horse sense is the thing a horse has which 
          keeps it from betting on people."""
          """I’ve often said there is nothing better for 
          the inside of the man, than the outside of the horse.""",
          """A man on a horse is spiritually, as well as physically, 
          bigger then a man on foot.""",
          """No heaven can heaven be, if my horse isn’t there 
          to welcome me."""]

cv = CountVectorizer(min_df=2)
count_vector = cv.fit_transform(corpus)
cv.vocabulary_

cv.stop_words_

print("number of docus, size of vocabulary, stop_words list size")
for i in range(1, len(corpus)):
    cv = CountVectorizer(min_df=i)
    count_vector = cv.fit_transform(corpus)
    len_voc = len(cv.vocabulary_)
    len_stop_words = len(cv.stop_words_)
    print(f"{i:10d} {len_voc:15d} {len_stop_words:19d}")
    

cv = CountVectorizer(max_df=0.20)
count_vector = cv.fit_transform(corpus)
cv.stop_words_

quotes = ["A horse, a horse, my kingdom for a horse!",
          "Horse sense is the thing a horse has which keeps it from betting on people."
          "I’ve often said there is nothing better for the inside of the man, than the outside of the horse.",
          "A man on a horse is spiritually, as well as physically, bigger then a man on foot.",
          "No heaven can heaven be, if my horse isn’t there to welcome me."]

corpus = []
books = ["night_and_day_virginia_woolf.txt",
         "the_way_of_all_flash_butler.txt",
         "moby_dick_melville.txt",
         "sons_and_lovers_lawrence.txt",
         "robinson_crusoe_defoe.txt",
         "james_joyce_ulysses.txt"]
path = "books"

corpus = []
for book in books:
    txt = open(path + "/" + book).read()
    corpus.append(txt)
    
[book[:30] for book in corpus]

from sklearn.feature_extraction import text
import re

corpus = []
books = ["night_and_day_virginia_woolf.txt",
         "the_way_of_all_flash_butler.txt",
         "moby_dick_melville.txt",
         "sons_and_lovers_lawrence.txt",
         "robinson_crusoe_defoe.txt",
         "james_joyce_ulysses.txt"]
path = "books"

corpus = []
for book in books:
    txt = open(path + "/" + book).read()
    text_begin = re.search(r"\*\*\* ?START OF (THE|THIS) PROJECT.*?\*\*\*", txt, re.DOTALL)
    text_end = re.search(r"\*\*\* ?END OF (THE|THIS) PROJECT.*?\*\*\*", txt, re.DOTALL)
    corpus.append(txt[text_begin.end():text_end.start()])

vectorizer = text.CountVectorizer()
vectorizer.fit(corpus)
token_count_matrix = vectorizer.transform(corpus)
print(token_count_matrix)

print("Number of words in vocabulary: ", len(vectorizer.vocabulary_))

token_count_matrix.toarray()

from sklearn.feature_extraction import text


# our corpus:
quotes = ["A horse, a horse, my kingdom for a horse!",
          "Horse sense is the thing a horse has which keeps it from betting on people."
          "I’ve often said there is nothing better for the inside of the man, than the outside of the horse.",
          "A man on a horse is spiritually, as well as physically, bigger then a man on foot.",
          "No heaven can heaven be, if my horse isn’t there to welcome me."]

vectorizer = text.CountVectorizer()
vectorizer.fit(quotes)
vectorized_text = vectorizer.fit_transform(quotes)

tfidf_transformer = text.TfidfTransformer(smooth_idf=True,use_idf=True) 
tfidf_transformer.fit(vectorized_text)


"""
alternative way to output the data:
import pandas as pd
df_idf = pd.DataFrame(tfidf_transformer.idf_, 
                      index=vectorizer.get_feature_names(),
                      columns=["idf_weight"]) 
df_idf.sort_values(by=['idf_weights'])  # sorting data
print(df_idf)

"""
print(f"{'word':15s}: idf_weight")
word_weight_list = list(zip(vectorizer.get_feature_names_out(), tfidf_transformer.idf_))
word_weight_list.sort(key=lambda x:x[1])  # sort list by the weights (2nd component)
for word, idf_weight in word_weight_list:
    print(f"{word:15s}: {idf_weight:4.3f}")

import pandas as pd
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.ensemble import RandomForestClassifier
from sklearn.pipeline import Pipeline
from sklearn.metrics import classification_report
from sklearn.model_selection import train_test_split

# Load the training and testing data from CSV files
train_data = pd.read_csv("../data/IMDB/Train.csv")
test_data = pd.read_csv("../data/IMDB/Test.csv")

# Split the data into features (reviews) and labels (sentiments)
X_train, y_train = train_data['text'], train_data['label']
X_test, y_test = test_data['text'], test_data['label']

# Initialize TfidfVectorizer
vectorizer = TfidfVectorizer(max_features=5000, stop_words='english')

# Initialize Random Forest classifier
classifier = RandomForestClassifier(n_estimators=100, random_state=42)

# Create a pipeline with TfidfVectorizer and Random Forest
pipeline = Pipeline([
    ('vectorizer', vectorizer),
    ('classifier', classifier)
])

# Train the pipeline
pipeline.fit(X_train, y_train)

# Predict on test data
predicted = pipeline.predict(X_test)

# Print classification report
print(classification_report(y_test, predicted))