A_Self_Reported_BC_Traditional_ML_Classifier.ipynb

{"cells":[{"cell_type":"markdown","metadata":{},"source":["[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/swati-rajwal/BreastCancer_tweets_project/blob/main/A_Self_Reported_BC_Traditional_ML_Classifier.ipynb)"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":3765,"status":"ok","timestamp":1701050868266,"user":{"displayName":"Avinash kumar Pandey","userId":"07918242858559780186"},"user_tz":300},"id":"0LV0M09X2n1u","outputId":"72541a63-ae95-4ccd-a995-7981ec526da5"},"outputs":[{"name":"stderr","output_type":"stream","text":["[nltk_data] Downloading package stopwords to /root/nltk_data...\n","[nltk_data]   Unzipping corpora/stopwords.zip.\n","[nltk_data] Downloading package wordnet to /root/nltk_data...\n","[nltk_data] Downloading package punkt to /root/nltk_data...\n","[nltk_data]   Unzipping tokenizers/punkt.zip.\n","[nltk_data] Downloading package vader_lexicon to /root/nltk_data...\n"]},{"data":{"text/plain":["True"]},"execution_count":1,"metadata":{},"output_type":"execute_result"}],"source":["# Import Libraries\n","import pandas as pd\n","import numpy as np\n","from sklearn.model_selection import train_test_split, GridSearchCV\n","from sklearn.naive_bayes import MultinomialNB\n","from sklearn.linear_model import LogisticRegression\n","from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier\n","from sklearn.svm import SVC\n","from xgboost import XGBClassifier\n","from sklearn.model_selection import train_test_split, cross_val_score, GridSearchCV\n","from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n","from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier, VotingClassifier\n","from sklearn.linear_model import LogisticRegression\n","from sklearn.svm import SVC\n","from sklearn.metrics import accuracy_score, f1_score, make_scorer\n","import matplotlib.pyplot as plt\n","from sklearn.tree import DecisionTreeClassifier\n","import nltk\n","from nltk.corpus import stopwords\n","from nltk.stem import WordNetLemmatizer\n","from nltk.sentiment import SentimentIntensityAnalyzer\n","from nltk.tokenize import word_tokenize\n","from sklearn.metrics import accuracy_score, f1_score\n","from sklearn.neighbors import KNeighborsClassifier\n","from sklearn.tree import DecisionTreeClassifier\n","from sklearn.ensemble import AdaBoostClassifier\n","import gensim\n","from sklearn.cluster import KMeans\n","from nltk.tokenize import word_tokenize\n","\n","nltk.download('stopwords')\n","nltk.download('wordnet')\n","nltk.download('punkt')\n","nltk.download('vader_lexicon')"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":3,"status":"ok","timestamp":1701050868266,"user":{"displayName":"Avinash kumar Pandey","userId":"07918242858559780186"},"user_tz":300},"id":"V9B8uc_UyOpE","outputId":"3a1ef316-3ffb-49f1-88c8-33cd4595d6fd"},"outputs":[{"name":"stderr","output_type":"stream","text":["<ipython-input-2-a92e0564eb27>:4: FutureWarning: Passing a negative integer is deprecated in version 1.0 and will not be supported in future version. Instead, use None to not limit the column width.\n","  pd.set_option('display.max_colwidth', -1)  # For pandas versions >= 1.0\n"]}],"source":["# Set the display options to ensure all columns are shown\n","pd.set_option('display.max_columns', None)  # Display all columns\n","pd.set_option('display.width', None)        # No max width\n","pd.set_option('display.max_colwidth', -1)  # For pandas versions >= 1.0"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":14985,"status":"ok","timestamp":1701050884937,"user":{"displayName":"Avinash kumar Pandey","userId":"07918242858559780186"},"user_tz":300},"id":"ETFMCJHMaq6h","outputId":"fba630a9-7e6d-4894-bf79-7539a6b49dbc"},"outputs":[{"name":"stdout","output_type":"stream","text":["Mounted at /content/drive\n"]}],"source":["# Connecting the Python Code with the google drive\n","from google.colab import drive\n","drive.mount(\"/content/drive\")"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"qVYm1tBhC9HV"},"outputs":[],"source":["# IMPORTANT FUNCTIONS\n","\n","# Preprocessing Function\n","\n","def preprocess(text):\n","    if pd.isnull(text):\n","        return ''\n","    tokens = nltk.word_tokenize(text)\n","    filtered = [lemmatizer.lemmatize(word.lower()) for word in tokens if word not in stop_words and word.isalpha()]\n","    return ' '.join(filtered)\n","\n","# Bootstrapping Function for computing confidence intervals of F1-score\n","\n","def bootstrap_f1_macro(clf, X, y, n_iterations=1000, ci=0.95):\n","    scores = []\n","    for _ in range(n_iterations):\n","        # Sampling with replacement\n","        indices = np.random.choice(len(X), size=len(X), replace=True)\n","        X_resampled = X[indices]\n","        y_resampled = y[indices]\n","\n","        # Predict and compute F1 macro score\n","        y_pred = clf.predict(X_resampled)\n","        score = f1_score(y_resampled, y_pred, average='macro')\n","        scores.append(score)\n","\n","    # Sort scores and get percentiles for confidence interval\n","    lower_bound = (1.0 - ci) / 2.0\n","    upper_bound = 1.0 - lower_bound\n","    lower, upper = np.percentile(scores, [lower_bound*100, upper_bound*100])\n","\n","    return lower, upper\n","\n","\n","# Functions for word cluster feature\n","\n","def fit_word_clusters(df):\n","    # Tokenize and train Word2Vec\n","    sentences = df['text'].apply(word_tokenize).tolist()\n","    model = gensim.models.Word2Vec(sentences, vector_size=100, window=5, min_count=1, workers=4)\n","\n","    # Get word vectors for all words in vocabulary\n","    word_vectors = {word: model.wv[word] for word in model.wv.index_to_key}\n","\n","    # Cluster words using K-means\n","    num_clusters = 10\n","    kmeans = KMeans(n_clusters=num_clusters, random_state=0)\n","    kmeans.fit(list(word_vectors.values()))\n","\n","    return model, kmeans\n","\n","def transform_word_clusters(df, model, kmeans):\n","    word_vectors = {word: model.wv[word] for word in model.wv.index_to_key}\n","    num_clusters = kmeans.n_clusters\n","\n","    # Get the cluster assignment for each word\n","    word_cluster_assignment = {word: kmeans.predict([vector])[0] for word, vector in word_vectors.items()}\n","\n","    # Define the transform function\n","    def get_cluster_representation(text):\n","        tokens = word_tokenize(text)\n","        clusters = [word_cluster_assignment.get(token, -1) for token in tokens if token in word_cluster_assignment]\n","        representation = np.bincount(clusters, minlength=num_clusters)\n","        return representation[:10]\n","\n","    # Transform the dataframe\n","    cluster_features = df['text'].apply(get_cluster_representation)\n","    array_representation = np.array(cluster_features.tolist())\n","\n","    return array_representation"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":16672,"status":"ok","timestamp":1701050901761,"user":{"displayName":"Avinash kumar Pandey","userId":"07918242858559780186"},"user_tz":300},"id":"BAB4GbxGxVMs","outputId":"0c6d1c2f-62fc-4e2e-ac30-1111b40804e7"},"outputs":[{"name":"stderr","output_type":"stream","text":["/usr/local/lib/python3.10/dist-packages/sklearn/cluster/_kmeans.py:870: FutureWarning: The default value of `n_init` will change from 10 to 'auto' in 1.4. Set the value of `n_init` explicitly to suppress the warning\n","  warnings.warn(\n"]}],"source":["## HYPERPARAMETER TUNING & PERFORMANCE COMPARISON (On training and Dev Data)\n","\n","# Training Data Upload\n","df_train = pd.read_csv(\"/content/drive/MyDrive/Emory/BMI NLP/Breast Cancer Data/Classification_ data/train.csv\")\n","df_train = df_train.dropna(subset=['text'])\n","\n","# Testing Data Upload\n","df_test = pd.read_csv(\"/content/drive/MyDrive/Emory/BMI NLP/Breast Cancer Data/Classification_ data/dev_.csv\")\n","df_test = df_test.dropna(subset = [\"text\"])\n","\n","# Sentiment (Computed before Pre-processing)\n","sia = SentimentIntensityAnalyzer()\n","df_train['sentiment'] = df_train['text'].apply(lambda x: (sia.polarity_scores(x)['compound'] + 1) / 2)\n","df_test['sentiment'] = df_test['text'].apply(lambda x: (sia.polarity_scores(x)['compound'] + 1) / 2)\n","\n","# Pre-processing\n","lemmatizer = WordNetLemmatizer()\n","stop_words = set(stopwords.words('english'))\n","df_train['text'] = df_train['text'].apply(preprocess)\n","df_test[\"text\"] = df_test[\"text\"].apply(preprocess)\n","\n","# Preprocessed word length\n","df_train['processed_text_length'] = df_train['text'].apply(lambda x:len(word_tokenize(x)))\n","df_test['processed_text_length'] = df_test['text'].apply(lambda x:len(word_tokenize(x)))\n","\n","# Count of unique words (indicating vocabulary diversity)\n","df_train['unique_word_count'] = df_train['text'].apply(lambda x: len(set(word_tokenize(x))))\n","df_test['unique_word_count'] = df_test['text'].apply(lambda x: len(set(word_tokenize(x))))\n","\n","# Hyper parameter Tuning\n","df_train = df_train.dropna().reset_index(drop = True)\n","df_test = df_test.dropna().reset_index(drop = True)\n","train_df, test_df = df_train.copy(), df_test.copy()\n","\n","# TF-IDF Vector\n","vectorizer = TfidfVectorizer(ngram_range=(1, 2),stop_words = \"english\", max_features = 1000)\n","X_train_tfidf = vectorizer.fit_transform(train_df['text'])\n","X_test_tfidf = vectorizer.transform(test_df['text'])\n","\n","# Word Cluster Represenattion\n","model, kmeans = fit_word_clusters(train_df)  # Only on training data\n","train_clusters = transform_word_clusters(train_df, model, kmeans)\n","test_clusters = transform_word_clusters(test_df, model, kmeans)\n","\n","# In training the models, variables from columns_to_use, word_cluster feature and TF-IDF vectors are used (combined using np.hstack)\n","columns_to_use = ['unique_word_count','processed_text_length','sentiment']\n","\n","# Combining Vectors & Features\n","X_train = np.hstack((X_train_tfidf.toarray(), train_df[columns_to_use].values))\n","X_test = np.hstack((X_test_tfidf.toarray(), test_df[columns_to_use].values))\n","\n","# Initializing Final Features\n","y_train = train_df['Class']\n","y_test = test_df[\"Class\"]"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"background_save":true,"base_uri":"https://localhost:8080/"},"id":"3uiEbSqx6RvB","outputId":"3013bf6d-700c-4f3f-80bb-1e9b473a2819"},"outputs":[{"name":"stderr","output_type":"stream","text":["/usr/local/lib/python3.10/dist-packages/sklearn/cluster/_kmeans.py:870: FutureWarning: The default value of `n_init` will change from 10 to 'auto' in 1.4. Set the value of `n_init` explicitly to suppress the warning\n","  warnings.warn(\n"]},{"name":"stdout","output_type":"stream","text":["Naive Bayes\n","Best Parameters: {'alpha': 0.1}\n","Accuracy: 0.7483443708609272\n","Micro F1: 0.7483443708609272\n","Macro F1: 0.44057716681290826\n","------------\n","Decision Tree\n","Best Parameters: {'max_depth': 10}\n","Accuracy: 0.7814569536423841\n","Micro F1: 0.7814569536423841\n","Macro F1: 0.6478942905596382\n","------------\n","Logistic Regression\n","Best Parameters: {'C': 10, 'penalty': 'l1'}\n","Accuracy: 0.8013245033112583\n","Micro F1: 0.8013245033112583\n","Macro F1: 0.7449324324324325\n","------------\n","Random Forest\n","Best Parameters: {'max_depth': None, 'n_estimators': 100}\n","Accuracy: 0.7582781456953642\n","Micro F1: 0.7582781456953643\n","Macro F1: 0.5002606823076051\n","------------\n"]}],"source":["# Setting up classifiers and parameters for GridSearch\n","models = {\n","    'Naive Bayes': MultinomialNB(),\n","    'Decision Tree': DecisionTreeClassifier(),\n","    'Logistic Regression': LogisticRegression(solver='liblinear') ,\n","    'Random Forest': RandomForestClassifier(),\n","}\n","\n","# Range of paramaters for Hyperparameter tuning\n","params = {\n","    'Naive Bayes': {\n","        'alpha': [0.01, 0.1, 1, 10, 100]\n","    },\n","    'Decision Tree': {\n","        'max_depth': [None, 3, 5, 10, 15, 20],\n","\n","    },\n","    'Random Forest': {\n","        'n_estimators': [10, 50, 100],\n","        'max_depth': [None, 5, 10],\n","    },\n","\n","\n","    'Logistic Regression': {'C': [ 0.01, 0.1, 1, 10], 'penalty': ['l1', 'l2']}\n","    }\n","\n","# Model Performance Print\n","for model_name, model in models.items():\n","\n","  try:\n","    grid_search = GridSearchCV(model, params[model_name], cv=5)\n","    grid_search.fit(X_train, y_train)\n","    models[model_name].set_params(**grid_search.best_params_)\n","    predictions = grid_search.predict(X_test)\n","    print(model_name)\n","    print(\"Best Parameters:\", grid_search.best_params_)\n","    print(\"Accuracy:\", accuracy_score(y_test, predictions))\n","    print(\"Micro F1:\", f1_score(y_test, predictions, average='micro'))\n","    print(\"Macro F1:\", f1_score(y_test, predictions, average='macro'))\n","    print(\"------------\")\n","\n","  except :\n","    print(f\"Error with model {model_name}\")"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":6244,"status":"ok","timestamp":1701019431533,"user":{"displayName":"Avinash kumar Pandey","userId":"07918242858559780186"},"user_tz":300},"id":"wKdRAsQdLccN","outputId":"08a2d2b6-1f64-44d3-9c89-f0756aa9a34d"},"outputs":[{"name":"stdout","output_type":"stream","text":["-------------------------------------------------------------\n","Performance of Naive Bayes on Test.csv\n","Accuracy: 0.7649501661129569\n","Micro F1: 0.764950166112957\n","Macro F1: 0.5697257843527236\n","-------------------------------------------------------------\n","Performance of Decision Tree on Test.csv\n","Accuracy: 0.7425249169435216\n","Micro F1: 0.7425249169435216\n","Macro F1: 0.5902334057923281\n"]},{"name":"stderr","output_type":"stream","text":["/usr/local/lib/python3.10/dist-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n","STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n","\n","Increase the number of iterations (max_iter) or scale the data as shown in:\n","    https://scikit-learn.org/stable/modules/preprocessing.html\n","Please also refer to the documentation for alternative solver options:\n","    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n","  n_iter_i = _check_optimize_result(\n"]},{"name":"stdout","output_type":"stream","text":["-------------------------------------------------------------\n","Performance of Logistic Regression on Test.csv\n","Accuracy: 0.7857142857142857\n","Micro F1: 0.7857142857142857\n","Macro F1: 0.6879136777382603\n","-------------------------------------------------------------\n","Performance of Random Forest on Test.csv\n","Accuracy: 0.7732558139534884\n","Micro F1: 0.7732558139534884\n","Macro F1: 0.5882946063652431\n"]}],"source":["# Evaluate trained classifiers with optimal hyperparameters (On Test Data)\n","\n","# Test Data Upload\n","df1 = pd.read_csv(\"/content/drive/MyDrive/Emory/BMI NLP/Breast Cancer Data/Classification_ data/test.csv\")\n","df1 = df1.dropna(subset=['text'])\n","\n","# Fall Description Sentiment (Computed before Pre-processing)\n","sia = SentimentIntensityAnalyzer()\n","df1['sentiment'] = df1['text'].apply(lambda x: (sia.polarity_scores(x)['compound'] + 1) / 2)\n","\n","# Pre-processing\n","lemmatizer = WordNetLemmatizer()\n","stop_words = set(stopwords.words('english'))\n","df1['text'] = df1['text'].apply(preprocess)\n","\n","# Preprocessed word length of fall description\n","df1['processed_text_length'] = df1['text'].apply(lambda x:len(word_tokenize(x)))\n","\n","# Count of unique words (indicating vocabulary diversity)\n","df1['unique_word_count'] = df1['text'].apply(lambda x: len(set(word_tokenize(x))))\n","\n","df1 = df1.dropna().reset_index(drop = True)\n","test_df = df1\n","\n","# TF-IDF Vector\n","X_test_tfidf = vectorizer.transform(test_df['text'])\n","\n","# Word Cluster Represenattion\n","test_clusters = transform_word_clusters(test_df, model, kmeans)\n","columns_to_use = ['unique_word_count','processed_text_length','sentiment']\n","\n","# Combining Vectors & Features\n","X_test = np.hstack((X_test_tfidf.toarray(),test_df[columns_to_use].values))\n","\n","# Initializing Final Features\n","y_test = test_df[\"Class\"]\n","\n","# Using Optimal tuned Hyperparameters\n","classifiers = {\n","    \"Naive Bayes\": MultinomialNB(alpha=0.1),\n","    \"Decision Tree\": DecisionTreeClassifier(criterion='gini', max_depth=10),\n","    \"Logistic Regression\": LogisticRegression(C=10, penalty='l2'),\n","    \"Random Forest\": RandomForestClassifier(max_depth=None, n_estimators=50),\n","}\n","\n","# Output performance of the classifiers on test data\n","for clf_name, clf in classifiers.items() :\n","\n","  try :\n","    clf.fit(X_train, y_train)\n","    predictions = clf.predict(X_test)\n","    print(\"-------------------------------------------------------------\")\n","    print(f\"Performance of {clf_name} on Test.csv\")\n","    print(\"Accuracy:\", accuracy_score(y_test, predictions))\n","    print(\"Micro F1:\", f1_score(y_test, predictions, average='micro'))\n","    print(\"Macro F1:\", f1_score(y_test, predictions, average='macro'))\n","\n","  except :\n","    print(f\"Error with model {model_name}\")\n"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":123222,"status":"ok","timestamp":1699982092172,"user":{"displayName":"Avinash kumar Pandey","userId":"07918242858559780186"},"user_tz":300},"id":"B8QtJ_-0LZ0m","outputId":"9c8dc4ed-0710-40a8-cbca-7c34af36219a"},"outputs":[{"name":"stderr","output_type":"stream","text":["/usr/local/lib/python3.10/dist-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n","STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n","\n","Increase the number of iterations (max_iter) or scale the data as shown in:\n","    https://scikit-learn.org/stable/modules/preprocessing.html\n","Please also refer to the documentation for alternative solver options:\n","    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n","  n_iter_i = _check_optimize_result(\n"]},{"name":"stdout","output_type":"stream","text":["\n","Ensemble Classifier:\n","Accuracy: 0.7632890365448505\n","Micro F1: 0.7632890365448505\n","Macro F1: 0.5155408410337501\n"]}],"source":["# Ensemble Classifier (On Test Data)\n","\n","from sklearn.ensemble import VotingClassifier\n","\n","# Initializing the individual models with their best parameters\n","naive_bayes = MultinomialNB(alpha=0.1)\n","decision_tree = DecisionTreeClassifier(criterion='gini', max_depth=10)\n","logistic_regression = LogisticRegression(C=10, penalty='l2')\n","random_forest = RandomForestClassifier(max_depth=None, n_estimators=50)\n","\n","\n","# Create an ensemble of the models using soft voting\n","ensemble = VotingClassifier(estimators=[\n","    ('naive_bayes', naive_bayes),\n","    ('decision_tree', decision_tree),\n","    ('logistic_regression', logistic_regression),\n","    ('random_forest', random_forest)\n","\n","], voting='soft')\n","\n","# Train and evaluate the ensemble\n","ensemble.fit(X_train, y_train)\n","ensemble_predictions = ensemble.predict(X_test)\n","ensemble_accuracy = accuracy_score(y_test, ensemble_predictions)\n","ensemble_micro_f1 = f1_score(y_test, ensemble_predictions, average='micro')\n","ensemble_macro_f1 = f1_score(y_test, ensemble_predictions, average='macro')\n","\n","print(\"\\nEnsemble Classifier:\")\n","print(\"Accuracy:\", ensemble_accuracy)\n","print(\"Micro F1:\", ensemble_micro_f1)\n","print(\"Macro F1:\", ensemble_macro_f1)"]}],"metadata":{"accelerator":"GPU","colab":{"authorship_tag":"ABX9TyNX9XeRRaZj15QIyKbBfrAM","machine_shape":"hm","provenance":[{"file_id":"1ACd7t98h8MUqoFdyAhGR30QC5k6nGz4X","timestamp":1699843835171}]},"kernelspec":{"display_name":"Python 3","name":"python3"},"language_info":{"name":"python"}},"nbformat":4,"nbformat_minor":0}