E_Train_size_vs_performance.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/swati-rajwal/BreastCancer_tweets_project/blob/main/E_Train_size_vs_performance.ipynb)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2xmGoq4EhmWW",
        "outputId": "98a33389-033e-4243-d075-d53851768fa7"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Yq_0KXDO2Nge"
      },
      "outputs": [],
      "source": [
        "pip install -q transformers"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 315
        },
        "id": "iZ9L5gR23riJ",
        "outputId": "43219fda-2058-4f37-d75a-94e02cf244d5"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n",
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n",
            "<ipython-input-3-564121d0e846>:46: MatplotlibDeprecationWarning: The seaborn styles shipped by Matplotlib are deprecated since 3.6, as they no longer correspond to the styles shipped by seaborn. However, they will remain available as 'seaborn-v0_8-<style>'. Alternatively, directly use the seaborn API instead.\n",
            "  plt.style.use(\"seaborn-whitegrid\")\n",
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Package stopwords is already up-to-date!\n",
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]   Package wordnet is already up-to-date!\n",
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n",
            "[nltk_data] Downloading package vader_lexicon to /root/nltk_data...\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "execution_count": 3,
          "metadata": {},
          "output_type": "execute_result"
        },
        {
          "data": {
            "text/plain": [
              "<Figure size 640x480 with 0 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import tensorflow as tf\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "\n",
        "#data processing\n",
        "import re, string\n",
        "# import emoji\n",
        "import nltk\n",
        "nltk.download('punkt')\n",
        "nltk.download('stopwords')\n",
        "from nltk.corpus import stopwords\n",
        "from nltk.tokenize import word_tokenize\n",
        "\n",
        "from sklearn import preprocessing\n",
        "from imblearn.over_sampling import RandomOverSampler\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "#Naive Bayes\n",
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "from sklearn.feature_extraction.text import TfidfTransformer\n",
        "from sklearn.naive_bayes import MultinomialNB\n",
        "\n",
        "#transformers\n",
        "from transformers import BertTokenizerFast\n",
        "from transformers import TFBertModel\n",
        "from transformers import RobertaTokenizerFast\n",
        "from transformers import TFRobertaModel\n",
        "\n",
        "#keras\n",
        "import tensorflow as tf\n",
        "from tensorflow import keras\n",
        "\n",
        "\n",
        "#metrics\n",
        "from sklearn.metrics import accuracy_score, f1_score, make_scorer, fbeta_score\n",
        "from sklearn.metrics import classification_report, confusion_matrix\n",
        "\n",
        "#set seed for reproducibility\n",
        "seed=42\n",
        "\n",
        "#set style for plots\n",
        "sns.set_style(\"whitegrid\")\n",
        "sns.despine()\n",
        "plt.style.use(\"seaborn-whitegrid\")\n",
        "plt.rc(\"figure\", autolayout=True)\n",
        "plt.rc(\"axes\", labelweight=\"bold\", labelsize=\"large\", titleweight=\"bold\", titlepad=10)\n",
        "\n",
        "from nltk.stem import WordNetLemmatizer\n",
        "nltk.download('wordnet')\n",
        "lemmatizer = WordNetLemmatizer()\n",
        "\n",
        "# Name : Avinash Kumar Pandey\n",
        "# Final Project Task 1\n",
        "# BMI NLP Course\n",
        "\n",
        "# Import Libraries\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "from sklearn.model_selection import train_test_split, GridSearchCV\n",
        "from sklearn.naive_bayes import MultinomialNB\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier\n",
        "from sklearn.svm import SVC\n",
        "from xgboost import XGBClassifier\n",
        "from sklearn.model_selection import train_test_split, cross_val_score, GridSearchCV\n",
        "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
        "from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier, VotingClassifier\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.svm import SVC\n",
        "from sklearn.metrics import accuracy_score, f1_score, make_scorer, log_loss\n",
        "import matplotlib.pyplot as plt\n",
        "from sklearn.tree import DecisionTreeClassifier\n",
        "import nltk\n",
        "from nltk.corpus import stopwords\n",
        "from nltk.stem import WordNetLemmatizer\n",
        "from nltk.sentiment import SentimentIntensityAnalyzer\n",
        "from nltk.tokenize import word_tokenize\n",
        "from sklearn.metrics import accuracy_score, f1_score\n",
        "from sklearn.neighbors import KNeighborsClassifier\n",
        "from sklearn.tree import DecisionTreeClassifier\n",
        "from sklearn.ensemble import AdaBoostClassifier\n",
        "import gensim\n",
        "from sklearn.cluster import KMeans\n",
        "from nltk.tokenize import word_tokenize\n",
        "\n",
        "nltk.download('stopwords')\n",
        "nltk.download('wordnet')\n",
        "nltk.download('punkt')\n",
        "nltk.download('vader_lexicon')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "zI5kxpgX3P6g"
      },
      "outputs": [],
      "source": [
        "df_train = pd.read_csv('/content/drive/MyDrive/BioNLP Final project/Breast Cancer Data/Classification_ data/train.csv')\n",
        "df_dev = pd.read_csv('/content/drive/MyDrive/BioNLP Final project/Breast Cancer Data/Classification_ data/dev_.csv')\n",
        "df_test = pd.read_csv('/content/drive/MyDrive/BioNLP Final project/Breast Cancer Data/Classification_ data/test.csv')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5CQ-9qNTbAjK",
        "outputId": "1adc922e-8087-4b66-9979-c2cdb278e6e7"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Train set shape: (3513, 2)\n",
            "Dev set shape: (302, 2)\n",
            "Test set shape: (1204, 2)\n"
          ]
        }
      ],
      "source": [
        "print(f\"Train set shape: {df_train.shape}\")\n",
        "print(f\"Dev set shape: {df_dev.shape}\")\n",
        "print(f\"Test set shape: {df_test.shape}\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hsCsGg9qbhtX",
        "outputId": "79228c67-a537-4128-e789-63725f0139f5"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "Index(['text', 'Class'], dtype='object')"
            ]
          },
          "execution_count": 6,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df_train.columns"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 145,
          "referenced_widgets": [
            "60ffbe5df5a041fc879950ea7c5004bf",
            "f53bfa2e4c324490a7ca3f18d6ef1053",
            "cc576aedd0f44f8087bd1b7dbd53784d",
            "a85e9e0a7cd64cd7b2fc1283f618e4af",
            "f016b364387f4e708fdcf6487d18d698",
            "d9064c13ffd14a7e8d5e3b778b210d4d",
            "316ff69ef16c4494994fef46eaadfbf4",
            "a628bd1d43544640bfe25ae661ea8a7c",
            "c57d560e577b4dd2a87f4393c67d9063",
            "12b28312a0584a32a12f3916d77cb7f8",
            "73200837e1c347049ec865c24a923564",
            "1282255fd5db46daa976e6bffafa88f4",
            "56670582df7246b8b76eff2aad576265",
            "c1231df639c543d69ded72dc73d019b7",
            "5d974ed579be4696af147814999adb58",
            "6757da3f75de4f88a67973fef405ce58",
            "5066198fc8774f39a529b5a260bc2b1d",
            "0f0222c78f414d6295bb20e7dcaa6d8b",
            "9cc5848453e54cf9ae5fe9dd7fe64f41",
            "81f44880f3084670abc38eeb23dee176",
            "af73e7823d1a4158a2131ad4b1ea6d21",
            "128adf5e6f6c456bb0d22afca78d8a9a",
            "d652807b23e843258a447be5c359e80d",
            "74952293ee4949d9af72db325281b441",
            "dded31be4c1a4b919d009c2a8bbdd331",
            "68bf97233181443cb346b5f26816eb7e",
            "1c497309b70d43579b542b836e59ffd4",
            "6a75f6a9f5ef4d79b589a9dc752c61e3",
            "381c08e84f5d4f5b9ffe981e0d6f77a7",
            "a96ae2edc14f473590350dd2de9a2dd1",
            "a95b87fdc7c84c28b7fa203ebe3afb88",
            "eac479b597ef497e94625dce07e7b8d8",
            "21b2ae5667a44067a7d67987ee5a7897",
            "68a79118c69a4564b28ca67d631a719f",
            "e5972e9f088341ecaa72b51afd91fcd5",
            "093b4bd4f48e4acd8549da281b269fe9",
            "1028fc38fc954540a463d2e94493e332",
            "e1432fd10a8a4b4ca982d67c3a019ed7",
            "ce31928053c442d29e92d6d3d7d30eb4",
            "37366a210fae45788a7dff405051525c",
            "0f7b4bc5ebf24a87bbf5d37019b13166",
            "9a81396477ce4ccbac79725a40cef76f",
            "32184de396534b26b4011a827640c026",
            "22007d9a04964878a428d9aeec77e211"
          ]
        },
        "id": "L51Dm4bz3bWF",
        "outputId": "f3e3baa0-6c90-4950-afbc-e957598e2fdc"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "60ffbe5df5a041fc879950ea7c5004bf",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "vocab.json:   0%|          | 0.00/899k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "1282255fd5db46daa976e6bffafa88f4",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "merges.txt:   0%|          | 0.00/456k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "d652807b23e843258a447be5c359e80d",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "tokenizer.json:   0%|          | 0.00/1.36M [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "68a79118c69a4564b28ca67d631a719f",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "config.json:   0%|          | 0.00/481 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "tokenizer_roberta = RobertaTokenizerFast.from_pretrained(\"roberta-base\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 143
        },
        "id": "ivQClwioLYEG",
        "outputId": "d7d584cd-20a2-4a59-8b5c-631e3a4167be"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-2a784c61-2df6-4503-9413-ae3674dab237\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>Class</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>959</th>\n",
              "      <td>&lt;user&gt; i've enjoyed u all these years &amp; will m...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2068</th>\n",
              "      <td>&lt;user&gt; as a military wife, breast cancer survi...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1985</th>\n",
              "      <td>&lt;user&gt; wishing upon an &lt;hashtag&gt; alaskahappyho...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-2a784c61-2df6-4503-9413-ae3674dab237')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-2a784c61-2df6-4503-9413-ae3674dab237 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-2a784c61-2df6-4503-9413-ae3674dab237');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-ef130e8c-4640-4b16-8a9e-a8b837e28b7b\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-ef130e8c-4640-4b16-8a9e-a8b837e28b7b')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-ef130e8c-4640-4b16-8a9e-a8b837e28b7b button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "text/plain": [
              "                                                   text  Class\n",
              "959   <user> i've enjoyed u all these years & will m...      1\n",
              "2068  <user> as a military wife, breast cancer survi...      0\n",
              "1985  <user> wishing upon an <hashtag> alaskahappyho...      1"
            ]
          },
          "execution_count": 7,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df_train.sample(3)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4zvSkTZsyEf5"
      },
      "outputs": [],
      "source": [
        "# def preprocess(text):\n",
        "#     text = text.lower()\n",
        "#     tokens = nltk.word_tokenize(text)\n",
        "#     filtered = [lemmatizer.lemmatize(word) for word in tokens if word not in stopwords.words('english') and word not in string.punctuation]\n",
        "#     return ' '.join(filtered)\n",
        "\n",
        "\n",
        "# df_train['before_preprocess_length'] = df_train['medical_abstract'].apply(lambda x: len(str(x)))\n",
        "# print(\"############### Before preprocessing ##########\")\n",
        "# print(df_train['before_preprocess_length'].max())\n",
        "# print(df_train['before_preprocess_length'].min())\n",
        "\n",
        "\n",
        "\n",
        "  # Pre-load the stopwords\n",
        "STOPWORDS = set(stopwords.words('english'))\n",
        "\n",
        "def remove_emojis(text):\n",
        "    # Regular expression to match emojis\n",
        "    emoji_pattern = re.compile(\"[\"\n",
        "                           u\"\\U0001F600-\\U0001F64F\"  # emoticons\n",
        "                           u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n",
        "                           u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n",
        "                           u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n",
        "                           \"]+\", flags=re.UNICODE)\n",
        "    return emoji_pattern.sub(r'', text)\n",
        "\n",
        "def preprocess_text_optimized(text):\n",
        "    text = remove_emojis(text)\n",
        "    text = text.lower()\n",
        "    # Remove <word> patterns\n",
        "    text = re.sub(r'<[^>]+>', '', text)\n",
        "    text = text.translate(str.maketrans('', '', string.punctuation))\n",
        "    text = text.replace('@', '').replace('\\n', '')\n",
        "    tokens = word_tokenize(text)\n",
        "    # Filter out stopwords\n",
        "    # tokens = nltk.word_tokenize(text)\n",
        "    filtered = [lemmatizer.lemmatize(word) for word in tokens if word not in stopwords.words('english') and word not in string.punctuation]\n",
        "    return ' '.join(filtered)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "EX6epKgTcqCk",
        "outputId": "d9934ae8-8851-4cf8-b6c7-582b4276fbe2"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'!\"#$%&\\'()*+,-./:;<=>?@[\\\\]^_`{|}~'"
            ]
          },
          "execution_count": 16,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "string.punctuation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "gfw-RYtUjAns"
      },
      "outputs": [],
      "source": [
        "\n",
        "df_train['text']= df_train['text'].apply(preprocess_text_optimized)\n",
        "df_dev['text'] = df_dev['text'].apply(preprocess_text_optimized)\n",
        "df_test['text'] = df_test['text'].apply(preprocess_text_optimized)\n",
        "# print(\"############### Before preprocessing ##########\")\n",
        "# print(df_train['after_preprocess_length'].max())\n",
        "# print(df_train['after_preprocess_length'].min())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 143
        },
        "id": "ctw-c88l5cHp",
        "outputId": "401da568-8d87-416b-98c9-59844fb83d44"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-c382cc2b-4a6c-4a60-860a-fe20aeedd5f1\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>text</th>\n",
              "      <th>Class</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1002</th>\n",
              "      <td>’ breastcancerawareness month tomorrow gmsa su...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1619</th>\n",
              "      <td>following breastcancer diagnosis want yearofgr...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3237</th>\n",
              "      <td>’ honor participate pink patch project serve c...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-c382cc2b-4a6c-4a60-860a-fe20aeedd5f1')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-c382cc2b-4a6c-4a60-860a-fe20aeedd5f1 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-c382cc2b-4a6c-4a60-860a-fe20aeedd5f1');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-b299d189-190e-4a5c-a3d1-93abc5cce868\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-b299d189-190e-4a5c-a3d1-93abc5cce868')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-b299d189-190e-4a5c-a3d1-93abc5cce868 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "text/plain": [
              "                                                   text  Class\n",
              "1002  ’ breastcancerawareness month tomorrow gmsa su...      0\n",
              "1619  following breastcancer diagnosis want yearofgr...      1\n",
              "3237  ’ honor participate pink patch project serve c...      0"
            ]
          },
          "execution_count": 15,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df_train.sample(3)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5ife1Chg52bR",
        "outputId": "73fd3a71-24b2-4240-e930-398eb514de2e"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "text     ’ breastcancerawareness month tomorrow gmsa su...\n",
              "Class                                                    0\n",
              "Name: 1002, dtype: object"
            ]
          },
          "execution_count": 9,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df_train.iloc[1002]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DUHih5CK35lL",
        "outputId": "9ef8e070-25af-4acc-857a-bb6d6e4f3c1d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "MAX TOKENIZED SENTENCE LENGTH: 114\n"
          ]
        }
      ],
      "source": [
        "# Run this after preprocessing the texts\n",
        "token_lens = []\n",
        "\n",
        "for txt in df_train['text'].values:\n",
        "    tokens = tokenizer_roberta.encode(txt, max_length=512, truncation=True)\n",
        "    token_lens.append(len(tokens))\n",
        "\n",
        "max_len=np.max(token_lens)\n",
        "\n",
        "print(f\"MAX TOKENIZED SENTENCE LENGTH: {max_len}\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LongUewx57Gh",
        "outputId": "e240d8ba-7a3a-4f5d-9184-158ab365f1f1"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "*******************************************\n",
            "Currently running for size:  0.1\n",
            "Naive Bayes\n",
            "Decision Tree\n",
            "Logistic Regression\n",
            "Random Forest\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Some weights of the PyTorch model were not used when initializing the TF 2.0 model TFRobertaModel: ['lm_head.dense.bias', 'lm_head.bias', 'lm_head.layer_norm.weight', 'lm_head.layer_norm.bias', 'lm_head.dense.weight', 'roberta.embeddings.position_ids']\n",
            "- This IS expected if you are initializing TFRobertaModel from a PyTorch model trained on another task or with another architecture (e.g. initializing a TFBertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing TFRobertaModel from a PyTorch model that you expect to be exactly identical (e.g. initializing a TFBertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights or buffers of the TF 2.0 model TFRobertaModel were not initialized from the PyTorch model and are newly initialized: ['roberta.pooler.dense.weight', 'roberta.pooler.dense.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 1/7\n",
            "22/22 [==============================] - 58s 466ms/step - loss: 0.6053 - categorical_accuracy: 0.7407\n",
            "Epoch 2/7\n",
            "22/22 [==============================] - 9s 415ms/step - loss: 0.5821 - categorical_accuracy: 0.7436\n",
            "Epoch 3/7\n",
            "22/22 [==============================] - 9s 411ms/step - loss: 0.5534 - categorical_accuracy: 0.7436\n",
            "Epoch 4/7\n",
            "22/22 [==============================] - 9s 405ms/step - loss: 0.4980 - categorical_accuracy: 0.7578\n",
            "Epoch 5/7\n",
            "22/22 [==============================] - 11s 527ms/step - loss: 0.3902 - categorical_accuracy: 0.7920\n",
            "Epoch 6/7\n",
            "22/22 [==============================] - 9s 393ms/step - loss: 0.2522 - categorical_accuracy: 0.9031\n",
            "Epoch 7/7\n",
            "22/22 [==============================] - 9s 396ms/step - loss: 0.1338 - categorical_accuracy: 0.9516\n",
            "10/10 [==============================] - 5s 235ms/step\n",
            "10/10 [==============================] - 2s 237ms/step\n",
            "RoBERTa\n",
            "\n",
            "\n",
            "*******************************************\n",
            "Currently running for size:  0.2\n",
            "Naive Bayes\n",
            "Decision Tree\n",
            "Logistic Regression\n",
            "Random Forest\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Some weights of the PyTorch model were not used when initializing the TF 2.0 model TFRobertaModel: ['lm_head.dense.bias', 'lm_head.bias', 'lm_head.layer_norm.weight', 'lm_head.layer_norm.bias', 'lm_head.dense.weight', 'roberta.embeddings.position_ids']\n",
            "- This IS expected if you are initializing TFRobertaModel from a PyTorch model trained on another task or with another architecture (e.g. initializing a TFBertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing TFRobertaModel from a PyTorch model that you expect to be exactly identical (e.g. initializing a TFBertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights or buffers of the TF 2.0 model TFRobertaModel were not initialized from the PyTorch model and are newly initialized: ['roberta.pooler.dense.weight', 'roberta.pooler.dense.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 1/7\n",
            "44/44 [==============================] - 63s 453ms/step - loss: 0.5654 - categorical_accuracy: 0.7450\n",
            "Epoch 2/7\n",
            "44/44 [==============================] - 18s 403ms/step - loss: 0.5420 - categorical_accuracy: 0.7407\n",
            "Epoch 3/7\n",
            "44/44 [==============================] - 17s 395ms/step - loss: 0.4412 - categorical_accuracy: 0.7892\n",
            "Epoch 4/7\n",
            "44/44 [==============================] - 17s 390ms/step - loss: 0.3357 - categorical_accuracy: 0.8575\n",
            "Epoch 5/7\n",
            "44/44 [==============================] - 17s 395ms/step - loss: 0.1938 - categorical_accuracy: 0.9231\n",
            "Epoch 6/7\n",
            "44/44 [==============================] - 17s 396ms/step - loss: 0.1318 - categorical_accuracy: 0.9587\n",
            "Epoch 7/7\n",
            "44/44 [==============================] - 17s 395ms/step - loss: 0.1012 - categorical_accuracy: 0.9687\n",
            "10/10 [==============================] - 5s 229ms/step\n",
            "10/10 [==============================] - 2s 229ms/step\n",
            "RoBERTa\n",
            "\n",
            "\n",
            "*******************************************\n",
            "Currently running for size:  0.3\n",
            "Naive Bayes\n",
            "Decision Tree\n",
            "Logistic Regression\n",
            "Random Forest\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Some weights of the PyTorch model were not used when initializing the TF 2.0 model TFRobertaModel: ['lm_head.dense.bias', 'lm_head.bias', 'lm_head.layer_norm.weight', 'lm_head.layer_norm.bias', 'lm_head.dense.weight', 'roberta.embeddings.position_ids']\n",
            "- This IS expected if you are initializing TFRobertaModel from a PyTorch model trained on another task or with another architecture (e.g. initializing a TFBertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing TFRobertaModel from a PyTorch model that you expect to be exactly identical (e.g. initializing a TFBertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights or buffers of the TF 2.0 model TFRobertaModel were not initialized from the PyTorch model and are newly initialized: ['roberta.pooler.dense.weight', 'roberta.pooler.dense.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 1/7\n",
            "66/66 [==============================] - 71s 444ms/step - loss: 0.5731 - categorical_accuracy: 0.7407\n",
            "Epoch 2/7\n",
            "66/66 [==============================] - 26s 397ms/step - loss: 0.4451 - categorical_accuracy: 0.7863\n",
            "Epoch 3/7\n",
            "66/66 [==============================] - 26s 391ms/step - loss: 0.3077 - categorical_accuracy: 0.8765\n",
            "Epoch 4/7\n",
            "66/66 [==============================] - 26s 394ms/step - loss: 0.1959 - categorical_accuracy: 0.9259\n",
            "Epoch 5/7\n",
            "66/66 [==============================] - 26s 394ms/step - loss: 0.1453 - categorical_accuracy: 0.9402\n",
            "Epoch 6/7\n",
            "66/66 [==============================] - 26s 391ms/step - loss: 0.0904 - categorical_accuracy: 0.9658\n",
            "Epoch 7/7\n",
            "66/66 [==============================] - 26s 392ms/step - loss: 0.0652 - categorical_accuracy: 0.9763\n",
            "10/10 [==============================] - 6s 235ms/step\n",
            "10/10 [==============================] - 2s 232ms/step\n",
            "RoBERTa\n",
            "\n",
            "\n",
            "*******************************************\n",
            "Currently running for size:  0.4\n",
            "Naive Bayes\n",
            "Decision Tree\n",
            "Logistic Regression\n",
            "Random Forest\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Some weights of the PyTorch model were not used when initializing the TF 2.0 model TFRobertaModel: ['lm_head.dense.bias', 'lm_head.bias', 'lm_head.layer_norm.weight', 'lm_head.layer_norm.bias', 'lm_head.dense.weight', 'roberta.embeddings.position_ids']\n",
            "- This IS expected if you are initializing TFRobertaModel from a PyTorch model trained on another task or with another architecture (e.g. initializing a TFBertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing TFRobertaModel from a PyTorch model that you expect to be exactly identical (e.g. initializing a TFBertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights or buffers of the TF 2.0 model TFRobertaModel were not initialized from the PyTorch model and are newly initialized: ['roberta.pooler.dense.weight', 'roberta.pooler.dense.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 1/7\n",
            "88/88 [==============================] - 80s 435ms/step - loss: 0.5775 - categorical_accuracy: 0.7452\n",
            "Epoch 2/7\n",
            "88/88 [==============================] - 35s 395ms/step - loss: 0.4469 - categorical_accuracy: 0.7865\n",
            "Epoch 3/7\n",
            "88/88 [==============================] - 35s 396ms/step - loss: 0.3165 - categorical_accuracy: 0.8733\n",
            "Epoch 4/7\n",
            "88/88 [==============================] - 34s 391ms/step - loss: 0.2242 - categorical_accuracy: 0.9117\n",
            "Epoch 5/7\n",
            "88/88 [==============================] - 35s 392ms/step - loss: 0.1272 - categorical_accuracy: 0.9559\n",
            "Epoch 6/7\n",
            "88/88 [==============================] - 35s 392ms/step - loss: 0.1403 - categorical_accuracy: 0.9530\n",
            "Epoch 7/7\n",
            "88/88 [==============================] - 34s 391ms/step - loss: 0.0772 - categorical_accuracy: 0.9722\n",
            "10/10 [==============================] - 5s 229ms/step\n",
            "10/10 [==============================] - 2s 233ms/step\n",
            "RoBERTa\n",
            "\n",
            "\n",
            "*******************************************\n",
            "Currently running for size:  0.5\n",
            "Naive Bayes\n",
            "Decision Tree\n",
            "Logistic Regression\n",
            "Random Forest\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Some weights of the PyTorch model were not used when initializing the TF 2.0 model TFRobertaModel: ['lm_head.dense.bias', 'lm_head.bias', 'lm_head.layer_norm.weight', 'lm_head.layer_norm.bias', 'lm_head.dense.weight', 'roberta.embeddings.position_ids']\n",
            "- This IS expected if you are initializing TFRobertaModel from a PyTorch model trained on another task or with another architecture (e.g. initializing a TFBertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing TFRobertaModel from a PyTorch model that you expect to be exactly identical (e.g. initializing a TFBertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights or buffers of the TF 2.0 model TFRobertaModel were not initialized from the PyTorch model and are newly initialized: ['roberta.pooler.dense.weight', 'roberta.pooler.dense.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 1/7\n",
            "110/110 [==============================] - 86s 428ms/step - loss: 0.5853 - categorical_accuracy: 0.7329\n",
            "Epoch 2/7\n",
            "110/110 [==============================] - 43s 392ms/step - loss: 0.4844 - categorical_accuracy: 0.7864\n",
            "Epoch 3/7\n",
            "110/110 [==============================] - 43s 395ms/step - loss: 0.3296 - categorical_accuracy: 0.8713\n",
            "Epoch 4/7\n",
            "110/110 [==============================] - 43s 390ms/step - loss: 0.2269 - categorical_accuracy: 0.9186\n",
            "Epoch 5/7\n",
            "110/110 [==============================] - 43s 392ms/step - loss: 0.1508 - categorical_accuracy: 0.9419\n",
            "Epoch 6/7\n",
            "110/110 [==============================] - 43s 393ms/step - loss: 0.1065 - categorical_accuracy: 0.9567\n",
            "Epoch 7/7\n",
            "110/110 [==============================] - 43s 391ms/step - loss: 0.0498 - categorical_accuracy: 0.9812\n",
            "10/10 [==============================] - 7s 229ms/step\n",
            "10/10 [==============================] - 2s 230ms/step\n",
            "RoBERTa\n",
            "\n",
            "\n",
            "*******************************************\n",
            "Currently running for size:  0.6\n",
            "Naive Bayes\n",
            "Decision Tree\n",
            "Logistic Regression\n",
            "Random Forest\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Some weights of the PyTorch model were not used when initializing the TF 2.0 model TFRobertaModel: ['lm_head.dense.bias', 'lm_head.bias', 'lm_head.layer_norm.weight', 'lm_head.layer_norm.bias', 'lm_head.dense.weight', 'roberta.embeddings.position_ids']\n",
            "- This IS expected if you are initializing TFRobertaModel from a PyTorch model trained on another task or with another architecture (e.g. initializing a TFBertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing TFRobertaModel from a PyTorch model that you expect to be exactly identical (e.g. initializing a TFBertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights or buffers of the TF 2.0 model TFRobertaModel were not initialized from the PyTorch model and are newly initialized: ['roberta.pooler.dense.weight', 'roberta.pooler.dense.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 1/7\n",
            "132/132 [==============================] - 99s 419ms/step - loss: 0.5629 - categorical_accuracy: 0.7356\n",
            "Epoch 2/7\n",
            "132/132 [==============================] - 52s 395ms/step - loss: 0.4109 - categorical_accuracy: 0.8282\n",
            "Epoch 3/7\n",
            "132/132 [==============================] - 52s 393ms/step - loss: 0.2861 - categorical_accuracy: 0.8889\n",
            "Epoch 4/7\n",
            "132/132 [==============================] - 52s 392ms/step - loss: 0.1977 - categorical_accuracy: 0.9250\n",
            "Epoch 5/7\n",
            "132/132 [==============================] - 52s 391ms/step - loss: 0.1284 - categorical_accuracy: 0.9549\n",
            "Epoch 6/7\n",
            "132/132 [==============================] - 52s 392ms/step - loss: 0.0912 - categorical_accuracy: 0.9696\n",
            "Epoch 7/7\n",
            "132/132 [==============================] - 52s 391ms/step - loss: 0.0584 - categorical_accuracy: 0.9815\n",
            "10/10 [==============================] - 5s 230ms/step\n",
            "10/10 [==============================] - 2s 233ms/step\n",
            "RoBERTa\n",
            "\n",
            "\n",
            "*******************************************\n",
            "Currently running for size:  0.7\n",
            "Naive Bayes\n",
            "Decision Tree\n",
            "Logistic Regression\n",
            "Random Forest\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Some weights of the PyTorch model were not used when initializing the TF 2.0 model TFRobertaModel: ['lm_head.dense.bias', 'lm_head.bias', 'lm_head.layer_norm.weight', 'lm_head.layer_norm.bias', 'lm_head.dense.weight', 'roberta.embeddings.position_ids']\n",
            "- This IS expected if you are initializing TFRobertaModel from a PyTorch model trained on another task or with another architecture (e.g. initializing a TFBertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing TFRobertaModel from a PyTorch model that you expect to be exactly identical (e.g. initializing a TFBertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights or buffers of the TF 2.0 model TFRobertaModel were not initialized from the PyTorch model and are newly initialized: ['roberta.pooler.dense.weight', 'roberta.pooler.dense.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 1/7\n",
            "154/154 [==============================] - 107s 417ms/step - loss: 0.5360 - categorical_accuracy: 0.7556\n",
            "Epoch 2/7\n",
            "154/154 [==============================] - 61s 396ms/step - loss: 0.3632 - categorical_accuracy: 0.8406\n",
            "Epoch 3/7\n",
            "154/154 [==============================] - 61s 393ms/step - loss: 0.2612 - categorical_accuracy: 0.8979\n",
            "Epoch 4/7\n",
            "154/154 [==============================] - 60s 392ms/step - loss: 0.1855 - categorical_accuracy: 0.9325\n",
            "Epoch 5/7\n",
            "154/154 [==============================] - 60s 392ms/step - loss: 0.1281 - categorical_accuracy: 0.9528\n",
            "Epoch 6/7\n",
            "154/154 [==============================] - 60s 392ms/step - loss: 0.0839 - categorical_accuracy: 0.9711\n",
            "Epoch 7/7\n",
            "154/154 [==============================] - 60s 392ms/step - loss: 0.0668 - categorical_accuracy: 0.9780\n",
            "10/10 [==============================] - 6s 239ms/step\n",
            "10/10 [==============================] - 2s 232ms/step\n",
            "RoBERTa\n",
            "\n",
            "\n",
            "*******************************************\n",
            "Currently running for size:  0.8\n",
            "Naive Bayes\n",
            "Decision Tree\n",
            "Logistic Regression\n",
            "Random Forest\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Some weights of the PyTorch model were not used when initializing the TF 2.0 model TFRobertaModel: ['lm_head.dense.bias', 'lm_head.bias', 'lm_head.layer_norm.weight', 'lm_head.layer_norm.bias', 'lm_head.dense.weight', 'roberta.embeddings.position_ids']\n",
            "- This IS expected if you are initializing TFRobertaModel from a PyTorch model trained on another task or with another architecture (e.g. initializing a TFBertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing TFRobertaModel from a PyTorch model that you expect to be exactly identical (e.g. initializing a TFBertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights or buffers of the TF 2.0 model TFRobertaModel were not initialized from the PyTorch model and are newly initialized: ['roberta.pooler.dense.weight', 'roberta.pooler.dense.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 1/7\n",
            "176/176 [==============================] - 116s 416ms/step - loss: 0.4900 - categorical_accuracy: 0.7851\n",
            "Epoch 2/7\n",
            "176/176 [==============================] - 69s 394ms/step - loss: 0.3387 - categorical_accuracy: 0.8630\n",
            "Epoch 3/7\n",
            "176/176 [==============================] - 69s 393ms/step - loss: 0.2426 - categorical_accuracy: 0.9064\n",
            "Epoch 4/7\n",
            "176/176 [==============================] - 69s 392ms/step - loss: 0.1812 - categorical_accuracy: 0.9356\n",
            "Epoch 5/7\n",
            "176/176 [==============================] - 69s 392ms/step - loss: 0.1224 - categorical_accuracy: 0.9559\n",
            "Epoch 6/7\n",
            "176/176 [==============================] - 69s 392ms/step - loss: 0.1038 - categorical_accuracy: 0.9641\n",
            "Epoch 7/7\n",
            "176/176 [==============================] - 69s 392ms/step - loss: 0.0679 - categorical_accuracy: 0.9726\n",
            "10/10 [==============================] - 5s 230ms/step\n",
            "10/10 [==============================] - 2s 231ms/step\n",
            "RoBERTa\n",
            "\n",
            "\n"
          ]
        }
      ],
      "source": [
        "results_df = pd.DataFrame(columns=['Model name', 'Train set size', 'Accuracy', 'f1_micro', 'f1_macro', 'f2_micro', 'f2_macro', 'logloss'])\n",
        "\n",
        "def create_model(bert_model, max_len):\n",
        "\n",
        "      opt = tf.keras.optimizers.Adam(learning_rate=1e-5)\n",
        "      loss = tf.keras.losses.CategoricalCrossentropy()\n",
        "      accuracy = tf.keras.metrics.CategoricalAccuracy()\n",
        "\n",
        "      input_ids = tf.keras.Input(shape=(max_len,),dtype='int32')\n",
        "      attention_masks = tf.keras.Input(shape=(max_len,),dtype='int32')\n",
        "      output = bert_model([input_ids,attention_masks])\n",
        "      output = output[1]\n",
        "      output = tf.keras.layers.Dense(2, activation=tf.nn.softmax)(output)\n",
        "      model = tf.keras.models.Model(inputs = [input_ids,attention_masks],outputs = output)\n",
        "      model.compile(opt, loss=loss, metrics=accuracy)\n",
        "      return model\n",
        "\n",
        "def tokenize_roberta(data, max_len):\n",
        "    input_ids = []\n",
        "    attention_masks = []\n",
        "    for i in range(len(data)):\n",
        "        encoded = tokenizer_roberta.encode_plus(\n",
        "            data[i],\n",
        "            add_special_tokens=True,\n",
        "            max_length=max_len,\n",
        "            truncation=True,  # Explicitly activate truncation\n",
        "            padding='max_length',  # Update padding argument\n",
        "            return_attention_mask=True\n",
        "        )\n",
        "        input_ids.append(encoded['input_ids'])\n",
        "        attention_masks.append(encoded['attention_mask'])\n",
        "    return np.array(input_ids), np.array(attention_masks)\n",
        "\n",
        "def conf_matrix(y, y_pred, title):\n",
        "      fig, ax =plt.subplots(figsize=(5,5))\n",
        "      labels=[0, 1]\n",
        "      ax=sns.heatmap(confusion_matrix(y, y_pred), annot=True, cmap=\"Blues\", fmt='g', cbar=False, annot_kws={\"size\":25})\n",
        "      plt.title(title, fontsize=20)\n",
        "      ax.xaxis.set_ticklabels(labels, fontsize=17)\n",
        "      ax.yaxis.set_ticklabels(labels, fontsize=17)\n",
        "      ax.set_ylabel('Test', fontsize=20)\n",
        "      ax.set_xlabel('Predicted', fontsize=20)\n",
        "      plt.show()\n",
        "\n",
        "\n",
        "for size in [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8]:\n",
        "  print(\"*******************************************\")\n",
        "  print(\"Currently running for size: \",size)\n",
        "  df = pd.read_csv('/content/drive/MyDrive/BioNLP Final project/Breast Cancer Data/Classification_ data/train.csv')\n",
        "  df_test = pd.read_csv('/content/drive/MyDrive/BioNLP Final project/Breast Cancer Data/Classification_ data/dev_.csv')\n",
        "\n",
        "  # df_train, temp = train_test_split(df, train_size= size, random_state= 20)\n",
        "\n",
        "  df['text'] = df['text'].apply(preprocess_text_optimized)\n",
        "  df['text'] = df['text'].apply(preprocess_text_optimized)\n",
        "\n",
        "  X = df['text'].values\n",
        "  y = df['Class'].values\n",
        "  X_train, X_valid, y_train, y_valid = train_test_split(X, y, train_size=size, stratify=y, random_state=seed)\n",
        "\n",
        "  X_test = df_test['text'].values\n",
        "  y_test = df_test['Class'].values\n",
        "\n",
        "  y_train_le = y_train.copy()\n",
        "  y_valid_le = y_valid.copy()\n",
        "  y_test_le = y_test.copy()\n",
        "\n",
        "  ohe = preprocessing.OneHotEncoder()\n",
        "  y_train = ohe.fit_transform(np.array(y_train).reshape(-1, 1)).toarray()\n",
        "  y_valid = ohe.fit_transform(np.array(y_valid).reshape(-1, 1)).toarray()\n",
        "  y_test = ohe.fit_transform(np.array(y_test).reshape(-1, 1)).toarray()\n",
        "\n",
        "  clf = CountVectorizer()\n",
        "  X_train_cv =  clf.fit_transform(X_train)\n",
        "  X_test_cv = clf.transform(X_test)\n",
        "\n",
        "  tf_transformer = TfidfTransformer(use_idf=True).fit(X_train_cv)\n",
        "  X_train_tf = tf_transformer.transform(X_train_cv)\n",
        "  X_test_tf = tf_transformer.transform(X_test_cv)\n",
        "\n",
        "  # NAIVE BAYES\n",
        "  nb_clf = MultinomialNB()\n",
        "  nb_clf.fit(X_train_tf, y_train_le)\n",
        "  nb_pred = nb_clf.predict(X_test_tf)\n",
        "  results_df.loc[len(results_df.index)] = ['Naive bayes', size, accuracy_score(y_test.argmax(1), nb_pred),\n",
        "                                           f1_score(y_test.argmax(1), nb_pred, average='micro'),\n",
        "                                           f1_score(y_test.argmax(1), nb_pred, average='macro'),\n",
        "                                           fbeta_score(y_test.argmax(1), nb_pred, beta=2, average='micro'),\n",
        "                                           fbeta_score(y_test.argmax(1), nb_pred, beta=2, average='macro'),\n",
        "                                           log_loss(y_test, nb_clf.predict_proba(X_test_tf))]\n",
        "  print(\"Naive Bayes\")\n",
        "  # DECISION TREE\n",
        "  d_model = DecisionTreeClassifier(criterion='gini', max_depth=10)\n",
        "  d_model.fit(X_train_tf, y_train_le)\n",
        "  d_pred = d_model.predict(X_test_tf)\n",
        "  results_df.loc[len(results_df.index)] = ['Decision Tree', size, accuracy_score(y_test.argmax(1), d_pred),\n",
        "                                           f1_score(y_test.argmax(1), d_pred, average='micro'),\n",
        "                                           f1_score(y_test.argmax(1), d_pred, average='macro'),\n",
        "                                           fbeta_score(y_test.argmax(1), d_pred, beta=2, average='micro'),\n",
        "                                           fbeta_score(y_test.argmax(1), d_pred, beta=2, average='macro'),\n",
        "                                           log_loss(y_test, d_model.predict_proba(X_test_tf))]\n",
        "  print(\"Decision Tree\")\n",
        "\n",
        "  # LOGISTIC REGRESSION\n",
        "  l_model = LogisticRegression(C=10, penalty='l2')\n",
        "  l_model.fit(X_train_tf, y_train_le)\n",
        "  l_pred = l_model.predict(X_test_tf)\n",
        "  results_df.loc[len(results_df.index)] = ['Logistic Regression', size, accuracy_score(y_test.argmax(1), l_pred),\n",
        "                                           f1_score(y_test.argmax(1), l_pred, average='micro'),\n",
        "                                           f1_score(y_test.argmax(1), l_pred, average='macro'),\n",
        "                                           fbeta_score(y_test.argmax(1), l_pred, beta=2, average='micro'),\n",
        "                                           fbeta_score(y_test.argmax(1), l_pred, beta=2, average='macro'),\n",
        "                                           log_loss(y_test, l_model.predict_proba(X_test_tf))]\n",
        "  print(\"Logistic Regression\")\n",
        "\n",
        "  # RANDOM FOREST\n",
        "  r_model = RandomForestClassifier(max_depth=None, n_estimators=50)\n",
        "  r_model.fit(X_train_tf, y_train_le)\n",
        "  r_pred = r_model.predict(X_test_tf)\n",
        "  results_df.loc[len(results_df.index)] = ['Random Forest', size, accuracy_score(y_test.argmax(1), r_pred),\n",
        "                                           f1_score(y_test.argmax(1), r_pred, average='micro'),\n",
        "                                           f1_score(y_test.argmax(1), r_pred, average='macro'),\n",
        "                                           fbeta_score(y_test.argmax(1), r_pred, beta=2, average='micro'),\n",
        "                                           fbeta_score(y_test.argmax(1), r_pred, beta=2, average='macro'),\n",
        "                                           log_loss(y_test, r_model.predict_proba(X_test_tf))]\n",
        "\n",
        "  print(\"Random Forest\")\n",
        "\n",
        "  # ROBERTA\n",
        "  tokenizer_roberta = RobertaTokenizerFast.from_pretrained(\"roberta-base\")\n",
        "  token_lens = []\n",
        "  for txt in X_train:\n",
        "      tokens = tokenizer_roberta.encode(txt, max_length=512, truncation=True)\n",
        "      token_lens.append(len(tokens))\n",
        "  max_length=np.max(token_lens)\n",
        "  MAX_LEN=max_length\n",
        "\n",
        "  train_input_ids, train_attention_masks = tokenize_roberta(X_train, MAX_LEN)\n",
        "  val_input_ids, val_attention_masks = tokenize_roberta(X_valid, MAX_LEN)\n",
        "  test_input_ids, test_attention_masks = tokenize_roberta(X_test, MAX_LEN)\n",
        "  roberta_model = TFRobertaModel.from_pretrained('roberta-base')\n",
        "\n",
        "  model = create_model(roberta_model, MAX_LEN)\n",
        "  # model.summary()\n",
        "  # history =\n",
        "  # model.fit([train_input_ids,train_attention_masks], y_train, validation_data=([val_input_ids,val_attention_masks], y_valid),\n",
        "                        # epochs=7, batch_size=16)\n",
        "  model.fit([train_input_ids,train_attention_masks], y_train, epochs=7, batch_size=16)\n",
        "\n",
        "  result_roberta = model.predict([test_input_ids,test_attention_masks])\n",
        "  y_pred_roberta =  np.zeros_like(result_roberta)\n",
        "  y_pred_roberta[np.arange(len(y_pred_roberta)), result_roberta.argmax(1)] = 1\n",
        "\n",
        "  # conf_matrix(y_test.argmax(1),y_pred_roberta.argmax(1),'RoBERTa Confusion Matrix')\n",
        "  # print('\\tClassification Report for RoBERTa:\\n\\n',classification_report(y_test,y_pred_roberta, target_names=['Not self identified', 'Self-identified']))\n",
        "  results_df.loc[len(results_df.index)] = ['RoBERTa', size, accuracy_score(y_test.argmax(1),y_pred_roberta.argmax(1)),\n",
        "                                           f1_score(y_test.argmax(1),y_pred_roberta.argmax(1), average='micro'),\n",
        "                                           f1_score(y_test.argmax(1),y_pred_roberta.argmax(1), average='macro'),\n",
        "                                           fbeta_score(y_test.argmax(1),y_pred_roberta.argmax(1), beta=2, average='micro'),\n",
        "                                           fbeta_score(y_test.argmax(1),y_pred_roberta.argmax(1), beta=2, average='macro'),\n",
        "                                           log_loss(y_test, model.predict([test_input_ids,test_attention_masks]))]\n",
        "  print(\"RoBERTa\")\n",
        "  print(\"\\n\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "OhsCiUxhsIO9"
      },
      "outputs": [],
      "source": [
        "results_df.to_csv('result_dev.csv',index=False)"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "093b4bd4f48e4acd8549da281b269fe9": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0f7b4bc5ebf24a87bbf5d37019b13166",
            "max": 481,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_9a81396477ce4ccbac79725a40cef76f",
            "value": 481
          }
        },
        "0f0222c78f414d6295bb20e7dcaa6d8b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0f7b4bc5ebf24a87bbf5d37019b13166": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1028fc38fc954540a463d2e94493e332": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_32184de396534b26b4011a827640c026",
            "placeholder": "​",
            "style": "IPY_MODEL_22007d9a04964878a428d9aeec77e211",
            "value": " 481/481 [00:00&lt;00:00, 6.83kB/s]"
          }
        },
        "1282255fd5db46daa976e6bffafa88f4": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_56670582df7246b8b76eff2aad576265",
              "IPY_MODEL_c1231df639c543d69ded72dc73d019b7",
              "IPY_MODEL_5d974ed579be4696af147814999adb58"
            ],
            "layout": "IPY_MODEL_6757da3f75de4f88a67973fef405ce58"
          }
        },
        "128adf5e6f6c456bb0d22afca78d8a9a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "12b28312a0584a32a12f3916d77cb7f8": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1c497309b70d43579b542b836e59ffd4": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "21b2ae5667a44067a7d67987ee5a7897": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "22007d9a04964878a428d9aeec77e211": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "316ff69ef16c4494994fef46eaadfbf4": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "32184de396534b26b4011a827640c026": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "37366a210fae45788a7dff405051525c": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "381c08e84f5d4f5b9ffe981e0d6f77a7": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5066198fc8774f39a529b5a260bc2b1d": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "56670582df7246b8b76eff2aad576265": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5066198fc8774f39a529b5a260bc2b1d",
            "placeholder": "​",
            "style": "IPY_MODEL_0f0222c78f414d6295bb20e7dcaa6d8b",
            "value": "merges.txt: 100%"
          }
        },
        "5d974ed579be4696af147814999adb58": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_af73e7823d1a4158a2131ad4b1ea6d21",
            "placeholder": "​",
            "style": "IPY_MODEL_128adf5e6f6c456bb0d22afca78d8a9a",
            "value": " 456k/456k [00:00&lt;00:00, 2.24MB/s]"
          }
        },
        "60ffbe5df5a041fc879950ea7c5004bf": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_f53bfa2e4c324490a7ca3f18d6ef1053",
              "IPY_MODEL_cc576aedd0f44f8087bd1b7dbd53784d",
              "IPY_MODEL_a85e9e0a7cd64cd7b2fc1283f618e4af"
            ],
            "layout": "IPY_MODEL_f016b364387f4e708fdcf6487d18d698"
          }
        },
        "6757da3f75de4f88a67973fef405ce58": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "68a79118c69a4564b28ca67d631a719f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_e5972e9f088341ecaa72b51afd91fcd5",
              "IPY_MODEL_093b4bd4f48e4acd8549da281b269fe9",
              "IPY_MODEL_1028fc38fc954540a463d2e94493e332"
            ],
            "layout": "IPY_MODEL_e1432fd10a8a4b4ca982d67c3a019ed7"
          }
        },
        "68bf97233181443cb346b5f26816eb7e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_eac479b597ef497e94625dce07e7b8d8",
            "placeholder": "​",
            "style": "IPY_MODEL_21b2ae5667a44067a7d67987ee5a7897",
            "value": " 1.36M/1.36M [00:00&lt;00:00, 5.08MB/s]"
          }
        },
        "6a75f6a9f5ef4d79b589a9dc752c61e3": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "73200837e1c347049ec865c24a923564": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "74952293ee4949d9af72db325281b441": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6a75f6a9f5ef4d79b589a9dc752c61e3",
            "placeholder": "​",
            "style": "IPY_MODEL_381c08e84f5d4f5b9ffe981e0d6f77a7",
            "value": "tokenizer.json: 100%"
          }
        },
        "81f44880f3084670abc38eeb23dee176": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "9a81396477ce4ccbac79725a40cef76f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "9cc5848453e54cf9ae5fe9dd7fe64f41": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a628bd1d43544640bfe25ae661ea8a7c": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a85e9e0a7cd64cd7b2fc1283f618e4af": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_12b28312a0584a32a12f3916d77cb7f8",
            "placeholder": "​",
            "style": "IPY_MODEL_73200837e1c347049ec865c24a923564",
            "value": " 899k/899k [00:00&lt;00:00, 4.29MB/s]"
          }
        },
        "a95b87fdc7c84c28b7fa203ebe3afb88": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "a96ae2edc14f473590350dd2de9a2dd1": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "af73e7823d1a4158a2131ad4b1ea6d21": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c1231df639c543d69ded72dc73d019b7": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9cc5848453e54cf9ae5fe9dd7fe64f41",
            "max": 456318,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_81f44880f3084670abc38eeb23dee176",
            "value": 456318
          }
        },
        "c57d560e577b4dd2a87f4393c67d9063": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "cc576aedd0f44f8087bd1b7dbd53784d": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a628bd1d43544640bfe25ae661ea8a7c",
            "max": 898823,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_c57d560e577b4dd2a87f4393c67d9063",
            "value": 898823
          }
        },
        "ce31928053c442d29e92d6d3d7d30eb4": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d652807b23e843258a447be5c359e80d": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_74952293ee4949d9af72db325281b441",
              "IPY_MODEL_dded31be4c1a4b919d009c2a8bbdd331",
              "IPY_MODEL_68bf97233181443cb346b5f26816eb7e"
            ],
            "layout": "IPY_MODEL_1c497309b70d43579b542b836e59ffd4"
          }
        },
        "d9064c13ffd14a7e8d5e3b778b210d4d": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "dded31be4c1a4b919d009c2a8bbdd331": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a96ae2edc14f473590350dd2de9a2dd1",
            "max": 1355863,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_a95b87fdc7c84c28b7fa203ebe3afb88",
            "value": 1355863
          }
        },
        "e1432fd10a8a4b4ca982d67c3a019ed7": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e5972e9f088341ecaa72b51afd91fcd5": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ce31928053c442d29e92d6d3d7d30eb4",
            "placeholder": "​",
            "style": "IPY_MODEL_37366a210fae45788a7dff405051525c",
            "value": "config.json: 100%"
          }
        },
        "eac479b597ef497e94625dce07e7b8d8": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f016b364387f4e708fdcf6487d18d698": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f53bfa2e4c324490a7ca3f18d6ef1053": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d9064c13ffd14a7e8d5e3b778b210d4d",
            "placeholder": "​",
            "style": "IPY_MODEL_316ff69ef16c4494994fef46eaadfbf4",
            "value": "vocab.json: 100%"
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}