LbrAnalysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "c42d1f79",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import packages\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib as plt\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import sent_tokenize, word_tokenize\n",
    "from nltk.stem import PorterStemmer\n",
    "from nltk import RegexpParser\n",
    "from nltk.corpus import words\n",
    "import re\n",
    "from textblob import TextBlob\n",
    "from wordcloud import WordCloud"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "97251dcb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reading and subsetting the data\n",
    "\n",
    "df = pd.read_excel('C:/Users/User/Desktop/Python/Projects/Labour_Party_Conf_Tweets/MASTERLBRPTY.xlsx')\n",
    "\n",
    "twt = pd.DataFrame(df[\"tweet\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8b5610c9",
   "metadata": {},
   "source": [
    "**What is a tokenizer? Just a way\n",
    "to group language e.g words, sentences, paras**\n",
    "\n",
    "**What is a corpora - body of text. In this case, tweets**\n",
    "\n",
    "**lexicon - words and their means\n",
    "Defining word and sentence tokenizer**\n",
    "\n",
    "**a stopword is a word that is a filler / doesn't carry\n",
    "much meaning on its own**"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1b304ab0",
   "metadata": {},
   "source": [
    "## Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "611723c6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# -- Make tweet column string so it can be processed\n",
    "\n",
    "twt['tweet'] = twt['tweet'].to_string()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "663fdb05",
   "metadata": {},
   "outputs": [
    {
     "ename": "MemoryError",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mMemoryError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_14160/2623226797.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[1;31m# -- Remove @ tags\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 5\u001b[1;33m \u001b[0mtwt\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'tweet_proc'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtwt\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'tweet'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreplace\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34mr'@\\S+'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m''\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mregex\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minplace\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      6\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      7\u001b[0m \u001b[1;31m# -- Remove Urls\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Desktop\\Python\\Projects\\Labour_Party_Conf_Tweets\\venv\\lib\\site-packages\\pandas\\core\\series.py\u001b[0m in \u001b[0;36mreplace\u001b[1;34m(self, to_replace, value, inplace, limit, regex, method)\u001b[0m\n\u001b[0;32m   4866\u001b[0m         \u001b[0mmethod\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m\"pad\"\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   4867\u001b[0m     ):\n\u001b[1;32m-> 4868\u001b[1;33m         return super().replace(\n\u001b[0m\u001b[0;32m   4869\u001b[0m             \u001b[0mto_replace\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mto_replace\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   4870\u001b[0m             \u001b[0mvalue\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mvalue\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Desktop\\Python\\Projects\\Labour_Party_Conf_Tweets\\venv\\lib\\site-packages\\pandas\\core\\generic.py\u001b[0m in \u001b[0;36mreplace\u001b[1;34m(self, to_replace, value, inplace, limit, regex, method)\u001b[0m\n\u001b[0;32m   6607\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   6608\u001b[0m                 \u001b[1;32melif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mis_list_like\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m  \u001b[1;31m# NA -> 0\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 6609\u001b[1;33m                     new_data = self._mgr.replace(\n\u001b[0m\u001b[0;32m   6610\u001b[0m                         \u001b[0mto_replace\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mto_replace\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mvalue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minplace\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0minplace\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mregex\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mregex\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   6611\u001b[0m                     )\n",
      "\u001b[1;32m~\\Desktop\\Python\\Projects\\Labour_Party_Conf_Tweets\\venv\\lib\\site-packages\\pandas\\core\\internals\\managers.py\u001b[0m in \u001b[0;36mreplace\u001b[1;34m(self, to_replace, value, inplace, regex)\u001b[0m\n\u001b[0;32m    435\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mreplace\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mT\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mto_replace\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minplace\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mbool\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mregex\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mbool\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m->\u001b[0m \u001b[0mT\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    436\u001b[0m         \u001b[1;32massert\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mndim\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 437\u001b[1;33m         return self.apply(\n\u001b[0m\u001b[0;32m    438\u001b[0m             \u001b[1;34m\"replace\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mto_replace\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mto_replace\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mvalue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minplace\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0minplace\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mregex\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mregex\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    439\u001b[0m         )\n",
      "\u001b[1;32m~\\Desktop\\Python\\Projects\\Labour_Party_Conf_Tweets\\venv\\lib\\site-packages\\pandas\\core\\internals\\managers.py\u001b[0m in \u001b[0;36mapply\u001b[1;34m(self, f, align_keys, ignore_failures, **kwargs)\u001b[0m\n\u001b[0;32m    325\u001b[0m                     \u001b[0mapplied\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mb\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    326\u001b[0m                 \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 327\u001b[1;33m                     \u001b[0mapplied\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mb\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    328\u001b[0m             \u001b[1;32mexcept\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mTypeError\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mNotImplementedError\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    329\u001b[0m                 \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mignore_failures\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Desktop\\Python\\Projects\\Labour_Party_Conf_Tweets\\venv\\lib\\site-packages\\pandas\\core\\internals\\blocks.py\u001b[0m in \u001b[0;36mreplace\u001b[1;34m(self, to_replace, value, inplace, regex)\u001b[0m\n\u001b[0;32m    685\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    686\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mregex\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 687\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_replace_regex\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mto_replace\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minplace\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0minplace\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    688\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    689\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_can_hold_element\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mto_replace\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Desktop\\Python\\Projects\\Labour_Party_Conf_Tweets\\venv\\lib\\site-packages\\pandas\\core\\internals\\blocks.py\u001b[0m in \u001b[0;36m_replace_regex\u001b[1;34m(self, to_replace, value, inplace, convert, mask)\u001b[0m\n\u001b[0;32m    758\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    759\u001b[0m         \u001b[0mnew_values\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0minplace\u001b[0m \u001b[1;32melse\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 760\u001b[1;33m         \u001b[0mreplace_regex\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnew_values\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmask\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    761\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    762\u001b[0m         \u001b[0mblock\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmake_block\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnew_values\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Desktop\\Python\\Projects\\Labour_Party_Conf_Tweets\\venv\\lib\\site-packages\\pandas\\core\\array_algos\\replace.py\u001b[0m in \u001b[0;36mreplace_regex\u001b[1;34m(values, rx, value, mask)\u001b[0m\n\u001b[0;32m    153\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    154\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mmask\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 155\u001b[1;33m         \u001b[0mvalues\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    156\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    157\u001b[0m         \u001b[0mvalues\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mmask\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mmask\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Desktop\\Python\\Projects\\Labour_Party_Conf_Tweets\\venv\\lib\\site-packages\\numpy\\lib\\function_base.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m   2161\u001b[0m             \u001b[0mvargs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0m_n\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0m_n\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mnames\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2162\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2163\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_vectorize_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mfunc\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mvargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2164\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2165\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_get_ufunc_and_otypes\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Desktop\\Python\\Projects\\Labour_Party_Conf_Tweets\\venv\\lib\\site-packages\\numpy\\lib\\function_base.py\u001b[0m in \u001b[0;36m_vectorize_call\u001b[1;34m(self, func, args)\u001b[0m\n\u001b[0;32m   2244\u001b[0m             \u001b[0minputs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0masanyarray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ma\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mobject\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0ma\u001b[0m \u001b[1;32min\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2245\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2246\u001b[1;33m             \u001b[0moutputs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mufunc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0minputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2247\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2248\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mufunc\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnout\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Desktop\\Python\\Projects\\Labour_Party_Conf_Tweets\\venv\\lib\\site-packages\\pandas\\core\\array_algos\\replace.py\u001b[0m in \u001b[0;36mre_replacer\u001b[1;34m(s)\u001b[0m\n\u001b[0;32m    146\u001b[0m         \u001b[1;32mdef\u001b[0m \u001b[0mre_replacer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    147\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mis_re\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrx\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstr\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 148\u001b[1;33m                 \u001b[1;32mreturn\u001b[0m \u001b[0mrx\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msub\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0ms\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    149\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    150\u001b[0m                 \u001b[1;32mreturn\u001b[0m \u001b[0ms\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mMemoryError\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# -- Processing tweets\n",
    "\n",
    "# -- Remove @ tags\n",
    "\n",
    "twt['tweet_proc'] = twt['tweet'].replace(r'@\\S+', '',regex=True, inplace = True)\n",
    "\n",
    "# -- Remove Urls\n",
    "\n",
    "twt['tweet_proc'] = twt['tweet_proc'].replace(r'http\\S+', '',regex=True, inplace = True)\n",
    "\n",
    "# -- Make lower case\n",
    "twt['tweet_proc'] = twt['tweet'].str.lower()\n",
    "\n",
    "# -- Remove punctuation\n",
    "twt['tweet_proc'] = twt['tweet_proc'].str.replace('[^\\w\\s]','')\n",
    "\n",
    "# -- Remove underscores\n",
    "twt['tweet_proc'] = twt['tweet_proc'].str.replace(\"_\", \" \")\n",
    "\n",
    "# -- Remove numbers\n",
    "twt['tweet_proc'] = twt['tweet_proc'].str.replace('\\d+', '')\n",
    "\n",
    "# -- Define and remove stopwords\n",
    "\n",
    "stop_words = stopwords.words('english')\n",
    "custom_stop_words = ['labour', 'party', 'the', 'like', 'https', 'get','the','conference', 'im']\n",
    "\n",
    "for word in custom_stop_words:\n",
    "    stop_words.append(word)\n",
    "\n",
    "twt['tweet_proc'] = twt['tweet_proc'].apply(lambda x: ' '.join([word for word in x.split() if word not in (stop_words)]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b50b9b98",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a function to get the subjectivity\n",
    "def getSubjectivity(text):\n",
    "    return TextBlob(text).sentiment.subjectivity\n",
    "\n",
    "# Create a function to get the polarity\n",
    "\n",
    "def getPolarity(text):\n",
    "    return TextBlob(text).sentiment.polarity\n",
    "\n",
    "\n",
    "# Create two new columns\n",
    "\n",
    "twt['Subjectivity'] = twt['tweet_proc'].apply(getSubjectivity)\n",
    "twt['Polarity'] = twt['tweet_proc'].apply(getPolarity)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "40f49821",
   "metadata": {},
   "outputs": [],
   "source": [
    "allWords = ' '.join([tweets for tweets in twt['tweet_proc']])\n",
    "wordcloud = WordCloud(width = 500, height = 300, random_state = 21, max_font_size = 119).generate(allWords)\n",
    "\n",
    "plt.pyplot.imshow(wordcloud, interpolation = \"bilinear\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6a9dd34f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a function to compute the negative, neutral and positive analysis\n",
    "\n",
    "def getAnalysis(score):\n",
    "    if score <0:\n",
    "        return 'Negative'\n",
    "    elif score == 0:\n",
    "        return 'Neutral'\n",
    "    else:\n",
    "        return 'Positive'\n",
    "    \n",
    "twt['Polarity Analysis'] = twt['Polarity'].apply(getAnalysis)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ced72b4c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Count values in Polarity Analysis\n",
    "\n",
    "twt['Polarity Analysis'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ba8a66c6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# View most negative and most positive tweets\n",
    "\n",
    "twt.sort_values(by= 'Polarity')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cde48d74",
   "metadata": {},
   "outputs": [],
   "source": [
    "twt.to_csv('twt.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e019c1db",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "lbrkrn",
   "language": "python",
   "name": "lbrkrn"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}