DEMO.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CLASSIFICATION OF OPEN/CLOSED QUESTIONS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import math\n",
    "import nltk\n",
    "import re\n",
    "import string\n",
    "import textstat\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.metrics import precision_score\n",
    "from sklearn.metrics import recall_score\n",
    "from sklearn.metrics import f1_score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## READING TEST DATA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "demoData=pd.read_csv(\"demoData.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(100, 20)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "demoData.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Unnamed: 0', 'PostId', 'UserId', 'Comment', 'Body', 'Title', 'Tags',\n",
       "       'Reputation', 'experienceInTime', 'WebsiteUrl', 'Location', 'AboutMe',\n",
       "       'Views', 'Upvotes', 'Downvotes', 'ProfileImageUrl', 'tagCount',\n",
       "       'closed', 'numAns', 'numQues'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "demoData.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>PostId</th>\n",
       "      <th>UserId</th>\n",
       "      <th>Comment</th>\n",
       "      <th>Body</th>\n",
       "      <th>Title</th>\n",
       "      <th>Tags</th>\n",
       "      <th>Reputation</th>\n",
       "      <th>experienceInTime</th>\n",
       "      <th>WebsiteUrl</th>\n",
       "      <th>Location</th>\n",
       "      <th>AboutMe</th>\n",
       "      <th>Views</th>\n",
       "      <th>Upvotes</th>\n",
       "      <th>Downvotes</th>\n",
       "      <th>ProfileImageUrl</th>\n",
       "      <th>tagCount</th>\n",
       "      <th>closed</th>\n",
       "      <th>numAns</th>\n",
       "      <th>numQues</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>95161</td>\n",
       "      <td>34963087</td>\n",
       "      <td>5717469</td>\n",
       "      <td>102</td>\n",
       "      <td>&lt;p&gt;I have a problem with the code here.&lt;/p&gt;\\r\\...</td>\n",
       "      <td>'Var' could not be found C#</td>\n",
       "      <td>&lt;c#&gt;&lt;database&gt;&lt;visual-studio&gt;&lt;web-services&gt;</td>\n",
       "      <td>84</td>\n",
       "      <td>29</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>56</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>https://www.gravatar.com/avatar/24a8fb8eadb72a...</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>32355</td>\n",
       "      <td>24818505</td>\n",
       "      <td>188826</td>\n",
       "      <td>100</td>\n",
       "      <td>&lt;p&gt;I have set up Unity Bootstrapper for ASP.NE...</td>\n",
       "      <td>Unity PerRequestLifetimeManager SaveChanges</td>\n",
       "      <td>.net|asp.net-mvc|unity-container</td>\n",
       "      <td>5931</td>\n",
       "      <td>1739</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Russia</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1293</td>\n",
       "      <td>1558</td>\n",
       "      <td>18</td>\n",
       "      <td>https://i.stack.imgur.com/MMZTJ.jpg?s=128&amp;g=1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>36</td>\n",
       "      <td>479</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>122429</td>\n",
       "      <td>40822135</td>\n",
       "      <td>7214092</td>\n",
       "      <td>103</td>\n",
       "      <td>&lt;p&gt;In C++, the largest int value is 2147483647...</td>\n",
       "      <td>multiply two arrays like if they were number</td>\n",
       "      <td>&lt;c++&gt;&lt;arrays&gt;&lt;logic&gt;&lt;multiplication&gt;</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>https://graph.facebook.com/1842386039367885/pi...</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>50556</td>\n",
       "      <td>35023135</td>\n",
       "      <td>5083109</td>\n",
       "      <td>100</td>\n",
       "      <td>&lt;p&gt;I have started looking into Magento 2 grid....</td>\n",
       "      <td>Magento 2 grid structure</td>\n",
       "      <td>grid|magento2</td>\n",
       "      <td>81</td>\n",
       "      <td>205</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>&lt;p&gt;I am normal human being. \\r\\r\\r\\r\\r\\r\\r\\nSo...</td>\n",
       "      <td>20</td>\n",
       "      <td>30</td>\n",
       "      <td>0</td>\n",
       "      <td>https://www.gravatar.com/avatar/2bc11248162ab5...</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>58870</td>\n",
       "      <td>32220498</td>\n",
       "      <td>4920133</td>\n",
       "      <td>100</td>\n",
       "      <td>&lt;p&gt;On press of headset need to bring my applic...</td>\n",
       "      <td>registerMediaButtonEventReceiver Alternative s...</td>\n",
       "      <td>android|android-intent|android-activity|androi...</td>\n",
       "      <td>126</td>\n",
       "      <td>98</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>23</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>https://www.gravatar.com/avatar/e23a15fdcc50c0...</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0    PostId   UserId  Comment  \\\n",
       "0       95161  34963087  5717469      102   \n",
       "1       32355  24818505   188826      100   \n",
       "2      122429  40822135  7214092      103   \n",
       "3       50556  35023135  5083109      100   \n",
       "4       58870  32220498  4920133      100   \n",
       "\n",
       "                                                Body  \\\n",
       "0  <p>I have a problem with the code here.</p>\\r\\...   \n",
       "1  <p>I have set up Unity Bootstrapper for ASP.NE...   \n",
       "2  <p>In C++, the largest int value is 2147483647...   \n",
       "3  <p>I have started looking into Magento 2 grid....   \n",
       "4  <p>On press of headset need to bring my applic...   \n",
       "\n",
       "                                               Title  \\\n",
       "0                        'Var' could not be found C#   \n",
       "1        Unity PerRequestLifetimeManager SaveChanges   \n",
       "2       multiply two arrays like if they were number   \n",
       "3                           Magento 2 grid structure   \n",
       "4  registerMediaButtonEventReceiver Alternative s...   \n",
       "\n",
       "                                                Tags  Reputation  \\\n",
       "0        <c#><database><visual-studio><web-services>          84   \n",
       "1                   .net|asp.net-mvc|unity-container        5931   \n",
       "2               <c++><arrays><logic><multiplication>           3   \n",
       "3                                      grid|magento2          81   \n",
       "4  android|android-intent|android-activity|androi...         126   \n",
       "\n",
       "   experienceInTime WebsiteUrl Location  \\\n",
       "0                29        NaN      NaN   \n",
       "1              1739        NaN   Russia   \n",
       "2                 0        NaN      NaN   \n",
       "3               205        NaN      NaN   \n",
       "4                98        NaN      NaN   \n",
       "\n",
       "                                             AboutMe  Views  Upvotes  \\\n",
       "0                                                NaN     56        9   \n",
       "1                                                NaN   1293     1558   \n",
       "2                                                NaN      2        0   \n",
       "3  <p>I am normal human being. \\r\\r\\r\\r\\r\\r\\r\\nSo...     20       30   \n",
       "4                                                NaN     23        1   \n",
       "\n",
       "   Downvotes                                    ProfileImageUrl  tagCount  \\\n",
       "0          0  https://www.gravatar.com/avatar/24a8fb8eadb72a...         4   \n",
       "1         18      https://i.stack.imgur.com/MMZTJ.jpg?s=128&g=1         3   \n",
       "2          0  https://graph.facebook.com/1842386039367885/pi...         4   \n",
       "3          0  https://www.gravatar.com/avatar/2bc11248162ab5...         2   \n",
       "4          0  https://www.gravatar.com/avatar/e23a15fdcc50c0...         5   \n",
       "\n",
       "   closed  numAns  numQues  \n",
       "0       1       1       16  \n",
       "1       0      36      479  \n",
       "2       1       0        2  \n",
       "3       0      10        1  \n",
       "4       0       8        7  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "demoData.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## FEATURE ENGINEERING"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Credits: https://stackoverflow.com/questions/4576077/python-split-text-on-sentences\n",
    "\n",
    "alphabets= \"([A-Za-z])\"\n",
    "prefixes = \"(Mr|St|Mrs|Ms|Dr)[.]\"\n",
    "suffixes = \"(Inc|Ltd|Jr|Sr|Co)\"\n",
    "starters = \"(Mr|Mrs|Ms|Dr|He\\s|She\\s|It\\s|They\\s|Their\\s|Our\\s|We\\s|But\\s|However\\s|That\\s|This\\s|Wherever)\"\n",
    "acronyms = \"([A-Z][.][A-Z][.](?:[A-Z][.])?)\"\n",
    "websites = \"[.](com|net|org|io|gov)\"\n",
    "digits = \"([0-9])\" \n",
    "\n",
    "def split_into_sentences(text):\n",
    "    #text=body\n",
    "    text=striphtml(text)\n",
    "    text = \" \" + text + \"  \"\n",
    "    text = text.replace(\"\\n\",\" \")\n",
    "    text = re.sub(prefixes,\"\\\\1<prd>\",text)\n",
    "    text = re.sub(websites,\"<prd>\\\\1\",text)\n",
    "    if \"Ph.D\" in text: text = text.replace(\"Ph.D.\",\"Ph<prd>D<prd>\")\n",
    "    text = re.sub(\"\\s\" + alphabets + \"[.] \",\" \\\\1<prd> \",text)\n",
    "    text = re.sub(acronyms+\" \"+starters,\"\\\\1<stop> \\\\2\",text)\n",
    "    text = re.sub(alphabets + \"[.]\" + alphabets + \"[.]\" + alphabets + \"[.]\",\"\\\\1<prd>\\\\2<prd>\\\\3<prd>\",text)\n",
    "    text = re.sub(alphabets + \"[.]\" + alphabets + \"[.]\",\"\\\\1<prd>\\\\2<prd>\",text)\n",
    "    text = re.sub(\" \"+suffixes+\"[.] \"+starters,\" \\\\1<stop> \\\\2\",text)\n",
    "    text = re.sub(\" \"+suffixes+\"[.]\",\" \\\\1<prd>\",text)\n",
    "    text = re.sub(\" \" + alphabets + \"[.]\",\" \\\\1<prd>\",text)\n",
    "    text = re.sub(digits + \"[.]\" + digits,\"\\\\1<prd>\\\\2\",text)\n",
    "    if \"”\" in text: text = text.replace(\".”\",\"”.\")\n",
    "    if \"\\\"\" in text: text = text.replace(\".\\\"\",\"\\\".\")\n",
    "    if \"!\" in text: text = text.replace(\"!\\\"\",\"\\\"!\")\n",
    "    if \"?\" in text: text = text.replace(\"?\\\"\",\"\\\"?\")\n",
    "    text = text.replace(\".\",\".<stop>\")\n",
    "    text = text.replace(\"?\",\"?<stop>\")\n",
    "    text = text.replace(\"!\",\"!<stop>\")\n",
    "    text = text.replace(\"<prd>\",\".\")\n",
    "    sentences = text.split(\"<stop>\")\n",
    "    sentences = sentences[:-1]\n",
    "    sentences = [s.strip() for s in sentences]\n",
    "    #print(text)\n",
    "    return sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#METHODS FOR FEATURE CREATION\n",
    "\n",
    "#Removes all occurence of <code>    \n",
    "def removeCode(text):\n",
    "    #make sure you don't strip HTML beforehand\n",
    "    textWithoutCode=re.sub(r\"<code>(.*?)</code>\",\"\",text,flags= re.DOTALL)\n",
    "    #print(codes)\n",
    "    return textWithoutCode\n",
    "\n",
    "#Removes HTML from query data but doesn't remove the content bw opening and closing tag\n",
    "#Attributes are removed eg:<a href=\"this is all removed\">not removed</a>\n",
    "#https://stackoverflow.com/questions/3398852/using-python-remove-html-tags-formatting-from-a-string\n",
    "def striphtml(data):\n",
    "    p = re.compile(r'<.*?>')\n",
    "    return p.sub('', data)\n",
    "  \n",
    "    \n",
    "#Returns number of lowercase chars in the text\n",
    "def lowercaseCount(text):\n",
    "    text=removeCode(text)\n",
    "    text=striphtml(text)\n",
    "    lowercount=0\n",
    "    try:\n",
    "        for w in text:\n",
    "            for char in w:\n",
    "                if(char.islower()):\n",
    "                    lowercount+=1\n",
    "        return lowercount\n",
    "    except:\n",
    "        #print(type(text))\n",
    "        print(text)\n",
    "\n",
    "#Returns number of uppercase chars in the text\n",
    "def uppercaseCount(text):\n",
    "    text=removeCode(text)\n",
    "    text=striphtml(text)\n",
    "    uppercount=0\n",
    "    try:    \n",
    "        for w in text:\n",
    "            for char in w:\n",
    "                if(char.isupper()):\n",
    "                    uppercount+=1\n",
    "        return uppercount\n",
    "    except:\n",
    "        print(text)\n",
    "    \n",
    "    \n",
    "#Count of sentences in the post body\n",
    "def getSentenceCount(text):\n",
    "    text=removeCode(text)\n",
    "    return len(split_into_sentences(text))\n",
    "\n",
    "def getFirstLineLength(text):\n",
    "    text=removeCode(text)\n",
    "    sentences=split_into_sentences(text)\n",
    "    #print(len(sentences))\n",
    "    if(len(sentences)==0):\n",
    "        #print(text)\n",
    "        return 0\n",
    "    firstLine=sentences[0]\n",
    "    return len(firstLine)\n",
    " \n",
    "#Returns the length of total code present in the post\n",
    "def codeLength(text):\n",
    "    #make sure you don't strip HTML beforehand\n",
    "    codes=re.findall(r\"<code>(.*?)</code>\",text,flags= re.DOTALL)\n",
    "    #print(codes)\n",
    "    return len(''.join(codes))\n",
    "\n",
    "#https://stackoverflow.com/questions/6883049/regex-to-extract-urls-from-href-attribute-in-html-with-python\n",
    "#https://stackoverflow.com/questions/1374457/find-out-how-many-times-a-regex-matches-in-a-string-in-python\n",
    "#Returns count of url in a given post\n",
    "def urlCount(text):\n",
    "    urls=re.findall(r'https://?|ftp://',text)\n",
    "    #print(urls)\n",
    "    return len(urls)\n",
    "\n",
    "\n",
    "#Returns count of URls pointing to Stack Overflow \n",
    "def SOUrlCount(text):\n",
    "    SOUrls=re.findall(r'https://stackoverflow.com',text)\n",
    "    #print(SOUrls)\n",
    "    return len(SOUrls)\n",
    "       \n",
    "    \n",
    "def countOfInterrogativeSent(text):\n",
    "    text=text.lower()\n",
    "    #tokens=text.split() #1D list of words \n",
    "    interrogatives=re.findall(r'\\b[a-z]*[^.!][?]',text)\n",
    "    return len(interrogatives)\n",
    "    \n",
    "    \n",
    "\n",
    "def sentencesStartWithYouCount(text):\n",
    "    count=0\n",
    "    text=text.lower()\n",
    "    #text=re.sub('<.*>','',text)\n",
    "    text=removeCode(text)\n",
    "    text=striphtml(text)\n",
    "    sentences=split_into_sentences(text)\n",
    "    for sentence in sentences:\n",
    "        tokens=re.split('[\\s;\\',.\\-\\%]',sentence)\n",
    "        #print(tokens)\n",
    "        if(tokens[0]==\"you\"):\n",
    "            #print(\"token0=\"+tokens[0])\n",
    "            count+=1\n",
    "    return count;\n",
    "\n",
    "def sentencesStartWithICount(text):\n",
    "    count=0\n",
    "    text=text.lower()\n",
    "    text=removeCode(text)\n",
    "    text=striphtml(text)\n",
    "    sentences=split_into_sentences(text)\n",
    "    for sentence in sentences:\n",
    "        tokens=re.split('[\\s;\\',.\\-\\%]',sentence)\n",
    "        #print(tokens)\n",
    "        if(tokens[0]==\"i\"):\n",
    "            #print(\"token0=\"+tokens[0])\n",
    "            count+=1\n",
    "        if(len(tokens)>=2): #But I don't have, So I switched \n",
    "            if(tokens[1]==\"i\"):\n",
    "                count+=1\n",
    "        if(len(tokens)>=3):\n",
    "            if(tokens[2]==\"i\"): #As such I had amended...\n",
    "                count+=1\n",
    "    return count;\n",
    "\n",
    "\n",
    "#https://stackoverflow.com/questions/12628958/remove-small-words-using-python\n",
    "def countShortWords(text):\n",
    "    text=removeCode(text)\n",
    "    text=striphtml(text)\n",
    "    shortwords = re.findall(r'\\W*\\b\\w{1,3}\\b',text)\n",
    "    #print(shortwords)\n",
    "    return len(shortwords)\n",
    "\n",
    "   \n",
    "#Returns count of total number of words in the query\n",
    "#https://stackoverflow.com/questions/19410018/how-to-count-the-number-of-words-in-a-sentence-ignoring-numbers-punctuation-an\n",
    "def countWords(text):\n",
    "    text=removeCode(text)\n",
    "    text=striphtml(text)\n",
    "    return len(text.split())\n",
    "\n",
    "\n",
    "#Returns count of punctuations in the post body\n",
    "#https://stackoverflow.com/questions/265960/best-way-to-strip-punctuation-from-a-string-in-python\n",
    "def countPunctuations(text):\n",
    "    #remove all code in the post\n",
    "    text=re.sub('<code>.*?</code>','',text)\n",
    "    \n",
    "    text=striphtml(text)  #will also remove all URLs as URL is mentioned in href=\" \" attribute\n",
    "    puncs=re.findall(r'[^\\w\\s]',text)\n",
    "    #print(puncs)\n",
    "    return len(puncs)\n",
    "\n",
    "def makeBinary(text):\n",
    "    if(pd.isnull(text)):\n",
    "        return 0\n",
    "    return 1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Adding necessary feature cJolumns\n",
    "#https://stackoverflow.com/questions/40045632/adding-a-column-in-pandas-df-using-a-function\n",
    "\n",
    "demoData['lowerUpperRatio']=(demoData['Body'].apply(lowercaseCount))/(demoData['Body'].apply(uppercaseCount))\n",
    "\n",
    "demoData['sentenceCount']=demoData['Body'].apply(getSentenceCount)\n",
    "\n",
    "demoData['firstLineLength']=demoData['Body'].apply(getFirstLineLength)\n",
    "\n",
    "demoData['codeLength']=demoData['Body'].apply(codeLength)\n",
    "\n",
    "demoData['urlCount']=demoData['Body'].apply(urlCount)\n",
    "\n",
    "demoData['SOUrlCount']=demoData['Body'].apply(SOUrlCount)\n",
    "\n",
    "demoData['titleLengthInChars']=demoData['Title'].apply(len)\n",
    "\n",
    "demoData['countOfInterrogativeSent']=demoData['Body'].apply(countOfInterrogativeSent)\n",
    "\n",
    "demoData['sentencesStartWithYouCount']=demoData['Body'].apply(sentencesStartWithYouCount)\n",
    "demoData['sentencesStartWithICount']=demoData['Body'].apply(sentencesStartWithICount)\n",
    "\n",
    "demoData['shortWordCount']=demoData['Body'].apply(countShortWords)\n",
    "demoData['bodyWordCount']=demoData['Body'].apply(countWords)\n",
    "demoData['punctuationCount']=demoData['Body'].apply(countPunctuations)\n",
    "\n",
    "demoData['websiteUrlFilled']=demoData['WebsiteUrl'].apply(makeBinary)\n",
    "demoData['locationFilled']=demoData['Location'].apply(makeBinary) \n",
    "demoData['aboutMeFilled']=demoData['AboutMe'].apply(makeBinary)\n",
    "demoData['profileImageUrlFilled']=demoData['ProfileImageUrl'].apply(makeBinary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "tagWeightData=pd.read_csv(\"tagWeightData.csv\")\n",
    "#print(tagWeightData.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "idx=0\n",
    "def calculateTagWeight(tags):\n",
    "    forClosed=re.findall('<(.*?)>',tags)\n",
    "    forOpen=tags.split('|')\n",
    "    #print(forClosed)\n",
    "    #print(forOpen)\n",
    "    if(len(forClosed) ):  #if we pass tags for open in close it won't work, but vice versa will work hence \n",
    "        # this particular order of if-else \n",
    "        tagList=forClosed\n",
    "    elif(len(forOpen) ):\n",
    "        tagList=forOpen\n",
    "    #print(tagList)\n",
    "    wt=0\n",
    "#     for t in tagList:\n",
    "#         #print(t)\n",
    "#         try:\n",
    "            \n",
    "#         except:\n",
    "#             print(t)\n",
    "#         print(wt)\n",
    "#     print(\"check\")\n",
    "#     if(idx%1000==0)print(idx)\n",
    "#     idx+=1\n",
    "    try:\n",
    "        for t in tagList:\n",
    "            wt=wt+tagWeightData[tagWeightData['TagName']==t][['tagWeight']].values[0]\n",
    "    except:\n",
    "        #print(tagList)\n",
    "        pass\n",
    "    return wt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "demoData['tagWeightSum']=demoData['Tags'].apply(calculateTagWeight) #tagClosingWeightSum"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gunningFog(text):\n",
    "    text=removeCode(text)\n",
    "    text=striphtml(text)\n",
    "    return textstat.gunning_fog(text)\n",
    "\n",
    "def fleschReadingEase(text):\n",
    "    text=removeCode(text)\n",
    "    text=striphtml(text)\n",
    "    return textstat.flesch_reading_ease(text)\n",
    "\n",
    "def daleChallReadabilityScore(text):\n",
    "    text=removeCode(text)\n",
    "    text=striphtml(text)\n",
    "    return textstat.dale_chall_readability_score(text)\n",
    "\n",
    "\n",
    "def smogIndex(text):\n",
    "    text=removeCode(text)\n",
    "    text=striphtml(text)\n",
    "    return textstat.smog_index(text)\n",
    "\n",
    "def colemanLiauIndex(text):\n",
    "    text=removeCode(text)\n",
    "    text=striphtml(text)\n",
    "    return textstat.coleman_liau_index(text)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#https://en.wikipedia.org/wiki/Gunning_fog_index\n",
    "demoData['gunningFog']=demoData['Body'].apply(gunningFog)\n",
    "#https://en.wikipedia.org/wiki/Flesch%E2%80%93Kincaid_readability_tests#Flesch_reading_ease\n",
    "demoData['fleschReadingEase']=demoData['Body'].apply(fleschReadingEase)\n",
    "#https://en.wikipedia.org/wiki/Dale%E2%80%93Chall_readability_formula\n",
    "demoData['daleChallReadabilityScore']=demoData['Body'].apply(daleChallReadabilityScore)\n",
    "#https://en.wikipedia.org/wiki/SMOG\n",
    "demoData['smogIndex']=demoData['Body'].apply(smogIndex)\n",
    "#https://en.wikipedia.org/wiki/Coleman%E2%80%93Liau_index\n",
    "demoData['colemanLiauIndex']=demoData['Body'].apply(colemanLiauIndex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   Unnamed: 0    PostId   UserId  Comment  \\\n",
      "0       95161  34963087  5717469      102   \n",
      "1       32355  24818505   188826      100   \n",
      "2      122429  40822135  7214092      103   \n",
      "3       50556  35023135  5083109      100   \n",
      "4       58870  32220498  4920133      100   \n",
      "\n",
      "                                                Body  \\\n",
      "0  <p>I have a problem with the code here.</p>\\r\\...   \n",
      "1  <p>I have set up Unity Bootstrapper for ASP.NE...   \n",
      "2  <p>In C++, the largest int value is 2147483647...   \n",
      "3  <p>I have started looking into Magento 2 grid....   \n",
      "4  <p>On press of headset need to bring my applic...   \n",
      "\n",
      "                                               Title  \\\n",
      "0                        'Var' could not be found C#   \n",
      "1        Unity PerRequestLifetimeManager SaveChanges   \n",
      "2       multiply two arrays like if they were number   \n",
      "3                           Magento 2 grid structure   \n",
      "4  registerMediaButtonEventReceiver Alternative s...   \n",
      "\n",
      "                                                Tags  Reputation  \\\n",
      "0        <c#><database><visual-studio><web-services>          84   \n",
      "1                   .net|asp.net-mvc|unity-container        5931   \n",
      "2               <c++><arrays><logic><multiplication>           3   \n",
      "3                                      grid|magento2          81   \n",
      "4  android|android-intent|android-activity|androi...         126   \n",
      "\n",
      "   experienceInTime WebsiteUrl        ...         locationFilled  \\\n",
      "0                29        NaN        ...                      0   \n",
      "1              1739        NaN        ...                      1   \n",
      "2                 0        NaN        ...                      0   \n",
      "3               205        NaN        ...                      0   \n",
      "4                98        NaN        ...                      0   \n",
      "\n",
      "  aboutMeFilled  profileImageUrlFilled      tagWeightSum  gunningFog  \\\n",
      "0             0                      1  [-213.884824499]        6.02   \n",
      "1             0                      1  [-292.309497846]        9.29   \n",
      "2             0                      1  [-136.484954326]       13.93   \n",
      "3             1                      1   [-413.57561729]        8.28   \n",
      "4             0                      1  [-625.499637315]        9.87   \n",
      "\n",
      "  fleschReadingEase  daleChallReadabilityScore  smogIndex  colemanLiauIndex  \\\n",
      "0            102.00                       5.85        6.7              9.84   \n",
      "1             70.63                       7.21        9.5              7.60   \n",
      "2             41.23                       7.67       13.6             14.63   \n",
      "3             62.98                       6.36       11.9              6.09   \n",
      "4             53.21                       7.34        9.4              9.45   \n",
      "\n",
      "   userProfileFilled  \n",
      "0                  1  \n",
      "1                  2  \n",
      "2                  1  \n",
      "3                  2  \n",
      "4                  1  \n",
      "\n",
      "[5 rows x 44 columns]\n"
     ]
    }
   ],
   "source": [
    "demoData['userProfileFilled']=demoData['websiteUrlFilled']+demoData['locationFilled']+demoData['aboutMeFilled']+demoData['profileImageUrlFilled']\n",
    "print(demoData.head())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### FROM 20 Columns to 44 Columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Unnamed: 0', 'PostId', 'UserId', 'Comment', 'Body', 'Title', 'Tags',\n",
       "       'Reputation', 'experienceInTime', 'WebsiteUrl', 'Location', 'AboutMe',\n",
       "       'Views', 'Upvotes', 'Downvotes', 'ProfileImageUrl', 'tagCount',\n",
       "       'closed', 'numAns', 'numQues', 'lowerUpperRatio', 'sentenceCount',\n",
       "       'firstLineLength', 'codeLength', 'urlCount', 'SOUrlCount',\n",
       "       'titleLengthInChars', 'countOfInterrogativeSent',\n",
       "       'sentencesStartWithYouCount', 'sentencesStartWithICount',\n",
       "       'shortWordCount', 'bodyWordCount', 'punctuationCount',\n",
       "       'websiteUrlFilled', 'locationFilled', 'aboutMeFilled',\n",
       "       'profileImageUrlFilled', 'tagWeightSum', 'gunningFog',\n",
       "       'fleschReadingEase', 'daleChallReadabilityScore', 'smogIndex',\n",
       "       'colemanLiauIndex', 'userProfileFilled'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "demoData.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(100, 44)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "demoData.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = demoData.loc[:,['Reputation', 'experienceInTime', 'Views', 'Upvotes', 'Downvotes', 'closed',\n",
    "       'tagCount', 'lowerUpperRatio', 'sentenceCount', 'firstLineLength',\n",
    "       'codeLength', 'urlCount', 'SOUrlCount', 'titleLengthInChars',\n",
    "       'countOfInterrogativeSent', 'sentencesStartWithYouCount',\n",
    "       'sentencesStartWithICount', 'shortWordCount', 'bodyWordCount',\n",
    "       'punctuationCount', 'tagWeightSum', 'gunningFog',\n",
    "       'fleschReadingEase', 'daleChallReadabilityScore', 'smogIndex',\n",
    "       'colemanLiauIndex', 'userProfileFilled', 'numAns', 'numQues'] ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of X: (100, 28)\n",
      "Shape of y: (100,)\n"
     ]
    }
   ],
   "source": [
    "y = X.closed\n",
    "X.drop(['closed'],inplace=True,axis=1)\n",
    "print(\"Shape of X: \"+str(X.shape))\n",
    "print(\"Shape of y: \"+str(y.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['Reputation', 'experienceInTime', 'Views', 'Upvotes', 'Downvotes',\n",
      "       'tagCount', 'lowerUpperRatio', 'sentenceCount', 'firstLineLength',\n",
      "       'codeLength', 'urlCount', 'SOUrlCount', 'titleLengthInChars',\n",
      "       'countOfInterrogativeSent', 'sentencesStartWithYouCount',\n",
      "       'sentencesStartWithICount', 'shortWordCount', 'bodyWordCount',\n",
      "       'punctuationCount', 'tagWeightSum', 'gunningFog', 'fleschReadingEase',\n",
      "       'daleChallReadabilityScore', 'smogIndex', 'colemanLiauIndex',\n",
      "       'userProfileFilled', 'numAns', 'numQues'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "print(X.columns)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## FEATURE SCALING"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   Reputation  experienceInTime  Views  Upvotes  Downvotes  tagCount  \\\n",
      "0          84                29     56        9          0         4   \n",
      "1        5931              1739   1293     1558         18         3   \n",
      "2           3                 0      2        0          0         4   \n",
      "3          81               205     20       30          0         2   \n",
      "4         126                98     23        1          0         5   \n",
      "\n",
      "   lowerUpperRatio  sentenceCount  firstLineLength  codeLength   ...    \\\n",
      "0        25.375000              7               36        1340   ...     \n",
      "1         7.941176              7               41         140   ...     \n",
      "2        48.625000             13               44           0   ...     \n",
      "3        17.090909              5               43           0   ...     \n",
      "4        20.684211              9              175         648   ...     \n",
      "\n",
      "   punctuationCount      tagWeightSum  gunningFog  fleschReadingEase  \\\n",
      "0                95  [-213.884824499]        6.02             102.00   \n",
      "1                11  [-292.309497846]        9.29              70.63   \n",
      "2                28  [-136.484954326]       13.93              41.23   \n",
      "3                 7   [-413.57561729]        8.28              62.98   \n",
      "4                15  [-625.499637315]        9.87              53.21   \n",
      "\n",
      "   daleChallReadabilityScore  smogIndex  colemanLiauIndex  userProfileFilled  \\\n",
      "0                       5.85        6.7              9.84                  1   \n",
      "1                       7.21        9.5              7.60                  2   \n",
      "2                       7.67       13.6             14.63                  1   \n",
      "3                       6.36       11.9              6.09                  2   \n",
      "4                       7.34        9.4              9.45                  1   \n",
      "\n",
      "   numAns numQues  \n",
      "0       1      16  \n",
      "1      36     479  \n",
      "2       0       2  \n",
      "3      10       1  \n",
      "4       8       7  \n",
      "\n",
      "[5 rows x 28 columns]\n",
      "   Reputation  experienceInTime     Views   Upvotes  Downvotes  tagCount  \\\n",
      "0   -0.178673         -0.821868 -0.142746 -0.279562  -0.065531  0.892056   \n",
      "1    0.361482          2.099543  0.577799  1.714028  -0.011646  0.076813   \n",
      "2   -0.186156         -0.871412 -0.174200 -0.291145  -0.065531  0.892056   \n",
      "3   -0.178950         -0.521185 -0.163716 -0.252535  -0.065531 -0.738430   \n",
      "4   -0.174793         -0.703986 -0.161968 -0.289858  -0.065531  1.707299   \n",
      "\n",
      "   lowerUpperRatio  sentenceCount  firstLineLength  codeLength    ...     \\\n",
      "0        -0.178379       0.093750        -0.682281    0.535068    ...      \n",
      "1        -0.854698       0.093750        -0.628835   -0.297213    ...      \n",
      "2         0.723569       0.635033        -0.596767   -0.394312    ...      \n",
      "3        -0.499748      -0.086677        -0.607456   -0.394312    ...      \n",
      "4        -0.360351       0.274178         0.803518    0.055119    ...      \n",
      "\n",
      "   punctuationCount       tagWeightSum  gunningFog  fleschReadingEase  \\\n",
      "0          0.361896   [0.145015141573]   -1.164173           1.289365   \n",
      "1         -0.226899  [-0.220883121519]   -0.869478           0.442915   \n",
      "2         -0.107738   [0.506132081581]   -0.451316          -0.350380   \n",
      "3         -0.254937  [-0.786662494041]   -0.960500           0.236496   \n",
      "4         -0.198861   [-1.77541549135]   -0.817208          -0.027126   \n",
      "\n",
      "   daleChallReadabilityScore  smogIndex  colemanLiauIndex  userProfileFilled  \\\n",
      "0                  -1.017733   0.165816          0.003027          -0.486306   \n",
      "1                  -0.277197   0.659986         -0.384015           0.413572   \n",
      "2                  -0.026722   1.383592          0.830677          -0.486306   \n",
      "3                  -0.740032   1.083560         -0.644924           0.413572   \n",
      "4                  -0.206411   0.642337         -0.064360          -0.486306   \n",
      "\n",
      "     numAns   numQues  \n",
      "0 -0.166139 -0.256931  \n",
      "1  0.018325  3.493053  \n",
      "2 -0.171410 -0.370322  \n",
      "3 -0.118706 -0.378421  \n",
      "4 -0.129246 -0.329825  \n",
      "\n",
      "[5 rows x 28 columns]\n"
     ]
    }
   ],
   "source": [
    "#(x-u)/std\n",
    "mean=[2.01807416e+03,   5.10066908e+02,   3.01059583e+02,   2.26216980e+02,\n",
    "   2.18902723e+01,   2.90577889e+00,   2.99731747e+01,   5.96079902e+00,\n",
    "   9.98289770e+01,   5.68528057e+02,   1.91942299e-01,   1.54941374e-02,\n",
    "   5.12481187e+01,   1.19541739e+00,   8.63771594e-03,   2.02549439e+00,\n",
    "   4.09886685e+01,   8.53872034e+01,   4.33703530e+01,  -2.44966593e+02,\n",
    "   1.89378949e+01,   5.42152991e+01,   7.71907479e+00,   5.76047464e+00,\n",
    "   9.82247931e+00,   1.54041351e+00,   3.25230693e+01,   4.77225618e+01]\n",
    "\n",
    "\n",
    "var=[1.17173655e+08,   3.42615432e+05,   2.94724961e+06,   6.03714126e+05,\n",
    "   1.11587341e+05,   1.50461647e+00,   6.64480756e+02,   1.22872198e+02,\n",
    "   8.75204965e+03,   2.07884824e+06,   8.66455736e-01,   2.93294209e-02,\n",
    "   4.03492388e+02,   4.61051576e+00,   1.12006717e-02,   2.81215161e+00,\n",
    "   1.21652922e+03,   4.70156550e+03,   2.03530533e+04,   4.59393389e+04,\n",
    "   1.23125524e+02,   1.37349155e+03,   3.37276468e+00,   3.21043352e+01,\n",
    "   3.34948687e+01,   1.23490391e+00,   3.60008314e+04,   1.52441460e+04]\n",
    "\n",
    "\n",
    "print(X.head())\n",
    "X=(X-mean)/np.sqrt(var)\n",
    "X.replace(np.nan,0,inplace=True)  #replacing NaN to 0\n",
    "X.lowerUpperRatio.replace(np.inf,0,inplace=True)  #replacing infinity to 0\n",
    "print(X.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(mean)\n",
    "# print(np.sqrt(var))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## IMPORTING PRETRAINED MODEL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "from joblib import dump, load\n",
    "classifier1=load('rbf_svm_open_close.joblib') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SVC(C=12, cache_size=200, class_weight=None, coef0=0.0,\n",
      "  decision_function_shape='ovr', degree=3, gamma='auto', kernel='rbf',\n",
      "  max_iter=-1, probability=False, random_state=None, shrinking=True,\n",
      "  tol=0.001, verbose=False)\n"
     ]
    }
   ],
   "source": [
    "print(classifier1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PREDICTION"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Actual Predicted\n",
      "1\t1\n",
      "0\t0\n",
      "1\t1\n",
      "0\t0\n",
      "0\t0\n",
      "0\t0\n",
      "0\t0\n",
      "1\t1\n",
      "1\t1\n",
      "0\t0\n",
      "0\t0\n",
      "0\t0\n",
      "1\t1\n",
      "1\t0\n",
      "0\t0\n",
      "1\t1\n",
      "0\t0\n",
      "1\t0\n",
      "0\t0\n",
      "1\t1\n",
      "1\t0\n",
      "1\t1\n",
      "0\t1\n",
      "0\t0\n",
      "1\t1\n"
     ]
    }
   ],
   "source": [
    "y_pred=classifier1.predict(X)\n",
    "print(\"Actual Predicted\")\n",
    "for i in range(0,25):\n",
    "    print(str(y[i])+\"\\t\"+str(y_pred[i]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MODEL PERFORMANCE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.87\n",
      "Precision: 0.876409017713\n",
      "Recall: 0.877777777778\n",
      "F1 Score:0.8699869987\n"
     ]
    }
   ],
   "source": [
    "print(\"Accuracy: \"+str(np.mean(y_pred==y)))\n",
    "print(\"Precision: \"+str(precision_score(y, y_pred, average='macro')) )\n",
    "print(\"Recall: \"+str(recall_score(y, y_pred, average='macro')) )\n",
    "print(\"F1 Score:\"+str(f1_score(y, y_pred, average='macro')) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# wholeData=pd.read_csv(\"openCloseData9.csv\")\n",
    "# print(wholeData.shape)\n",
    "# sample=wholeData.sample(n=100)\n",
    "# print(sample.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sample = sample[['PostId', 'UserId', 'Comment', 'Body', 'Title', 'Tags',\n",
    "#        'Reputation', 'experienceInTime', 'WebsiteUrl', 'Location', 'AboutMe',\n",
    "#        'Views', 'Upvotes', 'Downvotes', 'ProfileImageUrl', 'tagCount',\n",
    "#        'closed', 'numAns', 'numQues']]\n",
    "# print(sample.shape)\n",
    "# sample.to_csv(\"demoData.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}