{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ***Setup***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.utils import shuffle\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "import contractions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import os\n",
    "parent_dir = os.path.abspath('..')\n",
    "sys.path.append(parent_dir)\n",
    "from constants import CONSTANTS\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(CONSTANTS.DATASET_PATH)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ***0- Check if dataset is loaded correctly***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert df.shape == (1200, 3), f\"Expected shape (1200, 3), but got {df.shape}\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>label</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Psoriasis</td>\n",
       "      <td>I have been experiencing a skin rash on my arm...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Psoriasis</td>\n",
       "      <td>My skin has been peeling, especially on my kne...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>Psoriasis</td>\n",
       "      <td>I have been experiencing joint pain in my fing...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Psoriasis</td>\n",
       "      <td>There is a silver like dusting on my skin, esp...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Psoriasis</td>\n",
       "      <td>My nails have small dents or pits in them, and...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0      label                                               text\n",
       "0           0  Psoriasis  I have been experiencing a skin rash on my arm...\n",
       "1           1  Psoriasis  My skin has been peeling, especially on my kne...\n",
       "2           2  Psoriasis  I have been experiencing joint pain in my fing...\n",
       "3           3  Psoriasis  There is a silver like dusting on my skin, esp...\n",
       "4           4  Psoriasis  My nails have small dents or pits in them, and..."
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ***1- Refactor dataset columns so we have text and label only***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>I have been experiencing a skin rash on my arm...</td>\n",
       "      <td>Psoriasis</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>My skin has been peeling, especially on my kne...</td>\n",
       "      <td>Psoriasis</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>I have been experiencing joint pain in my fing...</td>\n",
       "      <td>Psoriasis</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>There is a silver like dusting on my skin, esp...</td>\n",
       "      <td>Psoriasis</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>My nails have small dents or pits in them, and...</td>\n",
       "      <td>Psoriasis</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text      label\n",
       "0  I have been experiencing a skin rash on my arm...  Psoriasis\n",
       "1  My skin has been peeling, especially on my kne...  Psoriasis\n",
       "2  I have been experiencing joint pain in my fing...  Psoriasis\n",
       "3  There is a silver like dusting on my skin, esp...  Psoriasis\n",
       "4  My nails have small dents or pits in them, and...  Psoriasis"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df.drop(columns=['Unnamed: 0'])\n",
    "df = df[['text', 'label']]\n",
    "\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ***2- Check for duplicates***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of duplicates found: 47\n",
      "Shape of after removing duplicates: (1153, 2)\n"
     ]
    }
   ],
   "source": [
    "duplicates = df[df.duplicated()]\n",
    "print(f\"Number of duplicates found: {len(duplicates)}\")\n",
    "\n",
    "# Remove duplicates\n",
    "df = df.drop_duplicates()\n",
    "print(f\"Shape of after removing duplicates: {df.shape}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ***3- Check for nulls***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Count of null values in each column:\n",
      "text     0\n",
      "label    0\n",
      "dtype: int64\n",
      "Shape of the DataFrame after removing null values: (1153, 2)\n"
     ]
    }
   ],
   "source": [
    "null_values = df.isnull().sum()\n",
    "print(\"Count of null values in each column:\")\n",
    "print(null_values)\n",
    "\n",
    "# Remove rows with null values\n",
    "df = df.dropna()\n",
    "print(f\"Shape of the DataFrame after removing null values: {df.shape}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ***4- Shuffle dataset***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>I've had back pain, a cough that won't go away...</td>\n",
       "      <td>Cervical spondylosis</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Recently, I've been having problems using the ...</td>\n",
       "      <td>Dimorphic Hemorrhoids</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>I've been exhausted and experiencing nausea an...</td>\n",
       "      <td>Jaundice</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>I've had a nagging cough that won't go away, a...</td>\n",
       "      <td>Bronchial Asthma</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>I've been quite exhausted and ill. My throat h...</td>\n",
       "      <td>Common Cold</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text                  label\n",
       "0  I've had back pain, a cough that won't go away...   Cervical spondylosis\n",
       "1  Recently, I've been having problems using the ...  Dimorphic Hemorrhoids\n",
       "2  I've been exhausted and experiencing nausea an...               Jaundice\n",
       "3  I've had a nagging cough that won't go away, a...       Bronchial Asthma\n",
       "4  I've been quite exhausted and ill. My throat h...            Common Cold"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = shuffle(df, random_state=42).reset_index(drop=True)\n",
    "\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ***5- Count unique texts for each label***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Count of unique text values for each label\n",
      "label\n",
      "Jaundice                           38\n",
      "Dimorphic Hemorrhoids              41\n",
      "Malaria                            44\n",
      "Acne                               46\n",
      "Arthritis                          46\n",
      "Migraine                           47\n",
      "Pneumonia                          47\n",
      "gastroesophageal reflux disease    48\n",
      "Cervical spondylosis               49\n",
      "Chicken pox                        49\n",
      "Bronchial Asthma                   49\n",
      "Common Cold                        49\n",
      "Impetigo                           50\n",
      "peptic ulcer disease               50\n",
      "Fungal infection                   50\n",
      "Dengue                             50\n",
      "Psoriasis                          50\n",
      "Typhoid                            50\n",
      "Varicose Veins                     50\n",
      "allergy                            50\n",
      "diabetes                           50\n",
      "drug reaction                      50\n",
      "Hypertension                       50\n",
      "urinary tract infection            50\n",
      "Name: text, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "unique_text_counts = df.groupby('label')['text'].nunique()\n",
    "unique_text_counts = unique_text_counts.sort_values()\n",
    "\n",
    "print(\"Count of unique text values for each label\")\n",
    "print(unique_text_counts)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### You can validate that sum of previous counts is 1153"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ***6- Convert into lowercase***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>i've had back pain, a cough that won't go away...</td>\n",
       "      <td>cervical spondylosis</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>recently, i've been having problems using the ...</td>\n",
       "      <td>dimorphic hemorrhoids</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>i've been exhausted and experiencing nausea an...</td>\n",
       "      <td>jaundice</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>i've had a nagging cough that won't go away, a...</td>\n",
       "      <td>bronchial asthma</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>i've been quite exhausted and ill. my throat h...</td>\n",
       "      <td>common cold</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text                  label\n",
       "0  i've had back pain, a cough that won't go away...   cervical spondylosis\n",
       "1  recently, i've been having problems using the ...  dimorphic hemorrhoids\n",
       "2  i've been exhausted and experiencing nausea an...               jaundice\n",
       "3  i've had a nagging cough that won't go away, a...       bronchial asthma\n",
       "4  i've been quite exhausted and ill. my throat h...            common cold"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['text'] = df['text'].str.lower()\n",
    "df['label'] = df['label'].str.lower()\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ***7- Expand contractions***\n",
    "**i.e. I'm => I am**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>i have had back pain, a cough that will not go...</td>\n",
       "      <td>cervical spondylosis</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>recently, i have been having problems using th...</td>\n",
       "      <td>dimorphic hemorrhoids</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>i have been exhausted and experiencing nausea ...</td>\n",
       "      <td>jaundice</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>i have had a nagging cough that will not go aw...</td>\n",
       "      <td>bronchial asthma</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>i have been quite exhausted and ill. my throat...</td>\n",
       "      <td>common cold</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text                  label\n",
       "0  i have had back pain, a cough that will not go...   cervical spondylosis\n",
       "1  recently, i have been having problems using th...  dimorphic hemorrhoids\n",
       "2  i have been exhausted and experiencing nausea ...               jaundice\n",
       "3  i have had a nagging cough that will not go aw...       bronchial asthma\n",
       "4  i have been quite exhausted and ill. my throat...            common cold"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['text'] = df['text'].apply(contractions.fix)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ***8- Split into Train-Test and save in .csv files***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train set shape: (691, 2)\n",
      "Test set shaape: (462, 2)\n"
     ]
    }
   ],
   "source": [
    "train_df, test_df = train_test_split(df, test_size=0.4, random_state=42)\n",
    "print(f\"Train set shape: {train_df.shape}\")\n",
    "print(f\"Test set shaape: {test_df.shape}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df.to_csv(CONSTANTS.TRAIN_SET_PATH, index=False)\n",
    "test_df.to_csv(CONSTANTS.TEST_SET_PATH, index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.20"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
