{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ***Contents***\n",
    "**[Labels distribution](#1--Label-distribution)**  \n",
    "**[Tags distribution in texts](#2--Tags-distribution-in-texts)**  \n",
    "**[Texts length](#3--Texts-length)**  \n",
    "**[N-grams analysis](#4--N-grams-analysis)**   "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ***Setup***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk import pos_tag\n",
    "from nltk import ngrams\n",
    "\n",
    "import contractions\n",
    "\n",
    "import string\n",
    "\n",
    "from collections import Counter\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import os\n",
    "parent_dir = os.path.abspath('..')\n",
    "sys.path.append(parent_dir)\n",
    "from constants import CONSTANTS\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Now, we have to download nltk additional resources, try running following cell with running VPN, if it didn't work successfully then try to download model manually from:**\n",
    "- https://www.nltk.org/data.html"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Uncomment if you haven't already**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# nltk.download('punkt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# nltk.download('stopwords')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# nltk.download('punkt_tab')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# nltk.download('wordnet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# nltk.download('averaged_perceptron_tagger')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# nltk.download('averaged_perceptron_tagger_eng')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**You must see an output similar to the following output:**\n",
    "> `[nltk_data] Downloading package punkt to`  \n",
    "> `[nltk_data]     ...\\AppData\\Roaming\\nltk_data...`  \n",
    "> `[nltk_data]   Package punkt is already up-to-date!`  \n",
    "> `True`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ***Load dataset***\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(CONSTANTS.DATASET_PATH)\n",
    "assert df.shape == (1200, 3), f\"Expected shape (1200, 3), but got {df.shape}\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>label</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Psoriasis</td>\n",
       "      <td>I have been experiencing a skin rash on my arm...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Psoriasis</td>\n",
       "      <td>My skin has been peeling, especially on my kne...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>Psoriasis</td>\n",
       "      <td>I have been experiencing joint pain in my fing...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Psoriasis</td>\n",
       "      <td>There is a silver like dusting on my skin, esp...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Psoriasis</td>\n",
       "      <td>My nails have small dents or pits in them, and...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0      label                                               text\n",
       "0           0  Psoriasis  I have been experiencing a skin rash on my arm...\n",
       "1           1  Psoriasis  My skin has been peeling, especially on my kne...\n",
       "2           2  Psoriasis  I have been experiencing joint pain in my fing...\n",
       "3           3  Psoriasis  There is a silver like dusting on my skin, esp...\n",
       "4           4  Psoriasis  My nails have small dents or pits in them, and..."
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ***Some helper functions***\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "stop_words = set(stopwords.words('english'))\n",
    "\n",
    "def plot_common_words(label_value):\n",
    "    filtered_data = df[df['label'] == label_value]\n",
    "    all_texts = ' '.join(filtered_data['text']).lower()\n",
    "    \n",
    "    # Tokenize the text\n",
    "    words = word_tokenize(all_texts)\n",
    "    \n",
    "    filtered_words = [word for word in words if word.isalnum() and word not in stop_words]\n",
    "    word_counts = Counter(filtered_words)\n",
    "\n",
    "    most_common_words = word_counts.most_common(20)  # Get the 20 most common words\n",
    "    labels, sizes = zip(*most_common_words)  # Unzip into labels and sizes\n",
    "    \n",
    "    # Plotting bar chart section\n",
    "    plt.figure(figsize=(10, 6))\n",
    "    plt.bar(labels, sizes, color='skyblue')\n",
    "    plt.xlabel('Words')\n",
    "    plt.ylabel('Frequency')\n",
    "    plt.title(f'Most Common Words for Label: {label_value}')\n",
    "    plt.xticks(rotation=45, ha='right')\n",
    "    plt.tight_layout()\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function to fix contractions\n",
    "def fix_contractions(text):\n",
    "    return contractions.fix(text)\n",
    "\n",
    "# Function to remove punctuation\n",
    "def remove_punctuation(text):\n",
    "    return text.translate(str.maketrans('', '', string.punctuation))\n",
    "\n",
    "# Function to extract N-grams\n",
    "def get_ngrams(text, n):\n",
    "    tokens = nltk.word_tokenize(text)\n",
    "    stop_words = set(stopwords.words('english'))\n",
    "    filtered_tokens = [word for word in tokens if word.lower() not in stop_words]\n",
    "    return list(ngrams(filtered_tokens, n))\n",
    "\n",
    "def plot_common_n_grams(text, n):\n",
    "    # Fix contractions and remove punctuation in the text\n",
    "    fixed_text = text.apply(fix_contractions)\n",
    "    fixed_text = fixed_text.apply(remove_punctuation)\n",
    "\n",
    "    all_ngrams = []\n",
    "\n",
    "    for doc in fixed_text:\n",
    "        all_ngrams.extend(get_ngrams(doc, n))\n",
    "\n",
    "    ngram_counts = Counter(all_ngrams)\n",
    "\n",
    "    most_common_ngrams = ngram_counts.most_common(20)  # Get top 20 N-grams\n",
    "\n",
    "    # Bar chart section\n",
    "    ngram_labels = [' '.join(ngram) for ngram, count in most_common_ngrams]\n",
    "    ngram_values = [count for ngram, count in most_common_ngrams]\n",
    "    plt.figure(figsize=(10, 6))\n",
    "    plt.barh(ngram_labels, ngram_values, color='skyblue')\n",
    "    plt.xlabel('Frequency')\n",
    "    plt.title(f'Top {n}-grams in Symptoms')\n",
    "    plt.gca().invert_yaxis() \n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ***Remove nulls, and dupblicates***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset shape before removing nulls and duplicates (1200, 3)\n",
      "Dataset shape after removing nulls and duplicates (1153, 2)\n"
     ]
    }
   ],
   "source": [
    "print(f\"Dataset shape before removing nulls and duplicates {df.shape}\")\n",
    "\n",
    "df = df.drop(columns=['Unnamed: 0'])\n",
    "df = df[['text', 'label']]\n",
    "# Remove rows with null values\n",
    "df = df.dropna()\n",
    "# Remove duplicates\n",
    "df = df.drop_duplicates()\n",
    "\n",
    "print(f\"Dataset shape after removing nulls and duplicates {df.shape}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ***1- Label distribution***\n",
    "[Back to contents](#Contents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of values in label column:\n",
      "24\n"
     ]
    }
   ],
   "source": [
    "print(f\"Number of values in label column:\\n{df['label'].nunique()}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Values in label column:\n",
      "label\n",
      "Jaundice                           38\n",
      "Dimorphic Hemorrhoids              41\n",
      "Malaria                            44\n",
      "Acne                               46\n",
      "Arthritis                          46\n",
      "Migraine                           47\n",
      "Pneumonia                          47\n",
      "gastroesophageal reflux disease    48\n",
      "Bronchial Asthma                   49\n",
      "Chicken pox                        49\n",
      "Cervical spondylosis               49\n",
      "Common Cold                        49\n",
      "Psoriasis                          50\n",
      "Dengue                             50\n",
      "Fungal infection                   50\n",
      "diabetes                           50\n",
      "Hypertension                       50\n",
      "urinary tract infection            50\n",
      "allergy                            50\n",
      "drug reaction                      50\n",
      "peptic ulcer disease               50\n",
      "Varicose Veins                     50\n",
      "Impetigo                           50\n",
      "Typhoid                            50\n",
      "Name: count, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "# Count the occurrences of each label\n",
    "label_counts = df['label'].value_counts()\n",
    "print(f\"Values in label column:\\n{label_counts.sort_values()}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 165568 characters]",
      "text/plain": [
       "<Figure size 800x800 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Create a pie chart\n",
    "plt.figure(figsize=(8, 8))\n",
    "plt.pie(label_counts, labels=label_counts.index, autopct='%1.1f%%', startangle=90)\n",
    "plt.title('Distribution of Labels')\n",
    "plt.axis('equal')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ***Jaundice اليرقان***\n",
    "[Back to contents](#Contents)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 58088 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_common_words(\"Jaundice\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ***Malaria الملاريا***\n",
    "[Back to contents](#Contents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 54796 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_common_words(\"Malaria\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ***Acne حب الشباب***\n",
    "[Back to contents](#Contents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 57852 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_common_words(\"Acne\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ***Arthritis التهاب المفاصل***\n",
    "[Back to contents](#Contents)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 56192 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_common_words(\"Arthritis\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ***Migraine صداع نصفي***\n",
    "[Back to contents](#Contents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 61544 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_common_words(\"Migraine\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ***Pneumonia التهاب رئوي***\n",
    "[Back to contents](#Contents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 55208 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_common_words(\"Pneumonia\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ***Gastroesophageal reflux disease ارتجاع معدي مريئي***\n",
    "[Back to contents](#Contents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 60416 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_common_words(\"gastroesophageal reflux disease\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ***Bronchial Asthma الربو القصبي***\n",
    "[Back to contents](#Contents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 56220 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_common_words(\"Bronchial Asthma\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ***Chicken pox جدري***\n",
    "[Back to contents](#Contents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 49396 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_common_words(\"Chicken pox\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## **Cervical spondylosis التهاب الفقرات العنقية**\n",
    "[Back to contents](#Contents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 59952 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_common_words(\"Cervical spondylosis\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ***Common Cold الزكام***\n",
    "[Back to contents](#Contents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 50416 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_common_words(\"Common Cold\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ***Psoriasis صدفية***\n",
    "- مرض جلدي يسبب طفحًا جلديًا  \n",
    "\n",
    "[Back to contents](#Contents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 55128 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_common_words(\"Psoriasis\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ***Dengue حمى الضنك***\n",
    "- حمى شديدة وظهور أعراض شبيهة بأعراض الإنفلونزا\n",
    "\n",
    "[Back to contents](#Contents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 58052 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_common_words(\"Dengue\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ***Fungal infection عدوى فطرية***\n",
    "- مرض جلدي غالباً\n",
    "\n",
    "[Back to contents](#Contents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 51224 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_common_words(\"Fungal infection\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ***Diabetes داء السكري***\n",
    "[Back to contents](#Contents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 58284 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_common_words(\"diabetes\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ***Hypertension ارتفاع ضغط الدم***\n",
    "[Back to contents](#Contents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 58896 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_common_words(\"Hypertension\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ***Urinary tract infection التهاب الجهاز البولي***\n",
    "[Back to contents](#Contents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 57472 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_common_words(\"urinary tract infection\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ***Allergy الحساسية***\n",
    "[Back to contents](#Contents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 54060 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_common_words(\"allergy\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ***Drug reaction تفاعلات دوائية ضارة***\n",
    "[Back to contents](#Contents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 54388 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_common_words(\"drug reaction\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ***Peptic ulcer disease القرحة الهضمية***\n",
    "[Back to contents](#Contents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 56772 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_common_words(\"peptic ulcer disease\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ***Varicose Veins دوالي الأوردة***\n",
    "[Back to contents](#Contents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 56432 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_common_words(\"Varicose Veins\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ***Impetigo القوباء (مرض جلدي)***\n",
    "[Back to contents](#Contents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 50532 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_common_words(\"Impetigo\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ***Typhoid حمى التيفوئيد***\n",
    "[Back to contents](#Contents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 55624 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_common_words(\"Typhoid\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ***2- Tags distribution in texts***\n",
    "[Back to contents](#Contents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 46944 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pos_counts = {\n",
    "    'Noun': 0,\n",
    "    'Verb': 0,\n",
    "    'Adjective': 0,\n",
    "    'Adverb': 0,\n",
    "    'Other': 0\n",
    "}\n",
    "\n",
    "tag_mapping = {\n",
    "    'NN': 'Noun', 'NNS': 'Noun', 'NNP': 'Noun', 'NNPS': 'Noun',  # Nouns\n",
    "    'VB': 'Verb', 'VBD': 'Verb', 'VBG': 'Verb', 'VBN': 'Verb', 'VBP': 'Verb', 'VBZ': 'Verb',  # Verbs\n",
    "    'JJ': 'Adjective', 'JJR': 'Adjective', 'JJS': 'Adjective',  # Adjectives\n",
    "    'RB': 'Adverb', 'RBR': 'Adverb', 'RBS': 'Adverb'  # Adverbs\n",
    "}\n",
    "\n",
    "for text in df['text']:\n",
    "    tokens = word_tokenize(text)  # Tokenize the text\n",
    "    pos_tags = pos_tag(tokens)     # Get POS tags\n",
    "    for word, tag in pos_tags:\n",
    "        category = tag_mapping.get(tag, 'Other')  # Default to 'Other' if not found\n",
    "        pos_counts[category] += 1\n",
    "\n",
    "# Pie chart section\n",
    "labels = pos_counts.keys()\n",
    "sizes = pos_counts.values()\n",
    "plt.figure(figsize=(10, 6))\n",
    "plt.pie(sizes, labels=labels, autopct='%1.1f%%', startangle=140)\n",
    "plt.title('Distribution of Main POS Tags in Text')\n",
    "plt.axis('equal')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Others\n",
    "1. **Pronouns**: Words that replace nouns (e.g., he, she, it, they).  \n",
    "1. **Prepositions**: Words that indicate relationships between nouns (e.g., in, on, at, with).  \n",
    "1. **Conjunctions**: Words that connect clauses or sentences (e.g., and, but, or).  \n",
    "1. **Determiners**: Words that introduce nouns (e.g., the, a, an).  \n",
    "1. **Interjections**: Words that express emotion (e.g., oh, wow, ouch).  \n",
    "1. **Numerals**: Words that express quantities or numbers (e.g., one, two, first).  \n",
    "1. **Articles**: Words that define a noun as specific or unspecific (e.g., a, an, the).  \n",
    "1. **Punctuation** Marks: Any punctuation that is tokenized (e.g., commas, periods)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ***3- Texts length***\n",
    "[Back to contents](#Contents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average: 171.78949884420697\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 109960 characters]",
      "text/plain": [
       "<Figure size 1200x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df['text_length'] = df['text'].apply(len)\n",
    "\n",
    "average_length = df.groupby('label')['text_length'].mean()\n",
    "\n",
    "overall_average = average_length.mean()\n",
    "print(f\"Average: {overall_average}\")\n",
    "\n",
    "# Bar chart section\n",
    "plt.figure(figsize=(12, 6))\n",
    "average_length.plot(kind='bar', color='skyblue')\n",
    "plt.title('Average Length of Texts by Label')\n",
    "plt.xlabel('Label')\n",
    "plt.ylabel('Average Length of Texts')\n",
    "plt.xticks(rotation=45, ha='right', fontsize=10)\n",
    "plt.axhline(y=overall_average, color='red', linestyle='--', label='Overall Average')\n",
    "plt.legend()\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ***4- N-grams analysis***\n",
    "[Back to contents](#Contents)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## **1-Gram**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 37484 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_common_n_grams(df[\"text\"], 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## **2-Gram**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 56392 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_common_n_grams(df[\"text\"], 2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## **3-Gram**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 77884 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_common_n_grams(df[\"text\"], 3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## **4-Gram**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 94348 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_common_n_grams(df[\"text\"], 4)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "NLP",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.20"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
