{
    "cells": [
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "# ***Setup***"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 1,
            "metadata": {},
            "outputs": [],
            "source": [
                "import matplotlib.pyplot as plt\n",
                "\n",
                "from nltk.corpus import wordnet as wn\n",
                "\n",
                "import numpy as np\n",
                "\n",
                "import pandas as pd\n",
                "\n",
                "import pickle\n",
                "\n",
                "from sklearn.feature_selection import SelectKBest, chi2\n",
                "from sklearn.feature_extraction.text import TfidfVectorizer\n",
                "from sklearn.naive_bayes import MultinomialNB\n",
                "from sklearn.metrics import accuracy_score, classification_report\n",
                "\n",
                "from tabulate import tabulate"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 2,
            "metadata": {},
            "outputs": [
                {
                    "name": "stderr",
                    "output_type": "stream",
                    "text": [
                        "Warming up PyWSD (takes ~10 secs)... took 10.154893636703491 secs.\n"
                    ]
                }
            ],
            "source": [
                "from pywsd.lesk import simple_lesk"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 3,
            "metadata": {},
            "outputs": [],
            "source": [
                "import sys\n",
                "import os\n",
                "parent_dir = os.path.abspath('..')\n",
                "sys.path.append(parent_dir)\n",
                "from constants import CONSTANTS\n",
                "%load_ext autoreload\n",
                "%autoreload 2"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## ***Load dataset***"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 4,
            "metadata": {},
            "outputs": [],
            "source": [
                "train_df = pd.read_csv(CONSTANTS.AUGMENTED_TRAIN_SET_PATH)\n",
                "test_df = pd.read_csv(CONSTANTS.AUGMENTED_TEST_SET_PATH)"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## ***Load dep. parsing results***\n"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 5,
            "metadata": {},
            "outputs": [
                {
                    "name": "stderr",
                    "output_type": "stream",
                    "text": [
                        "C:\\Users\\TOSHIBA\\AppData\\Roaming\\Python\\Python39\\site-packages\\networkx\\utils\\backends.py:135: RuntimeWarning: networkx backend defined more than once: nx-loopback\n",
                        "  backends.update(_get_backends(\"networkx.backends\"))\n"
                    ]
                }
            ],
            "source": [
                "with open(CONSTANTS.DEP_PARSED_TEXTS_OBJECT_PATH, 'rb') as f:\n",
                "    loaded_data = pickle.load(f)"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## ***Helper functions***"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "### ***Get processed text by row id***"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 6,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "My skin has been peeling, especially on my knees, elbows, and scalp. This peeling is often accompanied by a burning or stinging sensation.\n"
                    ]
                }
            ],
            "source": [
                "# Each sentence in the dataset has an id, and a document contain its stanza processing\n",
                "def get_doc_by_id(target_id):\n",
                "    for obj in loaded_data:\n",
                "        if obj[\"id\"] == target_id:\n",
                "            return obj[\"processed_text\"]\n",
                "    return None  # Return None if not found\n",
                "print(get_doc_by_id(1).text)"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "### ***Get text tokens by row id***\n"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 7,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "My skin has been peeling, especially on my knees, elbows, and scalp. This peeling is often accompanied by a burning or stinging sensation.\n",
                        "['My', 'skin', 'has', 'been', 'peeling', ',', 'especially', 'on', 'my', 'knees', ',', 'elbows', ',', 'and', 'scalp', '.', 'This', 'peeling', 'is', 'often', 'accompanied', 'by', 'a', 'burning', 'or', 'stinging', 'sensation', '.']\n"
                    ]
                }
            ],
            "source": [
                "def get_text_tokens(text_id):\n",
                "    processed_text = get_doc_by_id(text_id)\n",
                "    tokens = [word.text for sent in processed_text.sentences for word in sent.words]\n",
                "    return tokens\n",
                "print(get_doc_by_id(1).text)\n",
                "print(get_text_tokens(1))"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "### ***Get wordnet POS tag***"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 8,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "My skin has been peeling, especially on my knees, elbows, and scalp. This peeling is often accompanied by a burning or stinging sensation.\n",
                        "[('skin', 'n'), ('peeling', 'v'), ('especially', 'r'), ('knees', 'n'), ('elbows', 'n'), ('scalp', 'n'), ('peeling', 'n'), ('often', 'r'), ('accompanied', 'v'), ('burning', 'n'), ('stinging', 'a'), ('sensation', 'n')]\n"
                    ]
                }
            ],
            "source": [
                "def get_wordnet_pos(text_id):\n",
                "    processed_text = get_doc_by_id(text_id)\n",
                "    pos_tags = [(word.text, word.upos) for sent in processed_text.sentences for word in sent.words]\n",
                "    pos_mapping = {\n",
                "        'NOUN': 'n',  # Noun\n",
                "        'VERB': 'v',  # Verb\n",
                "        'ADJ': 'a',   # Adjective\n",
                "        'ADV': 'r'    # Adverb\n",
                "    }    \n",
                "    pos_tags = [(word, pos_mapping.get(pos, None)) for word, pos in pos_tags if pos in pos_mapping]\n",
                "    return pos_tags\n",
                "print(get_doc_by_id(1).text)\n",
                "print(get_wordnet_pos(1))"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "### ***Extract synsets from text***"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 9,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "My skin has been peeling, especially on my knees, elbows, and scalp. This peeling is often accompanied by a burning or stinging sensation.\n",
                        "skin.n.01 skin.n.02 hide.n.02 skin.n.04 peel.n.02 skin.n.06 clamber.v.01 skin.v.02 bark.v.03 skin.v.04 hour_angle.n.02 have.v.01 have.v.02 experience.v.03 own.v.01 get.v.03 consume.v.02 have.v.07 hold.v.03 have.v.09 have.v.10 have.v.11 have.v.12 induce.v.02 accept.v.02 receive.v.01 suffer.v.02 have.v.17 give_birth.v.01 take.v.35 be.v.01 be.v.02 be.v.03 exist.v.01 be.v.05 equal.v.01 constitute.v.01 be.v.08 embody.v.02 be.v.10 be.v.11 be.v.12 cost.v.01 desquamation.n.01 skin.v.04 peel_off.v.04 undress.v.01 particularly.r.01 specially.r.01 on.a.01 on.a.02 along.r.01 on.r.02 on.r.03 knee.n.01 stifle.n.01 knee.n.03 elbow.n.01 elbow.n.02 elbow.n.03 elbow.n.04 elbow.n.05 elbow.v.01 elbow.v.02 scalp.n.01 scalp.v.01 scalp.v.02 desquamation.n.01 skin.v.04 peel_off.v.04 undress.v.01 be.v.01 be.v.02 be.v.03 exist.v.01 be.v.05 equal.v.01 constitute.v.01 be.v.08 embody.v.02 be.v.10 be.v.11 be.v.12 cost.v.01 frequently.r.01 much.r.05 often.r.03 attach_to.v.01 accompany.v.02 play_along.v.02 company.v.01 accompanied.a.01 accompanied.a.02 by.r.01 aside.r.06 angstrom.n.01 vitamin_a.n.01 deoxyadenosine_monophosphate.n.01 adenine.n.01 ampere.n.02 a.n.06 a.n.07 burning.n.01 burn.n.01 combustion.n.01 electrocution.n.01 burning.n.05 burning.n.06 burn.v.01 burn.v.02 burn.v.03 bite.v.02 burn.v.05 burn.v.06 burn.v.07 burn.v.08 burn.v.09 burn.v.10 cauterize.v.01 sunburn.v.01 cut.v.21 burn_off.v.01 burn.v.15 burning.s.01 oregon.n.01 operating_room.n.01 sting.n.01 bite.v.02 sting.v.02 stick.v.15 prick.v.02 sting.v.05 cutting.s.01 sensation.n.01 ace.n.03 sensation.n.03 sensation.n.04 sense.n.03\n"
                    ]
                }
            ],
            "source": [
                "def get_synsets(text_id):\n",
                "    words = get_text_tokens(text_id)\n",
                "    synsets = []\n",
                "    \n",
                "    for word in words:\n",
                "        syns = wn.synsets(word)\n",
                "        if syns:\n",
                "            synsets.extend([syn.name() for syn in syns])\n",
                "    \n",
                "    return \" \".join(synsets)\n",
                "\n",
                "print(get_doc_by_id(1).text)\n",
                "print(get_synsets(1))"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "### ***Filter synsets by POS***\n"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 10,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "My skin has been peeling, especially on my knees, elbows, and scalp. This peeling is often accompanied by a burning or stinging sensation.\n",
                        "skin.n.01 skin.n.02 hide.n.02 skin.n.04 peel.n.02 skin.n.06 skin.v.04 peel_off.v.04 undress.v.01 particularly.r.01 specially.r.01 knee.n.01 stifle.n.01 knee.n.03 elbow.n.01 elbow.n.02 elbow.n.03 elbow.n.04 elbow.n.05 scalp.n.01 desquamation.n.01 frequently.r.01 much.r.05 often.r.03 attach_to.v.01 accompany.v.02 play_along.v.02 company.v.01 burning.n.01 burn.n.01 combustion.n.01 electrocution.n.01 burning.n.05 burning.n.06 cutting.s.01 sensation.n.01 ace.n.03 sensation.n.03 sensation.n.04 sense.n.03\n"
                    ]
                }
            ],
            "source": [
                "def get_filtered_synsets(text_id):\n",
                "    words = get_text_tokens(text_id)\n",
                "    pos_tags = get_wordnet_pos(text_id)\n",
                "    synsets = []\n",
                "    for word, pos in pos_tags:\n",
                "        if pos:  # Filter by WordNet-compatible POS\n",
                "            syns = wn.synsets(word, pos=pos)\n",
                "            if syns:\n",
                "                synsets.extend([syn.name() for syn in syns])\n",
                "    return \" \".join(synsets)\n",
                "print(get_doc_by_id(1).text)\n",
                "print(get_filtered_synsets(1))"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "### ***Filter synsets using WSD***\n"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 11,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "My skin has been peeling, especially on my knees, elbows, and scalp. This peeling is often accompanied by a burning or stinging sensation.\n",
                        "skin.v.04 take.v.35 exist.v.01 skin.v.04 specially.r.01 on.r.03 stifle.n.01 elbow.v.02 scalp.n.01 skin.v.04 exist.v.01 often.r.03 play_along.v.02 by.r.01 vitamin_a.n.01 burning.n.06 oregon.n.01 sting.n.03 sense.n.03\n"
                    ]
                }
            ],
            "source": [
                "def get_wsd_synsets(text_id):\n",
                "    words = get_text_tokens(text_id)\n",
                "    text = get_doc_by_id(text_id).text\n",
                "    synsets = []\n",
                "    for word in words:\n",
                "        syn = simple_lesk(text, word)  # Disambiguate based on context\n",
                "        if syn:\n",
                "            synsets.append(syn.name())\n",
                "    return \" \".join(synsets)\n",
                "print(get_doc_by_id(1).text)\n",
                "print(get_wsd_synsets(1))"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "### ***Features selection***\n",
                "- Using `SelectKBest`"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 12,
            "metadata": {},
            "outputs": [],
            "source": [
                "def select_features(X_train, X_test, y_train, scorer, k_value):\n",
                "\n",
                "    # Apply features selection\n",
                "    selector = SelectKBest(score_func=scorer, k=k_value)\n",
                "    X_train_selected = selector.fit_transform(X_train, y_train)\n",
                "    X_test_selected = selector.transform(X_test)\n",
                "    return X_train_selected, X_test_selected    \n"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "### ***Plot train, and test accuracies vs number_of_features***\n"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 13,
            "metadata": {},
            "outputs": [],
            "source": [
                "def plot_accuracies(X_train, X_test, y_train, y_test, model, k_start=500, k_end=4500, step=250):\n",
                "    train_accuracies = []\n",
                "    test_accuracies = []\n",
                "    features_counts = []\n",
                "    for i in range(k_start, k_end + 1, step):\n",
                "        X_train_selected, X_test_selected = select_features(X_train, X_test, y_train, chi2, i)\n",
                "        model.fit(X_train_selected, y_train)\n",
                "        \n",
                "        # Training set acc\n",
                "        y_train_pred = model.predict(X_train_selected)\n",
                "        train_accuracy = accuracy_score(y_train, y_train_pred)\n",
                "        train_accuracies.append(train_accuracy)\n",
                "\n",
                "        # Testing set acc\n",
                "        y_pred = model.predict(X_test_selected)\n",
                "        test_accuracy = accuracy_score(y_test, y_pred)\n",
                "        test_accuracies.append(test_accuracy)\n",
                "\n",
                "        features_counts.append(i)\n",
                "\n",
                "    # Plotting the accuracies\n",
                "    plt.figure(figsize=(10, 6))\n",
                "    plt.plot(features_counts, train_accuracies, label='Train Accuracy', marker='.')\n",
                "    plt.plot(features_counts, test_accuracies, label='Test Accuracy', marker='.')\n",
                "    plt.title('Train and Test Accuracy vs Number of Features')\n",
                "    plt.xlabel('Number of Features Selected')\n",
                "    plt.ylabel('Accuracy')\n",
                "    plt.legend()\n",
                "    plt.grid()\n",
                "\n",
                "    # Finding closest points (3)\n",
                "    differences = np.abs(np.array(train_accuracies) - np.array(test_accuracies))\n",
                "    closest_indices = np.argsort(differences)[:3]  # indices of the three smallest differences\n",
                "    colors = ['darkgreen', 'mediumseagreen', 'lightgreen']\n",
                "\n",
                "    # Draw a rect\n",
                "    for i, idx in enumerate(closest_indices):\n",
                "        x = features_counts[idx] - 5\n",
                "        y_bottom = min(train_accuracies[idx], test_accuracies[idx])\n",
                "        y_top = max(train_accuracies[idx], test_accuracies[idx])\n",
                "        height = y_top - y_bottom\n",
                "\n",
                "        plt.gca().add_patch(plt.Rectangle(\n",
                "            (x, y_bottom), 10, height,\n",
                "            color=colors[i], alpha=0.5\n",
                "        ))\n",
                "        \n",
                "        # Print the number of selected features for each closest point\n",
                "        print(f\"Closest Point {i+1}: Number of Features = {features_counts[idx]}, Train Accuracy = {train_accuracies[idx]}, Test Accuracy = {test_accuracies[idx]}\")\n",
                "\n",
                "\n",
                "    plt.show()    "
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "### ***Evaluate model***\n"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 14,
            "metadata": {},
            "outputs": [],
            "source": [
                "def print_clf_report_as_table(report):\n",
                "    data = []\n",
                "    for key, value in report.items():\n",
                "        if key != 'accuracy' and key != 'macro avg' and key != 'weighted avg':\n",
                "            data.append([key, value['precision'], value['recall'], value['f1-score'], value['support']])\n",
                "\n",
                "    data.append(['accuracy', '', '', report['accuracy'], ''])\n",
                "\n",
                "    data.append(['macro avg', report['macro avg']['precision'], report['macro avg']['recall'], report['macro avg']['f1-score'], ''])\n",
                "\n",
                "    data.append(['weighted avg', report['weighted avg']['precision'], report['weighted avg']['recall'], report['weighted avg']['f1-score'], ''])\n",
                "\n",
                "    print(tabulate(data, headers=['Class', 'Precision', 'Recall', 'F1-score', 'Support'], tablefmt='psql'))"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 15,
            "metadata": {},
            "outputs": [],
            "source": [
                "def evaluate_model(X_train, X_test, y_train, y_test, scorer, k_value, model):\n",
                "    X_train_selected, X_test_selected = select_features(X_train, X_test, y_train, scorer, k_value)\n",
                "    model.fit(X_train_selected, y_train)\n",
                "        \n",
                "    # Training set acc\n",
                "    y_train_pred = model.predict(X_train_selected)\n",
                "    train_accuracy = accuracy_score(y_train, y_train_pred)\n",
                "\n",
                "    # Testing set acc\n",
                "    y_pred = model.predict(X_test_selected)\n",
                "    test_accuracy = accuracy_score(y_test, y_pred)\n",
                "\n",
                "    print(f'Train Accuracy: {train_accuracy}')\n",
                "    print(f'Test Accuracy: {test_accuracy}')\n",
                "    print(f'Difference: {train_accuracy-test_accuracy}')    \n",
                "    # Print classification report\n",
                "    report = classification_report(y_test, y_pred, output_dict=True)\n",
                "    print_clf_report_as_table(report)"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "# ***Use synsets as Features***"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## ***Apply features extraction on the dataset***"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 16,
            "metadata": {},
            "outputs": [],
            "source": [
                "train_df[\"features\"] = train_df[\"Id\"].apply(get_synsets)\n",
                "test_df[\"features\"] = test_df[\"Id\"].apply(get_synsets)"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "#### ***Vectorize the features***\n",
                "- **This will be done using `TF-IDF`**"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 17,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "Train set shape after features extraction: (4320, 4549)\n",
                        "Test set shape after features extraction: (480, 4549)\n"
                    ]
                }
            ],
            "source": [
                "vectorizer = TfidfVectorizer()\n",
                "X_train = vectorizer.fit_transform(train_df['features'])\n",
                "X_test = vectorizer.transform(test_df['features'])\n",
                "print(f\"Train set shape after features extraction: {X_train.shape}\")\n",
                "print(f\"Test set shape after features extraction: {X_test.shape}\")"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 18,
            "metadata": {},
            "outputs": [],
            "source": [
                "y_train = train_df[\"label\"]\n",
                "y_test = test_df[\"label\"]"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## ***Select best number of features***"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 19,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "Closest Point 1: Number of Features = 500, Train Accuracy = 0.8923611111111112, Test Accuracy = 0.8458333333333333\n",
                        "Closest Point 2: Number of Features = 750, Train Accuracy = 0.9064814814814814, Test Accuracy = 0.8520833333333333\n",
                        "Closest Point 3: Number of Features = 1250, Train Accuracy = 0.9185185185185185, Test Accuracy = 0.8583333333333333\n"
                    ]
                },
                {
                    "data": {
                        "image/png": "[encoded data, 57496 characters]",
                        "text/plain": [
                            "<Figure size 1000x600 with 1 Axes>"
                        ]
                    },
                    "metadata": {},
                    "output_type": "display_data"
                }
            ],
            "source": [
                "model = MultinomialNB()\n",
                "plot_accuracies(X_train, X_test, y_train, y_test, model)"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "### ***Tune model's hyperparameters to reduce OF***\n"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 20,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "Closest Point 1: Number of Features = 500, Train Accuracy = 0.9423611111111111, Test Accuracy = 0.89375\n",
                        "Closest Point 2: Number of Features = 1250, Train Accuracy = 0.9652777777777778, Test Accuracy = 0.9104166666666667\n",
                        "Closest Point 3: Number of Features = 1000, Train Accuracy = 0.9618055555555556, Test Accuracy = 0.9041666666666667\n"
                    ]
                },
                {
                    "data": {
                        "image/png": "[encoded data, 60268 characters]",
                        "text/plain": [
                            "<Figure size 1000x600 with 1 Axes>"
                        ]
                    },
                    "metadata": {},
                    "output_type": "display_data"
                }
            ],
            "source": [
                "model = MultinomialNB(alpha=0.1)\n",
                "plot_accuracies(X_train, X_test, y_train, y_test, model)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 21,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "Closest Point 1: Number of Features = 500, Train Accuracy = 0.9423611111111111, Test Accuracy = 0.89375\n",
                        "Closest Point 2: Number of Features = 1250, Train Accuracy = 0.9652777777777778, Test Accuracy = 0.9104166666666667\n",
                        "Closest Point 3: Number of Features = 1000, Train Accuracy = 0.9618055555555556, Test Accuracy = 0.9041666666666667\n"
                    ]
                },
                {
                    "data": {
                        "image/png": "[encoded data, 60268 characters]",
                        "text/plain": [
                            "<Figure size 1000x600 with 1 Axes>"
                        ]
                    },
                    "metadata": {},
                    "output_type": "display_data"
                }
            ],
            "source": [
                "model = MultinomialNB(alpha=0.01)\n",
                "plot_accuracies(X_train, X_test, y_train, y_test, model)"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## ***Evaluate best model***"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 22,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "Train Accuracy: 0.9782407407407407\n",
                        "Test Accuracy: 0.9333333333333333\n",
                        "Difference: 0.044907407407407396\n",
                        "+---------------------------------+--------------------+--------------------+------------+-----------+\n",
                        "| Class                           | Precision          | Recall             |   F1-score | Support   |\n",
                        "|---------------------------------+--------------------+--------------------+------------+-----------|\n",
                        "| Acne                            | 1.0                | 0.9523809523809523 |   0.97561  | 21.0      |\n",
                        "| Arthritis                       | 0.9090909090909091 | 1.0                |   0.952381 | 20.0      |\n",
                        "| Bronchial Asthma                | 0.9047619047619048 | 1.0                |   0.95     | 19.0      |\n",
                        "| Cervical spondylosis            | 1.0                | 1.0                |   1        | 21.0      |\n",
                        "| Chicken pox                     | 0.6521739130434783 | 1.0                |   0.789474 | 15.0      |\n",
                        "| Common Cold                     | 0.9090909090909091 | 0.9523809523809523 |   0.930233 | 21.0      |\n",
                        "| Dengue                          | 0.9285714285714286 | 0.5909090909090909 |   0.722222 | 22.0      |\n",
                        "| Dimorphic Hemorrhoids           | 0.95               | 1.0                |   0.974359 | 19.0      |\n",
                        "| Fungal infection                | 1.0                | 0.9615384615384616 |   0.980392 | 26.0      |\n",
                        "| Hypertension                    | 1.0                | 0.8333333333333334 |   0.909091 | 18.0      |\n",
                        "| Impetigo                        | 0.9583333333333334 | 1.0                |   0.978723 | 23.0      |\n",
                        "| Jaundice                        | 1.0                | 1.0                |   1        | 22.0      |\n",
                        "| Malaria                         | 1.0                | 1.0                |   1        | 17.0      |\n",
                        "| Migraine                        | 1.0                | 0.9583333333333334 |   0.978723 | 24.0      |\n",
                        "| Pneumonia                       | 0.9565217391304348 | 1.0                |   0.977778 | 22.0      |\n",
                        "| Psoriasis                       | 0.8666666666666667 | 0.7647058823529411 |   0.8125   | 17.0      |\n",
                        "| Typhoid                         | 0.8947368421052632 | 0.9444444444444444 |   0.918919 | 18.0      |\n",
                        "| Varicose Veins                  | 0.9565217391304348 | 0.88               |   0.916667 | 25.0      |\n",
                        "| allergy                         | 0.8                | 0.8                |   0.8      | 15.0      |\n",
                        "| diabetes                        | 0.8947368421052632 | 1.0                |   0.944444 | 17.0      |\n",
                        "| drug reaction                   | 0.9285714285714286 | 0.8125             |   0.866667 | 16.0      |\n",
                        "| gastroesophageal reflux disease | 0.9130434782608695 | 1.0                |   0.954545 | 21.0      |\n",
                        "| peptic ulcer disease            | 1.0                | 0.8888888888888888 |   0.941176 | 18.0      |\n",
                        "| urinary tract infection         | 0.9583333333333334 | 1.0                |   0.978723 | 23.0      |\n",
                        "| accuracy                        |                    |                    |   0.933333 |           |\n",
                        "| macro avg                       | 0.932548102799819  | 0.9308089724817665 |   0.927193 |           |\n",
                        "| weighted avg                    | 0.9392592028490198 | 0.9333333333333333 |   0.932121 |           |\n",
                        "+---------------------------------+--------------------+--------------------+------------+-----------+\n"
                    ]
                }
            ],
            "source": [
                "model = MultinomialNB(alpha=0.01)\n",
                "evaluate_model(X_train, X_test, y_train, y_test, chi2, 1500, model)"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "# ***Use synsets as Features with filtering by POS***\n"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 23,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "Train set shape after features extraction: (4320, 3739)\n",
                        "Test set shape after features extraction: (480, 3739)\n"
                    ]
                }
            ],
            "source": [
                "train_df[\"features\"] = train_df[\"Id\"].apply(get_filtered_synsets)\n",
                "test_df[\"features\"] = test_df[\"Id\"].apply(get_filtered_synsets)\n",
                "\n",
                "vectorizer = TfidfVectorizer()\n",
                "X_train = vectorizer.fit_transform(train_df['features'])\n",
                "X_test = vectorizer.transform(test_df['features'])\n",
                "print(f\"Train set shape after features extraction: {X_train.shape}\")\n",
                "print(f\"Test set shape after features extraction: {X_test.shape}\")\n",
                "\n",
                "y_train = train_df[\"label\"]\n",
                "y_test = test_df[\"label\"]"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 24,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "Closest Point 1: Number of Features = 500, Train Accuracy = 0.9604166666666667, Test Accuracy = 0.9083333333333333\n",
                        "Closest Point 2: Number of Features = 1500, Train Accuracy = 0.9810185185185185, Test Accuracy = 0.9270833333333334\n",
                        "Closest Point 3: Number of Features = 1000, Train Accuracy = 0.9766203703703704, Test Accuracy = 0.91875\n"
                    ]
                },
                {
                    "data": {
                        "image/png": "[encoded data, 55764 characters]",
                        "text/plain": [
                            "<Figure size 1000x600 with 1 Axes>"
                        ]
                    },
                    "metadata": {},
                    "output_type": "display_data"
                }
            ],
            "source": [
                "model = MultinomialNB(alpha=0.01)\n",
                "plot_accuracies(X_train, X_test, y_train, y_test, model, k_end=3500)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 25,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "Train Accuracy: 0.9810185185185185\n",
                        "Test Accuracy: 0.9270833333333334\n",
                        "Difference: 0.05393518518518514\n",
                        "+---------------------------------+--------------------+--------------------+------------+-----------+\n",
                        "| Class                           | Precision          | Recall             |   F1-score | Support   |\n",
                        "|---------------------------------+--------------------+--------------------+------------+-----------|\n",
                        "| Acne                            | 1.0                | 0.9523809523809523 |   0.97561  | 21.0      |\n",
                        "| Arthritis                       | 0.9090909090909091 | 1.0                |   0.952381 | 20.0      |\n",
                        "| Bronchial Asthma                | 1.0                | 1.0                |   1        | 19.0      |\n",
                        "| Cervical spondylosis            | 1.0                | 0.9523809523809523 |   0.97561  | 21.0      |\n",
                        "| Chicken pox                     | 0.5909090909090909 | 0.8666666666666667 |   0.702703 | 15.0      |\n",
                        "| Common Cold                     | 0.8695652173913043 | 0.9523809523809523 |   0.909091 | 21.0      |\n",
                        "| Dengue                          | 0.9230769230769231 | 0.5454545454545454 |   0.685714 | 22.0      |\n",
                        "| Dimorphic Hemorrhoids           | 0.95               | 1.0                |   0.974359 | 19.0      |\n",
                        "| Fungal infection                | 1.0                | 0.9615384615384616 |   0.980392 | 26.0      |\n",
                        "| Hypertension                    | 1.0                | 0.8888888888888888 |   0.941176 | 18.0      |\n",
                        "| Impetigo                        | 0.92               | 1.0                |   0.958333 | 23.0      |\n",
                        "| Jaundice                        | 1.0                | 1.0                |   1        | 22.0      |\n",
                        "| Malaria                         | 1.0                | 1.0                |   1        | 17.0      |\n",
                        "| Migraine                        | 1.0                | 0.9583333333333334 |   0.978723 | 24.0      |\n",
                        "| Pneumonia                       | 0.9565217391304348 | 1.0                |   0.977778 | 22.0      |\n",
                        "| Psoriasis                       | 0.9285714285714286 | 0.7647058823529411 |   0.83871  | 17.0      |\n",
                        "| Typhoid                         | 0.85               | 0.9444444444444444 |   0.894737 | 18.0      |\n",
                        "| Varicose Veins                  | 0.9565217391304348 | 0.88               |   0.916667 | 25.0      |\n",
                        "| allergy                         | 0.8125             | 0.8666666666666667 |   0.83871  | 15.0      |\n",
                        "| diabetes                        | 0.8947368421052632 | 1.0                |   0.944444 | 17.0      |\n",
                        "| drug reaction                   | 0.9375             | 0.9375             |   0.9375   | 16.0      |\n",
                        "| gastroesophageal reflux disease | 0.9090909090909091 | 0.9523809523809523 |   0.930233 | 21.0      |\n",
                        "| peptic ulcer disease            | 1.0                | 0.8333333333333334 |   0.909091 | 18.0      |\n",
                        "| urinary tract infection         | 0.88               | 0.9565217391304348 |   0.916667 | 23.0      |\n",
                        "| accuracy                        |                    |                    |   0.927083 |           |\n",
                        "| macro avg                       | 0.9286701999373624 | 0.9255657404722303 |   0.922443 |           |\n",
                        "| weighted avg                    | 0.9345733080206406 | 0.9270833333333334 |   0.926302 |           |\n",
                        "+---------------------------------+--------------------+--------------------+------------+-----------+\n"
                    ]
                }
            ],
            "source": [
                "model = MultinomialNB(alpha=0.01)\n",
                "evaluate_model(X_train, X_test, y_train, y_test, chi2, 1500, model) "
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "# ***Use synsets as Features with filtering using lesk's algorithm for WSD***\n"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 26,
            "metadata": {},
            "outputs": [],
            "source": [
                "train_df[\"features\"] = train_df[\"Id\"].apply(get_wsd_synsets)\n",
                "test_df[\"features\"] = test_df[\"Id\"].apply(get_wsd_synsets)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 27,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "Train set shape after features extraction: (4320, 2573)\n",
                        "Test set shape after features extraction: (480, 2573)\n"
                    ]
                }
            ],
            "source": [
                "vectorizer = TfidfVectorizer()\n",
                "X_train = vectorizer.fit_transform(train_df['features'])\n",
                "X_test = vectorizer.transform(test_df['features'])\n",
                "print(f\"Train set shape after features extraction: {X_train.shape}\")\n",
                "print(f\"Test set shape after features extraction: {X_test.shape}\")\n",
                "y_train = train_df[\"label\"]\n",
                "y_test = test_df[\"label\"]"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 28,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "Closest Point 1: Number of Features = 500, Train Accuracy = 0.9805555555555555, Test Accuracy = 0.95625\n",
                        "Closest Point 2: Number of Features = 2000, Train Accuracy = 0.9942129629629629, Test Accuracy = 0.9645833333333333\n",
                        "Closest Point 3: Number of Features = 1750, Train Accuracy = 0.9946759259259259, Test Accuracy = 0.9645833333333333\n"
                    ]
                },
                {
                    "data": {
                        "image/png": "[encoded data, 61148 characters]",
                        "text/plain": [
                            "<Figure size 1000x600 with 1 Axes>"
                        ]
                    },
                    "metadata": {},
                    "output_type": "display_data"
                }
            ],
            "source": [
                "model = MultinomialNB(alpha=0.01)\n",
                "plot_accuracies(X_train, X_test, y_train, y_test, model, k_end=2500)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 29,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "Train Accuracy: 0.9960648148148148\n",
                        "Test Accuracy: 0.95625\n",
                        "Difference: 0.03981481481481475\n",
                        "+---------------------------------+--------------------+--------------------+------------+-----------+\n",
                        "| Class                           | Precision          | Recall             |   F1-score | Support   |\n",
                        "|---------------------------------+--------------------+--------------------+------------+-----------|\n",
                        "| Acne                            | 1.0                | 1.0                |   1        | 21.0      |\n",
                        "| Arthritis                       | 0.9523809523809523 | 1.0                |   0.97561  | 20.0      |\n",
                        "| Bronchial Asthma                | 1.0                | 0.9473684210526315 |   0.972973 | 19.0      |\n",
                        "| Cervical spondylosis            | 1.0                | 1.0                |   1        | 21.0      |\n",
                        "| Chicken pox                     | 0.6363636363636364 | 0.9333333333333333 |   0.756757 | 15.0      |\n",
                        "| Common Cold                     | 0.9545454545454546 | 1.0                |   0.976744 | 21.0      |\n",
                        "| Dengue                          | 1.0                | 0.6363636363636364 |   0.777778 | 22.0      |\n",
                        "| Dimorphic Hemorrhoids           | 1.0                | 1.0                |   1        | 19.0      |\n",
                        "| Fungal infection                | 1.0                | 0.9615384615384616 |   0.980392 | 26.0      |\n",
                        "| Hypertension                    | 1.0                | 0.9444444444444444 |   0.971429 | 18.0      |\n",
                        "| Impetigo                        | 0.8846153846153846 | 1.0                |   0.938776 | 23.0      |\n",
                        "| Jaundice                        | 1.0                | 1.0                |   1        | 22.0      |\n",
                        "| Malaria                         | 1.0                | 1.0                |   1        | 17.0      |\n",
                        "| Migraine                        | 1.0                | 1.0                |   1        | 24.0      |\n",
                        "| Pneumonia                       | 0.9565217391304348 | 1.0                |   0.977778 | 22.0      |\n",
                        "| Psoriasis                       | 0.9333333333333333 | 0.8235294117647058 |   0.875    | 17.0      |\n",
                        "| Typhoid                         | 0.9                | 1.0                |   0.947368 | 18.0      |\n",
                        "| Varicose Veins                  | 0.9565217391304348 | 0.88               |   0.916667 | 25.0      |\n",
                        "| allergy                         | 1.0                | 1.0                |   1        | 15.0      |\n",
                        "| diabetes                        | 1.0                | 1.0                |   1        | 17.0      |\n",
                        "| drug reaction                   | 0.9375             | 0.9375             |   0.9375   | 16.0      |\n",
                        "| gastroesophageal reflux disease | 0.9545454545454546 | 1.0                |   0.976744 | 21.0      |\n",
                        "| peptic ulcer disease            | 1.0                | 0.8888888888888888 |   0.941176 | 18.0      |\n",
                        "| urinary tract infection         | 0.9583333333333334 | 1.0                |   0.978723 | 23.0      |\n",
                        "| accuracy                        |                    |                    |   0.95625  |           |\n",
                        "| macro avg                       | 0.9593608761407674 | 0.9563736082244209 |   0.954226 |           |\n",
                        "| weighted avg                    | 0.962697899172084  | 0.95625            |   0.955957 |           |\n",
                        "+---------------------------------+--------------------+--------------------+------------+-----------+\n"
                    ]
                }
            ],
            "source": [
                "model = MultinomialNB(alpha=0.01)\n",
                "evaluate_model(X_train, X_test, y_train, y_test, chi2, 1750, model) "
            ]
        }
    ],
    "metadata": {
        "kernelspec": {
            "display_name": "NLP",
            "language": "python",
            "name": "python3"
        },
        "language_info": {
            "codemirror_mode": {
                "name": "ipython",
                "version": 3
            },
            "file_extension": ".py",
            "mimetype": "text/x-python",
            "name": "python",
            "nbconvert_exporter": "python",
            "pygments_lexer": "ipython3",
            "version": "3.9.20"
        }
    },
    "nbformat": 4,
    "nbformat_minor": 2
}