{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "07e239bf",
   "metadata": {},
   "source": [
    "Imports and setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "1638b7b97e3bd6f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-11-22T11:40:21.711998Z",
     "start_time": "2025-11-22T11:40:20.129376Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded 29 test pairs\n"
     ]
    }
   ],
   "source": [
    "import spacy\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "from sentence_transformers import SentenceTransformer\n",
    "\n",
    "# Load models\n",
    "nlp = spacy.load(\"en_core_web_lg\")\n",
    "model_bert = SentenceTransformer(\"all-MiniLM-L6-v2\")\n",
    "\n",
    "test_pairs = [\n",
    "## STANDARD TEST PAIRS (from previous notebooks)\n",
    "    # Direct copies and near-copies\n",
    "    (\"The cat sat on the mat.\", \"The cat sat on the mat.\"),           # Exact copy\n",
    "    (\"The cat sat on the mat.\", \"The cat sat on the mat\"),            # No punctuation\n",
    "    (\"The cat sat on the mat.\", \"The  cat  sat  on  the  mat.\"),      # Extra spaces\n",
    "    \n",
    "    # Paraphrases with same meaning\n",
    "    (\"The cat sat on the mat.\", \"On the mat, the cat was sitting.\"),  # Structural change\n",
    "    (\"The cat sat on the mat.\", \"The feline rested on the rug.\"),     # Synonym replacement\n",
    "    (\"The quick brown fox jumps.\", \"A fast brown fox leaps.\"),        # Partial synonym\n",
    "    \n",
    "    # Different sentences\n",
    "    (\"The cat sat on the mat.\", \"The dog ran in the park.\"),          # Different content\n",
    "    (\"I love programming.\", \"She enjoys reading books.\"),             # Completely different\n",
    "    (\"The weather is nice today.\", \"It's raining outside.\"),          # Opposite meaning\n",
    "    \n",
    "    # Edge cases\n",
    "    (\"Short.\", \"Short.\"),                                             # Very short\n",
    "    (\"A B C D E F G\", \"A B C D E F G\"),                              # Repeated words\n",
    "    (\"\", \"\"),                                                         # Empty strings\n",
    "## ADDITIONAL TEST PAIRS (for semantic evaluation)\n",
    "    # Polysemy (word sense ambiguity) - Critical for BERT vs SpaCy\n",
    "    (\"He went to the bank to deposit money.\", \"He went to the river bank to fish.\"),           # Bank: financial vs. riverbank\n",
    "    (\"I saw the bat in the cave.\", \"The baseball bat is broken.\"),                             # Bat: animal vs. sports equipment\n",
    "    (\"The light is very bright.\", \"Can you carry this light package?\"),                        # Light: brightness vs. weight\n",
    "    \n",
    "    # Synonymy (different words, same meaning) - Tests semantic understanding\n",
    "    (\"The car is fast.\", \"The automobile is quick.\"),                                          # car/automobile, fast/quick\n",
    "    (\"He commenced the project.\", \"He started the work.\"),                                     # commenced/started, project/work\n",
    "    (\"The substantial building is ornate.\", \"The large building is decorated.\"),               # substantial/large, ornate/decorated\n",
    "    \n",
    "    # Negation (opposite meaning) - Tests if methods catch semantic inversion\n",
    "    (\"The weather is good.\", \"The weather is bad.\"),                                           # Direct negation\n",
    "    (\"The solution is simple.\", \"The problem is complex.\"),                                    # Opposite adjectives\n",
    "    (\"I like this movie.\", \"I dislike this movie.\"),                                           # Explicit negation\n",
    "    \n",
    "    # Semantic similarity without word overlap - Pure meaning test\n",
    "    (\"A man walks down the street.\", \"A person strolls along the road.\"),                      # Very few shared words, similar meaning\n",
    "    (\"The student studied the textbook.\", \"The pupil learned from the book.\"),                 # Different words, same semantic content\n",
    "    \n",
    "    # Partial overlap (high word similarity, low semantic similarity)\n",
    "    (\"The bank is on the river.\", \"The bank account is overdrawn.\"),                           # \"bank\" is only shared word; different meanings\n",
    "    (\"I read the book last night.\", \"The book was read by many people.\"),                      # Same key words (book, read) but different focus/meaning\n",
    "    \n",
    "    # Length variation (tests robustness to sentence length)\n",
    "    (\"Cat sat.\", \"The cat sat on the mat for hours and hours.\"),                               # Very different lengths, similar core meaning\n",
    "    (\"Go.\", \"You should go to the store and buy milk immediately.\"),                           # Minimal vs. detailed\n",
    "    \n",
    "    # Metaphor/Figurative language\n",
    "    (\"Time is money.\", \"Time has value.\"),                                                     # Metaphor vs. literal\n",
    "    (\"The world is a stage.\", \"Life is like a theater performance.\"),                          # Figurative expressions\n",
    "]\n",
    "\n",
    "print(f\"Loaded {len(test_pairs)} test pairs\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b79941bf4553fd6",
   "metadata": {},
   "source": [
    "## Method 1: SIF smooth inverse frequency\n",
    "Weighting word vectors based on frequency, emphasize information-dense words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "8a3c4314a90086fe",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-11-22T11:47:39.286432Z",
     "start_time": "2025-11-22T11:47:39.271377Z"
    }
   },
   "outputs": [],
   "source": [
    "def sentence_similarity_sif(sent1, sent2, a=0.001):\n",
    "    \"\"\"Compute similarity using SIF-weighted word vectors.\"\"\"\n",
    "    def get_weighted_vector(text):\n",
    "        doc = nlp(text)\n",
    "        vectors = []\n",
    "        weights = []\n",
    "        \n",
    "        for token in doc:\n",
    "            if token.has_vector and token.is_alpha and not token.is_stop:\n",
    "                prob = np.exp(token.prob) if token.prob > -19 else 1e-6\n",
    "                weight = a / (a + prob)\n",
    "                vectors.append(token.vector)\n",
    "                weights.append(weight)\n",
    "        \n",
    "        if not vectors:\n",
    "            return np.zeros(nlp.vocab.vectors_length)\n",
    "        \n",
    "        return np.average(vectors, axis=0, weights=weights)\n",
    "\n",
    "    v1 = get_weighted_vector(sent1)\n",
    "    v2 = get_weighted_vector(sent2)\n",
    "    \n",
    "    return cosine_similarity(v1.reshape(1, -1), v2.reshape(1, -1))[0][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "c51a0436",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SIF Similarity: 0.609\n"
     ]
    }
   ],
   "source": [
    "s1 = \"The cat ate the food.\"\n",
    "s2 = \"The feline devoured the sustenance.\"\n",
    "\n",
    "print(f\"SIF Similarity: {sentence_similarity_sif(s1, s2):.3f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8f32b5695f554268",
   "metadata": {},
   "source": [
    "## Method 2: BERT - Transformer Embedding\n",
    "Use contectual embeddings to resolve polysemy issues"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "68a6757447e4a1c7",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-11-18T23:45:03.085563Z",
     "start_time": "2025-11-18T23:45:03.082190Z"
    }
   },
   "outputs": [],
   "source": [
    "def sentence_similarity_bert(sent1, sent2):\n",
    "    \"\"\"Compute similarity using SBERT contextual embeddings.\"\"\"\n",
    "    embeddings = model_bert.encode([sent1, sent2])\n",
    "    return cosine_similarity([embeddings[0]], [embeddings[1]])[0][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "420df430",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bank (Money) vs Bank (River): 0.590\n",
      "Bank (Money) vs Institution:  0.641\n"
     ]
    }
   ],
   "source": [
    "sent_a = \"He went to the bank to deposit money.\"\n",
    "sent_b = \"He went to the river bank to fish.\"\n",
    "sent_c = \"He visited the financial institution.\"\n",
    "\n",
    "print(f\"Bank (Money) vs Bank (River): {sentence_similarity_bert(sent_a, sent_b):.3f}\")\n",
    "print(f\"Bank (Money) vs Institution:  {sentence_similarity_bert(sent_a, sent_c):.3f}\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a9c3aa050f5bc0fe",
   "metadata": {},
   "source": [
    "### Semantic Evaluation\n",
    "Comparing simple average baseline, SIF and BERT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c100956f89d9b581",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Semantic Method Results:\n",
      "======================================================================\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pair_id</th>\n",
       "      <th>Spacy Avg</th>\n",
       "      <th>Spacy SIF</th>\n",
       "      <th>BERT</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>1.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0.984</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.995</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>1.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>1.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>0.965</td>\n",
       "      <td>0.954</td>\n",
       "      <td>0.930</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>0.901</td>\n",
       "      <td>0.696</td>\n",
       "      <td>0.549</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>0.942</td>\n",
       "      <td>0.930</td>\n",
       "      <td>0.945</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>6</td>\n",
       "      <td>0.857</td>\n",
       "      <td>0.642</td>\n",
       "      <td>0.067</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>7</td>\n",
       "      <td>0.702</td>\n",
       "      <td>0.529</td>\n",
       "      <td>0.154</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>8</td>\n",
       "      <td>0.845</td>\n",
       "      <td>0.603</td>\n",
       "      <td>0.468</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>9</td>\n",
       "      <td>1.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>1.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>10</td>\n",
       "      <td>1.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>1.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>11</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>1.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>12</td>\n",
       "      <td>0.908</td>\n",
       "      <td>0.707</td>\n",
       "      <td>0.590</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>13</td>\n",
       "      <td>0.844</td>\n",
       "      <td>0.716</td>\n",
       "      <td>0.472</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>14</td>\n",
       "      <td>0.803</td>\n",
       "      <td>0.694</td>\n",
       "      <td>0.310</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>15</td>\n",
       "      <td>0.939</td>\n",
       "      <td>0.803</td>\n",
       "      <td>0.831</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>16</td>\n",
       "      <td>0.909</td>\n",
       "      <td>0.619</td>\n",
       "      <td>0.778</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>17</td>\n",
       "      <td>0.934</td>\n",
       "      <td>0.838</td>\n",
       "      <td>0.819</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>18</td>\n",
       "      <td>0.979</td>\n",
       "      <td>0.930</td>\n",
       "      <td>0.822</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>19</td>\n",
       "      <td>0.885</td>\n",
       "      <td>0.623</td>\n",
       "      <td>0.620</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>20</td>\n",
       "      <td>0.967</td>\n",
       "      <td>0.873</td>\n",
       "      <td>0.721</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>21</td>\n",
       "      <td>0.912</td>\n",
       "      <td>0.780</td>\n",
       "      <td>0.603</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>22</td>\n",
       "      <td>0.874</td>\n",
       "      <td>0.712</td>\n",
       "      <td>0.652</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>23</td>\n",
       "      <td>0.813</td>\n",
       "      <td>0.632</td>\n",
       "      <td>0.506</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>24</td>\n",
       "      <td>0.891</td>\n",
       "      <td>0.891</td>\n",
       "      <td>0.620</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>25</td>\n",
       "      <td>0.761</td>\n",
       "      <td>0.755</td>\n",
       "      <td>0.628</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>26</td>\n",
       "      <td>0.806</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.202</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>27</td>\n",
       "      <td>0.890</td>\n",
       "      <td>0.764</td>\n",
       "      <td>0.735</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>28</td>\n",
       "      <td>0.897</td>\n",
       "      <td>0.721</td>\n",
       "      <td>0.442</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    pair_id  Spacy Avg  Spacy SIF   BERT\n",
       "0         0      1.000      1.000  1.000\n",
       "1         1      0.984      1.000  0.995\n",
       "2         2      1.000      1.000  1.000\n",
       "3         3      0.965      0.954  0.930\n",
       "4         4      0.901      0.696  0.549\n",
       "5         5      0.942      0.930  0.945\n",
       "6         6      0.857      0.642  0.067\n",
       "7         7      0.702      0.529  0.154\n",
       "8         8      0.845      0.603  0.468\n",
       "9         9      1.000      1.000  1.000\n",
       "10       10      1.000      1.000  1.000\n",
       "11       11      0.000      0.000  1.000\n",
       "12       12      0.908      0.707  0.590\n",
       "13       13      0.844      0.716  0.472\n",
       "14       14      0.803      0.694  0.310\n",
       "15       15      0.939      0.803  0.831\n",
       "16       16      0.909      0.619  0.778\n",
       "17       17      0.934      0.838  0.819\n",
       "18       18      0.979      0.930  0.822\n",
       "19       19      0.885      0.623  0.620\n",
       "20       20      0.967      0.873  0.721\n",
       "21       21      0.912      0.780  0.603\n",
       "22       22      0.874      0.712  0.652\n",
       "23       23      0.813      0.632  0.506\n",
       "24       24      0.891      0.891  0.620\n",
       "25       25      0.761      0.755  0.628\n",
       "26       26      0.806      0.000  0.202\n",
       "27       27      0.890      0.764  0.735\n",
       "28       28      0.897      0.721  0.442"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def sentence_similarity_avg(s1, s2):\n",
    "    d1, d2 = nlp(s1), nlp(s2)\n",
    "    v1 = np.mean([t.vector for t in d1 if t.has_vector], axis=0) if len(d1) > 0 else np.zeros(300)\n",
    "    v2 = np.mean([t.vector for t in d2 if t.has_vector], axis=0) if len(d2) > 0 else np.zeros(300)\n",
    "    return cosine_similarity([v1], [v2])[0][0]\n",
    "\n",
    "def evaluate_semantic_methods(pairs):\n",
    "    \"\"\"Evaluate semantic methods on test pairs without verbose text output.\"\"\"\n",
    "    methods = {\n",
    "        \"Spacy Avg\": sentence_similarity_avg,\n",
    "        \"Spacy SIF\": sentence_similarity_sif,\n",
    "        \"BERT\": sentence_similarity_bert\n",
    "    }\n",
    "\n",
    "    rows = []\n",
    "    for i, (sent1, sent2) in enumerate(pairs):\n",
    "        row = {\"pair_id\": i, \"sentence_1\": sent1, \"sentence_2\": sent2}\n",
    "        for name, func in methods.items():\n",
    "            score = func(sent1, sent2)\n",
    "            row[name] = score\n",
    "        rows.append(row)\n",
    "\n",
    "    return pd.DataFrame(rows)\n",
    "\n",
    "results_df = evaluate_semantic_methods(test_pairs)\n",
    "\n",
    "# Display only the scores (not sentences) for a cleaner comparison\n",
    "print(\"Semantic Method Results:\")\n",
    "print(\"=\" * 70)\n",
    "display(results_df[[\"pair_id\", \"Spacy Avg\", \"Spacy SIF\", \"BERT\"]].round(3))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e5192493",
   "metadata": {},
   "source": [
    "### Visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "5e586947",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 35328 characters]",
      "text/plain": [
       "<Figure size 1000x500 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(10, 5))\n",
    "melted = results_df.melt(\n",
    "    id_vars=[\"pair_id\"], \n",
    "    value_vars=[\"Spacy Avg\", \"Spacy SIF\", \"BERT\"], \n",
    "    var_name=\"Method\", \n",
    "    value_name=\"Similarity\"\n",
    ")\n",
    "\n",
    "sns.barplot(data=melted, x=\"pair_id\", y=\"Similarity\", hue=\"Method\")\n",
    "plt.title(\"Semantic Similarity Scores by Method\")\n",
    "plt.xlabel(\"Pair ID\")\n",
    "plt.ylabel(\"Similarity\")\n",
    "plt.ylim(0, 1)\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d3b42bc4",
   "metadata": {},
   "source": [
    "Category Comparison"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "57f4e206",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Performance by Category:\n",
      "================================================================================\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\">Spacy Avg</th>\n",
       "      <th colspan=\"2\" halign=\"left\">Spacy SIF</th>\n",
       "      <th colspan=\"2\" halign=\"left\">BERT</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>category</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Different</th>\n",
       "      <td>0.801</td>\n",
       "      <td>0.086</td>\n",
       "      <td>0.591</td>\n",
       "      <td>0.057</td>\n",
       "      <td>0.229</td>\n",
       "      <td>0.211</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Edge Case</th>\n",
       "      <td>0.667</td>\n",
       "      <td>0.577</td>\n",
       "      <td>0.667</td>\n",
       "      <td>0.577</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Exact Copy</th>\n",
       "      <td>0.995</td>\n",
       "      <td>0.009</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.998</td>\n",
       "      <td>0.003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Figurative</th>\n",
       "      <td>0.893</td>\n",
       "      <td>0.005</td>\n",
       "      <td>0.742</td>\n",
       "      <td>0.031</td>\n",
       "      <td>0.588</td>\n",
       "      <td>0.208</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Length Var</th>\n",
       "      <td>0.784</td>\n",
       "      <td>0.032</td>\n",
       "      <td>0.377</td>\n",
       "      <td>0.534</td>\n",
       "      <td>0.415</td>\n",
       "      <td>0.302</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Negation</th>\n",
       "      <td>0.944</td>\n",
       "      <td>0.051</td>\n",
       "      <td>0.809</td>\n",
       "      <td>0.163</td>\n",
       "      <td>0.721</td>\n",
       "      <td>0.101</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Paraphrase</th>\n",
       "      <td>0.936</td>\n",
       "      <td>0.032</td>\n",
       "      <td>0.860</td>\n",
       "      <td>0.143</td>\n",
       "      <td>0.808</td>\n",
       "      <td>0.224</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Partial Overlap</th>\n",
       "      <td>0.852</td>\n",
       "      <td>0.055</td>\n",
       "      <td>0.762</td>\n",
       "      <td>0.183</td>\n",
       "      <td>0.563</td>\n",
       "      <td>0.080</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Polysemy</th>\n",
       "      <td>0.851</td>\n",
       "      <td>0.053</td>\n",
       "      <td>0.706</td>\n",
       "      <td>0.011</td>\n",
       "      <td>0.457</td>\n",
       "      <td>0.140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Semantic Sim</th>\n",
       "      <td>0.893</td>\n",
       "      <td>0.027</td>\n",
       "      <td>0.746</td>\n",
       "      <td>0.048</td>\n",
       "      <td>0.628</td>\n",
       "      <td>0.035</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Synonymy</th>\n",
       "      <td>0.927</td>\n",
       "      <td>0.016</td>\n",
       "      <td>0.754</td>\n",
       "      <td>0.117</td>\n",
       "      <td>0.809</td>\n",
       "      <td>0.028</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                Spacy Avg        Spacy SIF          BERT       \n",
       "                     mean    std      mean    std   mean    std\n",
       "category                                                       \n",
       "Different           0.801  0.086     0.591  0.057  0.229  0.211\n",
       "Edge Case           0.667  0.577     0.667  0.577  1.000  0.000\n",
       "Exact Copy          0.995  0.009     1.000  0.000  0.998  0.003\n",
       "Figurative          0.893  0.005     0.742  0.031  0.588  0.208\n",
       "Length Var          0.784  0.032     0.377  0.534  0.415  0.302\n",
       "Negation            0.944  0.051     0.809  0.163  0.721  0.101\n",
       "Paraphrase          0.936  0.032     0.860  0.143  0.808  0.224\n",
       "Partial Overlap     0.852  0.055     0.762  0.183  0.563  0.080\n",
       "Polysemy            0.851  0.053     0.706  0.011  0.457  0.140\n",
       "Semantic Sim        0.893  0.027     0.746  0.048  0.628  0.035\n",
       "Synonymy            0.927  0.016     0.754  0.117  0.809  0.028"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Manually assign categories to each pair\n",
    "categories = [\n",
    "    # Direct copies (0-2)\n",
    "    'Exact Copy', 'Exact Copy', 'Exact Copy',\n",
    "    # Paraphrases (3-5)\n",
    "    'Paraphrase', 'Paraphrase', 'Paraphrase',\n",
    "    # Different (6-8)\n",
    "    'Different', 'Different', 'Different',\n",
    "    # Edge cases (9-11)\n",
    "    'Edge Case', 'Edge Case', 'Edge Case',\n",
    "    # Polysemy (12-14)\n",
    "    'Polysemy', 'Polysemy', 'Polysemy',\n",
    "    # Synonymy (15-17)\n",
    "    'Synonymy', 'Synonymy', 'Synonymy',\n",
    "    # Negation (18-20)\n",
    "    'Negation', 'Negation', 'Negation',\n",
    "    # Semantic Similarity (21-22)\n",
    "    'Semantic Sim', 'Semantic Sim',\n",
    "    # Partial Overlap (23-24)\n",
    "    'Partial Overlap', 'Partial Overlap',\n",
    "    # Length Variation (25-26)\n",
    "    'Length Var', 'Length Var',\n",
    "    # Figurative (27-28)\n",
    "    'Figurative', 'Figurative'\n",
    "]\n",
    "\n",
    "results_df['category'] = categories\n",
    "\n",
    "# Group by category and show average scores\n",
    "print(\"\\nPerformance by Category:\")\n",
    "print(\"=\" * 80)\n",
    "category_stats = results_df.groupby('category')[[\"Spacy Avg\", \"Spacy SIF\", \"BERT\"]].agg(['mean', 'std']).round(3)\n",
    "display(category_stats)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cf836250",
   "metadata": {},
   "source": [
    "Comparison Heatmap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "66a4bb52",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 120384 characters]",
      "text/plain": [
       "<Figure size 1400x800 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(14, 8))\n",
    "\n",
    "# Create a pivot table for heatmap (pairs vs methods)\n",
    "heatmap_data = results_df[[\"pair_id\", \"Spacy Avg\", \"Spacy SIF\", \"BERT\"]].set_index(\"pair_id\")\n",
    "\n",
    "sns.heatmap(heatmap_data, annot=True, fmt='.2f', cmap='RdYlGn', cbar_kws={'label': 'Similarity'}, \n",
    "            vmin=0, vmax=1, linewidths=0.5)\n",
    "plt.title('Semantic Similarity Heatmap: All Pairs vs Methods', fontsize=14, fontweight='bold')\n",
    "plt.xlabel('Method')\n",
    "plt.ylabel('Pair ID')\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "89686f33",
   "metadata": {},
   "source": [
    "A method with higher varience it may be less reliable while all 3 methods agreeing indicates that a pair is confident"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "8c0d18f9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Method Variance (higher = more disagreement):\n",
      "Spacy Avg    0.033659\n",
      "Spacy SIF    0.061698\n",
      "BERT         0.073351\n",
      "dtype: float64\n",
      "\n",
      "Method Std Dev:\n",
      "Spacy Avg    0.183464\n",
      "Spacy SIF    0.248391\n",
      "BERT         0.270834\n",
      "dtype: float64\n"
     ]
    }
   ],
   "source": [
    "\n",
    "method_variance = results_df[[\"Spacy Avg\", \"Spacy SIF\", \"BERT\"]].var()\n",
    "method_std = results_df[[\"Spacy Avg\", \"Spacy SIF\", \"BERT\"]].std()\n",
    "\n",
    "print(\"Method Variance (higher = more disagreement):\")\n",
    "print(method_variance)\n",
    "print(\"\\nMethod Std Dev:\")\n",
    "print(method_std)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a05b9608",
   "metadata": {},
   "source": [
    "Two methods may give similar absoloute scores when compared but rank individual pairs differently"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "9cf007fb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    pair_id  Spacy Avg_rank  Spacy SIF_rank  BERT_rank\n",
      "0         0            28.5            26.5       27.5\n",
      "1         1            25.0            26.5       24.0\n",
      "2         2            28.5            26.5       27.5\n",
      "3         3            22.0            24.0       22.0\n",
      "4         4            15.0            10.0        9.0\n",
      "5         5            21.0            22.0       23.0\n",
      "6         6             9.0             8.0        1.0\n",
      "7         7             2.0             3.0        2.0\n",
      "8         8             8.0             4.0        6.0\n",
      "9         9            27.0            29.0       27.5\n",
      "10       10            26.0            26.5       27.5\n",
      "11       11             1.0             1.5       25.0\n",
      "12       12            16.0            11.0       10.0\n",
      "13       13             7.0            13.0        7.0\n",
      "14       14             4.0             9.0        4.0\n",
      "15       15            20.0            18.0       21.0\n",
      "16       16            17.0             5.0       18.0\n",
      "17       17            19.0            19.0       19.0\n",
      "18       18            24.0            23.0       20.0\n",
      "19       19            11.0             6.0       13.0\n",
      "20       20            23.0            20.0       16.0\n",
      "21       21            18.0            17.0       11.0\n",
      "22       22            10.0            12.0       15.0\n",
      "23       23             6.0             7.0        8.0\n",
      "24       24            13.0            21.0       12.0\n",
      "25       25             3.0            15.0       14.0\n",
      "26       26             5.0             1.5        3.0\n",
      "27       27            12.0            16.0       17.0\n",
      "28       28            14.0            14.0        5.0\n"
     ]
    }
   ],
   "source": [
    "for method in [\"Spacy Avg\", \"Spacy SIF\", \"BERT\"]:\n",
    "    results_df[f'{method}_rank'] = results_df[method].rank()\n",
    "\n",
    "print(results_df[[\"pair_id\", \"Spacy Avg_rank\", \"Spacy SIF_rank\", \"BERT_rank\"]])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "df98b471",
   "metadata": {},
   "source": [
    "Test methods against the polysemy problem"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "04f40f67",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Polysemy Test (Should prefer higher for synonyms, lower for different meanings):\n",
      "   pair_id               sentence_1  Spacy Avg  Spacy SIF      BERT\n",
      "6        6  The cat sat on the mat.   0.856538   0.641634  0.067119\n",
      "7        7      I love programming.   0.702400   0.528659  0.153708\n"
     ]
    }
   ],
   "source": [
    "polysemy_pairs = results_df.iloc[6:8]  # Rows 6-7 are the \"bank\" examples\n",
    "print(\"Polysemy Test (Should prefer higher for synonyms, lower for different meanings):\")\n",
    "print(polysemy_pairs[[\"pair_id\", \"sentence_1\", \"Spacy Avg\", \"Spacy SIF\", \"BERT\"]])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.14.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
