notebook/FAQIR/09.Evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import required libraries\n",
    "import sys\n",
    "sys.path.insert(0, '../../../BERT-FAQ/')\n",
    "\n",
    "from evaluation import Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "rank_results_filepath=\"../../../BERT-FAQ/data/FAQIR/rank_results\"\n",
    "\n",
    "ev = Evaluation()\n",
    "df = ev.get_eval_df(rank_results_filepath)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Method</th>\n",
       "      <th>Matching Field</th>\n",
       "      <th>Ranking Field</th>\n",
       "      <th>Loss</th>\n",
       "      <th>Training Data</th>\n",
       "      <th>Negative Sampling</th>\n",
       "      <th>NDCG@3</th>\n",
       "      <th>NDCG@5</th>\n",
       "      <th>NDCG@10</th>\n",
       "      <th>P@3</th>\n",
       "      <th>P@5</th>\n",
       "      <th>P@10</th>\n",
       "      <th>MAP</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Unsupervised</td>\n",
       "      <td>answer</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>0.2667</td>\n",
       "      <td>0.1431</td>\n",
       "      <td>0.2905</td>\n",
       "      <td>0.1212</td>\n",
       "      <td>0.3069</td>\n",
       "      <td>0.0916</td>\n",
       "      <td>0.1884</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Unsupervised</td>\n",
       "      <td>question</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>0.3798</td>\n",
       "      <td>0.2060</td>\n",
       "      <td>0.4045</td>\n",
       "      <td>0.1726</td>\n",
       "      <td>0.4258</td>\n",
       "      <td>0.1303</td>\n",
       "      <td>0.2781</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Unsupervised</td>\n",
       "      <td>question_answer</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>0.3761</td>\n",
       "      <td>0.2029</td>\n",
       "      <td>0.4008</td>\n",
       "      <td>0.1708</td>\n",
       "      <td>0.4225</td>\n",
       "      <td>0.1327</td>\n",
       "      <td>0.2664</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Unsupervised</td>\n",
       "      <td>question_answer_concat</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>0.4318</td>\n",
       "      <td>0.2482</td>\n",
       "      <td>0.4573</td>\n",
       "      <td>0.2130</td>\n",
       "      <td>0.4783</td>\n",
       "      <td>0.1606</td>\n",
       "      <td>0.3106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>answer</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>triplet</td>\n",
       "      <td>faq</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.3036</td>\n",
       "      <td>0.1679</td>\n",
       "      <td>0.3264</td>\n",
       "      <td>0.1473</td>\n",
       "      <td>0.3513</td>\n",
       "      <td>0.1136</td>\n",
       "      <td>0.2321</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>triplet</td>\n",
       "      <td>faq</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.4126</td>\n",
       "      <td>0.2316</td>\n",
       "      <td>0.4403</td>\n",
       "      <td>0.1988</td>\n",
       "      <td>0.4625</td>\n",
       "      <td>0.1499</td>\n",
       "      <td>0.3156</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>triplet</td>\n",
       "      <td>faq</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.4136</td>\n",
       "      <td>0.2299</td>\n",
       "      <td>0.4442</td>\n",
       "      <td>0.2039</td>\n",
       "      <td>0.4633</td>\n",
       "      <td>0.1550</td>\n",
       "      <td>0.3124</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer_concat</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>triplet</td>\n",
       "      <td>faq</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.4687</td>\n",
       "      <td>0.2787</td>\n",
       "      <td>0.4940</td>\n",
       "      <td>0.2399</td>\n",
       "      <td>0.5127</td>\n",
       "      <td>0.1809</td>\n",
       "      <td>0.3487</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>answer</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>triplet</td>\n",
       "      <td>faq</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.2989</td>\n",
       "      <td>0.1629</td>\n",
       "      <td>0.3216</td>\n",
       "      <td>0.1354</td>\n",
       "      <td>0.3444</td>\n",
       "      <td>0.1052</td>\n",
       "      <td>0.2191</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>triplet</td>\n",
       "      <td>faq</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.4031</td>\n",
       "      <td>0.2220</td>\n",
       "      <td>0.4309</td>\n",
       "      <td>0.1888</td>\n",
       "      <td>0.4493</td>\n",
       "      <td>0.1399</td>\n",
       "      <td>0.3022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>triplet</td>\n",
       "      <td>faq</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.4082</td>\n",
       "      <td>0.2246</td>\n",
       "      <td>0.4330</td>\n",
       "      <td>0.1937</td>\n",
       "      <td>0.4538</td>\n",
       "      <td>0.1460</td>\n",
       "      <td>0.2973</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer_concat</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>triplet</td>\n",
       "      <td>faq</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.4622</td>\n",
       "      <td>0.2719</td>\n",
       "      <td>0.4866</td>\n",
       "      <td>0.2292</td>\n",
       "      <td>0.5069</td>\n",
       "      <td>0.1731</td>\n",
       "      <td>0.3390</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>answer</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>triplet</td>\n",
       "      <td>user_query</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.3213</td>\n",
       "      <td>0.1795</td>\n",
       "      <td>0.3409</td>\n",
       "      <td>0.1540</td>\n",
       "      <td>0.3687</td>\n",
       "      <td>0.1199</td>\n",
       "      <td>0.2461</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>triplet</td>\n",
       "      <td>user_query</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.4394</td>\n",
       "      <td>0.2527</td>\n",
       "      <td>0.4669</td>\n",
       "      <td>0.2140</td>\n",
       "      <td>0.4869</td>\n",
       "      <td>0.1620</td>\n",
       "      <td>0.3430</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>triplet</td>\n",
       "      <td>user_query</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.4387</td>\n",
       "      <td>0.2477</td>\n",
       "      <td>0.4691</td>\n",
       "      <td>0.2156</td>\n",
       "      <td>0.4860</td>\n",
       "      <td>0.1680</td>\n",
       "      <td>0.3345</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer_concat</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>triplet</td>\n",
       "      <td>user_query</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.4960</td>\n",
       "      <td>0.3009</td>\n",
       "      <td>0.5237</td>\n",
       "      <td>0.2568</td>\n",
       "      <td>0.5387</td>\n",
       "      <td>0.1924</td>\n",
       "      <td>0.3771</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>answer</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>triplet</td>\n",
       "      <td>user_query</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.3545</td>\n",
       "      <td>0.2079</td>\n",
       "      <td>0.3748</td>\n",
       "      <td>0.1768</td>\n",
       "      <td>0.3967</td>\n",
       "      <td>0.1329</td>\n",
       "      <td>0.2760</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>triplet</td>\n",
       "      <td>user_query</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.4766</td>\n",
       "      <td>0.2778</td>\n",
       "      <td>0.4978</td>\n",
       "      <td>0.2323</td>\n",
       "      <td>0.5173</td>\n",
       "      <td>0.1771</td>\n",
       "      <td>0.3727</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>triplet</td>\n",
       "      <td>user_query</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.4828</td>\n",
       "      <td>0.2843</td>\n",
       "      <td>0.5071</td>\n",
       "      <td>0.2482</td>\n",
       "      <td>0.5239</td>\n",
       "      <td>0.1891</td>\n",
       "      <td>0.3767</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer_concat</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>triplet</td>\n",
       "      <td>user_query</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.5430</td>\n",
       "      <td>0.3404</td>\n",
       "      <td>0.5652</td>\n",
       "      <td>0.2857</td>\n",
       "      <td>0.5784</td>\n",
       "      <td>0.2178</td>\n",
       "      <td>0.4197</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>answer</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>softmax</td>\n",
       "      <td>faq</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.2931</td>\n",
       "      <td>0.1620</td>\n",
       "      <td>0.3146</td>\n",
       "      <td>0.1385</td>\n",
       "      <td>0.3402</td>\n",
       "      <td>0.1087</td>\n",
       "      <td>0.2195</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>softmax</td>\n",
       "      <td>faq</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.4019</td>\n",
       "      <td>0.2223</td>\n",
       "      <td>0.4280</td>\n",
       "      <td>0.1858</td>\n",
       "      <td>0.4494</td>\n",
       "      <td>0.1434</td>\n",
       "      <td>0.2997</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>softmax</td>\n",
       "      <td>faq</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.4005</td>\n",
       "      <td>0.2223</td>\n",
       "      <td>0.4248</td>\n",
       "      <td>0.1909</td>\n",
       "      <td>0.4487</td>\n",
       "      <td>0.1467</td>\n",
       "      <td>0.2918</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer_concat</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>softmax</td>\n",
       "      <td>faq</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.4507</td>\n",
       "      <td>0.2629</td>\n",
       "      <td>0.4761</td>\n",
       "      <td>0.2235</td>\n",
       "      <td>0.4984</td>\n",
       "      <td>0.1711</td>\n",
       "      <td>0.3304</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>answer</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>softmax</td>\n",
       "      <td>faq</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.2669</td>\n",
       "      <td>0.1440</td>\n",
       "      <td>0.2909</td>\n",
       "      <td>0.1219</td>\n",
       "      <td>0.3068</td>\n",
       "      <td>0.0915</td>\n",
       "      <td>0.1887</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>softmax</td>\n",
       "      <td>faq</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.3794</td>\n",
       "      <td>0.2060</td>\n",
       "      <td>0.4051</td>\n",
       "      <td>0.1726</td>\n",
       "      <td>0.4265</td>\n",
       "      <td>0.1304</td>\n",
       "      <td>0.2785</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>softmax</td>\n",
       "      <td>faq</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.3772</td>\n",
       "      <td>0.2034</td>\n",
       "      <td>0.4019</td>\n",
       "      <td>0.1711</td>\n",
       "      <td>0.4239</td>\n",
       "      <td>0.1331</td>\n",
       "      <td>0.2676</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer_concat</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>softmax</td>\n",
       "      <td>faq</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.4324</td>\n",
       "      <td>0.2488</td>\n",
       "      <td>0.4576</td>\n",
       "      <td>0.2128</td>\n",
       "      <td>0.4793</td>\n",
       "      <td>0.1608</td>\n",
       "      <td>0.3117</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>answer</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>softmax</td>\n",
       "      <td>user_query</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.3104</td>\n",
       "      <td>0.1708</td>\n",
       "      <td>0.3339</td>\n",
       "      <td>0.1488</td>\n",
       "      <td>0.3565</td>\n",
       "      <td>0.1125</td>\n",
       "      <td>0.2330</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>softmax</td>\n",
       "      <td>user_query</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.4293</td>\n",
       "      <td>0.2403</td>\n",
       "      <td>0.4507</td>\n",
       "      <td>0.2014</td>\n",
       "      <td>0.4745</td>\n",
       "      <td>0.1509</td>\n",
       "      <td>0.3255</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>softmax</td>\n",
       "      <td>user_query</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.4294</td>\n",
       "      <td>0.2426</td>\n",
       "      <td>0.4532</td>\n",
       "      <td>0.2085</td>\n",
       "      <td>0.4740</td>\n",
       "      <td>0.1583</td>\n",
       "      <td>0.3183</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer_concat</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>softmax</td>\n",
       "      <td>user_query</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.4900</td>\n",
       "      <td>0.2925</td>\n",
       "      <td>0.5140</td>\n",
       "      <td>0.2487</td>\n",
       "      <td>0.5311</td>\n",
       "      <td>0.1855</td>\n",
       "      <td>0.3639</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>answer</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>softmax</td>\n",
       "      <td>user_query</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.3668</td>\n",
       "      <td>0.2178</td>\n",
       "      <td>0.3910</td>\n",
       "      <td>0.1844</td>\n",
       "      <td>0.4106</td>\n",
       "      <td>0.1410</td>\n",
       "      <td>0.2917</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>softmax</td>\n",
       "      <td>user_query</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.4936</td>\n",
       "      <td>0.2888</td>\n",
       "      <td>0.5165</td>\n",
       "      <td>0.2426</td>\n",
       "      <td>0.5311</td>\n",
       "      <td>0.1842</td>\n",
       "      <td>0.3910</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>softmax</td>\n",
       "      <td>user_query</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.5035</td>\n",
       "      <td>0.3043</td>\n",
       "      <td>0.5237</td>\n",
       "      <td>0.2602</td>\n",
       "      <td>0.5412</td>\n",
       "      <td>0.2001</td>\n",
       "      <td>0.3970</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer_concat</td>\n",
       "      <td>BERT-Q-a</td>\n",
       "      <td>softmax</td>\n",
       "      <td>user_query</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.5658</td>\n",
       "      <td>0.3601</td>\n",
       "      <td>0.5823</td>\n",
       "      <td>0.2994</td>\n",
       "      <td>0.5971</td>\n",
       "      <td>0.2274</td>\n",
       "      <td>0.4426</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>answer</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>triplet</td>\n",
       "      <td>faq</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.2667</td>\n",
       "      <td>0.1431</td>\n",
       "      <td>0.2905</td>\n",
       "      <td>0.1212</td>\n",
       "      <td>0.3069</td>\n",
       "      <td>0.0916</td>\n",
       "      <td>0.1884</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>triplet</td>\n",
       "      <td>faq</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.3798</td>\n",
       "      <td>0.2060</td>\n",
       "      <td>0.4045</td>\n",
       "      <td>0.1726</td>\n",
       "      <td>0.4258</td>\n",
       "      <td>0.1303</td>\n",
       "      <td>0.2781</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>triplet</td>\n",
       "      <td>faq</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.3761</td>\n",
       "      <td>0.2029</td>\n",
       "      <td>0.4008</td>\n",
       "      <td>0.1708</td>\n",
       "      <td>0.4225</td>\n",
       "      <td>0.1327</td>\n",
       "      <td>0.2664</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer_concat</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>triplet</td>\n",
       "      <td>faq</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.4318</td>\n",
       "      <td>0.2482</td>\n",
       "      <td>0.4573</td>\n",
       "      <td>0.2130</td>\n",
       "      <td>0.4783</td>\n",
       "      <td>0.1606</td>\n",
       "      <td>0.3106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>answer</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>triplet</td>\n",
       "      <td>faq</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.2667</td>\n",
       "      <td>0.1431</td>\n",
       "      <td>0.2905</td>\n",
       "      <td>0.1212</td>\n",
       "      <td>0.3069</td>\n",
       "      <td>0.0916</td>\n",
       "      <td>0.1884</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>triplet</td>\n",
       "      <td>faq</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.3798</td>\n",
       "      <td>0.2060</td>\n",
       "      <td>0.4045</td>\n",
       "      <td>0.1726</td>\n",
       "      <td>0.4258</td>\n",
       "      <td>0.1303</td>\n",
       "      <td>0.2781</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>triplet</td>\n",
       "      <td>faq</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.3761</td>\n",
       "      <td>0.2029</td>\n",
       "      <td>0.4008</td>\n",
       "      <td>0.1708</td>\n",
       "      <td>0.4225</td>\n",
       "      <td>0.1327</td>\n",
       "      <td>0.2664</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer_concat</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>triplet</td>\n",
       "      <td>faq</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.4318</td>\n",
       "      <td>0.2482</td>\n",
       "      <td>0.4573</td>\n",
       "      <td>0.2130</td>\n",
       "      <td>0.4783</td>\n",
       "      <td>0.1606</td>\n",
       "      <td>0.3106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>answer</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>triplet</td>\n",
       "      <td>user_query</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.2667</td>\n",
       "      <td>0.1431</td>\n",
       "      <td>0.2905</td>\n",
       "      <td>0.1212</td>\n",
       "      <td>0.3069</td>\n",
       "      <td>0.0916</td>\n",
       "      <td>0.1884</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>triplet</td>\n",
       "      <td>user_query</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.3798</td>\n",
       "      <td>0.2060</td>\n",
       "      <td>0.4045</td>\n",
       "      <td>0.1726</td>\n",
       "      <td>0.4258</td>\n",
       "      <td>0.1303</td>\n",
       "      <td>0.2781</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>triplet</td>\n",
       "      <td>user_query</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.3761</td>\n",
       "      <td>0.2029</td>\n",
       "      <td>0.4008</td>\n",
       "      <td>0.1708</td>\n",
       "      <td>0.4225</td>\n",
       "      <td>0.1327</td>\n",
       "      <td>0.2664</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer_concat</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>triplet</td>\n",
       "      <td>user_query</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.4318</td>\n",
       "      <td>0.2482</td>\n",
       "      <td>0.4573</td>\n",
       "      <td>0.2130</td>\n",
       "      <td>0.4783</td>\n",
       "      <td>0.1606</td>\n",
       "      <td>0.3106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>answer</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>triplet</td>\n",
       "      <td>user_query</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.2667</td>\n",
       "      <td>0.1431</td>\n",
       "      <td>0.2905</td>\n",
       "      <td>0.1212</td>\n",
       "      <td>0.3069</td>\n",
       "      <td>0.0916</td>\n",
       "      <td>0.1884</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>triplet</td>\n",
       "      <td>user_query</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.3798</td>\n",
       "      <td>0.2060</td>\n",
       "      <td>0.4045</td>\n",
       "      <td>0.1726</td>\n",
       "      <td>0.4258</td>\n",
       "      <td>0.1303</td>\n",
       "      <td>0.2781</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>triplet</td>\n",
       "      <td>user_query</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.3761</td>\n",
       "      <td>0.2029</td>\n",
       "      <td>0.4008</td>\n",
       "      <td>0.1708</td>\n",
       "      <td>0.4225</td>\n",
       "      <td>0.1327</td>\n",
       "      <td>0.2664</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer_concat</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>triplet</td>\n",
       "      <td>user_query</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.4318</td>\n",
       "      <td>0.2482</td>\n",
       "      <td>0.4573</td>\n",
       "      <td>0.2130</td>\n",
       "      <td>0.4783</td>\n",
       "      <td>0.1606</td>\n",
       "      <td>0.3106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>answer</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>softmax</td>\n",
       "      <td>faq</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.2667</td>\n",
       "      <td>0.1431</td>\n",
       "      <td>0.2905</td>\n",
       "      <td>0.1212</td>\n",
       "      <td>0.3069</td>\n",
       "      <td>0.0916</td>\n",
       "      <td>0.1884</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>softmax</td>\n",
       "      <td>faq</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.3798</td>\n",
       "      <td>0.2060</td>\n",
       "      <td>0.4045</td>\n",
       "      <td>0.1726</td>\n",
       "      <td>0.4258</td>\n",
       "      <td>0.1303</td>\n",
       "      <td>0.2781</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>softmax</td>\n",
       "      <td>faq</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.3761</td>\n",
       "      <td>0.2029</td>\n",
       "      <td>0.4008</td>\n",
       "      <td>0.1708</td>\n",
       "      <td>0.4225</td>\n",
       "      <td>0.1327</td>\n",
       "      <td>0.2664</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer_concat</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>softmax</td>\n",
       "      <td>faq</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.4318</td>\n",
       "      <td>0.2482</td>\n",
       "      <td>0.4573</td>\n",
       "      <td>0.2130</td>\n",
       "      <td>0.4783</td>\n",
       "      <td>0.1606</td>\n",
       "      <td>0.3106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>answer</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>softmax</td>\n",
       "      <td>faq</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.2667</td>\n",
       "      <td>0.1431</td>\n",
       "      <td>0.2905</td>\n",
       "      <td>0.1212</td>\n",
       "      <td>0.3069</td>\n",
       "      <td>0.0916</td>\n",
       "      <td>0.1884</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>softmax</td>\n",
       "      <td>faq</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.3798</td>\n",
       "      <td>0.2060</td>\n",
       "      <td>0.4045</td>\n",
       "      <td>0.1726</td>\n",
       "      <td>0.4258</td>\n",
       "      <td>0.1303</td>\n",
       "      <td>0.2781</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>softmax</td>\n",
       "      <td>faq</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.3761</td>\n",
       "      <td>0.2029</td>\n",
       "      <td>0.4008</td>\n",
       "      <td>0.1708</td>\n",
       "      <td>0.4225</td>\n",
       "      <td>0.1327</td>\n",
       "      <td>0.2664</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer_concat</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>softmax</td>\n",
       "      <td>faq</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.4318</td>\n",
       "      <td>0.2482</td>\n",
       "      <td>0.4573</td>\n",
       "      <td>0.2130</td>\n",
       "      <td>0.4783</td>\n",
       "      <td>0.1606</td>\n",
       "      <td>0.3106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>answer</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>softmax</td>\n",
       "      <td>user_query</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.2667</td>\n",
       "      <td>0.1431</td>\n",
       "      <td>0.2905</td>\n",
       "      <td>0.1212</td>\n",
       "      <td>0.3069</td>\n",
       "      <td>0.0916</td>\n",
       "      <td>0.1884</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>softmax</td>\n",
       "      <td>user_query</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.3798</td>\n",
       "      <td>0.2060</td>\n",
       "      <td>0.4045</td>\n",
       "      <td>0.1726</td>\n",
       "      <td>0.4258</td>\n",
       "      <td>0.1303</td>\n",
       "      <td>0.2781</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>softmax</td>\n",
       "      <td>user_query</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.3761</td>\n",
       "      <td>0.2029</td>\n",
       "      <td>0.4008</td>\n",
       "      <td>0.1708</td>\n",
       "      <td>0.4225</td>\n",
       "      <td>0.1327</td>\n",
       "      <td>0.2664</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer_concat</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>softmax</td>\n",
       "      <td>user_query</td>\n",
       "      <td>simple</td>\n",
       "      <td>0.4318</td>\n",
       "      <td>0.2482</td>\n",
       "      <td>0.4573</td>\n",
       "      <td>0.2130</td>\n",
       "      <td>0.4783</td>\n",
       "      <td>0.1606</td>\n",
       "      <td>0.3106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>64</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>answer</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>softmax</td>\n",
       "      <td>user_query</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.2667</td>\n",
       "      <td>0.1431</td>\n",
       "      <td>0.2905</td>\n",
       "      <td>0.1212</td>\n",
       "      <td>0.3069</td>\n",
       "      <td>0.0916</td>\n",
       "      <td>0.1884</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>softmax</td>\n",
       "      <td>user_query</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.3798</td>\n",
       "      <td>0.2060</td>\n",
       "      <td>0.4045</td>\n",
       "      <td>0.1726</td>\n",
       "      <td>0.4258</td>\n",
       "      <td>0.1303</td>\n",
       "      <td>0.2781</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>66</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>softmax</td>\n",
       "      <td>user_query</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.3761</td>\n",
       "      <td>0.2029</td>\n",
       "      <td>0.4008</td>\n",
       "      <td>0.1708</td>\n",
       "      <td>0.4225</td>\n",
       "      <td>0.1327</td>\n",
       "      <td>0.2664</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>67</th>\n",
       "      <td>Supervised</td>\n",
       "      <td>question_answer_concat</td>\n",
       "      <td>BERT-Q-q</td>\n",
       "      <td>softmax</td>\n",
       "      <td>user_query</td>\n",
       "      <td>hard</td>\n",
       "      <td>0.4318</td>\n",
       "      <td>0.2482</td>\n",
       "      <td>0.4573</td>\n",
       "      <td>0.2130</td>\n",
       "      <td>0.4783</td>\n",
       "      <td>0.1606</td>\n",
       "      <td>0.3106</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          Method          Matching Field Ranking Field     Loss Training Data  \\\n",
       "0   Unsupervised                  answer                                        \n",
       "1   Unsupervised                question                                        \n",
       "2   Unsupervised         question_answer                                        \n",
       "3   Unsupervised  question_answer_concat                                        \n",
       "4     Supervised                  answer      BERT-Q-a  triplet           faq   \n",
       "5     Supervised                question      BERT-Q-a  triplet           faq   \n",
       "6     Supervised         question_answer      BERT-Q-a  triplet           faq   \n",
       "7     Supervised  question_answer_concat      BERT-Q-a  triplet           faq   \n",
       "8     Supervised                  answer      BERT-Q-a  triplet           faq   \n",
       "9     Supervised                question      BERT-Q-a  triplet           faq   \n",
       "10    Supervised         question_answer      BERT-Q-a  triplet           faq   \n",
       "11    Supervised  question_answer_concat      BERT-Q-a  triplet           faq   \n",
       "12    Supervised                  answer      BERT-Q-a  triplet    user_query   \n",
       "13    Supervised                question      BERT-Q-a  triplet    user_query   \n",
       "14    Supervised         question_answer      BERT-Q-a  triplet    user_query   \n",
       "15    Supervised  question_answer_concat      BERT-Q-a  triplet    user_query   \n",
       "16    Supervised                  answer      BERT-Q-a  triplet    user_query   \n",
       "17    Supervised                question      BERT-Q-a  triplet    user_query   \n",
       "18    Supervised         question_answer      BERT-Q-a  triplet    user_query   \n",
       "19    Supervised  question_answer_concat      BERT-Q-a  triplet    user_query   \n",
       "20    Supervised                  answer      BERT-Q-a  softmax           faq   \n",
       "21    Supervised                question      BERT-Q-a  softmax           faq   \n",
       "22    Supervised         question_answer      BERT-Q-a  softmax           faq   \n",
       "23    Supervised  question_answer_concat      BERT-Q-a  softmax           faq   \n",
       "24    Supervised                  answer      BERT-Q-a  softmax           faq   \n",
       "25    Supervised                question      BERT-Q-a  softmax           faq   \n",
       "26    Supervised         question_answer      BERT-Q-a  softmax           faq   \n",
       "27    Supervised  question_answer_concat      BERT-Q-a  softmax           faq   \n",
       "28    Supervised                  answer      BERT-Q-a  softmax    user_query   \n",
       "29    Supervised                question      BERT-Q-a  softmax    user_query   \n",
       "30    Supervised         question_answer      BERT-Q-a  softmax    user_query   \n",
       "31    Supervised  question_answer_concat      BERT-Q-a  softmax    user_query   \n",
       "32    Supervised                  answer      BERT-Q-a  softmax    user_query   \n",
       "33    Supervised                question      BERT-Q-a  softmax    user_query   \n",
       "34    Supervised         question_answer      BERT-Q-a  softmax    user_query   \n",
       "35    Supervised  question_answer_concat      BERT-Q-a  softmax    user_query   \n",
       "36    Supervised                  answer      BERT-Q-q  triplet           faq   \n",
       "37    Supervised                question      BERT-Q-q  triplet           faq   \n",
       "38    Supervised         question_answer      BERT-Q-q  triplet           faq   \n",
       "39    Supervised  question_answer_concat      BERT-Q-q  triplet           faq   \n",
       "40    Supervised                  answer      BERT-Q-q  triplet           faq   \n",
       "41    Supervised                question      BERT-Q-q  triplet           faq   \n",
       "42    Supervised         question_answer      BERT-Q-q  triplet           faq   \n",
       "43    Supervised  question_answer_concat      BERT-Q-q  triplet           faq   \n",
       "44    Supervised                  answer      BERT-Q-q  triplet    user_query   \n",
       "45    Supervised                question      BERT-Q-q  triplet    user_query   \n",
       "46    Supervised         question_answer      BERT-Q-q  triplet    user_query   \n",
       "47    Supervised  question_answer_concat      BERT-Q-q  triplet    user_query   \n",
       "48    Supervised                  answer      BERT-Q-q  triplet    user_query   \n",
       "49    Supervised                question      BERT-Q-q  triplet    user_query   \n",
       "50    Supervised         question_answer      BERT-Q-q  triplet    user_query   \n",
       "51    Supervised  question_answer_concat      BERT-Q-q  triplet    user_query   \n",
       "52    Supervised                  answer      BERT-Q-q  softmax           faq   \n",
       "53    Supervised                question      BERT-Q-q  softmax           faq   \n",
       "54    Supervised         question_answer      BERT-Q-q  softmax           faq   \n",
       "55    Supervised  question_answer_concat      BERT-Q-q  softmax           faq   \n",
       "56    Supervised                  answer      BERT-Q-q  softmax           faq   \n",
       "57    Supervised                question      BERT-Q-q  softmax           faq   \n",
       "58    Supervised         question_answer      BERT-Q-q  softmax           faq   \n",
       "59    Supervised  question_answer_concat      BERT-Q-q  softmax           faq   \n",
       "60    Supervised                  answer      BERT-Q-q  softmax    user_query   \n",
       "61    Supervised                question      BERT-Q-q  softmax    user_query   \n",
       "62    Supervised         question_answer      BERT-Q-q  softmax    user_query   \n",
       "63    Supervised  question_answer_concat      BERT-Q-q  softmax    user_query   \n",
       "64    Supervised                  answer      BERT-Q-q  softmax    user_query   \n",
       "65    Supervised                question      BERT-Q-q  softmax    user_query   \n",
       "66    Supervised         question_answer      BERT-Q-q  softmax    user_query   \n",
       "67    Supervised  question_answer_concat      BERT-Q-q  softmax    user_query   \n",
       "\n",
       "   Negative Sampling  NDCG@3  NDCG@5 NDCG@10     P@3     P@5    P@10     MAP  \n",
       "0                     0.2667  0.1431  0.2905  0.1212  0.3069  0.0916  0.1884  \n",
       "1                     0.3798  0.2060  0.4045  0.1726  0.4258  0.1303  0.2781  \n",
       "2                     0.3761  0.2029  0.4008  0.1708  0.4225  0.1327  0.2664  \n",
       "3                     0.4318  0.2482  0.4573  0.2130  0.4783  0.1606  0.3106  \n",
       "4             simple  0.3036  0.1679  0.3264  0.1473  0.3513  0.1136  0.2321  \n",
       "5             simple  0.4126  0.2316  0.4403  0.1988  0.4625  0.1499  0.3156  \n",
       "6             simple  0.4136  0.2299  0.4442  0.2039  0.4633  0.1550  0.3124  \n",
       "7             simple  0.4687  0.2787  0.4940  0.2399  0.5127  0.1809  0.3487  \n",
       "8               hard  0.2989  0.1629  0.3216  0.1354  0.3444  0.1052  0.2191  \n",
       "9               hard  0.4031  0.2220  0.4309  0.1888  0.4493  0.1399  0.3022  \n",
       "10              hard  0.4082  0.2246  0.4330  0.1937  0.4538  0.1460  0.2973  \n",
       "11              hard  0.4622  0.2719  0.4866  0.2292  0.5069  0.1731  0.3390  \n",
       "12            simple  0.3213  0.1795  0.3409  0.1540  0.3687  0.1199  0.2461  \n",
       "13            simple  0.4394  0.2527  0.4669  0.2140  0.4869  0.1620  0.3430  \n",
       "14            simple  0.4387  0.2477  0.4691  0.2156  0.4860  0.1680  0.3345  \n",
       "15            simple  0.4960  0.3009  0.5237  0.2568  0.5387  0.1924  0.3771  \n",
       "16              hard  0.3545  0.2079  0.3748  0.1768  0.3967  0.1329  0.2760  \n",
       "17              hard  0.4766  0.2778  0.4978  0.2323  0.5173  0.1771  0.3727  \n",
       "18              hard  0.4828  0.2843  0.5071  0.2482  0.5239  0.1891  0.3767  \n",
       "19              hard  0.5430  0.3404  0.5652  0.2857  0.5784  0.2178  0.4197  \n",
       "20            simple  0.2931  0.1620  0.3146  0.1385  0.3402  0.1087  0.2195  \n",
       "21            simple  0.4019  0.2223  0.4280  0.1858  0.4494  0.1434  0.2997  \n",
       "22            simple  0.4005  0.2223  0.4248  0.1909  0.4487  0.1467  0.2918  \n",
       "23            simple  0.4507  0.2629  0.4761  0.2235  0.4984  0.1711  0.3304  \n",
       "24              hard  0.2669  0.1440  0.2909  0.1219  0.3068  0.0915  0.1887  \n",
       "25              hard  0.3794  0.2060  0.4051  0.1726  0.4265  0.1304  0.2785  \n",
       "26              hard  0.3772  0.2034  0.4019  0.1711  0.4239  0.1331  0.2676  \n",
       "27              hard  0.4324  0.2488  0.4576  0.2128  0.4793  0.1608  0.3117  \n",
       "28            simple  0.3104  0.1708  0.3339  0.1488  0.3565  0.1125  0.2330  \n",
       "29            simple  0.4293  0.2403  0.4507  0.2014  0.4745  0.1509  0.3255  \n",
       "30            simple  0.4294  0.2426  0.4532  0.2085  0.4740  0.1583  0.3183  \n",
       "31            simple  0.4900  0.2925  0.5140  0.2487  0.5311  0.1855  0.3639  \n",
       "32              hard  0.3668  0.2178  0.3910  0.1844  0.4106  0.1410  0.2917  \n",
       "33              hard  0.4936  0.2888  0.5165  0.2426  0.5311  0.1842  0.3910  \n",
       "34              hard  0.5035  0.3043  0.5237  0.2602  0.5412  0.2001  0.3970  \n",
       "35              hard  0.5658  0.3601  0.5823  0.2994  0.5971  0.2274  0.4426  \n",
       "36            simple  0.2667  0.1431  0.2905  0.1212  0.3069  0.0916  0.1884  \n",
       "37            simple  0.3798  0.2060  0.4045  0.1726  0.4258  0.1303  0.2781  \n",
       "38            simple  0.3761  0.2029  0.4008  0.1708  0.4225  0.1327  0.2664  \n",
       "39            simple  0.4318  0.2482  0.4573  0.2130  0.4783  0.1606  0.3106  \n",
       "40              hard  0.2667  0.1431  0.2905  0.1212  0.3069  0.0916  0.1884  \n",
       "41              hard  0.3798  0.2060  0.4045  0.1726  0.4258  0.1303  0.2781  \n",
       "42              hard  0.3761  0.2029  0.4008  0.1708  0.4225  0.1327  0.2664  \n",
       "43              hard  0.4318  0.2482  0.4573  0.2130  0.4783  0.1606  0.3106  \n",
       "44            simple  0.2667  0.1431  0.2905  0.1212  0.3069  0.0916  0.1884  \n",
       "45            simple  0.3798  0.2060  0.4045  0.1726  0.4258  0.1303  0.2781  \n",
       "46            simple  0.3761  0.2029  0.4008  0.1708  0.4225  0.1327  0.2664  \n",
       "47            simple  0.4318  0.2482  0.4573  0.2130  0.4783  0.1606  0.3106  \n",
       "48              hard  0.2667  0.1431  0.2905  0.1212  0.3069  0.0916  0.1884  \n",
       "49              hard  0.3798  0.2060  0.4045  0.1726  0.4258  0.1303  0.2781  \n",
       "50              hard  0.3761  0.2029  0.4008  0.1708  0.4225  0.1327  0.2664  \n",
       "51              hard  0.4318  0.2482  0.4573  0.2130  0.4783  0.1606  0.3106  \n",
       "52            simple  0.2667  0.1431  0.2905  0.1212  0.3069  0.0916  0.1884  \n",
       "53            simple  0.3798  0.2060  0.4045  0.1726  0.4258  0.1303  0.2781  \n",
       "54            simple  0.3761  0.2029  0.4008  0.1708  0.4225  0.1327  0.2664  \n",
       "55            simple  0.4318  0.2482  0.4573  0.2130  0.4783  0.1606  0.3106  \n",
       "56              hard  0.2667  0.1431  0.2905  0.1212  0.3069  0.0916  0.1884  \n",
       "57              hard  0.3798  0.2060  0.4045  0.1726  0.4258  0.1303  0.2781  \n",
       "58              hard  0.3761  0.2029  0.4008  0.1708  0.4225  0.1327  0.2664  \n",
       "59              hard  0.4318  0.2482  0.4573  0.2130  0.4783  0.1606  0.3106  \n",
       "60            simple  0.2667  0.1431  0.2905  0.1212  0.3069  0.0916  0.1884  \n",
       "61            simple  0.3798  0.2060  0.4045  0.1726  0.4258  0.1303  0.2781  \n",
       "62            simple  0.3761  0.2029  0.4008  0.1708  0.4225  0.1327  0.2664  \n",
       "63            simple  0.4318  0.2482  0.4573  0.2130  0.4783  0.1606  0.3106  \n",
       "64              hard  0.2667  0.1431  0.2905  0.1212  0.3069  0.0916  0.1884  \n",
       "65              hard  0.3798  0.2060  0.4045  0.1726  0.4258  0.1303  0.2781  \n",
       "66              hard  0.3761  0.2029  0.4008  0.1708  0.4225  0.1327  0.2664  \n",
       "67              hard  0.4318  0.2482  0.4573  0.2130  0.4783  0.1606  0.3106  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}