{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a762c678",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/home/jovyan/conda-envs/SATD_empirical_py3.10/bin/python'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import sys\n",
    "sys.executable"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1aafbec6",
   "metadata": {},
   "source": [
    "# Download Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "4914ab6f",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting gdown\n",
      "  Using cached gdown-5.2.0-py3-none-any.whl.metadata (5.8 kB)\n",
      "Collecting beautifulsoup4 (from gdown)\n",
      "  Using cached beautifulsoup4-4.12.3-py3-none-any.whl.metadata (3.8 kB)\n",
      "Collecting filelock (from gdown)\n",
      "  Downloading filelock-3.16.1-py3-none-any.whl.metadata (2.9 kB)\n",
      "Collecting requests[socks] (from gdown)\n",
      "  Downloading requests-2.32.3-py3-none-any.whl.metadata (4.6 kB)\n",
      "Requirement already satisfied: tqdm in /home/jovyan/conda-envs/SATD_empirical_py3.10/lib/python3.10/site-packages (from gdown) (4.67.0)\n",
      "Collecting soupsieve>1.2 (from beautifulsoup4->gdown)\n",
      "  Using cached soupsieve-2.6-py3-none-any.whl.metadata (4.6 kB)\n",
      "Collecting charset-normalizer<4,>=2 (from requests[socks]->gdown)\n",
      "  Downloading charset_normalizer-3.4.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (35 kB)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /home/jovyan/conda-envs/SATD_empirical_py3.10/lib/python3.10/site-packages (from requests[socks]->gdown) (3.10)\n",
      "Collecting urllib3<3,>=1.21.1 (from requests[socks]->gdown)\n",
      "  Downloading urllib3-2.3.0-py3-none-any.whl.metadata (6.5 kB)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /home/jovyan/conda-envs/SATD_empirical_py3.10/lib/python3.10/site-packages (from requests[socks]->gdown) (2024.8.30)\n",
      "Collecting PySocks!=1.5.7,>=1.5.6 (from requests[socks]->gdown)\n",
      "  Using cached PySocks-1.7.1-py3-none-any.whl.metadata (13 kB)\n",
      "Using cached gdown-5.2.0-py3-none-any.whl (18 kB)\n",
      "Using cached beautifulsoup4-4.12.3-py3-none-any.whl (147 kB)\n",
      "Downloading filelock-3.16.1-py3-none-any.whl (16 kB)\n",
      "Downloading charset_normalizer-3.4.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (146 kB)\n",
      "Using cached PySocks-1.7.1-py3-none-any.whl (16 kB)\n",
      "Using cached soupsieve-2.6-py3-none-any.whl (36 kB)\n",
      "Downloading urllib3-2.3.0-py3-none-any.whl (128 kB)\n",
      "Downloading requests-2.32.3-py3-none-any.whl (64 kB)\n",
      "Installing collected packages: urllib3, soupsieve, PySocks, filelock, charset-normalizer, requests, beautifulsoup4, gdown\n",
      "Successfully installed PySocks-1.7.1 beautifulsoup4-4.12.3 charset-normalizer-3.4.1 filelock-3.16.1 gdown-5.2.0 requests-2.32.3 soupsieve-2.6 urllib3-2.3.0\n"
     ]
    }
   ],
   "source": [
    "# !pip install gdown"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "992c0e11",
   "metadata": {},
   "outputs": [],
   "source": [
    "import gdown\n",
    "import os\n",
    "\n",
    "def download_from_google_drive(fileId, filePath):\n",
    "    download_url = f\"https://drive.google.com/uc?id={fileId}\"\n",
    "    gdown.download(download_url, filePath, quiet=False) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "645f39c4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Datasets/Python/df0.pkl.gz already exists.\n",
      "Datasets/Python/df1.pkl.gz already exists.\n",
      "Datasets/Python/df3.pkl.gz already exists.\n"
     ]
    }
   ],
   "source": [
    "# Download Python dataset\n",
    "\n",
    "os.makedirs(\"Datasets/Python\", exist_ok=True)\n",
    "\n",
    "fileId_filePath = {\n",
    "    '1LFQarTywxsox1WAtD5BwGJl6GKIkRnrp': 'Datasets/Python/df0.pkl.gz',\n",
    "    '1GAW2DdF9bddxhPahNlf8VvqyWIHM9IbG': 'Datasets/Python/df1.pkl.gz',\n",
    "    '1aQ5Rpe-GI-Vqb5BCg8cz8bXXEKzKmguX': 'Datasets/Python/df3.pkl.gz',\n",
    "}\n",
    "\n",
    "for fileId, filePath in fileId_filePath.items():\n",
    "    if not os.path.exists(filePath):\n",
    "        download_from_google_drive(fileId, filePath)\n",
    "    else:\n",
    "        print(filePath, 'already exists.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "48484c26",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Datasets/Java/df0.pkl.gz already exists.\n",
      "Datasets/Java/df1.pkl.gz already exists.\n",
      "Datasets/Java/df3.pkl.gz already exists.\n"
     ]
    }
   ],
   "source": [
    "# Download Java dataset\n",
    "\n",
    "os.makedirs(\"Datasets/Java\", exist_ok=True)\n",
    "\n",
    "fileId_filePath = {\n",
    "    '142ZPmt-RuAWrsxvEmNEsB0-vUn0fpvX9': 'Datasets/Java/df0.pkl.gz',\n",
    "    '1LH2HF5HkzpkuJJOsDkBivsqtZpFKtEmL': 'Datasets/Java/df1.pkl.gz',\n",
    "    '1t5Pf0f8NSygdNBgTxtdsPPmvGbmFlbFe': 'Datasets/Java/df3.pkl.gz',\n",
    "}\n",
    "\n",
    "for fileId, filePath in fileId_filePath.items():\n",
    "    if not os.path.exists(filePath):\n",
    "        download_from_google_drive(fileId, filePath)\n",
    "    else:\n",
    "        print(filePath, 'already exists.')  "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d3443026",
   "metadata": {},
   "source": [
    "# Apply Filtering Steps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "b2574958",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import gzip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "57df0aa5",
   "metadata": {},
   "outputs": [],
   "source": [
    "from enum import Enum\n",
    "\n",
    "class MethodRetrievalApproach(Enum):\n",
    "    WITH_AST = 1\n",
    "    WITHOUT_AST = 2\n",
    "    AST_FALLBACK = 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "56be30f0",
   "metadata": {},
   "outputs": [],
   "source": [
    "DATASET = 'Python' # Java or Python"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "11eb62b0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1607408\n",
      "Index(['user', 'project', 'created_in_file', 'last_appeared_in_file',\n",
      "       'created_in_line', 'last_appeared_in_line', 'created_in_commit',\n",
      "       'deleted_in_commit', 'created_at_date', 'deleted_at_date', 'content',\n",
      "       'deleted_in_lines', 'created_in_lines', 'updated_in_commits',\n",
      "       'last_content', 'SATD_comment'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_pickle(gzip.open(f'Datasets/{DATASET}/df0.pkl.gz', 'rb'))\n",
    "print(len(df))\n",
    "print(df.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "35eb3bf4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of repositories in the Python SATD dataset: 11754\n",
      "Note: Repositories in which the SATD Tracker did not find any SATD in their code have no rows in this dataset.\n"
     ]
    }
   ],
   "source": [
    "print(f'Number of repositories in the {DATASET} SATD dataset:',len(set(df['project'])))\n",
    "print(\"Note: Repositories in which the SATD Tracker did not find any SATD in their code have no rows in this dataset.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "bf2b4aa5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1059299\n"
     ]
    }
   ],
   "source": [
    "# Filter 1: Keep SATDs that are deleted (potentially repaid), and remove others\n",
    "df = df.dropna(subset=['deleted_in_commit'])\n",
    "print(len(df))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "f3ec1e4b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "949188\n"
     ]
    }
   ],
   "source": [
    "# Filter 2: The length of SATD comment should be at least three words\n",
    "df = df[df['SATD_comment'].apply(lambda x: len(x.split()) > 2)]\n",
    "print(len(df))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "358c32ef",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "949188\n",
      "Index(['user', 'project', 'created_in_file', 'last_appeared_in_file',\n",
      "       'created_in_line', 'last_appeared_in_line', 'created_in_commit',\n",
      "       'deleted_in_commit', 'created_at_date', 'deleted_at_date', 'content',\n",
      "       'deleted_in_lines', 'created_in_lines', 'updated_in_commits',\n",
      "       'last_content', 'SATD_comment', 'containing_method_applied_approach',\n",
      "       'containing_method_before_repayment',\n",
      "       'containing_method_after_repayment', 'method_is_updated',\n",
      "       'SATD_count_before_repayment', 'SATD_count_after_repayment'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "# Load df1: This version includes some more columns that are required for the next filters\n",
    "df1 = pd.read_pickle(gzip.open(f'Datasets/{DATASET}/df1.pkl.gz', 'rb'))\n",
    "print(len(df1))\n",
    "print(df1.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "ed459f49",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "723258\n"
     ]
    }
   ],
   "source": [
    "# Filter 3: Keep SATDs that are inside methods, and remove others\n",
    "df2 = df1[df1[\"containing_method_before_repayment\"].str.len()>0]\n",
    "print(len(df2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "c030ed01",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "325031\n"
     ]
    }
   ],
   "source": [
    "# Filter 4: Keep SATDs that the containing method’s name still exist after repayment\n",
    "df2 = df2[df2[\"containing_method_after_repayment\"].str.len()>0]\n",
    "print(len(df2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "4241d1a4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "288007\n"
     ]
    }
   ],
   "source": [
    "# Filter 5: Keep SATDs that the containing method is updated after repayment\n",
    "df2 = df2[df2[\"method_is_updated\"]]\n",
    "print(len(df2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "4c3313d8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "171825\n"
     ]
    }
   ],
   "source": [
    "# Filter 6: Keep SATDs that no other SATDs existed in that method before repayment. Also, no SATD exists in the method after repayment.\n",
    "df2 = df2[(df2[\"SATD_count_before_repayment\"] == 1) & (df2[\"SATD_count_after_repayment\"] == 0)]\n",
    "print(len(df2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "073466ef",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "143341\n"
     ]
    }
   ],
   "source": [
    "# Filter 7: Removing duplicates\n",
    "df2 = df2.drop_duplicates(subset=['containing_method_before_repayment', 'containing_method_after_repayment'])\n",
    "print(len(df2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "9654e6ed",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "140929\n"
     ]
    }
   ],
   "source": [
    "# Filter 8: Remove SATDs that having non-ASCII characters\n",
    "def contains_non_ascii(s: str) -> bool:\n",
    "    return not all(ord(char) < 128 for char in s)\n",
    "df2 = df2[df2['containing_method_before_repayment'].apply(contains_non_ascii)==False]\n",
    "df2 = df2[df2['containing_method_after_repayment'].apply(contains_non_ascii)==False]\n",
    "print(len(df2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "f26a8238",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "131945\n"
     ]
    }
   ],
   "source": [
    "# Filter 9: Keep SATDs that the number of tokens in before and after SATD repayment is less than 1,024\n",
    "import re\n",
    "def split_to_tokens(text):\n",
    "    # Split by word boundaries and include punctuation as separate tokens\n",
    "    tokens = re.findall(r\"\\w+|[^\\w\\s]\", text, re.UNICODE)\n",
    "    return tokens\n",
    "\n",
    "df2['method_tokens_before_repayment'] = df2['containing_method_before_repayment'].apply(lambda x: len(split_to_tokens(x)))\n",
    "df2['method_tokens_after_repayment'] = df2['containing_method_after_repayment'].apply(lambda x: len(split_to_tokens(x)))\n",
    "df2 = df2[(df2['method_tokens_before_repayment'] <= 1024) & (df2['method_tokens_after_repayment'] <= 1024)]\n",
    "print(len(df2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "cc311154",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "131945\n",
      "Index(['rand_index', 'user', 'project', 'created_in_file',\n",
      "       'last_appeared_in_file', 'created_in_line', 'last_appeared_in_line',\n",
      "       'created_in_commit', 'deleted_in_commit', 'created_at_date',\n",
      "       'deleted_at_date', 'content', 'deleted_in_lines', 'created_in_lines',\n",
      "       'updated_in_commits', 'last_content', 'SATD_comment',\n",
      "       'containing_method_before_repayment',\n",
      "       'containing_method_applied_approach',\n",
      "       'containing_method_after_repayment', 'method_is_updated',\n",
      "       'SATD_count_before_repayment', 'SATD_count_after_repayment',\n",
      "       'method_tokens_before_repayment', 'method_tokens_after_repayment',\n",
      "       'prompt', 'is_repayment_llama3'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "# Load df3: This version includes the Llama3-70b label column that is required for the last filter\n",
    "df3 = pd.read_pickle(gzip.open(f'Datasets/{DATASET}/df3.pkl.gz', 'rb'))\n",
    "print(len(df3))\n",
    "print(df3.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "83c5c765",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "is_repayment_llama3\n",
      "yes        58722\n",
      "no         47394\n",
      "unclear    24429\n",
      "Name: count, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "# show the value counts in is_repayment_llama3 column\n",
    "print(df3['is_repayment_llama3'].value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "4347b7fd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "58722\n"
     ]
    }
   ],
   "source": [
    "# Filter 10: Remove items that the method update is not related to SATD repayment\n",
    "df3 = df3[df3['is_repayment_llama3'] == 'yes']\n",
    "print(len(df3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "a51e791f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7219"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Number of repositories having at least one SATD in the filtered dataset\n",
    "df3[['user', 'project']].drop_duplicates().shape[0]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:SATD_empirical_py3.10]",
   "language": "python",
   "name": "conda-env-SATD_empirical_py3.10-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
