{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "d90f65e9",
   "metadata": {},
   "source": [
    "# pandas: Handle Missing Values"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1ddd3556",
   "metadata": {},
   "source": [
    "## Package Import"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "48ff8eec",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-09-02T18:58:25.120547Z",
     "iopub.status.busy": "2025-09-02T18:58:25.120352Z",
     "iopub.status.idle": "2025-09-02T18:58:25.348181Z",
     "shell.execute_reply": "2025-09-02T18:58:25.347654Z"
    },
    "tags": [
     "hide-output",
     "scroll-output"
    ]
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "171736d0",
   "metadata": {},
   "source": [
    "## Dataset Import"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c58ea7aa",
   "metadata": {},
   "source": [
    "The dataset used in this notebook is from [Kaggle - Pokemon](https://www.kaggle.com/datasets/abcsds/pokemon)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "cc6cb203",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-09-02T18:58:25.349998Z",
     "iopub.status.busy": "2025-09-02T18:58:25.349819Z",
     "iopub.status.idle": "2025-09-02T18:58:25.355121Z",
     "shell.execute_reply": "2025-09-02T18:58:25.354677Z"
    },
    "tags": [
     "hide-output",
     "scroll-output"
    ]
   },
   "outputs": [],
   "source": [
    "data = pd.read_csv('data/Pokemon.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "cf9535bf",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-09-02T18:58:25.356747Z",
     "iopub.status.busy": "2025-09-02T18:58:25.356607Z",
     "iopub.status.idle": "2025-09-02T18:58:25.366216Z",
     "shell.execute_reply": "2025-09-02T18:58:25.365793Z"
    },
    "tags": [
     "hide-output",
     "scroll-output"
    ]
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>#</th>\n",
       "      <th>Name</th>\n",
       "      <th>Type 1</th>\n",
       "      <th>Type 2</th>\n",
       "      <th>Total</th>\n",
       "      <th>HP</th>\n",
       "      <th>Attack</th>\n",
       "      <th>Defense</th>\n",
       "      <th>Sp. Atk</th>\n",
       "      <th>Sp. Def</th>\n",
       "      <th>Speed</th>\n",
       "      <th>Generation</th>\n",
       "      <th>Legendary</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Bulbasaur</td>\n",
       "      <td>Grass</td>\n",
       "      <td>Poison</td>\n",
       "      <td>318</td>\n",
       "      <td>45</td>\n",
       "      <td>49</td>\n",
       "      <td>49</td>\n",
       "      <td>65</td>\n",
       "      <td>65</td>\n",
       "      <td>45</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>Ivysaur</td>\n",
       "      <td>Grass</td>\n",
       "      <td>Poison</td>\n",
       "      <td>405</td>\n",
       "      <td>60</td>\n",
       "      <td>62</td>\n",
       "      <td>63</td>\n",
       "      <td>80</td>\n",
       "      <td>80</td>\n",
       "      <td>60</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Venusaur</td>\n",
       "      <td>Grass</td>\n",
       "      <td>Poison</td>\n",
       "      <td>525</td>\n",
       "      <td>80</td>\n",
       "      <td>82</td>\n",
       "      <td>83</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>80</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>VenusaurMega Venusaur</td>\n",
       "      <td>Grass</td>\n",
       "      <td>Poison</td>\n",
       "      <td>625</td>\n",
       "      <td>80</td>\n",
       "      <td>100</td>\n",
       "      <td>123</td>\n",
       "      <td>122</td>\n",
       "      <td>120</td>\n",
       "      <td>80</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Charmander</td>\n",
       "      <td>Fire</td>\n",
       "      <td>NaN</td>\n",
       "      <td>309</td>\n",
       "      <td>39</td>\n",
       "      <td>52</td>\n",
       "      <td>43</td>\n",
       "      <td>60</td>\n",
       "      <td>50</td>\n",
       "      <td>65</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>795</th>\n",
       "      <td>719</td>\n",
       "      <td>Diancie</td>\n",
       "      <td>Rock</td>\n",
       "      <td>Fairy</td>\n",
       "      <td>600</td>\n",
       "      <td>50</td>\n",
       "      <td>100</td>\n",
       "      <td>150</td>\n",
       "      <td>100</td>\n",
       "      <td>150</td>\n",
       "      <td>50</td>\n",
       "      <td>6</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>796</th>\n",
       "      <td>719</td>\n",
       "      <td>DiancieMega Diancie</td>\n",
       "      <td>Rock</td>\n",
       "      <td>Fairy</td>\n",
       "      <td>700</td>\n",
       "      <td>50</td>\n",
       "      <td>160</td>\n",
       "      <td>110</td>\n",
       "      <td>160</td>\n",
       "      <td>110</td>\n",
       "      <td>110</td>\n",
       "      <td>6</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>797</th>\n",
       "      <td>720</td>\n",
       "      <td>HoopaHoopa Confined</td>\n",
       "      <td>Psychic</td>\n",
       "      <td>Ghost</td>\n",
       "      <td>600</td>\n",
       "      <td>80</td>\n",
       "      <td>110</td>\n",
       "      <td>60</td>\n",
       "      <td>150</td>\n",
       "      <td>130</td>\n",
       "      <td>70</td>\n",
       "      <td>6</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>798</th>\n",
       "      <td>720</td>\n",
       "      <td>HoopaHoopa Unbound</td>\n",
       "      <td>Psychic</td>\n",
       "      <td>Dark</td>\n",
       "      <td>680</td>\n",
       "      <td>80</td>\n",
       "      <td>160</td>\n",
       "      <td>60</td>\n",
       "      <td>170</td>\n",
       "      <td>130</td>\n",
       "      <td>80</td>\n",
       "      <td>6</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>799</th>\n",
       "      <td>721</td>\n",
       "      <td>Volcanion</td>\n",
       "      <td>Fire</td>\n",
       "      <td>Water</td>\n",
       "      <td>600</td>\n",
       "      <td>80</td>\n",
       "      <td>110</td>\n",
       "      <td>120</td>\n",
       "      <td>130</td>\n",
       "      <td>90</td>\n",
       "      <td>70</td>\n",
       "      <td>6</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>800 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       #                   Name   Type 1  Type 2  Total  HP  Attack  Defense  \\\n",
       "0      1              Bulbasaur    Grass  Poison    318  45      49       49   \n",
       "1      2                Ivysaur    Grass  Poison    405  60      62       63   \n",
       "2      3               Venusaur    Grass  Poison    525  80      82       83   \n",
       "3      3  VenusaurMega Venusaur    Grass  Poison    625  80     100      123   \n",
       "4      4             Charmander     Fire     NaN    309  39      52       43   \n",
       "..   ...                    ...      ...     ...    ...  ..     ...      ...   \n",
       "795  719                Diancie     Rock   Fairy    600  50     100      150   \n",
       "796  719    DiancieMega Diancie     Rock   Fairy    700  50     160      110   \n",
       "797  720    HoopaHoopa Confined  Psychic   Ghost    600  80     110       60   \n",
       "798  720     HoopaHoopa Unbound  Psychic    Dark    680  80     160       60   \n",
       "799  721              Volcanion     Fire   Water    600  80     110      120   \n",
       "\n",
       "     Sp. Atk  Sp. Def  Speed  Generation  Legendary  \n",
       "0         65       65     45           1      False  \n",
       "1         80       80     60           1      False  \n",
       "2        100      100     80           1      False  \n",
       "3        122      120     80           1      False  \n",
       "4         60       50     65           1      False  \n",
       "..       ...      ...    ...         ...        ...  \n",
       "795      100      150     50           6       True  \n",
       "796      160      110    110           6       True  \n",
       "797      150      130     70           6       True  \n",
       "798      170      130     80           6       True  \n",
       "799      130       90     70           6       True  \n",
       "\n",
       "[800 rows x 13 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "67648eb6",
   "metadata": {},
   "source": [
    "## Handle missing values"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "910e3327",
   "metadata": {},
   "source": [
    "`DataFrame.isna()` return a same-sized object:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "baceaed4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-09-02T18:58:25.368072Z",
     "iopub.status.busy": "2025-09-02T18:58:25.367906Z",
     "iopub.status.idle": "2025-09-02T18:58:25.376670Z",
     "shell.execute_reply": "2025-09-02T18:58:25.376226Z"
    },
    "tags": [
     "hide-output",
     "scroll-output"
    ]
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>#</th>\n",
       "      <th>Name</th>\n",
       "      <th>Type 1</th>\n",
       "      <th>Type 2</th>\n",
       "      <th>Total</th>\n",
       "      <th>HP</th>\n",
       "      <th>Attack</th>\n",
       "      <th>Defense</th>\n",
       "      <th>Sp. Atk</th>\n",
       "      <th>Sp. Def</th>\n",
       "      <th>Speed</th>\n",
       "      <th>Generation</th>\n",
       "      <th>Legendary</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       #   Name  Type 1  Type 2  Total     HP  Attack  Defense  Sp. Atk  \\\n",
       "0  False  False   False   False  False  False   False    False    False   \n",
       "1  False  False   False   False  False  False   False    False    False   \n",
       "2  False  False   False   False  False  False   False    False    False   \n",
       "3  False  False   False   False  False  False   False    False    False   \n",
       "4  False  False   False    True  False  False   False    False    False   \n",
       "\n",
       "   Sp. Def  Speed  Generation  Legendary  \n",
       "0    False  False       False      False  \n",
       "1    False  False       False      False  \n",
       "2    False  False       False      False  \n",
       "3    False  False       False      False  \n",
       "4    False  False       False      False  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.isna().head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d916b34a",
   "metadata": {},
   "source": [
    "`isna().sum()` to check total missing values for each column:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "7ce2805c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-09-02T18:58:25.378376Z",
     "iopub.status.busy": "2025-09-02T18:58:25.378206Z",
     "iopub.status.idle": "2025-09-02T18:58:25.383044Z",
     "shell.execute_reply": "2025-09-02T18:58:25.382400Z"
    },
    "tags": [
     "hide-output",
     "scroll-output"
    ]
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "#               0\n",
       "Name            0\n",
       "Type 1          0\n",
       "Type 2        386\n",
       "Total           0\n",
       "HP              0\n",
       "Attack          0\n",
       "Defense         0\n",
       "Sp. Atk         0\n",
       "Sp. Def         0\n",
       "Speed           0\n",
       "Generation      0\n",
       "Legendary       0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.isna().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8fbaa85a",
   "metadata": {},
   "source": [
    "`isna().mean()` to check the proportion of missing values:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "6bb0f804",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-09-02T18:58:25.384739Z",
     "iopub.status.busy": "2025-09-02T18:58:25.384594Z",
     "iopub.status.idle": "2025-09-02T18:58:25.389832Z",
     "shell.execute_reply": "2025-09-02T18:58:25.389249Z"
    },
    "tags": [
     "hide-output",
     "scroll-output"
    ]
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "#             0.0000\n",
       "Name          0.0000\n",
       "Type 1        0.0000\n",
       "Type 2        0.4825\n",
       "Total         0.0000\n",
       "HP            0.0000\n",
       "Attack        0.0000\n",
       "Defense       0.0000\n",
       "Sp. Atk       0.0000\n",
       "Sp. Def       0.0000\n",
       "Speed         0.0000\n",
       "Generation    0.0000\n",
       "Legendary     0.0000\n",
       "dtype: float64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.isna().mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "431abdf5",
   "metadata": {},
   "source": [
    "We can choose to delete rows/columns that have missing values:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "270bf56b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-09-02T18:58:25.391198Z",
     "iopub.status.busy": "2025-09-02T18:58:25.391080Z",
     "iopub.status.idle": "2025-09-02T18:58:25.398205Z",
     "shell.execute_reply": "2025-09-02T18:58:25.397707Z"
    },
    "tags": [
     "hide-output",
     "scroll-output"
    ]
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>#</th>\n",
       "      <th>Name</th>\n",
       "      <th>Type 1</th>\n",
       "      <th>Type 2</th>\n",
       "      <th>Total</th>\n",
       "      <th>HP</th>\n",
       "      <th>Attack</th>\n",
       "      <th>Defense</th>\n",
       "      <th>Sp. Atk</th>\n",
       "      <th>Sp. Def</th>\n",
       "      <th>Speed</th>\n",
       "      <th>Generation</th>\n",
       "      <th>Legendary</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Bulbasaur</td>\n",
       "      <td>Grass</td>\n",
       "      <td>Poison</td>\n",
       "      <td>318</td>\n",
       "      <td>45</td>\n",
       "      <td>49</td>\n",
       "      <td>49</td>\n",
       "      <td>65</td>\n",
       "      <td>65</td>\n",
       "      <td>45</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>Ivysaur</td>\n",
       "      <td>Grass</td>\n",
       "      <td>Poison</td>\n",
       "      <td>405</td>\n",
       "      <td>60</td>\n",
       "      <td>62</td>\n",
       "      <td>63</td>\n",
       "      <td>80</td>\n",
       "      <td>80</td>\n",
       "      <td>60</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Venusaur</td>\n",
       "      <td>Grass</td>\n",
       "      <td>Poison</td>\n",
       "      <td>525</td>\n",
       "      <td>80</td>\n",
       "      <td>82</td>\n",
       "      <td>83</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>80</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>VenusaurMega Venusaur</td>\n",
       "      <td>Grass</td>\n",
       "      <td>Poison</td>\n",
       "      <td>625</td>\n",
       "      <td>80</td>\n",
       "      <td>100</td>\n",
       "      <td>123</td>\n",
       "      <td>122</td>\n",
       "      <td>120</td>\n",
       "      <td>80</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>6</td>\n",
       "      <td>Charizard</td>\n",
       "      <td>Fire</td>\n",
       "      <td>Flying</td>\n",
       "      <td>534</td>\n",
       "      <td>78</td>\n",
       "      <td>84</td>\n",
       "      <td>78</td>\n",
       "      <td>109</td>\n",
       "      <td>85</td>\n",
       "      <td>100</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   #                   Name Type 1  Type 2  Total  HP  Attack  Defense  \\\n",
       "0  1              Bulbasaur  Grass  Poison    318  45      49       49   \n",
       "1  2                Ivysaur  Grass  Poison    405  60      62       63   \n",
       "2  3               Venusaur  Grass  Poison    525  80      82       83   \n",
       "3  3  VenusaurMega Venusaur  Grass  Poison    625  80     100      123   \n",
       "6  6              Charizard   Fire  Flying    534  78      84       78   \n",
       "\n",
       "   Sp. Atk  Sp. Def  Speed  Generation  Legendary  \n",
       "0       65       65     45           1      False  \n",
       "1       80       80     60           1      False  \n",
       "2      100      100     80           1      False  \n",
       "3      122      120     80           1      False  \n",
       "6      109       85    100           1      False  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.dropna(axis='index').head() # Drop rows"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1b99adb5",
   "metadata": {},
   "source": [
    "- We can use `thresh=` to keep the rows/columns that has at least `thresh` non-missing values:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "aeeb6657",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-09-02T18:58:25.399633Z",
     "iopub.status.busy": "2025-09-02T18:58:25.399506Z",
     "iopub.status.idle": "2025-09-02T18:58:25.406051Z",
     "shell.execute_reply": "2025-09-02T18:58:25.405572Z"
    },
    "tags": [
     "hide-output",
     "scroll-output"
    ]
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>#</th>\n",
       "      <th>Name</th>\n",
       "      <th>Type 1</th>\n",
       "      <th>Total</th>\n",
       "      <th>HP</th>\n",
       "      <th>Attack</th>\n",
       "      <th>Defense</th>\n",
       "      <th>Sp. Atk</th>\n",
       "      <th>Sp. Def</th>\n",
       "      <th>Speed</th>\n",
       "      <th>Generation</th>\n",
       "      <th>Legendary</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Bulbasaur</td>\n",
       "      <td>Grass</td>\n",
       "      <td>318</td>\n",
       "      <td>45</td>\n",
       "      <td>49</td>\n",
       "      <td>49</td>\n",
       "      <td>65</td>\n",
       "      <td>65</td>\n",
       "      <td>45</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>Ivysaur</td>\n",
       "      <td>Grass</td>\n",
       "      <td>405</td>\n",
       "      <td>60</td>\n",
       "      <td>62</td>\n",
       "      <td>63</td>\n",
       "      <td>80</td>\n",
       "      <td>80</td>\n",
       "      <td>60</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Venusaur</td>\n",
       "      <td>Grass</td>\n",
       "      <td>525</td>\n",
       "      <td>80</td>\n",
       "      <td>82</td>\n",
       "      <td>83</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>80</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>VenusaurMega Venusaur</td>\n",
       "      <td>Grass</td>\n",
       "      <td>625</td>\n",
       "      <td>80</td>\n",
       "      <td>100</td>\n",
       "      <td>123</td>\n",
       "      <td>122</td>\n",
       "      <td>120</td>\n",
       "      <td>80</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Charmander</td>\n",
       "      <td>Fire</td>\n",
       "      <td>309</td>\n",
       "      <td>39</td>\n",
       "      <td>52</td>\n",
       "      <td>43</td>\n",
       "      <td>60</td>\n",
       "      <td>50</td>\n",
       "      <td>65</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   #                   Name Type 1  Total  HP  Attack  Defense  Sp. Atk  \\\n",
       "0  1              Bulbasaur  Grass    318  45      49       49       65   \n",
       "1  2                Ivysaur  Grass    405  60      62       63       80   \n",
       "2  3               Venusaur  Grass    525  80      82       83      100   \n",
       "3  3  VenusaurMega Venusaur  Grass    625  80     100      123      122   \n",
       "4  4             Charmander   Fire    309  39      52       43       60   \n",
       "\n",
       "   Sp. Def  Speed  Generation  Legendary  \n",
       "0       65     45           1      False  \n",
       "1       80     60           1      False  \n",
       "2      100     80           1      False  \n",
       "3      120     80           1      False  \n",
       "4       50     65           1      False  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.dropna(axis='columns', thresh=len(data)*0.6).head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "a7e93619",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-09-02T18:58:25.407458Z",
     "iopub.status.busy": "2025-09-02T18:58:25.407336Z",
     "iopub.status.idle": "2025-09-02T18:58:25.413907Z",
     "shell.execute_reply": "2025-09-02T18:58:25.413500Z"
    },
    "tags": [
     "hide-output",
     "scroll-output"
    ]
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>#</th>\n",
       "      <th>Name</th>\n",
       "      <th>Type 1</th>\n",
       "      <th>Type 2</th>\n",
       "      <th>Total</th>\n",
       "      <th>HP</th>\n",
       "      <th>Attack</th>\n",
       "      <th>Defense</th>\n",
       "      <th>Sp. Atk</th>\n",
       "      <th>Sp. Def</th>\n",
       "      <th>Speed</th>\n",
       "      <th>Generation</th>\n",
       "      <th>Legendary</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Bulbasaur</td>\n",
       "      <td>Grass</td>\n",
       "      <td>Poison</td>\n",
       "      <td>318</td>\n",
       "      <td>45</td>\n",
       "      <td>49</td>\n",
       "      <td>49</td>\n",
       "      <td>65</td>\n",
       "      <td>65</td>\n",
       "      <td>45</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>Ivysaur</td>\n",
       "      <td>Grass</td>\n",
       "      <td>Poison</td>\n",
       "      <td>405</td>\n",
       "      <td>60</td>\n",
       "      <td>62</td>\n",
       "      <td>63</td>\n",
       "      <td>80</td>\n",
       "      <td>80</td>\n",
       "      <td>60</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Venusaur</td>\n",
       "      <td>Grass</td>\n",
       "      <td>Poison</td>\n",
       "      <td>525</td>\n",
       "      <td>80</td>\n",
       "      <td>82</td>\n",
       "      <td>83</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>80</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>VenusaurMega Venusaur</td>\n",
       "      <td>Grass</td>\n",
       "      <td>Poison</td>\n",
       "      <td>625</td>\n",
       "      <td>80</td>\n",
       "      <td>100</td>\n",
       "      <td>123</td>\n",
       "      <td>122</td>\n",
       "      <td>120</td>\n",
       "      <td>80</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Charmander</td>\n",
       "      <td>Fire</td>\n",
       "      <td>NaN</td>\n",
       "      <td>309</td>\n",
       "      <td>39</td>\n",
       "      <td>52</td>\n",
       "      <td>43</td>\n",
       "      <td>60</td>\n",
       "      <td>50</td>\n",
       "      <td>65</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   #                   Name Type 1  Type 2  Total  HP  Attack  Defense  \\\n",
       "0  1              Bulbasaur  Grass  Poison    318  45      49       49   \n",
       "1  2                Ivysaur  Grass  Poison    405  60      62       63   \n",
       "2  3               Venusaur  Grass  Poison    525  80      82       83   \n",
       "3  3  VenusaurMega Venusaur  Grass  Poison    625  80     100      123   \n",
       "4  4             Charmander   Fire     NaN    309  39      52       43   \n",
       "\n",
       "   Sp. Atk  Sp. Def  Speed  Generation  Legendary  \n",
       "0       65       65     45           1      False  \n",
       "1       80       80     60           1      False  \n",
       "2      100      100     80           1      False  \n",
       "3      122      120     80           1      False  \n",
       "4       60       50     65           1      False  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.dropna(axis='columns', thresh=len(data)*0.4).head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0b3ff4a4",
   "metadata": {},
   "source": [
    "`DataFrame.fillna()` to fill missing values:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "5bf8f760",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-09-02T18:58:25.415393Z",
     "iopub.status.busy": "2025-09-02T18:58:25.415220Z",
     "iopub.status.idle": "2025-09-02T18:58:25.421892Z",
     "shell.execute_reply": "2025-09-02T18:58:25.421363Z"
    },
    "tags": [
     "hide-output",
     "scroll-output"
    ]
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>#</th>\n",
       "      <th>Name</th>\n",
       "      <th>Type 1</th>\n",
       "      <th>Type 2</th>\n",
       "      <th>Total</th>\n",
       "      <th>HP</th>\n",
       "      <th>Attack</th>\n",
       "      <th>Defense</th>\n",
       "      <th>Sp. Atk</th>\n",
       "      <th>Sp. Def</th>\n",
       "      <th>Speed</th>\n",
       "      <th>Generation</th>\n",
       "      <th>Legendary</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Bulbasaur</td>\n",
       "      <td>Grass</td>\n",
       "      <td>Poison</td>\n",
       "      <td>318</td>\n",
       "      <td>45</td>\n",
       "      <td>49</td>\n",
       "      <td>49</td>\n",
       "      <td>65</td>\n",
       "      <td>65</td>\n",
       "      <td>45</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>Ivysaur</td>\n",
       "      <td>Grass</td>\n",
       "      <td>Poison</td>\n",
       "      <td>405</td>\n",
       "      <td>60</td>\n",
       "      <td>62</td>\n",
       "      <td>63</td>\n",
       "      <td>80</td>\n",
       "      <td>80</td>\n",
       "      <td>60</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Venusaur</td>\n",
       "      <td>Grass</td>\n",
       "      <td>Poison</td>\n",
       "      <td>525</td>\n",
       "      <td>80</td>\n",
       "      <td>82</td>\n",
       "      <td>83</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>80</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>VenusaurMega Venusaur</td>\n",
       "      <td>Grass</td>\n",
       "      <td>Poison</td>\n",
       "      <td>625</td>\n",
       "      <td>80</td>\n",
       "      <td>100</td>\n",
       "      <td>123</td>\n",
       "      <td>122</td>\n",
       "      <td>120</td>\n",
       "      <td>80</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Charmander</td>\n",
       "      <td>Fire</td>\n",
       "      <td>hehehehe</td>\n",
       "      <td>309</td>\n",
       "      <td>39</td>\n",
       "      <td>52</td>\n",
       "      <td>43</td>\n",
       "      <td>60</td>\n",
       "      <td>50</td>\n",
       "      <td>65</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   #                   Name Type 1    Type 2  Total  HP  Attack  Defense  \\\n",
       "0  1              Bulbasaur  Grass    Poison    318  45      49       49   \n",
       "1  2                Ivysaur  Grass    Poison    405  60      62       63   \n",
       "2  3               Venusaur  Grass    Poison    525  80      82       83   \n",
       "3  3  VenusaurMega Venusaur  Grass    Poison    625  80     100      123   \n",
       "4  4             Charmander   Fire  hehehehe    309  39      52       43   \n",
       "\n",
       "   Sp. Atk  Sp. Def  Speed  Generation  Legendary  \n",
       "0       65       65     45           1      False  \n",
       "1       80       80     60           1      False  \n",
       "2      100      100     80           1      False  \n",
       "3      122      120     80           1      False  \n",
       "4       60       50     65           1      False  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.fillna('hehehehe').head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f52cd262",
   "metadata": {},
   "source": [
    "Or `DataFrame.interpolate()` to fill wrt surrounding values:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "610ad8ff",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-09-02T18:58:25.423978Z",
     "iopub.status.busy": "2025-09-02T18:58:25.423822Z",
     "iopub.status.idle": "2025-09-02T18:58:25.431582Z",
     "shell.execute_reply": "2025-09-02T18:58:25.431141Z"
    },
    "tags": [
     "hide-output",
     "scroll-output"
    ]
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_5867/3876796089.py:1: FutureWarning: DataFrame.interpolate with object dtype is deprecated and will raise in a future version. Call obj.infer_objects(copy=False) before interpolating instead.\n",
      "  data.infer_objects(copy=False).interpolate().head()\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>#</th>\n",
       "      <th>Name</th>\n",
       "      <th>Type 1</th>\n",
       "      <th>Type 2</th>\n",
       "      <th>Total</th>\n",
       "      <th>HP</th>\n",
       "      <th>Attack</th>\n",
       "      <th>Defense</th>\n",
       "      <th>Sp. Atk</th>\n",
       "      <th>Sp. Def</th>\n",
       "      <th>Speed</th>\n",
       "      <th>Generation</th>\n",
       "      <th>Legendary</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Bulbasaur</td>\n",
       "      <td>Grass</td>\n",
       "      <td>Poison</td>\n",
       "      <td>318</td>\n",
       "      <td>45</td>\n",
       "      <td>49</td>\n",
       "      <td>49</td>\n",
       "      <td>65</td>\n",
       "      <td>65</td>\n",
       "      <td>45</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>Ivysaur</td>\n",
       "      <td>Grass</td>\n",
       "      <td>Poison</td>\n",
       "      <td>405</td>\n",
       "      <td>60</td>\n",
       "      <td>62</td>\n",
       "      <td>63</td>\n",
       "      <td>80</td>\n",
       "      <td>80</td>\n",
       "      <td>60</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Venusaur</td>\n",
       "      <td>Grass</td>\n",
       "      <td>Poison</td>\n",
       "      <td>525</td>\n",
       "      <td>80</td>\n",
       "      <td>82</td>\n",
       "      <td>83</td>\n",
       "      <td>100</td>\n",
       "      <td>100</td>\n",
       "      <td>80</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>VenusaurMega Venusaur</td>\n",
       "      <td>Grass</td>\n",
       "      <td>Poison</td>\n",
       "      <td>625</td>\n",
       "      <td>80</td>\n",
       "      <td>100</td>\n",
       "      <td>123</td>\n",
       "      <td>122</td>\n",
       "      <td>120</td>\n",
       "      <td>80</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Charmander</td>\n",
       "      <td>Fire</td>\n",
       "      <td>NaN</td>\n",
       "      <td>309</td>\n",
       "      <td>39</td>\n",
       "      <td>52</td>\n",
       "      <td>43</td>\n",
       "      <td>60</td>\n",
       "      <td>50</td>\n",
       "      <td>65</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   #                   Name Type 1  Type 2  Total  HP  Attack  Defense  \\\n",
       "0  1              Bulbasaur  Grass  Poison    318  45      49       49   \n",
       "1  2                Ivysaur  Grass  Poison    405  60      62       63   \n",
       "2  3               Venusaur  Grass  Poison    525  80      82       83   \n",
       "3  3  VenusaurMega Venusaur  Grass  Poison    625  80     100      123   \n",
       "4  4             Charmander   Fire     NaN    309  39      52       43   \n",
       "\n",
       "   Sp. Atk  Sp. Def  Speed  Generation  Legendary  \n",
       "0       65       65     45           1      False  \n",
       "1       80       80     60           1      False  \n",
       "2      100      100     80           1      False  \n",
       "3      122      120     80           1      False  \n",
       "4       60       50     65           1      False  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.infer_objects(copy=False).interpolate().head()"
   ]
  }
 ],
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}