{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "e312113e",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "import pandas as pd\n",
    "import matplotlib as plt\n",
    "import seaborn as sns\n",
    "import numpy as np\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "de2c028d",
   "metadata": {},
   "source": [
    "# Загрузка и знакомство с данными"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "id": "5cd00195",
   "metadata": {},
   "outputs": [],
   "source": [
    "# dataset https://www.kaggle.com/datasets/mrdaniilak/russia-real-estate-20182021/data \n",
    "\n",
    "df = pd.read_csv('data/all_v2.csv')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "05b57100",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>price</th>\n",
       "      <th>date</th>\n",
       "      <th>time</th>\n",
       "      <th>geo_lat</th>\n",
       "      <th>geo_lon</th>\n",
       "      <th>region</th>\n",
       "      <th>building_type</th>\n",
       "      <th>level</th>\n",
       "      <th>levels</th>\n",
       "      <th>rooms</th>\n",
       "      <th>area</th>\n",
       "      <th>kitchen_area</th>\n",
       "      <th>object_type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6050000</td>\n",
       "      <td>2018-02-19</td>\n",
       "      <td>20:00:21</td>\n",
       "      <td>59.805808</td>\n",
       "      <td>30.376141</td>\n",
       "      <td>2661</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>10</td>\n",
       "      <td>3</td>\n",
       "      <td>82.6</td>\n",
       "      <td>10.8</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>8650000</td>\n",
       "      <td>2018-02-27</td>\n",
       "      <td>12:04:54</td>\n",
       "      <td>55.683807</td>\n",
       "      <td>37.297405</td>\n",
       "      <td>81</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>24</td>\n",
       "      <td>2</td>\n",
       "      <td>69.1</td>\n",
       "      <td>12.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4000000</td>\n",
       "      <td>2018-02-28</td>\n",
       "      <td>15:44:00</td>\n",
       "      <td>56.295250</td>\n",
       "      <td>44.061637</td>\n",
       "      <td>2871</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "      <td>3</td>\n",
       "      <td>66.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1850000</td>\n",
       "      <td>2018-03-01</td>\n",
       "      <td>11:24:52</td>\n",
       "      <td>44.996132</td>\n",
       "      <td>39.074783</td>\n",
       "      <td>2843</td>\n",
       "      <td>4</td>\n",
       "      <td>12</td>\n",
       "      <td>16</td>\n",
       "      <td>2</td>\n",
       "      <td>38.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5450000</td>\n",
       "      <td>2018-03-01</td>\n",
       "      <td>17:42:43</td>\n",
       "      <td>55.918767</td>\n",
       "      <td>37.984642</td>\n",
       "      <td>81</td>\n",
       "      <td>3</td>\n",
       "      <td>13</td>\n",
       "      <td>14</td>\n",
       "      <td>2</td>\n",
       "      <td>60.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>3300000</td>\n",
       "      <td>2018-03-02</td>\n",
       "      <td>21:18:42</td>\n",
       "      <td>55.908253</td>\n",
       "      <td>37.726448</td>\n",
       "      <td>81</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>32.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>4704280</td>\n",
       "      <td>2018-03-04</td>\n",
       "      <td>12:35:25</td>\n",
       "      <td>55.621097</td>\n",
       "      <td>37.431002</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>25</td>\n",
       "      <td>1</td>\n",
       "      <td>31.7</td>\n",
       "      <td>6.0</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>3600000</td>\n",
       "      <td>2018-03-04</td>\n",
       "      <td>20:52:38</td>\n",
       "      <td>59.875526</td>\n",
       "      <td>30.395457</td>\n",
       "      <td>2661</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>31.1</td>\n",
       "      <td>6.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>3390000</td>\n",
       "      <td>2018-03-05</td>\n",
       "      <td>07:07:05</td>\n",
       "      <td>53.195031</td>\n",
       "      <td>50.106952</td>\n",
       "      <td>3106</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>24</td>\n",
       "      <td>2</td>\n",
       "      <td>64.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>2800000</td>\n",
       "      <td>2018-03-06</td>\n",
       "      <td>09:57:10</td>\n",
       "      <td>55.736972</td>\n",
       "      <td>38.846457</td>\n",
       "      <td>81</td>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>55.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     price        date      time    geo_lat    geo_lon  region  building_type  \\\n",
       "0  6050000  2018-02-19  20:00:21  59.805808  30.376141    2661              1   \n",
       "1  8650000  2018-02-27  12:04:54  55.683807  37.297405      81              3   \n",
       "2  4000000  2018-02-28  15:44:00  56.295250  44.061637    2871              1   \n",
       "3  1850000  2018-03-01  11:24:52  44.996132  39.074783    2843              4   \n",
       "4  5450000  2018-03-01  17:42:43  55.918767  37.984642      81              3   \n",
       "5  3300000  2018-03-02  21:18:42  55.908253  37.726448      81              1   \n",
       "6  4704280  2018-03-04  12:35:25  55.621097  37.431002       3              2   \n",
       "7  3600000  2018-03-04  20:52:38  59.875526  30.395457    2661              1   \n",
       "8  3390000  2018-03-05  07:07:05  53.195031  50.106952    3106              2   \n",
       "9  2800000  2018-03-06  09:57:10  55.736972  38.846457      81              1   \n",
       "\n",
       "   level  levels  rooms  area  kitchen_area  object_type  \n",
       "0      8      10      3  82.6          10.8            1  \n",
       "1      5      24      2  69.1          12.0            1  \n",
       "2      5       9      3  66.0          10.0            1  \n",
       "3     12      16      2  38.0           5.0           11  \n",
       "4     13      14      2  60.0          10.0            1  \n",
       "5      4       5      1  32.0           6.0            1  \n",
       "6      1      25      1  31.7           6.0           11  \n",
       "7      2       5      1  31.1           6.0            1  \n",
       "8      4      24      2  64.0          13.0           11  \n",
       "9      9      10      2  55.0           8.0            1  "
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "id": "701c5e26",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 5477006 entries, 0 to 5477005\n",
      "Data columns (total 13 columns):\n",
      " #   Column         Non-Null Count    Dtype  \n",
      "---  ------         --------------    -----  \n",
      " 0   price          5477006 non-null  int64  \n",
      " 1   date           5477006 non-null  object \n",
      " 2   time           5477006 non-null  object \n",
      " 3   geo_lat        5477006 non-null  float64\n",
      " 4   geo_lon        5477006 non-null  float64\n",
      " 5   region         5477006 non-null  int64  \n",
      " 6   building_type  5477006 non-null  int64  \n",
      " 7   level          5477006 non-null  int64  \n",
      " 8   levels         5477006 non-null  int64  \n",
      " 9   rooms          5477006 non-null  int64  \n",
      " 10  area           5477006 non-null  float64\n",
      " 11  kitchen_area   5477006 non-null  float64\n",
      " 12  object_type    5477006 non-null  int64  \n",
      "dtypes: float64(4), int64(7), object(2)\n",
      "memory usage: 543.2+ MB\n"
     ]
    }
   ],
   "source": [
    "df.info(show_counts=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "c085b0c6",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['region'] = df['region'].astype('category')\n",
    "df['building_type'] = df['building_type'].astype('category')\n",
    "df['object_type'] = df['object_type'].astype('category')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "id": "73fc1a17",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['level'] = df['level'].astype('int8')\n",
    "df['levels'] = df['levels'].astype('int8')\n",
    "df['rooms'] = df['rooms'].astype('int8')\n",
    "df['rooms'] = df['rooms'].astype('int8')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "id": "870155e0",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "df['area'] = df['area'].astype('float16')\n",
    "df['kitchen_area'] = df['kitchen_area'].astype('float16')\n",
    "df['kitchen_area'] = df['kitchen_area'].astype('float16')\n",
    "df['geo_lat'] = df['geo_lat'].astype('float32')\n",
    "df['geo_lon'] = df['geo_lon'].astype('float32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "id": "8e7a34e7",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/andrey/work/institute/MLE/assets/eda/.venv_eda/lib/python3.10/site-packages/pandas/core/nanops.py:1487: RuntimeWarning: overflow encountered in cast\n",
      "  return dtype.type(n)\n",
      "/home/andrey/work/institute/MLE/assets/eda/.venv_eda/lib/python3.10/site-packages/numpy/_core/_methods.py:53: RuntimeWarning: overflow encountered in reduce\n",
      "  return umr_sum(a, axis, dtype, out, keepdims, initial, where)\n",
      "/home/andrey/work/institute/MLE/assets/eda/.venv_eda/lib/python3.10/site-packages/pandas/core/nanops.py:731: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  the_mean = the_sum / count if count > 0 else np.nan\n",
      "/home/andrey/work/institute/MLE/assets/eda/.venv_eda/lib/python3.10/site-packages/pandas/core/nanops.py:1487: RuntimeWarning: overflow encountered in cast\n",
      "  return dtype.type(n)\n",
      "/home/andrey/work/institute/MLE/assets/eda/.venv_eda/lib/python3.10/site-packages/numpy/_core/_methods.py:53: RuntimeWarning: overflow encountered in reduce\n",
      "  return umr_sum(a, axis, dtype, out, keepdims, initial, where)\n",
      "/home/andrey/work/institute/MLE/assets/eda/.venv_eda/lib/python3.10/site-packages/pandas/core/nanops.py:731: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  the_mean = the_sum / count if count > 0 else np.nan\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>price</th>\n",
       "      <th>geo_lat</th>\n",
       "      <th>geo_lon</th>\n",
       "      <th>level</th>\n",
       "      <th>levels</th>\n",
       "      <th>rooms</th>\n",
       "      <th>area</th>\n",
       "      <th>kitchen_area</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>5.477006e+06</td>\n",
       "      <td>5.477006e+06</td>\n",
       "      <td>5.477006e+06</td>\n",
       "      <td>5.477006e+06</td>\n",
       "      <td>5.477006e+06</td>\n",
       "      <td>5.477006e+06</td>\n",
       "      <td>5.477006e+06</td>\n",
       "      <td>5.477006e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>4.422029e+06</td>\n",
       "      <td>5.403825e+01</td>\n",
       "      <td>5.324431e+01</td>\n",
       "      <td>6.214530e+00</td>\n",
       "      <td>1.139892e+01</td>\n",
       "      <td>1.726173e+00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>2.150752e+07</td>\n",
       "      <td>4.622758e+00</td>\n",
       "      <td>2.074763e+01</td>\n",
       "      <td>4.957419e+00</td>\n",
       "      <td>6.535734e+00</td>\n",
       "      <td>1.082133e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>-2.144967e+09</td>\n",
       "      <td>4.145906e+01</td>\n",
       "      <td>1.989020e+01</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>-2.000000e+00</td>\n",
       "      <td>7.000732e-02</td>\n",
       "      <td>1.000214e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.950000e+06</td>\n",
       "      <td>5.337767e+01</td>\n",
       "      <td>3.777790e+01</td>\n",
       "      <td>2.000000e+00</td>\n",
       "      <td>5.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>3.800000e+01</td>\n",
       "      <td>7.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>2.990000e+06</td>\n",
       "      <td>5.517139e+01</td>\n",
       "      <td>4.306774e+01</td>\n",
       "      <td>5.000000e+00</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>2.000000e+00</td>\n",
       "      <td>4.803125e+01</td>\n",
       "      <td>9.703125e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>4.802000e+06</td>\n",
       "      <td>5.622613e+01</td>\n",
       "      <td>6.564895e+01</td>\n",
       "      <td>9.000000e+00</td>\n",
       "      <td>1.600000e+01</td>\n",
       "      <td>2.000000e+00</td>\n",
       "      <td>6.312500e+01</td>\n",
       "      <td>1.270312e+01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>2.147484e+09</td>\n",
       "      <td>7.198040e+01</td>\n",
       "      <td>1.625361e+02</td>\n",
       "      <td>3.900000e+01</td>\n",
       "      <td>3.900000e+01</td>\n",
       "      <td>1.000000e+01</td>\n",
       "      <td>7.856000e+03</td>\n",
       "      <td>1.000000e+04</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              price       geo_lat       geo_lon         level        levels  \\\n",
       "count  5.477006e+06  5.477006e+06  5.477006e+06  5.477006e+06  5.477006e+06   \n",
       "mean   4.422029e+06  5.403825e+01  5.324431e+01  6.214530e+00  1.139892e+01   \n",
       "std    2.150752e+07  4.622758e+00  2.074763e+01  4.957419e+00  6.535734e+00   \n",
       "min   -2.144967e+09  4.145906e+01  1.989020e+01  1.000000e+00  1.000000e+00   \n",
       "25%    1.950000e+06  5.337767e+01  3.777790e+01  2.000000e+00  5.000000e+00   \n",
       "50%    2.990000e+06  5.517139e+01  4.306774e+01  5.000000e+00  1.000000e+01   \n",
       "75%    4.802000e+06  5.622613e+01  6.564895e+01  9.000000e+00  1.600000e+01   \n",
       "max    2.147484e+09  7.198040e+01  1.625361e+02  3.900000e+01  3.900000e+01   \n",
       "\n",
       "              rooms          area  kitchen_area  \n",
       "count  5.477006e+06  5.477006e+06  5.477006e+06  \n",
       "mean   1.726173e+00           NaN           NaN  \n",
       "std    1.082133e+00  0.000000e+00  0.000000e+00  \n",
       "min   -2.000000e+00  7.000732e-02  1.000214e-02  \n",
       "25%    1.000000e+00  3.800000e+01  7.000000e+00  \n",
       "50%    2.000000e+00  4.803125e+01  9.703125e+00  \n",
       "75%    2.000000e+00  6.312500e+01  1.270312e+01  \n",
       "max    1.000000e+01  7.856000e+03  1.000000e+04  "
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "id": "10867be6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['region', 'building_type', 'object_type']"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat_features = df.select_dtypes(include=['category']).columns.to_list()\n",
    "cat_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "id": "88249b5c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['price',\n",
       " 'geo_lat',\n",
       " 'geo_lon',\n",
       " 'level',\n",
       " 'levels',\n",
       " 'rooms',\n",
       " 'area',\n",
       " 'kitchen_area']"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_features = df.select_dtypes(include=['number']).columns.to_list()\n",
    "num_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "id": "637c7446",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.drop(columns=['date', 'time'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "id": "1ab16b00",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 5477006 entries, 0 to 5477005\n",
      "Data columns (total 11 columns):\n",
      " #   Column         Non-Null Count    Dtype   \n",
      "---  ------         --------------    -----   \n",
      " 0   price          5477006 non-null  int64   \n",
      " 1   geo_lat        5477006 non-null  float32 \n",
      " 2   geo_lon        5477006 non-null  float32 \n",
      " 3   region         5477006 non-null  category\n",
      " 4   building_type  5477006 non-null  category\n",
      " 5   level          5477006 non-null  int8    \n",
      " 6   levels         5477006 non-null  int8    \n",
      " 7   rooms          5477006 non-null  int8    \n",
      " 8   area           5477006 non-null  float16 \n",
      " 9   kitchen_area   5477006 non-null  float16 \n",
      " 10  object_type    5477006 non-null  category\n",
      "dtypes: category(3), float16(2), float32(2), int64(1), int8(3)\n",
      "memory usage: 135.8 MB\n"
     ]
    }
   ],
   "source": [
    "df.info(show_counts=True) # уменьшили размер в 3 раза (135 Мб против 543)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "id": "76a6c9a9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "region - numer of unique = 84\n",
      "building_type - numer of unique = 6\n",
      "object_type - numer of unique = 2\n"
     ]
    }
   ],
   "source": [
    "for cat in cat_features:\n",
    "    print(f'{cat} - numer of unique = {df[cat].nunique()}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "id": "b76cc3d5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique categories in region: region\n",
      "9654     1049435\n",
      "2843      637224\n",
      "81        500368\n",
      "2661      461820\n",
      "3         439511\n",
      "          ...   \n",
      "16705        139\n",
      "69            77\n",
      "4963          65\n",
      "1901          12\n",
      "61888          5\n",
      "Name: count, Length: 84, dtype: int64\n",
      "Unique categories in building_type: building_type\n",
      "1    1955661\n",
      "3    1892756\n",
      "2    1130731\n",
      "0     307165\n",
      "4     174356\n",
      "5      16337\n",
      "Name: count, dtype: int64\n",
      "Unique categories in object_type: object_type\n",
      "1     3863809\n",
      "11    1613197\n",
      "Name: count, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "for col in cat_features:\n",
    "    print(f'Unique categories in {col}: {df[col].value_counts()}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6c892b3e",
   "metadata": {},
   "source": [
    "# Очистка данных"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "id": "e9d29872",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.loc[df['price'] > 50000]\n",
    "df = df.query('area >= 8')\n",
    "df = df.query('kitchen_area >= 3')\n",
    "df = df.query('kitchen_area < area')\n",
    "df = df.query('rooms != -2')\n",
    "df = df.query('level <= levels')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "id": "67dd3a9c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# меняем признак \"студии\" -1 на 0\n",
    "df.loc[df['rooms'] == -1, 'rooms'] = 0 "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a3d3ad69",
   "metadata": {},
   "source": [
    "# Анализ признаков для модели\n",
    "\n",
    "https://seaborn.pydata.org/examples/index.html - галерея примеров"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "78845b8b",
   "metadata": {},
   "source": [
    "## displot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "8f5ee97a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<seaborn.axisgrid.FacetGrid at 0x7df3e5371c00>"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 58636 characters]",
      "text/plain": [
       "<Figure size 604.986x500 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.displot(df, x='price', bins=100, hue='building_type', kde=True, log_scale=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "a872743b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Создаем новый признак - уровень высоты этажа в здании\n",
    "df['floor_level'] = df.apply(lambda x: 'low' if x['level']/x['levels'] <= 0.3 \\\n",
    "                             else 'hi' if x['level']/x['levels'] > 0.7 \\\n",
    "                             else 'mid', axis = 1) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "fa63fc34",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['floor_level'] = df.apply(lambda x: 'first' if x['level'] == 1 \\\n",
    "                            else 'last' if x['level'] == x['levels'] \\\n",
    "                            else x['floor_level'], axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "374d184b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "floor_level\n",
       "mid      2092890\n",
       "hi       1208629\n",
       "low       740915\n",
       "last      697099\n",
       "first     654013\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['floor_level'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "c03175ff",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<seaborn.axisgrid.FacetGrid at 0x7df3e55f4be0>"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 57152 characters]",
      "text/plain": [
       "<Figure size 582.736x500 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.displot(df, x='price', bins=100, hue='floor_level', kde=True, log_scale=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9318a819",
   "metadata": {},
   "source": [
    "## heatmap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "7ae43e0e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: >"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 81472 characters]",
      "text/plain": [
       "<Figure size 640x480 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "feature_correlation = df[num_features].corr()\n",
    "sns.heatmap(feature_correlation, annot=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f4ab2ef2",
   "metadata": {},
   "source": [
    "# Групповые операции"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "11e4da4e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def flat_index(df_stats): \n",
    "    df_stats.columns = df_stats.columns.get_level_values(0) + '_' +  df_stats.columns.get_level_values(1) \n",
    "    df_stats.columns = df_stats.columns.to_flat_index() \n",
    "    df_stats.reset_index(inplace=True) \n",
    "    return df_stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "8fdbd526",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1524625/3084051676.py:1: FutureWarning: The default of observed=False is deprecated and will be changed to True in a future version of pandas. Pass observed=False to retain current behavior or observed=True to adopt the future default and silence this warning.\n",
      "  aggregated_df = df[num_features+['building_type']].groupby(by='building_type').agg(['mean', 'std'])\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>building_type</th>\n",
       "      <th>price_mean</th>\n",
       "      <th>price_std</th>\n",
       "      <th>geo_lat_mean</th>\n",
       "      <th>geo_lat_std</th>\n",
       "      <th>geo_lon_mean</th>\n",
       "      <th>geo_lon_std</th>\n",
       "      <th>level_mean</th>\n",
       "      <th>level_std</th>\n",
       "      <th>levels_mean</th>\n",
       "      <th>levels_std</th>\n",
       "      <th>rooms_mean</th>\n",
       "      <th>rooms_std</th>\n",
       "      <th>area_mean</th>\n",
       "      <th>area_std</th>\n",
       "      <th>kitchen_area_mean</th>\n",
       "      <th>kitchen_area_std</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>5.284833e+06</td>\n",
       "      <td>1.279574e+07</td>\n",
       "      <td>53.747444</td>\n",
       "      <td>5.098192</td>\n",
       "      <td>55.518322</td>\n",
       "      <td>22.517204</td>\n",
       "      <td>8.851292</td>\n",
       "      <td>5.923950</td>\n",
       "      <td>16.738440</td>\n",
       "      <td>6.499030</td>\n",
       "      <td>1.686900</td>\n",
       "      <td>0.934588</td>\n",
       "      <td>58.559349</td>\n",
       "      <td>37.077482</td>\n",
       "      <td>12.792236</td>\n",
       "      <td>6.599672</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>3.512978e+06</td>\n",
       "      <td>1.952254e+07</td>\n",
       "      <td>54.790298</td>\n",
       "      <td>3.357936</td>\n",
       "      <td>61.443077</td>\n",
       "      <td>21.923780</td>\n",
       "      <td>5.571202</td>\n",
       "      <td>4.060143</td>\n",
       "      <td>10.213742</td>\n",
       "      <td>4.642660</td>\n",
       "      <td>1.794023</td>\n",
       "      <td>0.966615</td>\n",
       "      <td>51.407722</td>\n",
       "      <td>25.746609</td>\n",
       "      <td>9.298869</td>\n",
       "      <td>4.062081</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>7.126816e+06</td>\n",
       "      <td>1.267507e+07</td>\n",
       "      <td>53.999779</td>\n",
       "      <td>5.346035</td>\n",
       "      <td>42.106968</td>\n",
       "      <td>13.565219</td>\n",
       "      <td>9.031732</td>\n",
       "      <td>6.130808</td>\n",
       "      <td>16.850670</td>\n",
       "      <td>6.862993</td>\n",
       "      <td>1.697287</td>\n",
       "      <td>0.928301</td>\n",
       "      <td>57.157982</td>\n",
       "      <td>32.994183</td>\n",
       "      <td>13.154365</td>\n",
       "      <td>6.146328</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>4.068649e+06</td>\n",
       "      <td>8.331654e+06</td>\n",
       "      <td>53.556213</td>\n",
       "      <td>4.910420</td>\n",
       "      <td>51.249592</td>\n",
       "      <td>19.097178</td>\n",
       "      <td>4.929223</td>\n",
       "      <td>3.997643</td>\n",
       "      <td>8.741839</td>\n",
       "      <td>5.626125</td>\n",
       "      <td>1.878324</td>\n",
       "      <td>0.944347</td>\n",
       "      <td>55.062199</td>\n",
       "      <td>36.979730</td>\n",
       "      <td>10.442657</td>\n",
       "      <td>6.912905</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>4.004994e+06</td>\n",
       "      <td>3.367351e+07</td>\n",
       "      <td>51.181896</td>\n",
       "      <td>5.909806</td>\n",
       "      <td>47.777088</td>\n",
       "      <td>17.570242</td>\n",
       "      <td>5.430987</td>\n",
       "      <td>4.308443</td>\n",
       "      <td>9.746289</td>\n",
       "      <td>5.743188</td>\n",
       "      <td>1.811687</td>\n",
       "      <td>0.892482</td>\n",
       "      <td>52.034588</td>\n",
       "      <td>42.411056</td>\n",
       "      <td>10.051356</td>\n",
       "      <td>4.793207</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>2.618608e+06</td>\n",
       "      <td>1.543122e+07</td>\n",
       "      <td>56.440563</td>\n",
       "      <td>3.981515</td>\n",
       "      <td>63.951946</td>\n",
       "      <td>28.464556</td>\n",
       "      <td>2.472882</td>\n",
       "      <td>3.850120</td>\n",
       "      <td>3.766808</td>\n",
       "      <td>5.931042</td>\n",
       "      <td>2.031180</td>\n",
       "      <td>0.907434</td>\n",
       "      <td>50.446354</td>\n",
       "      <td>52.361071</td>\n",
       "      <td>9.285330</td>\n",
       "      <td>4.603177</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  building_type    price_mean     price_std  geo_lat_mean  geo_lat_std  \\\n",
       "0             0  5.284833e+06  1.279574e+07     53.747444     5.098192   \n",
       "1             1  3.512978e+06  1.952254e+07     54.790298     3.357936   \n",
       "2             2  7.126816e+06  1.267507e+07     53.999779     5.346035   \n",
       "3             3  4.068649e+06  8.331654e+06     53.556213     4.910420   \n",
       "4             4  4.004994e+06  3.367351e+07     51.181896     5.909806   \n",
       "5             5  2.618608e+06  1.543122e+07     56.440563     3.981515   \n",
       "\n",
       "   geo_lon_mean  geo_lon_std  level_mean  level_std  levels_mean  levels_std  \\\n",
       "0     55.518322    22.517204    8.851292   5.923950    16.738440    6.499030   \n",
       "1     61.443077    21.923780    5.571202   4.060143    10.213742    4.642660   \n",
       "2     42.106968    13.565219    9.031732   6.130808    16.850670    6.862993   \n",
       "3     51.249592    19.097178    4.929223   3.997643     8.741839    5.626125   \n",
       "4     47.777088    17.570242    5.430987   4.308443     9.746289    5.743188   \n",
       "5     63.951946    28.464556    2.472882   3.850120     3.766808    5.931042   \n",
       "\n",
       "   rooms_mean  rooms_std  area_mean   area_std  kitchen_area_mean  \\\n",
       "0    1.686900   0.934588  58.559349  37.077482          12.792236   \n",
       "1    1.794023   0.966615  51.407722  25.746609           9.298869   \n",
       "2    1.697287   0.928301  57.157982  32.994183          13.154365   \n",
       "3    1.878324   0.944347  55.062199  36.979730          10.442657   \n",
       "4    1.811687   0.892482  52.034588  42.411056          10.051356   \n",
       "5    2.031180   0.907434  50.446354  52.361071           9.285330   \n",
       "\n",
       "   kitchen_area_std  \n",
       "0          6.599672  \n",
       "1          4.062081  \n",
       "2          6.146328  \n",
       "3          6.912905  \n",
       "4          4.793207  \n",
       "5          4.603177  "
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "aggregated_df = df[num_features+['building_type']].groupby(by='building_type').agg(['mean', 'std'])\n",
    "aggregated_df = flat_index(aggregated_df)\n",
    "aggregated_df "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "77fe1bda",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "0fbb62de",
   "metadata": {},
   "source": [
    "## lineplot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "acb4cdee",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: xlabel='building_type', ylabel='price_mean'>"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 29652 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "sns.lineplot(aggregated_df,x='building_type',y='price_mean', label='mean')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b8bc652d",
   "metadata": {},
   "source": [
    "## subplots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "20ca7cc9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: xlabel='building_type', ylabel='area'>"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 166784 characters]",
      "text/plain": [
       "<Figure size 1650x1400 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, axs = plt.pyplot.subplots(2,2)\n",
    "fig.tight_layout(pad=1)\n",
    "fig.set_size_inches(16.5, 14, forward=True)\n",
    "\n",
    "sns.lineplot(aggregated_df,x='building_type',y='price_mean', label='mean', ax=axs[0,0])\n",
    "\n",
    "sns.lineplot(aggregated_df,x='building_type',y='area_mean', label='area',ax=axs[0,1])\n",
    "sns.lineplot(aggregated_df,x='building_type',y='kitchen_area_mean', label='kitchen_area_mean',ax=axs[0,1])\n",
    "\n",
    "sns.lineplot(aggregated_df,x='building_type',y='level_mean', label='level_mean',ax=axs[1,0])\n",
    "\n",
    "# Группировка непосредственно в seaborn\n",
    "sns.lineplot(df, x='building_type', y='area', label='area', ax=axs[1,1])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5d64d58a",
   "metadata": {},
   "source": [
    "# Bokeh\n",
    "https://bokeh.org/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "fddb38a2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "    <style>\n",
       "        .bk-notebook-logo {\n",
       "            display: block;\n",
       "            width: 20px;\n",
       "            height: 20px;\n",
       "            background-image: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAUCAYAAACNiR0NAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAABx0RVh0U29mdHdhcmUAQWRvYmUgRmlyZXdvcmtzIENTNui8sowAAAOkSURBVDiNjZRtaJVlGMd/1/08zzln5zjP1LWcU9N0NkN8m2CYjpgQYQXqSs0I84OLIC0hkEKoPtiH3gmKoiJDU7QpLgoLjLIQCpEsNJ1vqUOdO7ppbuec5+V+rj4ctwzd8IIbbi6u+8f1539dt3A78eXC7QizUF7gyV1fD1Yqg4JWz84yffhm0qkFqBogB9rM8tZdtwVsPUhWhGcFJngGeWrPzHm5oaMmkfEg1usvLFyc8jLRqDOMru7AyC8saQr7GG7f5fvDeH7Ej8CM66nIF+8yngt6HWaKh7k49Soy9nXurCi1o3qUbS3zWfrYeQDTB/Qj6kX6Ybhw4B+bOYoLKCC9H3Nu/leUTZ1JdRWkkn2ldcCamzrcf47KKXdAJllSlxAOkRgyHsGC/zRday5Qld9DyoM4/q/rUoy/CXh3jzOu3bHUVZeU+DEn8FInkPBFlu3+nW3Nw0mk6vCDiWg8CeJaxEwuHS3+z5RgY+YBR6V1Z1nxSOfoaPa4LASWxxdNp+VWTk7+4vzaou8v8PN+xo+KY2xsw6une2frhw05CTYOmQvsEhjhWjn0bmXPjpE1+kplmmkP3suftwTubK9Vq22qKmrBhpY4jvd5afdRA3wGjFAgcnTK2s4hY0/GPNIb0nErGMCRxWOOX64Z8RAC4oCXdklmEvcL8o0BfkNK4lUg9HTl+oPlQxdNo3Mg4Nv175e/1LDGzZen30MEjRUtmXSfiTVu1kK8W4txyV6BMKlbgk3lMwYCiusNy9fVfvvwMxv8Ynl6vxoByANLTWplvuj/nF9m2+PDtt1eiHPBr1oIfhCChQMBw6Aw0UulqTKZdfVvfG7VcfIqLG9bcldL/+pdWTLxLUy8Qq38heUIjh4XlzZxzQm19lLFlr8vdQ97rjZVOLf8nclzckbcD4wxXMidpX30sFd37Fv/GtwwhzhxGVAprjbg0gCAEeIgwCZyTV2Z1REEW8O4py0wsjeloKoMr6iCY6dP92H6Vw/oTyICIthibxjm/DfN9lVz8IqtqKYLUXfoKVMVQVVJOElGjrnnUt9T9wbgp8AyYKaGlqingHZU/uG2NTZSVqwHQTWkx9hxjkpWDaCg6Ckj5qebgBVbT3V3NNXMSiWSDdGV3hrtzla7J+duwPOToIg42ChPQOQjspnSlp1V+Gjdged7+8UN5CRAV7a5EdFNwCjEaBR27b3W890TE7g24NAP/mMDXRWrGoFPQI9ls/MWO2dWFAar/xcOIImbbpA3zgAAAABJRU5ErkJggg==);\n",
       "        }\n",
       "    </style>\n",
       "    <div>\n",
       "        <a href=\"https://bokeh.org\" target=\"_blank\" class=\"bk-notebook-logo\"></a>\n",
       "        <span id=\"dec2ab1e-cda5-49ce-8f7d-85a26b4e3d5b\">Loading BokehJS ...</span>\n",
       "    </div>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "'use strict';\n",
       "(function(root) {\n",
       "  function now() {\n",
       "    return new Date();\n",
       "  }\n",
       "\n",
       "  const force = true;\n",
       "\n",
       "  if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n",
       "    root._bokeh_onload_callbacks = [];\n",
       "    root._bokeh_is_loading = undefined;\n",
       "  }\n",
       "\n",
       "const JS_MIME_TYPE = 'application/javascript';\n",
       "  const HTML_MIME_TYPE = 'text/html';\n",
       "  const EXEC_MIME_TYPE = 'application/vnd.bokehjs_exec.v0+json';\n",
       "  const CLASS_NAME = 'output_bokeh rendered_html';\n",
       "\n",
       "  /**\n",
       "   * Render data to the DOM node\n",
       "   */\n",
       "  function render(props, node) {\n",
       "    const script = document.createElement(\"script\");\n",
       "    node.appendChild(script);\n",
       "  }\n",
       "\n",
       "  /**\n",
       "   * Handle when an output is cleared or removed\n",
       "   */\n",
       "  function handleClearOutput(event, handle) {\n",
       "    function drop(id) {\n",
       "      const view = Bokeh.index.get_by_id(id)\n",
       "      if (view != null) {\n",
       "        view.model.document.clear()\n",
       "        Bokeh.index.delete(view)\n",
       "      }\n",
       "    }\n",
       "\n",
       "    const cell = handle.cell;\n",
       "\n",
       "    const id = cell.output_area._bokeh_element_id;\n",
       "    const server_id = cell.output_area._bokeh_server_id;\n",
       "\n",
       "    // Clean up Bokeh references\n",
       "    if (id != null) {\n",
       "      drop(id)\n",
       "    }\n",
       "\n",
       "    if (server_id !== undefined) {\n",
       "      // Clean up Bokeh references\n",
       "      const cmd_clean = \"from bokeh.io.state import curstate; print(curstate().uuid_to_server['\" + server_id + \"'].get_sessions()[0].document.roots[0]._id)\";\n",
       "      cell.notebook.kernel.execute(cmd_clean, {\n",
       "        iopub: {\n",
       "          output: function(msg) {\n",
       "            const id = msg.content.text.trim()\n",
       "            drop(id)\n",
       "          }\n",
       "        }\n",
       "      });\n",
       "      // Destroy server and session\n",
       "      const cmd_destroy = \"import bokeh.io.notebook as ion; ion.destroy_server('\" + server_id + \"')\";\n",
       "      cell.notebook.kernel.execute(cmd_destroy);\n",
       "    }\n",
       "  }\n",
       "\n",
       "  /**\n",
       "   * Handle when a new output is added\n",
       "   */\n",
       "  function handleAddOutput(event, handle) {\n",
       "    const output_area = handle.output_area;\n",
       "    const output = handle.output;\n",
       "\n",
       "    // limit handleAddOutput to display_data with EXEC_MIME_TYPE content only\n",
       "    if ((output.output_type != \"display_data\") || (!Object.prototype.hasOwnProperty.call(output.data, EXEC_MIME_TYPE))) {\n",
       "      return\n",
       "    }\n",
       "\n",
       "    const toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n",
       "\n",
       "    if (output.metadata[EXEC_MIME_TYPE][\"id\"] !== undefined) {\n",
       "      toinsert[toinsert.length - 1].firstChild.textContent = output.data[JS_MIME_TYPE];\n",
       "      // store reference to embed id on output_area\n",
       "      output_area._bokeh_element_id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n",
       "    }\n",
       "    if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n",
       "      const bk_div = document.createElement(\"div\");\n",
       "      bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n",
       "      const script_attrs = bk_div.children[0].attributes;\n",
       "      for (let i = 0; i < script_attrs.length; i++) {\n",
       "        toinsert[toinsert.length - 1].firstChild.setAttribute(script_attrs[i].name, script_attrs[i].value);\n",
       "        toinsert[toinsert.length - 1].firstChild.textContent = bk_div.children[0].textContent\n",
       "      }\n",
       "      // store reference to server id on output_area\n",
       "      output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n",
       "    }\n",
       "  }\n",
       "\n",
       "  function register_renderer(events, OutputArea) {\n",
       "\n",
       "    function append_mime(data, metadata, element) {\n",
       "      // create a DOM node to render to\n",
       "      const toinsert = this.create_output_subarea(\n",
       "        metadata,\n",
       "        CLASS_NAME,\n",
       "        EXEC_MIME_TYPE\n",
       "      );\n",
       "      this.keyboard_manager.register_events(toinsert);\n",
       "      // Render to node\n",
       "      const props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n",
       "      render(props, toinsert[toinsert.length - 1]);\n",
       "      element.append(toinsert);\n",
       "      return toinsert\n",
       "    }\n",
       "\n",
       "    /* Handle when an output is cleared or removed */\n",
       "    events.on('clear_output.CodeCell', handleClearOutput);\n",
       "    events.on('delete.Cell', handleClearOutput);\n",
       "\n",
       "    /* Handle when a new output is added */\n",
       "    events.on('output_added.OutputArea', handleAddOutput);\n",
       "\n",
       "    /**\n",
       "     * Register the mime type and append_mime function with output_area\n",
       "     */\n",
       "    OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n",
       "      /* Is output safe? */\n",
       "      safe: true,\n",
       "      /* Index of renderer in `output_area.display_order` */\n",
       "      index: 0\n",
       "    });\n",
       "  }\n",
       "\n",
       "  // register the mime type if in Jupyter Notebook environment and previously unregistered\n",
       "  if (root.Jupyter !== undefined) {\n",
       "    const events = require('base/js/events');\n",
       "    const OutputArea = require('notebook/js/outputarea').OutputArea;\n",
       "\n",
       "    if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n",
       "      register_renderer(events, OutputArea);\n",
       "    }\n",
       "  }\n",
       "  if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n",
       "    root._bokeh_timeout = Date.now() + 5000;\n",
       "    root._bokeh_failed_load = false;\n",
       "  }\n",
       "\n",
       "  const NB_LOAD_WARNING = {'data': {'text/html':\n",
       "     \"<div style='background-color: #fdd'>\\n\"+\n",
       "     \"<p>\\n\"+\n",
       "     \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n",
       "     \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n",
       "     \"</p>\\n\"+\n",
       "     \"<ul>\\n\"+\n",
       "     \"<li>re-rerun `output_notebook()` to attempt to load from CDN again, or</li>\\n\"+\n",
       "     \"<li>use INLINE resources instead, as so:</li>\\n\"+\n",
       "     \"</ul>\\n\"+\n",
       "     \"<code>\\n\"+\n",
       "     \"from bokeh.resources import INLINE\\n\"+\n",
       "     \"output_notebook(resources=INLINE)\\n\"+\n",
       "     \"</code>\\n\"+\n",
       "     \"</div>\"}};\n",
       "\n",
       "  function display_loaded(error = null) {\n",
       "    const el = document.getElementById(\"dec2ab1e-cda5-49ce-8f7d-85a26b4e3d5b\");\n",
       "    if (el != null) {\n",
       "      const html = (() => {\n",
       "        if (typeof root.Bokeh === \"undefined\") {\n",
       "          if (error == null) {\n",
       "            return \"BokehJS is loading ...\";\n",
       "          } else {\n",
       "            return \"BokehJS failed to load.\";\n",
       "          }\n",
       "        } else {\n",
       "          const prefix = `BokehJS ${root.Bokeh.version}`;\n",
       "          if (error == null) {\n",
       "            return `${prefix} successfully loaded.`;\n",
       "          } else {\n",
       "            return `${prefix} <b>encountered errors</b> while loading and may not function as expected.`;\n",
       "          }\n",
       "        }\n",
       "      })();\n",
       "      el.innerHTML = html;\n",
       "\n",
       "      if (error != null) {\n",
       "        const wrapper = document.createElement(\"div\");\n",
       "        wrapper.style.overflow = \"auto\";\n",
       "        wrapper.style.height = \"5em\";\n",
       "        wrapper.style.resize = \"vertical\";\n",
       "        const content = document.createElement(\"div\");\n",
       "        content.style.fontFamily = \"monospace\";\n",
       "        content.style.whiteSpace = \"pre-wrap\";\n",
       "        content.style.backgroundColor = \"rgb(255, 221, 221)\";\n",
       "        content.textContent = error.stack ?? error.toString();\n",
       "        wrapper.append(content);\n",
       "        el.append(wrapper);\n",
       "      }\n",
       "    } else if (Date.now() < root._bokeh_timeout) {\n",
       "      setTimeout(() => display_loaded(error), 100);\n",
       "    }\n",
       "  }\n",
       "\n",
       "  function run_callbacks() {\n",
       "    try {\n",
       "      root._bokeh_onload_callbacks.forEach(function(callback) {\n",
       "        if (callback != null)\n",
       "          callback();\n",
       "      });\n",
       "    } finally {\n",
       "      delete root._bokeh_onload_callbacks\n",
       "    }\n",
       "    console.debug(\"Bokeh: all callbacks have finished\");\n",
       "  }\n",
       "\n",
       "  function load_libs(css_urls, js_urls, callback) {\n",
       "    if (css_urls == null) css_urls = [];\n",
       "    if (js_urls == null) js_urls = [];\n",
       "\n",
       "    root._bokeh_onload_callbacks.push(callback);\n",
       "    if (root._bokeh_is_loading > 0) {\n",
       "      console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n",
       "      return null;\n",
       "    }\n",
       "    if (js_urls == null || js_urls.length === 0) {\n",
       "      run_callbacks();\n",
       "      return null;\n",
       "    }\n",
       "    console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n",
       "    root._bokeh_is_loading = css_urls.length + js_urls.length;\n",
       "\n",
       "    function on_load() {\n",
       "      root._bokeh_is_loading--;\n",
       "      if (root._bokeh_is_loading === 0) {\n",
       "        console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n",
       "        run_callbacks()\n",
       "      }\n",
       "    }\n",
       "\n",
       "    function on_error(url) {\n",
       "      console.error(\"failed to load \" + url);\n",
       "    }\n",
       "\n",
       "    for (let i = 0; i < css_urls.length; i++) {\n",
       "      const url = css_urls[i];\n",
       "      const element = document.createElement(\"link\");\n",
       "      element.onload = on_load;\n",
       "      element.onerror = on_error.bind(null, url);\n",
       "      element.rel = \"stylesheet\";\n",
       "      element.type = \"text/css\";\n",
       "      element.href = url;\n",
       "      console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n",
       "      document.body.appendChild(element);\n",
       "    }\n",
       "\n",
       "    for (let i = 0; i < js_urls.length; i++) {\n",
       "      const url = js_urls[i];\n",
       "      const element = document.createElement('script');\n",
       "      element.onload = on_load;\n",
       "      element.onerror = on_error.bind(null, url);\n",
       "      element.async = false;\n",
       "      element.src = url;\n",
       "      console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n",
       "      document.head.appendChild(element);\n",
       "    }\n",
       "  };\n",
       "\n",
       "  function inject_raw_css(css) {\n",
       "    const element = document.createElement(\"style\");\n",
       "    element.appendChild(document.createTextNode(css));\n",
       "    document.body.appendChild(element);\n",
       "  }\n",
       "\n",
       "  const js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-3.5.2.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-3.5.2.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-3.5.2.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-3.5.2.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-mathjax-3.5.2.min.js\"];\n",
       "  const css_urls = [];\n",
       "\n",
       "  const inline_js = [    function(Bokeh) {\n",
       "      Bokeh.set_log_level(\"info\");\n",
       "    },\n",
       "function(Bokeh) {\n",
       "    }\n",
       "  ];\n",
       "\n",
       "  function run_inline_js() {\n",
       "    if (root.Bokeh !== undefined || force === true) {\n",
       "      try {\n",
       "            for (let i = 0; i < inline_js.length; i++) {\n",
       "      inline_js[i].call(root, root.Bokeh);\n",
       "    }\n",
       "\n",
       "      } catch (error) {display_loaded(error);throw error;\n",
       "      }if (force === true) {\n",
       "        display_loaded();\n",
       "      }} else if (Date.now() < root._bokeh_timeout) {\n",
       "      setTimeout(run_inline_js, 100);\n",
       "    } else if (!root._bokeh_failed_load) {\n",
       "      console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n",
       "      root._bokeh_failed_load = true;\n",
       "    } else if (force !== true) {\n",
       "      const cell = $(document.getElementById(\"dec2ab1e-cda5-49ce-8f7d-85a26b4e3d5b\")).parents('.cell').data().cell;\n",
       "      cell.output_area.append_execute_result(NB_LOAD_WARNING)\n",
       "    }\n",
       "  }\n",
       "\n",
       "  if (root._bokeh_is_loading === 0) {\n",
       "    console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n",
       "    run_inline_js();\n",
       "  } else {\n",
       "    load_libs(css_urls, js_urls, function() {\n",
       "      console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n",
       "      run_inline_js();\n",
       "    });\n",
       "  }\n",
       "}(window));"
      ],
      "application/vnd.bokehjs_load.v0+json": ""
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from bokeh.plotting import figure, show\n",
    "from bokeh.models import ColumnDataSource,  HoverTool, Legend\n",
    "from bokeh.io import output_notebook \n",
    "output_notebook()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "75e6c73b",
   "metadata": {},
   "outputs": [],
   "source": [
    "source = ColumnDataSource(data=df.sample(frac=0.1))\n",
    "p = figure(width=1000)\n",
    "p.scatter(source=source, x='area', y='price' )\n",
    "hover = HoverTool(tooltips=[('flat area ', '@area'),\n",
    "                              ('price', '@price'),\n",
    "                              ('level', '@level')])\n",
    "\n",
    "p.add_tools(hover)\n",
    "show(p)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "15c89262",
   "metadata": {},
   "outputs": [],
   "source": [
    "from bokeh.transform import factor_cmap\n",
    "\n",
    "types = df['floor_level'].unique().tolist()\n",
    "source = ColumnDataSource(data=df.sample(frac=0.1))\n",
    "p = figure(width=1000)\n",
    "p.scatter(source=source, x='area', y='level', color=factor_cmap('floor_level', 'Category10_5', types) )\n",
    "hover = HoverTool(tooltips=[('area', '@area'),\n",
    "                              ('price', '@price'),\n",
    "                              ('level', '@level')])\n",
    "\n",
    "p.add_tools(hover)\n",
    "\n",
    "show(p)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "082960cc",
   "metadata": {},
   "source": [
    "# Save clean dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "id": "ec654599",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Сохраняем обработанный датафрейм, чтобы на следующих этапах не проводить повторно ту же обработку.\n",
    "# Лучше сохранить в pickle формате, чтобы сохранились все типы данных, в т.ч. category\n",
    "df.to_pickle('./data/clean_data.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "id": "d410d06a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Index: 5393546 entries, 0 to 5477005\n",
      "Data columns (total 11 columns):\n",
      " #   Column         Dtype   \n",
      "---  ------         -----   \n",
      " 0   price          int64   \n",
      " 1   geo_lat        float32 \n",
      " 2   geo_lon        float32 \n",
      " 3   region         category\n",
      " 4   building_type  category\n",
      " 5   level          int8    \n",
      " 6   levels         int8    \n",
      " 7   rooms          int8    \n",
      " 8   area           float16 \n",
      " 9   kitchen_area   float16 \n",
      " 10  object_type    category\n",
      "dtypes: category(3), float16(2), float32(2), int64(1), int8(3)\n",
      "memory usage: 174.9 MB\n"
     ]
    }
   ],
   "source": [
    "# Считать можно так:\n",
    "df = pd.read_pickle('./data/clean_data.pkl')\n",
    "df.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3a7cbaaa",
   "metadata": {},
   "source": [
    "# Выводы после EDA"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "695334e5",
   "metadata": {},
   "source": [
    "Выводы, полученные в ходе анализа:\n",
    "* перечислить все действия, проведенные на этапе очистки данных\n",
    "* Создавались ли новые признаки? \n",
    "* Какие закономерности выявлены по графикам, которые могут быть полезны в дальнейшем для решния задачи?"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
