diff --git a/hw0/boriskin/hw0.ipynb b/hw0/boriskin/hw0.ipynb
new file mode 100644
index 0000000..e42cba7
--- /dev/null
+++ b/hw0/boriskin/hw0.ipynb
@@ -0,0 +1,1760 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "\u001b[32mimport \u001b[39m\u001b[36m$ivy.$                                  \u001b[39m"
+      ]
+     },
+     "execution_count": 1,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import $ivy.`org.apache.spark::spark-sql:2.4.3`"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "\u001b[32mimport \u001b[39m\u001b[36morg.apache.spark.sql._\u001b[39m"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import org.apache.spark.sql._"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "\u001b[32mimport \u001b[39m\u001b[36morg.apache.log4j.{Level, Logger}\n",
+       "\u001b[39m"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import org.apache.log4j.{Level, Logger}\n",
+    "Logger.getLogger(\"org\").setLevel(Level.OFF)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Loading spark-stubs\n",
+      "Creating SparkSession\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<a target=\"_blank\" href=\"http://192.168.1.111:4040\">Spark UI</a>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "\u001b[36mspark\u001b[39m: \u001b[32mSparkSession\u001b[39m = org.apache.spark.sql.SparkSession@6731a7cf"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "val spark = {\n",
+    "  NotebookSparkSession.builder()\n",
+    "    .master(\"local[*]\")\n",
+    "    .getOrCreate()\n",
+    "}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "defined \u001b[32mfunction\u001b[39m \u001b[36msc\u001b[39m"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "def sc = spark.sparkContext"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "\u001b[36mrdd\u001b[39m: \u001b[32morg\u001b[39m.\u001b[32mapache\u001b[39m.\u001b[32mspark\u001b[39m.\u001b[32mrdd\u001b[39m.\u001b[32mRDD\u001b[39m[\u001b[32mInt\u001b[39m] = ParallelCollectionRDD[0] at parallelize at cmd5.sc:1"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "val rdd = sc.parallelize(1 to 100000000, 100)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "\u001b[32mimport \u001b[39m\u001b[36mspark.implicits._\n",
+       "\n",
+       "\u001b[39m"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import spark.implicits._\n",
+    "\n",
+    "org.apache.spark.sql.catalyst.encoders.OuterScopes.addOuterScope(this);"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<script>\n",
+       "var comm = Jupyter.notebook.kernel.comm_manager.new_comm('cancel-stage-83287be9-9cff-4a4d-af4d-96dc7455e29e', {});\n",
+       "\n",
+       "function cancelStage(stageId) {\n",
+       "  console.log('Cancelling stage ' + stageId);\n",
+       "  comm.send({ 'stageId': stageId });\n",
+       "}\n",
+       "</script>\n",
+       "          "
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "  <span style=\"float: left;\">csv at cmd7.sc:1</span>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div class=\"progress\">\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: blue; width: 100%; word-wrap: normal; white-space: nowrap; text-align: center; color: white\" aria-valuenow=\"100\" aria-valuemin=\"0\" aria-valuemax=\"100\">\n",
+       "    1 / 1\n",
+       "  </div>\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: red; width: 0%\" aria-valuenow=\"0\" aria-valuemin=\"0\" aria-valuemax=\"100\"></div>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div class=\"jp-RenderedText\">\n",
+       "<pre><code><span style=\"color: rgb(0, 187, 187)\"><span class=\"ansi-cyan-fg\">data</span></span>: <span style=\"color: rgb(0, 187, 0)\"><span class=\"ansi-green-fg\">DataFrame</span></span> = <style>@keyframes fadein { from { opacity: 0; } to { opacity: 1; } }</style><span style=\"animation: fadein 2s;\">[id: string, name: string ... 14 more fields]</span></code></pre>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "\u001b[36mdata\u001b[39m: \u001b[32mDataFrame\u001b[39m = [id: string, name: string ... 14 more fields]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "var data = spark.read.option(\"header\", \"true\").option(\"mode\", \"DROPMALFORMED\").option(\"escape\", \"\\\"\").csv(\"AB_NYC_2019.csv\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "defined \u001b[32mclass\u001b[39m \u001b[36mRichDF\u001b[39m"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "implicit class RichDF(val df: DataFrame) {\n",
+    "  def showHTML(limit:Int = 20, truncate: Int = 20) = {\n",
+    "    import xml.Utility.escape\n",
+    "    val data = df.take(limit)\n",
+    "    val header = df.schema.fieldNames.toSeq\n",
+    "    val rows: Seq[Seq[String]] = data.map { row =>\n",
+    "      row.toSeq.map { cell =>\n",
+    "        val str = cell match {\n",
+    "          case null => \"null\"\n",
+    "          case binary: Array[Byte] => binary.map(\"%02X\".format(_)).mkString(\"[\", \" \", \"]\")\n",
+    "          case array: Array[_] => array.mkString(\"[\", \", \", \"]\")\n",
+    "          case seq: Seq[_] => seq.mkString(\"[\", \", \", \"]\")\n",
+    "          case _ => cell.toString\n",
+    "        }\n",
+    "        if (truncate > 0 && str.length > truncate) {\n",
+    "          // do not show ellipses for strings shorter than 4 characters.\n",
+    "          if (truncate < 4) str.substring(0, truncate)\n",
+    "          else str.substring(0, truncate - 3) + \"...\"\n",
+    "        } else {\n",
+    "          str\n",
+    "        }\n",
+    "      }: Seq[String]\n",
+    "    }\n",
+    "\n",
+    "    publish.html(s\"\"\"\n",
+    "      <table class=\"table\">\n",
+    "        <tr>\n",
+    "        ${header.map(h => s\"<th>${escape(h)}</th>\").mkString}\n",
+    "        </tr>\n",
+    "        ${rows.map { row =>\n",
+    "          s\"<tr>${row.map { c => s\"<td>${escape(c)}</td>\" }.mkString}</tr>\"\n",
+    "        }.mkString\n",
+    "        }\n",
+    "      </table>\"\"\")\n",
+    "  }\n",
+    "}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {
+    "scrolled": false
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "  <span style=\"float: left;\">take at cmd8.sc:4</span>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div class=\"progress\">\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: blue; width: 100%; word-wrap: normal; white-space: nowrap; text-align: center; color: white\" aria-valuenow=\"100\" aria-valuemin=\"0\" aria-valuemax=\"100\">\n",
+       "    1 / 1\n",
+       "  </div>\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: red; width: 0%\" aria-valuenow=\"0\" aria-valuemin=\"0\" aria-valuemax=\"100\"></div>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "      <table class=\"table\">\n",
+       "        <tr>\n",
+       "        <th>id</th><th>name</th><th>host_id</th><th>host_name</th><th>neighbourhood_group</th><th>neighbourhood</th><th>latitude</th><th>longitude</th><th>room_type</th><th>price</th><th>minimum_nights</th><th>number_of_reviews</th><th>last_review</th><th>reviews_per_month</th><th>calculated_host_listings_count</th><th>availability_365</th>\n",
+       "        </tr>\n",
+       "        <tr><td>2539</td><td>Clean &amp; quiet apt...</td><td>2787</td><td>John</td><td>Brooklyn</td><td>Kensington</td><td>40.64749</td><td>-73.97237</td><td>Private room</td><td>149</td><td>1</td><td>9</td><td>2018-10-19</td><td>0.21</td><td>6</td><td>365</td></tr><tr><td>2595</td><td>Skylit Midtown Ca...</td><td>2845</td><td>Jennifer</td><td>Manhattan</td><td>Midtown</td><td>40.75362</td><td>-73.98377</td><td>Entire home/apt</td><td>225</td><td>1</td><td>45</td><td>2019-05-21</td><td>0.38</td><td>2</td><td>355</td></tr><tr><td>3647</td><td>THE VILLAGE OF HA...</td><td>4632</td><td>Elisabeth</td><td>Manhattan</td><td>Harlem</td><td>40.80902</td><td>-73.9419</td><td>Private room</td><td>150</td><td>3</td><td>0</td><td>null</td><td>null</td><td>1</td><td>365</td></tr><tr><td>3831</td><td>Cozy Entire Floor...</td><td>4869</td><td>LisaRoxanne</td><td>Brooklyn</td><td>Clinton Hill</td><td>40.68514</td><td>-73.95976</td><td>Entire home/apt</td><td>89</td><td>1</td><td>270</td><td>2019-07-05</td><td>4.64</td><td>1</td><td>194</td></tr><tr><td>5022</td><td>Entire Apt: Spaci...</td><td>7192</td><td>Laura</td><td>Manhattan</td><td>East Harlem</td><td>40.79851</td><td>-73.94399</td><td>Entire home/apt</td><td>80</td><td>10</td><td>9</td><td>2018-11-19</td><td>0.10</td><td>1</td><td>0</td></tr><tr><td>5099</td><td>Large Cozy 1 BR A...</td><td>7322</td><td>Chris</td><td>Manhattan</td><td>Murray Hill</td><td>40.74767</td><td>-73.975</td><td>Entire home/apt</td><td>200</td><td>3</td><td>74</td><td>2019-06-22</td><td>0.59</td><td>1</td><td>129</td></tr><tr><td>5121</td><td>BlissArtsSpace!</td><td>7356</td><td>Garon</td><td>Brooklyn</td><td>Bedford-Stuyvesant</td><td>40.68688</td><td>-73.95596</td><td>Private room</td><td>60</td><td>45</td><td>49</td><td>2017-10-05</td><td>0.40</td><td>1</td><td>0</td></tr><tr><td>5178</td><td>Large Furnished R...</td><td>8967</td><td>Shunichi</td><td>Manhattan</td><td>Hell's Kitchen</td><td>40.76489</td><td>-73.98493</td><td>Private room</td><td>79</td><td>2</td><td>430</td><td>2019-06-24</td><td>3.47</td><td>1</td><td>220</td></tr><tr><td>5203</td><td>Cozy Clean Guest ...</td><td>7490</td><td>MaryEllen</td><td>Manhattan</td><td>Upper West Side</td><td>40.80178</td><td>-73.96723</td><td>Private room</td><td>79</td><td>2</td><td>118</td><td>2017-07-21</td><td>0.99</td><td>1</td><td>0</td></tr><tr><td>5238</td><td>Cute &amp; Cozy Lower...</td><td>7549</td><td>Ben</td><td>Manhattan</td><td>Chinatown</td><td>40.71344</td><td>-73.99037</td><td>Entire home/apt</td><td>150</td><td>1</td><td>160</td><td>2019-06-09</td><td>1.33</td><td>4</td><td>188</td></tr><tr><td>5295</td><td>Beautiful 1br on ...</td><td>7702</td><td>Lena</td><td>Manhattan</td><td>Upper West Side</td><td>40.80316</td><td>-73.96545</td><td>Entire home/apt</td><td>135</td><td>5</td><td>53</td><td>2019-06-22</td><td>0.43</td><td>1</td><td>6</td></tr><tr><td>5441</td><td>Central Manhattan...</td><td>7989</td><td>Kate</td><td>Manhattan</td><td>Hell's Kitchen</td><td>40.76076</td><td>-73.98867</td><td>Private room</td><td>85</td><td>2</td><td>188</td><td>2019-06-23</td><td>1.50</td><td>1</td><td>39</td></tr><tr><td>5803</td><td>Lovely Room 1, Ga...</td><td>9744</td><td>Laurie</td><td>Brooklyn</td><td>South Slope</td><td>40.66829</td><td>-73.98779</td><td>Private room</td><td>89</td><td>4</td><td>167</td><td>2019-06-24</td><td>1.34</td><td>3</td><td>314</td></tr><tr><td>6021</td><td>Wonderful Guest B...</td><td>11528</td><td>Claudio</td><td>Manhattan</td><td>Upper West Side</td><td>40.79826</td><td>-73.96113</td><td>Private room</td><td>85</td><td>2</td><td>113</td><td>2019-07-05</td><td>0.91</td><td>1</td><td>333</td></tr><tr><td>6090</td><td>West Village Nest...</td><td>11975</td><td>Alina</td><td>Manhattan</td><td>West Village</td><td>40.7353</td><td>-74.00525</td><td>Entire home/apt</td><td>120</td><td>90</td><td>27</td><td>2018-10-31</td><td>0.22</td><td>1</td><td>0</td></tr><tr><td>6848</td><td>Only 2 stops to M...</td><td>15991</td><td>Allen &amp; Irina</td><td>Brooklyn</td><td>Williamsburg</td><td>40.70837</td><td>-73.95352</td><td>Entire home/apt</td><td>140</td><td>2</td><td>148</td><td>2019-06-29</td><td>1.20</td><td>1</td><td>46</td></tr><tr><td>7097</td><td>Perfect for Your ...</td><td>17571</td><td>Jane</td><td>Brooklyn</td><td>Fort Greene</td><td>40.69169</td><td>-73.97185</td><td>Entire home/apt</td><td>215</td><td>2</td><td>198</td><td>2019-06-28</td><td>1.72</td><td>1</td><td>321</td></tr><tr><td>7322</td><td>Chelsea Perfect</td><td>18946</td><td>Doti</td><td>Manhattan</td><td>Chelsea</td><td>40.74192</td><td>-73.99501</td><td>Private room</td><td>140</td><td>1</td><td>260</td><td>2019-07-01</td><td>2.12</td><td>1</td><td>12</td></tr><tr><td>7726</td><td>Hip Historic Brow...</td><td>20950</td><td>Adam And Charity</td><td>Brooklyn</td><td>Crown Heights</td><td>40.67592</td><td>-73.94694</td><td>Entire home/apt</td><td>99</td><td>3</td><td>53</td><td>2019-06-22</td><td>4.44</td><td>1</td><td>21</td></tr><tr><td>7750</td><td>Huge 2 BR Upper E...</td><td>17985</td><td>Sing</td><td>Manhattan</td><td>East Harlem</td><td>40.79685</td><td>-73.94872</td><td>Entire home/apt</td><td>190</td><td>7</td><td>0</td><td>null</td><td>null</td><td>2</td><td>249</td></tr>\n",
+       "      </table>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "data.toDF.showHTML()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "\u001b[32mimport \u001b[39m\u001b[36morg.apache.spark.sql.expressions.Window\n",
+       "\u001b[39m\n",
+       "\u001b[32mimport \u001b[39m\u001b[36morg.apache.spark.sql.functions.{avg, callUDF, col, corr, desc, lit, row_number, udf, variance}\u001b[39m"
+      ]
+     },
+     "execution_count": 11,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import org.apache.spark.sql.expressions.Window\n",
+    "import org.apache.spark.sql.functions.{avg, callUDF, col, corr, desc, lit, row_number, udf, variance}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "  <span style=\"float: left;\">take at cmd8.sc:4</span>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div class=\"progress\">\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: blue; width: 100%; word-wrap: normal; white-space: nowrap; text-align: center; color: white\" aria-valuenow=\"100\" aria-valuemin=\"0\" aria-valuemax=\"100\">\n",
+       "    1 / 1\n",
+       "  </div>\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: red; width: 0%\" aria-valuenow=\"0\" aria-valuemin=\"0\" aria-valuemax=\"100\"></div>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "      <table class=\"table\">\n",
+       "        <tr>\n",
+       "        <th>id</th><th>name</th><th>host_id</th><th>host_name</th><th>neighbourhood_group</th><th>neighbourhood</th><th>latitude</th><th>longitude</th><th>room_type</th><th>price</th><th>minimum_nights</th><th>number_of_reviews</th><th>last_review</th><th>reviews_per_month</th><th>calculated_host_listings_count</th><th>availability_365</th>\n",
+       "        </tr>\n",
+       "        <tr><td>2539</td><td>Clean &amp; quiet apt...</td><td>2787</td><td>John</td><td>Brooklyn</td><td>Kensington</td><td>40.64749</td><td>-73.97237</td><td>Private room</td><td>149</td><td>1</td><td>9</td><td>2018-10-19</td><td>0.21</td><td>6</td><td>365</td></tr><tr><td>2595</td><td>Skylit Midtown Ca...</td><td>2845</td><td>Jennifer</td><td>Manhattan</td><td>Midtown</td><td>40.75362</td><td>-73.98377</td><td>Entire home/apt</td><td>225</td><td>1</td><td>45</td><td>2019-05-21</td><td>0.38</td><td>2</td><td>355</td></tr><tr><td>3831</td><td>Cozy Entire Floor...</td><td>4869</td><td>LisaRoxanne</td><td>Brooklyn</td><td>Clinton Hill</td><td>40.68514</td><td>-73.95976</td><td>Entire home/apt</td><td>89</td><td>1</td><td>270</td><td>2019-07-05</td><td>4.64</td><td>1</td><td>194</td></tr><tr><td>5022</td><td>Entire Apt: Spaci...</td><td>7192</td><td>Laura</td><td>Manhattan</td><td>East Harlem</td><td>40.79851</td><td>-73.94399</td><td>Entire home/apt</td><td>80</td><td>10</td><td>9</td><td>2018-11-19</td><td>0.10</td><td>1</td><td>0</td></tr><tr><td>5099</td><td>Large Cozy 1 BR A...</td><td>7322</td><td>Chris</td><td>Manhattan</td><td>Murray Hill</td><td>40.74767</td><td>-73.975</td><td>Entire home/apt</td><td>200</td><td>3</td><td>74</td><td>2019-06-22</td><td>0.59</td><td>1</td><td>129</td></tr><tr><td>5121</td><td>BlissArtsSpace!</td><td>7356</td><td>Garon</td><td>Brooklyn</td><td>Bedford-Stuyvesant</td><td>40.68688</td><td>-73.95596</td><td>Private room</td><td>60</td><td>45</td><td>49</td><td>2017-10-05</td><td>0.40</td><td>1</td><td>0</td></tr><tr><td>5178</td><td>Large Furnished R...</td><td>8967</td><td>Shunichi</td><td>Manhattan</td><td>Hell's Kitchen</td><td>40.76489</td><td>-73.98493</td><td>Private room</td><td>79</td><td>2</td><td>430</td><td>2019-06-24</td><td>3.47</td><td>1</td><td>220</td></tr><tr><td>5203</td><td>Cozy Clean Guest ...</td><td>7490</td><td>MaryEllen</td><td>Manhattan</td><td>Upper West Side</td><td>40.80178</td><td>-73.96723</td><td>Private room</td><td>79</td><td>2</td><td>118</td><td>2017-07-21</td><td>0.99</td><td>1</td><td>0</td></tr><tr><td>5238</td><td>Cute &amp; Cozy Lower...</td><td>7549</td><td>Ben</td><td>Manhattan</td><td>Chinatown</td><td>40.71344</td><td>-73.99037</td><td>Entire home/apt</td><td>150</td><td>1</td><td>160</td><td>2019-06-09</td><td>1.33</td><td>4</td><td>188</td></tr><tr><td>5295</td><td>Beautiful 1br on ...</td><td>7702</td><td>Lena</td><td>Manhattan</td><td>Upper West Side</td><td>40.80316</td><td>-73.96545</td><td>Entire home/apt</td><td>135</td><td>5</td><td>53</td><td>2019-06-22</td><td>0.43</td><td>1</td><td>6</td></tr><tr><td>5441</td><td>Central Manhattan...</td><td>7989</td><td>Kate</td><td>Manhattan</td><td>Hell's Kitchen</td><td>40.76076</td><td>-73.98867</td><td>Private room</td><td>85</td><td>2</td><td>188</td><td>2019-06-23</td><td>1.50</td><td>1</td><td>39</td></tr><tr><td>5803</td><td>Lovely Room 1, Ga...</td><td>9744</td><td>Laurie</td><td>Brooklyn</td><td>South Slope</td><td>40.66829</td><td>-73.98779</td><td>Private room</td><td>89</td><td>4</td><td>167</td><td>2019-06-24</td><td>1.34</td><td>3</td><td>314</td></tr><tr><td>6021</td><td>Wonderful Guest B...</td><td>11528</td><td>Claudio</td><td>Manhattan</td><td>Upper West Side</td><td>40.79826</td><td>-73.96113</td><td>Private room</td><td>85</td><td>2</td><td>113</td><td>2019-07-05</td><td>0.91</td><td>1</td><td>333</td></tr><tr><td>6090</td><td>West Village Nest...</td><td>11975</td><td>Alina</td><td>Manhattan</td><td>West Village</td><td>40.7353</td><td>-74.00525</td><td>Entire home/apt</td><td>120</td><td>90</td><td>27</td><td>2018-10-31</td><td>0.22</td><td>1</td><td>0</td></tr><tr><td>6848</td><td>Only 2 stops to M...</td><td>15991</td><td>Allen &amp; Irina</td><td>Brooklyn</td><td>Williamsburg</td><td>40.70837</td><td>-73.95352</td><td>Entire home/apt</td><td>140</td><td>2</td><td>148</td><td>2019-06-29</td><td>1.20</td><td>1</td><td>46</td></tr><tr><td>7097</td><td>Perfect for Your ...</td><td>17571</td><td>Jane</td><td>Brooklyn</td><td>Fort Greene</td><td>40.69169</td><td>-73.97185</td><td>Entire home/apt</td><td>215</td><td>2</td><td>198</td><td>2019-06-28</td><td>1.72</td><td>1</td><td>321</td></tr><tr><td>7322</td><td>Chelsea Perfect</td><td>18946</td><td>Doti</td><td>Manhattan</td><td>Chelsea</td><td>40.74192</td><td>-73.99501</td><td>Private room</td><td>140</td><td>1</td><td>260</td><td>2019-07-01</td><td>2.12</td><td>1</td><td>12</td></tr><tr><td>7726</td><td>Hip Historic Brow...</td><td>20950</td><td>Adam And Charity</td><td>Brooklyn</td><td>Crown Heights</td><td>40.67592</td><td>-73.94694</td><td>Entire home/apt</td><td>99</td><td>3</td><td>53</td><td>2019-06-22</td><td>4.44</td><td>1</td><td>21</td></tr><tr><td>7801</td><td>Sweet and Spaciou...</td><td>21207</td><td>Chaya</td><td>Brooklyn</td><td>Williamsburg</td><td>40.71842</td><td>-73.95718</td><td>Entire home/apt</td><td>299</td><td>3</td><td>9</td><td>2011-12-28</td><td>0.07</td><td>1</td><td>0</td></tr><tr><td>8024</td><td>CBG CtyBGd HelpsH...</td><td>22486</td><td>Lisel</td><td>Brooklyn</td><td>Park Slope</td><td>40.68069</td><td>-73.97706</td><td>Private room</td><td>130</td><td>2</td><td>130</td><td>2019-07-01</td><td>1.09</td><td>6</td><td>347</td></tr>\n",
+       "      </table>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "\u001b[36msqlContext\u001b[39m: \u001b[32mSQLContext\u001b[39m = org.apache.spark.sql.SQLContext@7d933b13"
+      ]
+     },
+     "execution_count": 12,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "// Фильтр по ячейкам, где есть Null-значения\n",
+    "\n",
+    "val sqlContext = spark.sqlContext\n",
+    "\n",
+    "sqlContext.createDataFrame(data.rdd.filter(x=> Range(0, x.length).count(x.isNullAt(_)) < 1 ), data.schema).showHTML()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "// Фильтр на предложения, где цена нулевая и где минимальное количество ночей больше 365:\n",
+    "\n",
+    "data = data.where(col(\"price\") > 0)\n",
+    "\n",
+    "data = data.where(col(\"minimum_nights\") < 366)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Медиана:\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "  <span style=\"float: left;\">take at cmd8.sc:4</span>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div class=\"progress\">\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: blue; width: 100%; word-wrap: normal; white-space: nowrap; text-align: center; color: white\" aria-valuenow=\"100\" aria-valuemin=\"0\" aria-valuemax=\"100\">\n",
+       "    2 / 2\n",
+       "  </div>\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: red; width: 0%\" aria-valuenow=\"0\" aria-valuemin=\"0\" aria-valuemax=\"100\"></div>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "  <span style=\"float: left;\">take at cmd8.sc:4</span>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div class=\"progress\">\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: blue; width: 100%; word-wrap: normal; white-space: nowrap; text-align: center; color: white\" aria-valuenow=\"100\" aria-valuemin=\"0\" aria-valuemax=\"100\">\n",
+       "    1 / 1\n",
+       "  </div>\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: red; width: 0%\" aria-valuenow=\"0\" aria-valuemin=\"0\" aria-valuemax=\"100\"></div>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "  <span style=\"float: left;\">take at cmd8.sc:4</span>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div class=\"progress\">\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: blue; width: 100%; word-wrap: normal; white-space: nowrap; text-align: center; color: white\" aria-valuenow=\"100\" aria-valuemin=\"0\" aria-valuemax=\"100\">\n",
+       "    4 / 4\n",
+       "  </div>\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: red; width: 0%\" aria-valuenow=\"0\" aria-valuemin=\"0\" aria-valuemax=\"100\"></div>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "  <span style=\"float: left;\">take at cmd8.sc:4</span>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div class=\"progress\">\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: blue; width: 100%; word-wrap: normal; white-space: nowrap; text-align: center; color: white\" aria-valuenow=\"100\" aria-valuemin=\"0\" aria-valuemax=\"100\">\n",
+       "    20 / 20\n",
+       "  </div>\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: red; width: 0%\" aria-valuenow=\"0\" aria-valuemin=\"0\" aria-valuemax=\"100\"></div>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "  <span style=\"float: left;\">take at cmd8.sc:4</span>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div class=\"progress\">\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: blue; width: 100%; word-wrap: normal; white-space: nowrap; text-align: center; color: white\" aria-valuenow=\"100\" aria-valuemin=\"0\" aria-valuemax=\"100\">\n",
+       "    25 / 25\n",
+       "  </div>\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: red; width: 0%\" aria-valuenow=\"0\" aria-valuemin=\"0\" aria-valuemax=\"100\"></div>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "      <table class=\"table\">\n",
+       "        <tr>\n",
+       "        <th>room_type</th><th>Медиана</th>\n",
+       "        </tr>\n",
+       "        <tr><td>51</td><td>1.0</td></tr><tr><td>205</td><td>4.0</td></tr><tr><td>54</td><td>7.0</td></tr><tr><td>200</td><td>1.0</td></tr><tr><td>279</td><td>2.0</td></tr><tr><td>138</td><td>2.0</td></tr><tr><td>69</td><td>2.0</td></tr><tr><td>42</td><td>5.0</td></tr><tr><td>Shared room</td><td>45.0</td></tr><tr><td>425</td><td>1.0</td></tr><tr><td>59</td><td>2.0</td></tr><tr><td>250</td><td>2.0</td></tr><tr><td>160</td><td>1.0</td></tr><tr><td>85</td><td>2.0</td></tr><tr><td>35</td><td>1.0</td></tr><tr><td>410</td><td>31.0</td></tr><tr><td>298</td><td>3.0</td></tr><tr><td>188</td><td>1.0</td></tr><tr><td>99</td><td>1.0</td></tr><tr><td>110</td><td>3.0</td></tr>\n",
+       "      </table>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "Мода:\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "  <span style=\"float: left;\">take at cmd8.sc:4</span>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div class=\"progress\">\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: blue; width: 100%; word-wrap: normal; white-space: nowrap; text-align: center; color: white\" aria-valuenow=\"100\" aria-valuemin=\"0\" aria-valuemax=\"100\">\n",
+       "    2 / 2\n",
+       "  </div>\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: red; width: 0%\" aria-valuenow=\"0\" aria-valuemin=\"0\" aria-valuemax=\"100\"></div>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "  <span style=\"float: left;\">take at cmd8.sc:4</span>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div class=\"progress\">\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: blue; width: 100%; word-wrap: normal; white-space: nowrap; text-align: center; color: white\" aria-valuenow=\"100\" aria-valuemin=\"0\" aria-valuemax=\"100\">\n",
+       "    200 / 200\n",
+       "  </div>\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: red; width: 0%\" aria-valuenow=\"0\" aria-valuemin=\"0\" aria-valuemax=\"100\"></div>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "  <span style=\"float: left;\">take at cmd8.sc:4</span>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div class=\"progress\">\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: blue; width: 100%; word-wrap: normal; white-space: nowrap; text-align: center; color: white\" aria-valuenow=\"100\" aria-valuemin=\"0\" aria-valuemax=\"100\">\n",
+       "    1 / 1\n",
+       "  </div>\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: red; width: 0%\" aria-valuenow=\"0\" aria-valuemin=\"0\" aria-valuemax=\"100\"></div>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "  <span style=\"float: left;\">take at cmd8.sc:4</span>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div class=\"progress\">\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: blue; width: 100%; word-wrap: normal; white-space: nowrap; text-align: center; color: white\" aria-valuenow=\"100\" aria-valuemin=\"0\" aria-valuemax=\"100\">\n",
+       "    4 / 4\n",
+       "  </div>\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: red; width: 0%\" aria-valuenow=\"0\" aria-valuemin=\"0\" aria-valuemax=\"100\"></div>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "  <span style=\"float: left;\">take at cmd8.sc:4</span>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div class=\"progress\">\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: blue; width: 100%; word-wrap: normal; white-space: nowrap; text-align: center; color: white\" aria-valuenow=\"100\" aria-valuemin=\"0\" aria-valuemax=\"100\">\n",
+       "    20 / 20\n",
+       "  </div>\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: red; width: 0%\" aria-valuenow=\"0\" aria-valuemin=\"0\" aria-valuemax=\"100\"></div>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "  <span style=\"float: left;\">take at cmd8.sc:4</span>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div class=\"progress\">\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: blue; width: 100%; word-wrap: normal; white-space: nowrap; text-align: center; color: white\" aria-valuenow=\"100\" aria-valuemin=\"0\" aria-valuemax=\"100\">\n",
+       "    25 / 25\n",
+       "  </div>\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: red; width: 0%\" aria-valuenow=\"0\" aria-valuemin=\"0\" aria-valuemax=\"100\"></div>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "      <table class=\"table\">\n",
+       "        <tr>\n",
+       "        <th>room_type</th><th>price</th>\n",
+       "        </tr>\n",
+       "        <tr><td>51</td><td>1</td></tr><tr><td>205</td><td>4</td></tr><tr><td>54</td><td>7</td></tr><tr><td>200</td><td>1</td></tr><tr><td>279</td><td>2</td></tr><tr><td>138</td><td>2</td></tr><tr><td>69</td><td>2</td></tr><tr><td>42</td><td>5</td></tr><tr><td>Shared room</td><td>35</td></tr><tr><td>425</td><td>1</td></tr><tr><td>59</td><td>2</td></tr><tr><td>250</td><td>2</td></tr><tr><td>160</td><td>1</td></tr><tr><td>85</td><td>2</td></tr><tr><td>35</td><td>5</td></tr><tr><td>410</td><td>31</td></tr><tr><td>188</td><td>1</td></tr><tr><td>298</td><td>3</td></tr><tr><td>99</td><td>1</td></tr><tr><td>110</td><td>3</td></tr>\n",
+       "      </table>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "Среднее:\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "  <span style=\"float: left;\">take at cmd8.sc:4</span>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div class=\"progress\">\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: blue; width: 100%; word-wrap: normal; white-space: nowrap; text-align: center; color: white\" aria-valuenow=\"100\" aria-valuemin=\"0\" aria-valuemax=\"100\">\n",
+       "    2 / 2\n",
+       "  </div>\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: red; width: 0%\" aria-valuenow=\"0\" aria-valuemin=\"0\" aria-valuemax=\"100\"></div>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "  <span style=\"float: left;\">take at cmd8.sc:4</span>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div class=\"progress\">\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: blue; width: 100%; word-wrap: normal; white-space: nowrap; text-align: center; color: white\" aria-valuenow=\"100\" aria-valuemin=\"0\" aria-valuemax=\"100\">\n",
+       "    1 / 1\n",
+       "  </div>\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: red; width: 0%\" aria-valuenow=\"0\" aria-valuemin=\"0\" aria-valuemax=\"100\"></div>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "  <span style=\"float: left;\">take at cmd8.sc:4</span>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div class=\"progress\">\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: blue; width: 100%; word-wrap: normal; white-space: nowrap; text-align: center; color: white\" aria-valuenow=\"100\" aria-valuemin=\"0\" aria-valuemax=\"100\">\n",
+       "    4 / 4\n",
+       "  </div>\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: red; width: 0%\" aria-valuenow=\"0\" aria-valuemin=\"0\" aria-valuemax=\"100\"></div>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "  <span style=\"float: left;\">take at cmd8.sc:4</span>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div class=\"progress\">\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: blue; width: 100%; word-wrap: normal; white-space: nowrap; text-align: center; color: white\" aria-valuenow=\"100\" aria-valuemin=\"0\" aria-valuemax=\"100\">\n",
+       "    20 / 20\n",
+       "  </div>\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: red; width: 0%\" aria-valuenow=\"0\" aria-valuemin=\"0\" aria-valuemax=\"100\"></div>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "  <span style=\"float: left;\">take at cmd8.sc:4</span>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div class=\"progress\">\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: blue; width: 100%; word-wrap: normal; white-space: nowrap; text-align: center; color: white\" aria-valuenow=\"100\" aria-valuemin=\"0\" aria-valuemax=\"100\">\n",
+       "    25 / 25\n",
+       "  </div>\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: red; width: 0%\" aria-valuenow=\"0\" aria-valuemin=\"0\" aria-valuemax=\"100\"></div>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "      <table class=\"table\">\n",
+       "        <tr>\n",
+       "        <th>room_type</th><th>Среднее</th>\n",
+       "        </tr>\n",
+       "        <tr><td>51</td><td>1.0</td></tr><tr><td>205</td><td>4.0</td></tr><tr><td>54</td><td>7.0</td></tr><tr><td>200</td><td>1.0</td></tr><tr><td>279</td><td>2.0</td></tr><tr><td>138</td><td>2.0</td></tr><tr><td>69</td><td>1.7142857142857142</td></tr><tr><td>42</td><td>5.0</td></tr><tr><td>Shared room</td><td>70.21991341991342</td></tr><tr><td>425</td><td>1.0</td></tr><tr><td>59</td><td>7.8</td></tr><tr><td>250</td><td>2.0</td></tr><tr><td>160</td><td>2.0</td></tr><tr><td>85</td><td>2.0</td></tr><tr><td>35</td><td>3.0</td></tr><tr><td>410</td><td>31.0</td></tr><tr><td>298</td><td>3.0</td></tr><tr><td>188</td><td>1.0</td></tr><tr><td>99</td><td>2.0</td></tr><tr><td>110</td><td>3.0</td></tr>\n",
+       "      </table>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "Дисперсия:\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "  <span style=\"float: left;\">take at cmd8.sc:4</span>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div class=\"progress\">\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: blue; width: 100%; word-wrap: normal; white-space: nowrap; text-align: center; color: white\" aria-valuenow=\"100\" aria-valuemin=\"0\" aria-valuemax=\"100\">\n",
+       "    2 / 2\n",
+       "  </div>\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: red; width: 0%\" aria-valuenow=\"0\" aria-valuemin=\"0\" aria-valuemax=\"100\"></div>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "  <span style=\"float: left;\">take at cmd8.sc:4</span>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div class=\"progress\">\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: blue; width: 100%; word-wrap: normal; white-space: nowrap; text-align: center; color: white\" aria-valuenow=\"100\" aria-valuemin=\"0\" aria-valuemax=\"100\">\n",
+       "    1 / 1\n",
+       "  </div>\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: red; width: 0%\" aria-valuenow=\"0\" aria-valuemin=\"0\" aria-valuemax=\"100\"></div>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "  <span style=\"float: left;\">take at cmd8.sc:4</span>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div class=\"progress\">\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: blue; width: 100%; word-wrap: normal; white-space: nowrap; text-align: center; color: white\" aria-valuenow=\"100\" aria-valuemin=\"0\" aria-valuemax=\"100\">\n",
+       "    4 / 4\n",
+       "  </div>\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: red; width: 0%\" aria-valuenow=\"0\" aria-valuemin=\"0\" aria-valuemax=\"100\"></div>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "  <span style=\"float: left;\">take at cmd8.sc:4</span>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div class=\"progress\">\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: blue; width: 100%; word-wrap: normal; white-space: nowrap; text-align: center; color: white\" aria-valuenow=\"100\" aria-valuemin=\"0\" aria-valuemax=\"100\">\n",
+       "    20 / 20\n",
+       "  </div>\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: red; width: 0%\" aria-valuenow=\"0\" aria-valuemin=\"0\" aria-valuemax=\"100\"></div>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "  <span style=\"float: left;\">take at cmd8.sc:4</span>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div class=\"progress\">\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: blue; width: 100%; word-wrap: normal; white-space: nowrap; text-align: center; color: white\" aria-valuenow=\"100\" aria-valuemin=\"0\" aria-valuemax=\"100\">\n",
+       "    25 / 25\n",
+       "  </div>\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: red; width: 0%\" aria-valuenow=\"0\" aria-valuemin=\"0\" aria-valuemax=\"100\"></div>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "      <table class=\"table\">\n",
+       "        <tr>\n",
+       "        <th>room_type</th><th>var_samp(price)</th>\n",
+       "        </tr>\n",
+       "        <tr><td>51</td><td>0.0</td></tr><tr><td>205</td><td>NaN</td></tr><tr><td>54</td><td>NaN</td></tr><tr><td>200</td><td>NaN</td></tr><tr><td>279</td><td>NaN</td></tr><tr><td>138</td><td>NaN</td></tr><tr><td>69</td><td>0.23809523809523805</td></tr><tr><td>42</td><td>NaN</td></tr><tr><td>Shared room</td><td>10382.929065850383</td></tr><tr><td>425</td><td>NaN</td></tr><tr><td>59</td><td>154.2</td></tr><tr><td>250</td><td>0.0</td></tr><tr><td>160</td><td>2.0</td></tr><tr><td>85</td><td>NaN</td></tr><tr><td>35</td><td>8.0</td></tr><tr><td>410</td><td>NaN</td></tr><tr><td>298</td><td>NaN</td></tr><tr><td>188</td><td>NaN</td></tr><tr><td>99</td><td>1.3333333333333333</td></tr><tr><td>110</td><td>NaN</td></tr>\n",
+       "      </table>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "// Посчитать медиану, моду и среднее, и дисперсию для каждого room_type\n",
+    "\n",
+    "println(\"Медиана:\")\n",
+    "\n",
+    "data.groupBy(\"room_type\").agg(callUDF(\"percentile_approx\", col(\"price\"), lit(0.5)).as(\"Медиана\")).toDF.showHTML()\n",
+    "\n",
+    "println(\"\")\n",
+    "println(\"\")\n",
+    "println(\"Мода:\")\n",
+    "\n",
+    "data.groupBy(\"room_type\", \"price\").count().withColumn(\"row_number\", row_number().over(Window.partitionBy(\"room_type\").orderBy(desc(\"count\")))).select(\"room_type\", \"price\").where(col(\"row_number\") === 1).toDF.showHTML()\n",
+    "\n",
+    "println(\"\")\n",
+    "println(\"\")\n",
+    "println(\"Среднее:\")\n",
+    "\n",
+    "data.groupBy(\"room_type\").agg(avg(\"price\").as(\"Среднее\")).toDF.showHTML()\n",
+    "\n",
+    "println(\"\")\n",
+    "println(\"\")\n",
+    "println(\"Дисперсия:\")\n",
+    "\n",
+    "data.select(\"room_type\", \"price\").groupBy(\"room_type\").agg(variance(\"price\")).as(\"Дисперсия\").toDF.showHTML()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Самое дешевое:\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "  <span style=\"float: left;\">take at cmd8.sc:4</span>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div class=\"progress\">\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: blue; width: 100%; word-wrap: normal; white-space: nowrap; text-align: center; color: white\" aria-valuenow=\"100\" aria-valuemin=\"0\" aria-valuemax=\"100\">\n",
+       "    2 / 2\n",
+       "  </div>\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: red; width: 0%\" aria-valuenow=\"0\" aria-valuemin=\"0\" aria-valuemax=\"100\"></div>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "      <table class=\"table\">\n",
+       "        <tr>\n",
+       "        <th>id</th><th>name</th><th>host_id</th><th>host_name</th><th>neighbourhood_group</th><th>neighbourhood</th><th>latitude</th><th>longitude</th><th>room_type</th><th>price</th><th>minimum_nights</th><th>number_of_reviews</th><th>last_review</th><th>reviews_per_month</th><th>calculated_host_listings_count</th><th>availability_365</th>\n",
+       "        </tr>\n",
+       "        <tr><td>24114389</td><td>Very Spacious bed...</td><td>180661875</td><td>Salim</td><td>Manhattan</td><td>Upper West Side</td><td>40.76844</td><td>-73.98333</td><td>Private room</td><td>10</td><td>1</td><td>2</td><td>2018-04-23</td><td>0.13</td><td>1</td><td>0</td></tr>\n",
+       "      </table>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "Самое дорогое:\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "  <span style=\"float: left;\">take at cmd8.sc:4</span>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div class=\"progress\">\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: blue; width: 100%; word-wrap: normal; white-space: nowrap; text-align: center; color: white\" aria-valuenow=\"100\" aria-valuemin=\"0\" aria-valuemax=\"100\">\n",
+       "    2 / 2\n",
+       "  </div>\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: red; width: 0%\" aria-valuenow=\"0\" aria-valuemin=\"0\" aria-valuemax=\"100\"></div>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "      <table class=\"table\">\n",
+       "        <tr>\n",
+       "        <th>id</th><th>name</th><th>host_id</th><th>host_name</th><th>neighbourhood_group</th><th>neighbourhood</th><th>latitude</th><th>longitude</th><th>room_type</th><th>price</th><th>minimum_nights</th><th>number_of_reviews</th><th>last_review</th><th>reviews_per_month</th><th>calculated_host_listings_count</th><th>availability_365</th>\n",
+       "        </tr>\n",
+       "        <tr><td>31340283</td><td>2br - The Heart o...</td><td>4382127</td><td>Matt</td><td>Manhattan</td><td>Lower East Side</td><td>40.7198</td><td>-73.98566</td><td>Entire home/apt</td><td>9999</td><td>30</td><td>0</td><td>null</td><td>null</td><td>1</td><td>365</td></tr>\n",
+       "      </table>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "// Найти самое дорогое и самое дешевое предложение\n",
+    "\n",
+    "println(\"Самое дешевое:\")\n",
+    "\n",
+    "data.orderBy(\"price\").showHTML(1)\n",
+    "\n",
+    "println(\"\")\n",
+    "println(\"\")\n",
+    "println(\"Самое дорогое:\")\n",
+    "\n",
+    "data.orderBy(desc(\"price\")).showHTML(1)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Корреляцию между ценой и минимальным количеством ночей:\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "  <span style=\"float: left;\">take at cmd8.sc:4</span>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div class=\"progress\">\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: blue; width: 100%; word-wrap: normal; white-space: nowrap; text-align: center; color: white\" aria-valuenow=\"100\" aria-valuemin=\"0\" aria-valuemax=\"100\">\n",
+       "    2 / 2\n",
+       "  </div>\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: red; width: 0%\" aria-valuenow=\"0\" aria-valuemin=\"0\" aria-valuemax=\"100\"></div>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "  <span style=\"float: left;\">take at cmd8.sc:4</span>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div class=\"progress\">\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: blue; width: 100%; word-wrap: normal; white-space: nowrap; text-align: center; color: white\" aria-valuenow=\"100\" aria-valuemin=\"0\" aria-valuemax=\"100\">\n",
+       "    1 / 1\n",
+       "  </div>\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: red; width: 0%\" aria-valuenow=\"0\" aria-valuemin=\"0\" aria-valuemax=\"100\"></div>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "      <table class=\"table\">\n",
+       "        <tr>\n",
+       "        <th>corr(price, minimum_nights)</th>\n",
+       "        </tr>\n",
+       "        <tr><td>0.049635888865656445</td></tr>\n",
+       "      </table>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "\n",
+      "Корреляцию между ценой и количеством отзывов:\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "  <span style=\"float: left;\">take at cmd8.sc:4</span>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div class=\"progress\">\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: blue; width: 100%; word-wrap: normal; white-space: nowrap; text-align: center; color: white\" aria-valuenow=\"100\" aria-valuemin=\"0\" aria-valuemax=\"100\">\n",
+       "    2 / 2\n",
+       "  </div>\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: red; width: 0%\" aria-valuenow=\"0\" aria-valuemin=\"0\" aria-valuemax=\"100\"></div>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "  <span style=\"float: left;\">take at cmd8.sc:4</span>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div class=\"progress\">\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: blue; width: 100%; word-wrap: normal; white-space: nowrap; text-align: center; color: white\" aria-valuenow=\"100\" aria-valuemin=\"0\" aria-valuemax=\"100\">\n",
+       "    1 / 1\n",
+       "  </div>\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: red; width: 0%\" aria-valuenow=\"0\" aria-valuemin=\"0\" aria-valuemax=\"100\"></div>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "      <table class=\"table\">\n",
+       "        <tr>\n",
+       "        <th>corr(price, number_of_reviews)</th>\n",
+       "        </tr>\n",
+       "        <tr><td>-0.04804569425619054</td></tr>\n",
+       "      </table>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "// Посчитать корреляцию между ценой и минимальным количеством ночей, кол-вом отзывов\n",
+    "\n",
+    "println(\"Корреляцию между ценой и минимальным количеством ночей:\")\n",
+    "\n",
+    "data.agg(corr(\"price\", \"minimum_nights\")).toDF.showHTML()\n",
+    "\n",
+    "println(\"\")\n",
+    "println(\"\")\n",
+    "println(\"Корреляцию между ценой и количеством отзывов:\")\n",
+    "\n",
+    "data.agg(corr(\"price\", \"number_of_reviews\")).toDF.showHTML()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "\u001b[36mencodeGeoHash\u001b[39m: (\u001b[32mDouble\u001b[39m, \u001b[32mDouble\u001b[39m, \u001b[32mInt\u001b[39m) => \u001b[32mString\u001b[39m = ammonite.$sess.cmd16$Helper$$Lambda$5564/213636581@362a7344"
+      ]
+     },
+     "execution_count": 17,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "// Нужно найти гео квадрат размером 5км на 5км с самой высокой средней стоимостью жилья\n",
+    "\n",
+    "val encodeGeoHash = (latitude: Double, longitude: Double, precision: Int) => {\n",
+    "    val base32 = \"0123456789bcdefghjkmnpqrstuvwxyz\"\n",
+    "    var (minLatitude, maxLatitude) = (-90.0, 90.0)\n",
+    "    var (minLongitude, maxLongitude) = (-180.0, 180.0)\n",
+    "    val bits = List(16, 8, 4, 2, 1)\n",
+    "\n",
+    "    (0 until precision).map { p => {\n",
+    "        base32 apply (0 until 5).map { i => {\n",
+    "            if (((5 * p) + i) % 2 == 0) {\n",
+    "                val mid = (minLongitude + maxLongitude) / 2.0\n",
+    "                if (longitude > mid) {\n",
+    "                    minLongitude = mid\n",
+    "                    bits(i)\n",
+    "                } else {\n",
+    "                    maxLongitude = mid\n",
+    "                    0\n",
+    "                }\n",
+    "            } else {\n",
+    "                val mid = (minLatitude + maxLatitude) / 2.0\n",
+    "                if (latitude > mid) {\n",
+    "                minLatitude = mid\n",
+    "                bits(i)\n",
+    "                } else {\n",
+    "                    maxLatitude = mid\n",
+    "                    0\n",
+    "                }\n",
+    "            }\n",
+    "        }\n",
+    "        }.reduceLeft((a, b) => a | b)\n",
+    "    }\n",
+    "    }.mkString(\"\")\n",
+    "}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "\u001b[36mgeoHash_udf\u001b[39m: \u001b[32mexpressions\u001b[39m.\u001b[32mUserDefinedFunction\u001b[39m = \u001b[33mUserDefinedFunction\u001b[39m(\n",
+       "  ammonite.$sess.cmd16$Helper$$Lambda$5564/213636581@362a7344,\n",
+       "  StringType,\n",
+       "  \u001b[33mSome\u001b[39m(\u001b[33mList\u001b[39m(DoubleType, DoubleType, IntegerType))\n",
+       ")"
+      ]
+     },
+     "execution_count": 18,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "val geoHash_udf = udf(encodeGeoHash)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "  <span style=\"float: left;\">take at cmd8.sc:4</span>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div class=\"progress\">\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: blue; width: 100%; word-wrap: normal; white-space: nowrap; text-align: center; color: white\" aria-valuenow=\"100\" aria-valuemin=\"0\" aria-valuemax=\"100\">\n",
+       "    2 / 2\n",
+       "  </div>\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: red; width: 0%\" aria-valuenow=\"0\" aria-valuemin=\"0\" aria-valuemax=\"100\"></div>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "  <span style=\"float: left;\">take at cmd8.sc:4</span>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div class=\"progress\">\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: blue; width: 100%; word-wrap: normal; white-space: nowrap; text-align: center; color: white\" aria-valuenow=\"100\" aria-valuemin=\"0\" aria-valuemax=\"100\">\n",
+       "    200 / 200\n",
+       "  </div>\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: red; width: 0%\" aria-valuenow=\"0\" aria-valuemin=\"0\" aria-valuemax=\"100\"></div>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "      <table class=\"table\">\n",
+       "        <tr>\n",
+       "        <th>geoHash</th><th>Средняя цена</th>\n",
+       "        </tr>\n",
+       "        <tr><td>dr5wf</td><td>350.0</td></tr>\n",
+       "      </table>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "\u001b[36mgeoHash\u001b[39m: \u001b[32mDataset\u001b[39m[\u001b[32mRow\u001b[39m] = [geoHash: string, Средняя цена: double]"
+      ]
+     },
+     "execution_count": 19,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "val geoHash = data.withColumn(\"geoHash\", geoHash_udf(col(\"latitude\"), col(\"longitude\"), lit(5))).groupBy(\"geoHash\").agg(avg(\"price\").as(\"Средняя цена\")).orderBy(desc(\"Средняя цена\"))\n",
+    "\n",
+    "geoHash.toDF.showHTML(1)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 31,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "defined \u001b[32mobject\u001b[39m \u001b[36mDecoder\u001b[39m"
+      ]
+     },
+     "execution_count": 31,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "// check source https://github.com/lifulong/geohash\n",
+    "\n",
+    "object Decoder {\n",
+    "    val BASE32\t\t\t    = \"0123456789bcdefghjkmnpqrstuvwxyz\"\n",
+    "    val MIN_LATITUDE :Double\t= -90\n",
+    "    val MAX_LATITUDE :Double\t= 90\n",
+    "    val MIN_LONGITUDE :Double\t= -180\n",
+    "    val MAX_LONGITUDE :Double\t= 180\n",
+    "\n",
+    "    val bits = Array[Byte](16, 8, 4, 2, 1)\n",
+    "    val base32 = BASE32.toCharArray\n",
+    "\n",
+    "    implicit def int2byte(int: Int) = {\n",
+    "        int.toByte\n",
+    "    }\n",
+    "\n",
+    "    def DecodeBounds(geohash: String): (Double, Double, Double, Double) = {\n",
+    "\n",
+    "        def toBitList(str: String) = str.flatMap {\n",
+    "            char => (\"00000\" + base32.indexOf(char).toBinaryString ).\n",
+    "                reverse.take(5).reverse.map('1' == ) } toList\n",
+    "\n",
+    "        def split(list: List[Boolean]): (List[Boolean], List[Boolean]) = {\n",
+    "\n",
+    "            list match{\n",
+    "                case Nil => (Nil,Nil)\n",
+    "                case x::Nil => (x::Nil,Nil)\n",
+    "                case x::y::zs => val (xs,ys) = split(zs); (x::xs,y::ys)\n",
+    "            }\n",
+    "        }\n",
+    "\n",
+    "        def dehash(xs: List[Boolean], min: Double, max: Double): (Double,Double) = {\n",
+    "\n",
+    "            ((min,max) /: xs ) {\n",
+    "                case ((min,max), bool) =>\n",
+    "                    if(bool) ((min + max)/2, max)\n",
+    "                    else (min, (min + max)/2)\n",
+    "            }\n",
+    "        }\n",
+    "\n",
+    "        val (xs, ys) = split(toBitList(geohash))\n",
+    "        val (minLat, maxLat) = dehash(ys, MIN_LATITUDE, MAX_LATITUDE)\n",
+    "        val (minLng, maxLng) = dehash(xs, MIN_LONGITUDE, MAX_LONGITUDE)\n",
+    "\n",
+    "        (minLat, maxLat, minLng, maxLng)\n",
+    "    }\n",
+    "}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 33,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "  <span style=\"float: left;\">collect at cmd32.sc:1</span>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div class=\"progress\">\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: blue; width: 100%; word-wrap: normal; white-space: nowrap; text-align: center; color: white\" aria-valuenow=\"100\" aria-valuemin=\"0\" aria-valuemax=\"100\">\n",
+       "    2 / 2\n",
+       "  </div>\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: red; width: 0%\" aria-valuenow=\"0\" aria-valuemin=\"0\" aria-valuemax=\"100\"></div>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "  <span style=\"float: left;\">collect at cmd32.sc:1</span>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div class=\"progress\">\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: blue; width: 100%; word-wrap: normal; white-space: nowrap; text-align: center; color: white\" aria-valuenow=\"100\" aria-valuemin=\"0\" aria-valuemax=\"100\">\n",
+       "    200 / 200\n",
+       "  </div>\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: red; width: 0%\" aria-valuenow=\"0\" aria-valuemin=\"0\" aria-valuemax=\"100\"></div>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "  <span style=\"float: left;\">collect at cmd32.sc:1</span>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div class=\"progress\">\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: blue; width: 100%; word-wrap: normal; white-space: nowrap; text-align: center; color: white\" aria-valuenow=\"100\" aria-valuemin=\"0\" aria-valuemax=\"100\">\n",
+       "    200 / 200\n",
+       "  </div>\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: red; width: 0%\" aria-valuenow=\"0\" aria-valuemin=\"0\" aria-valuemax=\"100\"></div>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "  <span style=\"float: left;\">collect at cmd32.sc:1</span>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div class=\"progress\">\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: blue; width: 100%; word-wrap: normal; white-space: nowrap; text-align: center; color: white\" aria-valuenow=\"100\" aria-valuemin=\"0\" aria-valuemax=\"100\">\n",
+       "    61 / 61\n",
+       "  </div>\n",
+       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: red; width: 0%\" aria-valuenow=\"0\" aria-valuemin=\"0\" aria-valuemax=\"100\"></div>\n",
+       "</div>\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "\u001b[36mcoordinates\u001b[39m: (\u001b[32mDouble\u001b[39m, \u001b[32mDouble\u001b[39m, \u001b[32mDouble\u001b[39m, \u001b[32mDouble\u001b[39m) = (\n",
+       "  \u001b[32m40.5615234375\u001b[39m,\n",
+       "  \u001b[32m40.60546875\u001b[39m,\n",
+       "  \u001b[32m-73.740234375\u001b[39m,\n",
+       "  \u001b[32m-73.6962890625\u001b[39m\n",
+       ")"
+      ]
+     },
+     "execution_count": 33,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "val coordinates = Decoder.DecodeBounds(geoHash.select(\"geoHash\").collect().map(_(0)).head.toString())"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Scala",
+   "language": "scala",
+   "name": "scala"
+  },
+  "language_info": {
+   "codemirror_mode": "text/x-scala",
+   "file_extension": ".sc",
+   "mimetype": "text/x-scala",
+   "name": "scala",
+   "nbconvert_exporter": "script",
+   "version": "2.12.11"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}