Skip to content

Create OPeNDAP_to_Zarr.ipynb #2

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
rsignell-usgs wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Create OPeNDAP_to_Zarr.ipynb #2

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
338 changes: 338 additions & 0 deletions notebooks/OPeNDAP_to_Zarr.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,338 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Write National Water Model (NWM) model data to Zarr"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"from dask.distributed import Client, progress, LocalCluster\n",
"import pandas as pd\n",
"import xarray as xr\n",
"import s3fs"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<table style=\"border: 2px solid white;\">\n",
"<tr>\n",
"<td style=\"vertical-align: top; border: 0px solid white\">\n",
"<h3>Client</h3>\n",
"<ul>\n",
" <li><b>Scheduler: </b>tcp://127.0.0.1:38546\n",
" <li><b>Dashboard: </b><a href='http://127.0.0.1:8787/status' target='_blank'>http://127.0.0.1:8787/status</a>\n",
"</ul>\n",
"</td>\n",
"<td style=\"vertical-align: top; border: 0px solid white\">\n",
"<h3>Cluster</h3>\n",
"<ul>\n",
" <li><b>Workers: </b>2</li>\n",
" <li><b>Cores: </b>2</li>\n",
" <li><b>Memory: </b>8.37 GB</li>\n",
"</ul>\n",
"</td>\n",
"</tr>\n",
"</table>"
],
"text/plain": [
"<Client: scheduler='tcp://127.0.0.1:38546' processes=2 cores=2>"
]
},
"execution_count": 2,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# depends on the machine you are using\n",
"cluster = LocalCluster()\n",
"client = Client(cluster)\n",
"\n",
"client "
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"root = 'http://tds.renci.org:8080/thredds/dodsC/nwm/forcing_short_range/'"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"dates = pd.date_range(start='2018-04-01T18:00', end='2018-04-02T04:00', freq='H')"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"urls = ['{}{}/nwm.t{}z.short_range.forcing.f001.conus.nc'.format(root,a.strftime('%Y%m%d'),a.strftime('%H')) for a in dates]"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"f_zarr = 'rsignell/nwm/test01'"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 488 ms, sys: 48 ms, total: 536 ms\n",
"Wall time: 7.22 s\n"
]
}
],
"source": [
"%%time \n",
"ds = xr.open_mfdataset(urls,concat_dim='time')"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {
"scrolled": true
},
"outputs": [
{
"data": {
"text/plain": [
"<xarray.Dataset>\n",
"Dimensions: (nv: 2, reference_time: 11, time: 11, x: 4608, y: 3840)\n",
"Coordinates:\n",
" * reference_time (reference_time) datetime64[ns] 2018-04-01T18:00:00 ...\n",
" * x (x) float64 -2.304e+06 -2.303e+06 -2.302e+06 ...\n",
" * y (y) float64 -1.92e+06 -1.919e+06 -1.918e+06 ...\n",
" * time (time) datetime64[ns] 2018-04-01T19:00:00 ...\n",
"Dimensions without coordinates: nv\n",
"Data variables:\n",
" time_bounds (time, nv) datetime64[ns] dask.array<shape=(11, 2), chunksize=(1, 2)>\n",
" ProjectionCoordinateSystem (time) |S64 b'' b'' b'' b'' b'' b'' b'' b'' ...\n",
" T2D (time, y, x) float64 dask.array<shape=(11, 3840, 4608), chunksize=(1, 3840, 4608)>\n",
" LWDOWN (time, y, x) float64 dask.array<shape=(11, 3840, 4608), chunksize=(1, 3840, 4608)>\n",
" Q2D (time, y, x) float64 dask.array<shape=(11, 3840, 4608), chunksize=(1, 3840, 4608)>\n",
" U2D (time, y, x) float64 dask.array<shape=(11, 3840, 4608), chunksize=(1, 3840, 4608)>\n",
" V2D (time, y, x) float64 dask.array<shape=(11, 3840, 4608), chunksize=(1, 3840, 4608)>\n",
" PSFC (time, y, x) float64 dask.array<shape=(11, 3840, 4608), chunksize=(1, 3840, 4608)>\n",
" RAINRATE (time, y, x) float32 dask.array<shape=(11, 3840, 4608), chunksize=(1, 3840, 4608)>\n",
" SWDOWN (time, y, x) float64 dask.array<shape=(11, 3840, 4608), chunksize=(1, 3840, 4608)>\n",
"Attributes:\n",
" model_initialization_time: 2018-04-01_18:00:00\n",
" model_output_valid_time: 2018-04-01_19:00:00\n",
" DODS.strlen: 0\n",
" DODS_EXTRA.Unlimited_Dimension: time"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"ds"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
"fs = s3fs.S3FileSystem(anon=False)"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [],
"source": [
"d = s3fs.S3Map(f_zarr, s3=fs)"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"CPU times: user 24.3 s, sys: 5.58 s, total: 29.9 s\n",
"Wall time: 12min 2s\n"
]
},
{
"data": {
"text/plain": [
"<xarray.backends.zarr.ZarrStore at 0x7f2b3c186b38>"
]
},
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"%time ds.to_zarr(store=d, mode='w')"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Test to see if we can read what we wrote"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [],
"source": [
"s3map = s3fs.S3Map(f_zarr, s3=fs)"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [],
"source": [
"# works if auto_chunk=False\n",
"ds2 = xr.open_zarr(s3map, auto_chunk=False)"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<xarray.Dataset>\n",
"Dimensions: (nv: 2, reference_time: 11, time: 11, x: 4608, y: 3840)\n",
"Coordinates:\n",
" * reference_time (reference_time) datetime64[ns] 2018-04-01T18:00:00 ...\n",
" * time (time) datetime64[ns] 2018-04-01T19:00:00 ...\n",
" * x (x) float64 -2.304e+06 -2.303e+06 -2.302e+06 ...\n",
" * y (y) float64 -1.92e+06 -1.919e+06 -1.918e+06 ...\n",
"Dimensions without coordinates: nv\n",
"Data variables:\n",
" LWDOWN (time, y, x) float64 ...\n",
" PSFC (time, y, x) float64 ...\n",
" ProjectionCoordinateSystem (time) |S64 ...\n",
" Q2D (time, y, x) float64 ...\n",
" RAINRATE (time, y, x) float32 ...\n",
" SWDOWN (time, y, x) float64 ...\n",
" T2D (time, y, x) float64 ...\n",
" U2D (time, y, x) float64 ...\n",
" V2D (time, y, x) float64 ...\n",
" time_bounds (time, nv) datetime64[ns] ...\n",
"Attributes:\n",
" DODS.strlen: 0\n",
" DODS_EXTRA.Unlimited_Dimension: time\n",
" model_initialization_time: 2018-04-01_18:00:00\n",
" model_output_valid_time: 2018-04-01_19:00:00"
]
},
"execution_count": 14,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"ds2"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [
{
"ename": "ValueError",
"evalue": "Chunks and shape must be of the same length/dimension. Got chunks=(11, 64), shape=(11,)",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-15-b3733a1ec58f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mds3\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mxr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen_zarr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms3map\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mauto_chunk\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
"\u001b[0;32m/opt/conda/lib/python3.6/site-packages/xarray/backends/zarr.py\u001b[0m in \u001b[0;36mopen_zarr\u001b[0;34m(store, group, synchronizer, auto_chunk, decode_cf, mask_and_scale, decode_times, concat_characters, decode_coords, drop_variables)\u001b[0m\n\u001b[1;32m 476\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 477\u001b[0m variables = OrderedDict([(k, maybe_chunk(k, v))\n\u001b[0;32m--> 478\u001b[0;31m for k, v in ds.variables.items()])\n\u001b[0m\u001b[1;32m 479\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mds\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_replace_vars_and_dims\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvariables\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 480\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/opt/conda/lib/python3.6/site-packages/xarray/backends/zarr.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 476\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 477\u001b[0m variables = OrderedDict([(k, maybe_chunk(k, v))\n\u001b[0;32m--> 478\u001b[0;31m for k, v in ds.variables.items()])\n\u001b[0m\u001b[1;32m 479\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mds\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_replace_vars_and_dims\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvariables\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 480\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/opt/conda/lib/python3.6/site-packages/xarray/backends/zarr.py\u001b[0m in \u001b[0;36mmaybe_chunk\u001b[0;34m(name, var)\u001b[0m\n\u001b[1;32m 471\u001b[0m \u001b[0mtoken2\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtokenize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvar\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_data\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 472\u001b[0m \u001b[0mname2\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'zarr-%s'\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mtoken2\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 473\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mvar\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mchunk\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchunks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mname2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlock\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 474\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 475\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mvar\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/opt/conda/lib/python3.6/site-packages/xarray/core/variable.py\u001b[0m in \u001b[0;36mchunk\u001b[0;34m(self, chunks, name, lock)\u001b[0m\n\u001b[1;32m 820\u001b[0m data = indexing.ImplicitToExplicitIndexingAdapter(\n\u001b[1;32m 821\u001b[0m data, indexing.OuterIndexer)\n\u001b[0;32m--> 822\u001b[0;31m \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mda\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_array\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mchunks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlock\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlock\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 823\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 824\u001b[0m return type(self)(self.dims, data, self._attrs, self._encoding,\n",
"\u001b[0;32m/opt/conda/lib/python3.6/site-packages/dask/array/core.py\u001b[0m in \u001b[0;36mfrom_array\u001b[0;34m(x, chunks, name, lock, asarray, fancy, getitem)\u001b[0m\n\u001b[1;32m 1977\u001b[0m \u001b[0;34m>>\u001b[0m\u001b[0;34m>\u001b[0m \u001b[0ma\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mda\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrom_array\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mchunks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1000\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1000\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlock\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# doctest: +SKIP\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1978\u001b[0m \"\"\"\n\u001b[0;32m-> 1979\u001b[0;31m \u001b[0mchunks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnormalize_chunks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchunks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1980\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32min\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1981\u001b[0m \u001b[0mtoken\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtokenize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mchunks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/opt/conda/lib/python3.6/site-packages/dask/array/core.py\u001b[0m in \u001b[0;36mnormalize_chunks\u001b[0;34m(chunks, shape)\u001b[0m\n\u001b[1;32m 1907\u001b[0m raise ValueError(\n\u001b[1;32m 1908\u001b[0m \u001b[0;34m\"Chunks and shape must be of the same length/dimension. \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1909\u001b[0;31m \"Got chunks=%s, shape=%s\" % (chunks, shape))\n\u001b[0m\u001b[1;32m 1910\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1911\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mshape\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mValueError\u001b[0m: Chunks and shape must be of the same length/dimension. Got chunks=(11, 64), shape=(11,)"
]
}
],
"source": [
"ds3 = xr.open_zarr(s3map, auto_chunk=True)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.4"
}
},
"nbformat": 4,
"nbformat_minor": 2
}