{ "cells": [ { "cell_type": "markdown", "id": "9e0e9928", "metadata": {}, "source": [ "# Time Based Utilities and Drift Removal" ] }, { "cell_type": "code", "execution_count": 1, "id": "10b03732", "metadata": {}, "outputs": [], "source": [ "%load_ext autoreload\n", "%autoreload 2\n", "\n", "import matplotlib.pyplot as plt\n", "import xarray as xr\n", "import numpy as np\n", "\n", "from xmip.utils import google_cmip_col\n", "from xmip.preprocessing import combined_preprocessing" ] }, { "cell_type": "code", "execution_count": 2, "id": "4c9b2a2d", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "[]\n", "[]\n" ] } ], "source": [ "from dask_gateway import Gateway\n", "g = Gateway()\n", "running_clusters = g.list_clusters()\n", "print(running_clusters)\n", "for c in running_clusters:\n", " # g.stop_cluster()\n", " cluster = g.connect(c.name)\n", " cluster.close()\n", "print(running_clusters)" ] }, { "cell_type": "code", "execution_count": 3, "id": "9f39a72c", "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "5576118f47db470c843f2d36fa25d2be", "version_major": 2, "version_minor": 0 }, "text/plain": [ "VBox(children=(HTML(value='
<xarray.Dataset>\n",
"Dimensions: (lev: 45, time: 6012, x: 360, y: 291)\n",
"Coordinates:\n",
" lat (y, x) float64 dask.array<chunksize=(291, 3), meta=np.ndarray>\n",
" * lev (lev) float64 3.047 9.454 16.36 ... 5.126e+03 5.375e+03 5.625e+03\n",
" lon (y, x) float64 dask.array<chunksize=(291, 3), meta=np.ndarray>\n",
" * time (time) object 5550-01-16 12:00:00 ... 6050-12-16 12:00:00\n",
" * x (x) int32 0 1 2 3 4 5 6 7 8 ... 351 352 353 354 355 356 357 358 359\n",
" * y (y) int32 0 1 2 3 4 5 6 7 8 ... 282 283 284 285 286 287 288 289 290\n",
"Data variables:\n",
" thetao (time, lev, y, x) float32 dask.array<chunksize=(6012, 1, 291, 3), meta=np.ndarray>\n",
"Attributes: (12/58)\n",
" CCCma_model_hash: 932b659de600c6a0e94f619abaf9cc79eabcd337\n",
" CCCma_parent_runid: canoecpl-007\n",
" CCCma_pycmor_hash: 3ecdc18eb7c1f7fbce0346850f41adf815d9fb66\n",
" CCCma_runid: c2-pictrl\n",
" Conventions: CF-1.7 CMIP-6.2\n",
" YMDH_branch_time_in_child: 5550:01:01:00\n",
" ... ...\n",
" title: CanESM5-CanOE output prepared for CMIP6\n",
" tracking_id: hdl:21.14100/8b3a90ee-6bf3-496a-ba59-648dded...\n",
" variable_id: thetao\n",
" variant_label: r1i1p2f1\n",
" version: v20190429\n",
" version_id: v20190429\n",
"
| \n",
"\n", "\n", " | \n", "
array([3.046773e+00, 9.454049e+00, 1.636397e+01, 2.389871e+01, 3.220929e+01,\n",
" 4.148185e+01, 5.194513e+01, 6.387905e+01, 7.762451e+01, 9.359412e+01,\n",
" 1.122835e+02, 1.342823e+02, 1.602840e+02, 1.910925e+02, 2.276233e+02,\n",
" 2.708962e+02, 3.220169e+02, 3.821444e+02, 4.524429e+02, 5.340197e+02,\n",
" 6.278525e+02, 7.347150e+02, 8.551112e+02, 9.892289e+02, 1.136922e+03,\n",
" 1.297724e+03, 1.470893e+03, 1.655472e+03, 1.850365e+03, 2.054414e+03,\n",
" 2.266454e+03, 2.485371e+03, 2.710133e+03, 2.939812e+03, 3.173588e+03,\n",
" 3.410756e+03, 3.650712e+03, 3.892950e+03, 4.137047e+03, 4.382654e+03,\n",
" 4.629485e+03, 4.877303e+03, 5.125919e+03, 5.375177e+03, 5.624952e+03])\n",
"
| \n",
"\n", "\n", " | \n", "
array([cftime.DatetimeNoLeap(5550, 1, 16, 12, 0, 0, 0),\n",
" cftime.DatetimeNoLeap(5550, 2, 15, 0, 0, 0, 0),\n",
" cftime.DatetimeNoLeap(5550, 3, 16, 12, 0, 0, 0), ...,\n",
" cftime.DatetimeNoLeap(6050, 10, 16, 12, 0, 0, 0),\n",
" cftime.DatetimeNoLeap(6050, 11, 16, 0, 0, 0, 0),\n",
" cftime.DatetimeNoLeap(6050, 12, 16, 12, 0, 0, 0)], dtype=object)array([ 0, 1, 2, ..., 357, 358, 359], dtype=int32)
array([ 0, 1, 2, ..., 288, 289, 290], dtype=int32)
\n",
"
| \n",
"\n", "\n", " | \n", "
<xarray.Dataset>\n",
"Dimensions: (bnds: 2, lev: 45, x: 360, y: 291)\n",
"Coordinates:\n",
" * lev (lev) float64 3.047 9.454 16.36 ... 5.375e+03 5.625e+03\n",
" * x (x) int32 0 1 2 3 4 5 6 7 ... 353 354 355 356 357 358 359\n",
" * y (y) int32 0 1 2 3 4 5 6 7 ... 284 285 286 287 288 289 290\n",
" trend_time_range (bnds) <U19 '5550-01-16 12:00:00' '5799-12-16 12:00:00'\n",
"Dimensions without coordinates: bnds\n",
"Data variables:\n",
" thetao (lev, y, x) float32 nan nan nan nan ... nan nan nan nan\n",
"Attributes: (12/58)\n",
" CCCma_model_hash: 932b659de600c6a0e94f619abaf9cc79eabcd337\n",
" CCCma_parent_runid: canoecpl-007\n",
" CCCma_pycmor_hash: 3ecdc18eb7c1f7fbce0346850f41adf815d9fb66\n",
" CCCma_runid: c2-his01\n",
" Conventions: CF-1.7 CMIP-6.2\n",
" YMDH_branch_time_in_child: 1850:01:01:00\n",
" ... ...\n",
" variant_label: r1i1p2f1\n",
" version: v20190429\n",
" netcdf_tracking_ids: hdl:21.14100/05f0fb20-3395-4112-af53-d1b9241...\n",
" version_id: v20190429\n",
" intake_esm_varname: None\n",
" intake_esm_dataset_key: CMIP.CCCma.CanESM5-CanOE.historical.r1i1p2f1...array([3.046773e+00, 9.454049e+00, 1.636397e+01, 2.389871e+01, 3.220929e+01,\n",
" 4.148185e+01, 5.194513e+01, 6.387905e+01, 7.762451e+01, 9.359412e+01,\n",
" 1.122835e+02, 1.342823e+02, 1.602840e+02, 1.910925e+02, 2.276233e+02,\n",
" 2.708962e+02, 3.220169e+02, 3.821444e+02, 4.524429e+02, 5.340197e+02,\n",
" 6.278525e+02, 7.347150e+02, 8.551112e+02, 9.892289e+02, 1.136922e+03,\n",
" 1.297724e+03, 1.470893e+03, 1.655472e+03, 1.850365e+03, 2.054414e+03,\n",
" 2.266454e+03, 2.485371e+03, 2.710133e+03, 2.939812e+03, 3.173588e+03,\n",
" 3.410756e+03, 3.650712e+03, 3.892950e+03, 4.137047e+03, 4.382654e+03,\n",
" 4.629485e+03, 4.877303e+03, 5.125919e+03, 5.375177e+03, 5.624952e+03])array([ 0, 1, 2, ..., 357, 358, 359], dtype=int32)
array([ 0, 1, 2, ..., 288, 289, 290], dtype=int32)
array(['5550-01-16 12:00:00', '5799-12-16 12:00:00'], dtype='<U19')
array([[[nan, nan, nan, ..., nan, nan, nan],\n",
" [nan, nan, nan, ..., nan, nan, nan],\n",
" [nan, nan, nan, ..., nan, nan, nan],\n",
" ...,\n",
" [nan, nan, nan, ..., nan, nan, nan],\n",
" [nan, nan, nan, ..., nan, nan, nan],\n",
" [nan, nan, nan, ..., nan, nan, nan]],\n",
"\n",
" [[nan, nan, nan, ..., nan, nan, nan],\n",
" [nan, nan, nan, ..., nan, nan, nan],\n",
" [nan, nan, nan, ..., nan, nan, nan],\n",
" ...,\n",
" [nan, nan, nan, ..., nan, nan, nan],\n",
" [nan, nan, nan, ..., nan, nan, nan],\n",
" [nan, nan, nan, ..., nan, nan, nan]],\n",
"\n",
" [[nan, nan, nan, ..., nan, nan, nan],\n",
" [nan, nan, nan, ..., nan, nan, nan],\n",
" [nan, nan, nan, ..., nan, nan, nan],\n",
" ...,\n",
"...\n",
" ...,\n",
" [nan, nan, nan, ..., nan, nan, nan],\n",
" [nan, nan, nan, ..., nan, nan, nan],\n",
" [nan, nan, nan, ..., nan, nan, nan]],\n",
"\n",
" [[nan, nan, nan, ..., nan, nan, nan],\n",
" [nan, nan, nan, ..., nan, nan, nan],\n",
" [nan, nan, nan, ..., nan, nan, nan],\n",
" ...,\n",
" [nan, nan, nan, ..., nan, nan, nan],\n",
" [nan, nan, nan, ..., nan, nan, nan],\n",
" [nan, nan, nan, ..., nan, nan, nan]],\n",
"\n",
" [[nan, nan, nan, ..., nan, nan, nan],\n",
" [nan, nan, nan, ..., nan, nan, nan],\n",
" [nan, nan, nan, ..., nan, nan, nan],\n",
" ...,\n",
" [nan, nan, nan, ..., nan, nan, nan],\n",
" [nan, nan, nan, ..., nan, nan, nan],\n",
" [nan, nan, nan, ..., nan, nan, nan]]], dtype=float32)<xarray.DataArray (time: 1980, lev: 45, y: 291, x: 360)>\n",
"dask.array<sub, shape=(1980, 45, 291, 360), dtype=float32, chunksize=(5, 45, 291, 360), chunktype=numpy.ndarray>\n",
"Coordinates:\n",
" * time (time) object 1850-01-16 12:00:00 ... 2014-12-16 12:00:00\n",
" * x (x) int32 0 1 2 3 4 5 6 7 8 ... 351 352 353 354 355 356 357 358 359\n",
" * y (y) int32 0 1 2 3 4 5 6 7 8 ... 282 283 284 285 286 287 288 289 290\n",
" lat (y, x) float64 dask.array<chunksize=(291, 360), meta=np.ndarray>\n",
" * lev (lev) float64 3.047 9.454 16.36 ... 5.126e+03 5.375e+03 5.625e+03\n",
" lon (y, x) float64 dask.array<chunksize=(291, 360), meta=np.ndarray>\n",
"Attributes:\n",
" cell_measures: area: areacello volume: volcello\n",
" cell_methods: area: mean where sea time: mean\n",
" comment: Diagnostic should be contributed even for models using co...\n",
" long_name: Sea Water Potential Temperature\n",
" original_name: votemper\n",
" standard_name: sea_water_potential_temperature\n",
" units: degC\n",
" drift_removed: linear_trend_CMIP.CCCma.CanESM5-CanOE.historical.r1i1p2f1...\n",
"
| \n",
"\n", "\n", " | \n", "
array([cftime.DatetimeNoLeap(1850, 1, 16, 12, 0, 0, 0),\n",
" cftime.DatetimeNoLeap(1850, 2, 15, 0, 0, 0, 0),\n",
" cftime.DatetimeNoLeap(1850, 3, 16, 12, 0, 0, 0), ...,\n",
" cftime.DatetimeNoLeap(2014, 10, 16, 12, 0, 0, 0),\n",
" cftime.DatetimeNoLeap(2014, 11, 16, 0, 0, 0, 0),\n",
" cftime.DatetimeNoLeap(2014, 12, 16, 12, 0, 0, 0)], dtype=object)array([ 0, 1, 2, ..., 357, 358, 359], dtype=int32)
array([ 0, 1, 2, ..., 288, 289, 290], dtype=int32)
\n",
"
| \n",
"\n", "\n", " | \n", "
array([3.046773e+00, 9.454049e+00, 1.636397e+01, 2.389871e+01, 3.220929e+01,\n",
" 4.148185e+01, 5.194513e+01, 6.387905e+01, 7.762451e+01, 9.359412e+01,\n",
" 1.122835e+02, 1.342823e+02, 1.602840e+02, 1.910925e+02, 2.276233e+02,\n",
" 2.708962e+02, 3.220169e+02, 3.821444e+02, 4.524429e+02, 5.340197e+02,\n",
" 6.278525e+02, 7.347150e+02, 8.551112e+02, 9.892289e+02, 1.136922e+03,\n",
" 1.297724e+03, 1.470893e+03, 1.655472e+03, 1.850365e+03, 2.054414e+03,\n",
" 2.266454e+03, 2.485371e+03, 2.710133e+03, 2.939812e+03, 3.173588e+03,\n",
" 3.410756e+03, 3.650712e+03, 3.892950e+03, 4.137047e+03, 4.382654e+03,\n",
" 4.629485e+03, 4.877303e+03, 5.125919e+03, 5.375177e+03, 5.624952e+03])\n",
"
| \n",
"\n", "\n", " | \n", "