In [1]:
#| default_exp feature_preprocessing.character_trajectories.tabular_to_timeseries
%load_ext autoreload
%autoreload 2
In [2]:
# declare a list tasks whose products you want to use as inputs
upstream = ['feature_preprocessing_character_trajectories']
In [3]:
# Parameters
upstream = {"feature_preprocessing_character_trajectories": {"nb": "/home/ubuntu/vitmtsc_nbdev/output/205_feature_preprocessing.character_trajectories.target_encoding.html", "CharacterTrajectories_TRAIN_TE": "/home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/target_encoding/train", "CharacterTrajectories_VALID_TE": "/home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/target_encoding/valid", "CharacterTrajectories_TEST_TE": "/home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/target_encoding/test", "CharacterTrajectories_workflow_dir": "/home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/target_encoding/nvtabular_workflow"}}
product = {"nb": "/home/ubuntu/vitmtsc_nbdev/output/305_feature_preprocessing.character_trajectories.tabular_to_timeseries.html", "CharacterTrajectories_TRAIN_MODEL_INPUT": "/home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/target_encoding-nn/train", "CharacterTrajectories_VALID_MODEL_INPUT": "/home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/target_encoding-nn/valid", "CharacterTrajectories_TEST_MODEL_INPUT": "/home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/target_encoding-nn/test"}
In [4]:
#| hide
from nbdev.showdoc import *
In [5]:
#| export
from vitmtsc import *
from vitmtsc.core import *
from vitmtsc.data.character_trajectories import *
from vitmtsc.feature_preprocessing.character_trajectories.target_encoding import *
import os
In [6]:
# |export
upstream = {
    "feature_preprocessing_character_trajectories": {
        "nb": "/home/ubuntu/vitmtsc_nbdev/output/205_feature_preprocessing.character_trajectories.target_encoding.html",
        "CharacterTrajectories_TRAIN_TE": "/home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/target_encoding/train",
        "CharacterTrajectories_VALID_TE": "/home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/target_encoding/valid",
        "CharacterTrajectories_TEST_TE": "/home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/target_encoding/test",
        "CharacterTrajectories_workflow_dir": "/home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/target_encoding/nvtabular_workflow",
    }
}
product = {
    "nb": "/home/ubuntu/vitmtsc_nbdev/output/305_feature_preprocessing.character_trajectories.tabular_to_timeseries.html",
    "CharacterTrajectories_TRAIN_MODEL_INPUT": "/home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/target_encoding-nn/train",
    "CharacterTrajectories_VALID_MODEL_INPUT": "/home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/target_encoding-nn/valid",
    "CharacterTrajectories_TEST_MODEL_INPUT": "/home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/target_encoding-nn/test",
}

Feature Preprocessing for Neural Networks - III¶

Convert Category Encoding data from tabular to time-series format

In [7]:
from dask.distributed import Client
from dask_cuda import LocalCUDACluster

cluster = LocalCUDACluster(memory_limit='auto', device_memory_limit=0.5, rmm_pool_size='20GB', rmm_managed_memory=True)
client = Client(cluster)
client
2022-09-23 19:02:06,330 - distributed.preloading - INFO - Creating preload: dask_cuda.initialize
2022-09-23 19:02:06,330 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize
2022-09-23 19:02:06,421 - distributed.preloading - INFO - Creating preload: dask_cuda.initialize
2022-09-23 19:02:06,421 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize
2022-09-23 19:02:06,426 - distributed.preloading - INFO - Creating preload: dask_cuda.initialize
2022-09-23 19:02:06,426 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize
2022-09-23 19:02:06,437 - distributed.preloading - INFO - Creating preload: dask_cuda.initialize
2022-09-23 19:02:06,437 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize
Out[7]:

Client

Client-380c9110-3b72-11ed-8103-0a4f0fdf7975

Connection method: Cluster object Cluster type: dask_cuda.LocalCUDACluster
Dashboard: http://127.0.0.1:8787/status

Cluster Info

LocalCUDACluster

5d00441e

Dashboard: http://127.0.0.1:8787/status Workers: 4
Total threads: 4 Total memory: 150.00 GiB
Status: running Using processes: True

Scheduler Info

Scheduler

Scheduler-0e2624ee-1240-4fd7-b571-d8c018eba423

Comm: tcp://127.0.0.1:39481 Workers: 4
Dashboard: http://127.0.0.1:8787/status Total threads: 4
Started: Just now Total memory: 150.00 GiB

Workers

Worker: 0

Comm: tcp://127.0.0.1:44667 Total threads: 1
Dashboard: http://127.0.0.1:46767/status Memory: 37.50 GiB
Nanny: tcp://127.0.0.1:33709
Local directory: /tmp/dask-worker-space/worker-isaxfxlx
GPU: Tesla T4 GPU memory: 14.76 GiB

Worker: 1

Comm: tcp://127.0.0.1:34099 Total threads: 1
Dashboard: http://127.0.0.1:44185/status Memory: 37.50 GiB
Nanny: tcp://127.0.0.1:35049
Local directory: /tmp/dask-worker-space/worker-_njv48sa
GPU: Tesla T4 GPU memory: 14.76 GiB

Worker: 2

Comm: tcp://127.0.0.1:46367 Total threads: 1
Dashboard: http://127.0.0.1:33237/status Memory: 37.50 GiB
Nanny: tcp://127.0.0.1:40065
Local directory: /tmp/dask-worker-space/worker-423vni88
GPU: Tesla T4 GPU memory: 14.76 GiB

Worker: 3

Comm: tcp://127.0.0.1:36545 Total threads: 1
Dashboard: http://127.0.0.1:42827/status Memory: 37.50 GiB
Nanny: tcp://127.0.0.1:44461
Local directory: /tmp/dask-worker-space/worker-fy1l03qn
GPU: Tesla T4 GPU memory: 14.76 GiB
In [8]:
#| export
DATASET_NAME = 'CharacterTrajectories'
SEQUENCE_LENGTH = 182
NUMBER_OF_FEATURES = 3
NUM_TARGET = 20

Convert from Tabular to Time-Series Format

In [9]:
#| export
MTSC_COLUMN_NAMES = [
'dim_0',
'dim_1',
'dim_2']
In [10]:
#| export
ALL_COLUMNS = ['case_id', 'case_id_seq', 'reading_id'] + MTSC_COLUMN_NAMES + ['class_vals']

Input Data Location

In [11]:
target_encoded_train_dir = os.path.join("./", upstream['feature_preprocessing_character_trajectories']['CharacterTrajectories_TRAIN_TE'])
target_encoded_valid_dir = os.path.join("./", upstream['feature_preprocessing_character_trajectories']['CharacterTrajectories_VALID_TE'])
target_encoded_test_dir = os.path.join("./", upstream['feature_preprocessing_character_trajectories']['CharacterTrajectories_TEST_TE'])

Output Data Location

In [12]:
output_train_dir = os.path.join("./", product['CharacterTrajectories_TRAIN_MODEL_INPUT'])
output_valid_dir = os.path.join("./", product['CharacterTrajectories_VALID_MODEL_INPUT'])
output_test_dir = os.path.join("./", product['CharacterTrajectories_TEST_MODEL_INPUT'])
In [13]:
!mkdir -p $output_train_dir
!mkdir -p $output_valid_dir
!mkdir -p $output_test_dir

Train Dataset Conversion
¶

Tabular to Time-Series format conversion

In [14]:
%%time
convert_from_tabular_to_timeseries_format(input_dir = target_encoded_train_dir, 
                                          output_dir = output_train_dir, 
                                          all_columns = ALL_COLUMNS,
                                          mtsc_column_names = MTSC_COLUMN_NAMES,
                                          chunk_size_processing = 50000,
                                          number_of_features = NUMBER_OF_FEATURES, 
                                          seq_len = SEQUENCE_LENGTH,
                                          chunk_size_file = 10000)
case_id_seq_min:  0 case_id_seq_max:  1421
Total number of chunks to be processed:  1
Started processing chunk:  0  with case_id_seq from :  0 to  1421
Before CumCount Min:  0 CumCount Max:  179
After CumCount Min:  0 CumCount Max:  179
sorted
flattened_gdf.shape:  (1137, 548)
Total number of files to be created:  1
Writing to output file:  /home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/target_encoding-nn/train/chunk_0_part_0.parquet with records from iloc:  0 to  1137
Finished processing chunk:  0  with case_id_seq from :  0 to  1421
CPU times: user 3.37 s, sys: 598 ms, total: 3.97 s
Wall time: 5.65 s

Valid Dataset Conversion
¶

Tabular to Time-Series format conversion

In [15]:
%%time
convert_from_tabular_to_timeseries_format(input_dir = target_encoded_valid_dir, 
                                          output_dir = output_valid_dir, 
                                          all_columns = ALL_COLUMNS,
                                          mtsc_column_names = MTSC_COLUMN_NAMES,
                                          chunk_size_processing = 50000,
                                          number_of_features = NUMBER_OF_FEATURES, 
                                          seq_len = SEQUENCE_LENGTH,
                                          chunk_size_file = 10000)
case_id_seq_min:  23 case_id_seq_max:  1418
Total number of chunks to be processed:  1
Started processing chunk:  0  with case_id_seq from :  0 to  1418
Before CumCount Min:  0 CumCount Max:  172
After CumCount Min:  0 CumCount Max:  172
sorted
flattened_gdf.shape:  (285, 548)
Total number of files to be created:  1
Writing to output file:  /home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/target_encoding-nn/valid/chunk_0_part_0.parquet with records from iloc:  0 to  285
Finished processing chunk:  0  with case_id_seq from :  0 to  1418
CPU times: user 740 ms, sys: 60.7 ms, total: 801 ms
Wall time: 818 ms

Test Dataset Conversion
¶

Tabular to Time-Series format conversion

In [16]:
%%time
convert_from_tabular_to_timeseries_format(input_dir = target_encoded_test_dir, 
                                          output_dir = output_test_dir, 
                                          all_columns = ALL_COLUMNS,
                                          mtsc_column_names = MTSC_COLUMN_NAMES,
                                          chunk_size_processing = 50000,
                                          number_of_features = NUMBER_OF_FEATURES, 
                                          seq_len = SEQUENCE_LENGTH,
                                          chunk_size_file = 10000)
case_id_seq_min:  0 case_id_seq_max:  1435
Total number of chunks to be processed:  1
Started processing chunk:  0  with case_id_seq from :  0 to  1435
Before CumCount Min:  0 CumCount Max:  181
After CumCount Min:  0 CumCount Max:  181
sorted
flattened_gdf.shape:  (1436, 548)
Total number of files to be created:  1
Writing to output file:  /home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/target_encoding-nn/test/chunk_0_part_0.parquet with records from iloc:  0 to  1436
Finished processing chunk:  0  with case_id_seq from :  0 to  1435
CPU times: user 800 ms, sys: 77.8 ms, total: 878 ms
Wall time: 898 ms

Verify Datasets
¶

In [17]:
%%time
import dask_cudf
train_gdf = dask_cudf.read_parquet(output_train_dir)
train_gdf.head()
CPU times: user 321 ms, sys: 20.1 ms, total: 341 ms
Wall time: 485 ms
Out[17]:
dim_0_0 dim_0_1 dim_0_2 dim_0_3 dim_0_4 dim_0_5 dim_0_6 dim_0_7 dim_0_8 dim_0_9 ... dim_2_174 dim_2_175 dim_2_176 dim_2_177 dim_2_178 dim_2_179 dim_2_180 dim_2_181 class_vals case_id
0 -0.107928 -0.101998 -0.100830 -0.107980 -0.125284 -0.153670 -0.193613 -0.243723 -0.299566 -0.354739 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 13.0 954.0
1 -0.241938 -0.282187 -0.316187 -0.344641 -0.373353 -0.409349 -0.457755 -0.520926 -0.598919 -0.690191 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.0 400.0
2 -0.285200 -0.343543 -0.400658 -0.459077 -0.524088 -0.599595 -0.686902 -0.785901 -0.896377 -1.018127 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.0 251.0
3 -0.258288 -0.268151 -0.247131 -0.196936 -0.124711 -0.037351 0.060679 0.167194 0.281728 0.404279 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.0 375.0
4 -0.149945 -0.154712 -0.150748 -0.133831 -0.100650 -0.050338 0.014458 0.089919 0.174723 0.270248 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.0 366.0

5 rows × 548 columns

In [18]:
train_gdf.tail()
Out[18]:
dim_0_0 dim_0_1 dim_0_2 dim_0_3 dim_0_4 dim_0_5 dim_0_6 dim_0_7 dim_0_8 dim_0_9 ... dim_2_174 dim_2_175 dim_2_176 dim_2_177 dim_2_178 dim_2_179 dim_2_180 dim_2_181 class_vals case_id
1132 -0.067803 -0.013844 0.067091 0.175923 0.311758 0.473402 0.659267 0.866019 1.087547 1.315279 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 19.0 1351.0
1133 -0.394238 -0.475264 -0.536898 -0.578857 -0.607846 -0.630126 -0.648238 -0.662181 -0.671709 -0.677333 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 11.0 828.0
1134 -0.102479 -0.110677 -0.127336 -0.150269 -0.177369 -0.207631 -0.240525 -0.275543 -0.312820 -0.353628 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 57.0
1135 0.143576 0.299206 0.488988 0.704623 0.937382 1.179030 1.421854 1.658054 1.878826 2.073959 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 19.0 1385.0
1136 -0.374473 -0.465755 -0.545642 -0.610448 -0.665223 -0.717764 -0.773475 -0.833931 -0.898080 -0.963991 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 10.0 779.0

5 rows × 548 columns

In [19]:
%%time
train_gdf['case_id'].nunique().compute(), train_gdf['class_vals'].nunique().compute()
CPU times: user 384 ms, sys: 20.5 ms, total: 405 ms
Wall time: 473 ms
Out[19]:
(1137, 20)
In [20]:
%%time
import dask_cudf
valid_gdf = dask_cudf.read_parquet(output_valid_dir)
valid_gdf.head()
CPU times: user 311 ms, sys: 17 ms, total: 328 ms
Wall time: 436 ms
Out[20]:
dim_0_0 dim_0_1 dim_0_2 dim_0_3 dim_0_4 dim_0_5 dim_0_6 dim_0_7 dim_0_8 dim_0_9 ... dim_2_174 dim_2_175 dim_2_176 dim_2_177 dim_2_178 dim_2_179 dim_2_180 dim_2_181 class_vals case_id
0 -0.067818 -0.110913 -0.200297 -0.329535 -0.484393 -0.652534 -0.828739 -1.013061 -1.205244 -1.400698 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 17.0 1247.0
1 -0.155226 -0.174237 -0.192516 -0.207862 -0.222582 -0.241315 -0.266942 -0.299259 -0.336715 -0.377834 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 15.0 1110.0
2 -0.197612 -0.200153 -0.181484 -0.140233 -0.077249 0.006616 0.109389 0.226975 0.355045 0.491321 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.0 351.0
3 -0.156387 -0.163468 -0.159924 -0.142182 -0.108299 -0.055478 0.019574 0.118058 0.238554 0.378835 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 19.0 1395.0
4 -0.298142 -0.325184 -0.321029 -0.287726 -0.234856 -0.171204 -0.100279 -0.021632 0.065202 0.158812 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.0 381.0

5 rows × 548 columns

In [21]:
valid_gdf.tail()
Out[21]:
dim_0_0 dim_0_1 dim_0_2 dim_0_3 dim_0_4 dim_0_5 dim_0_6 dim_0_7 dim_0_8 dim_0_9 ... dim_2_174 dim_2_175 dim_2_176 dim_2_177 dim_2_178 dim_2_179 dim_2_180 dim_2_181 class_vals case_id
280 -0.168708 -0.191889 -0.217918 -0.246967 -0.280014 -0.317423 -0.357981 -0.398777 -0.435821 -0.465152 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 15.0 1073.0
281 -0.254506 -0.270490 -0.259972 -0.226300 -0.179451 -0.127260 -0.070444 -0.004125 0.077391 0.177068 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.0 365.0
282 -0.133279 -0.144468 -0.161016 -0.181991 -0.205754 -0.230874 -0.256796 -0.284144 -0.314486 -0.349780 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 6.0 506.0
283 -0.095838 -0.094679 -0.095861 -0.098774 -0.105749 -0.121057 -0.147979 -0.187354 -0.238501 -0.301091 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 14.0 1034.0
284 -0.154512 -0.181448 -0.217386 -0.259778 -0.306061 -0.354698 -0.404998 -0.456535 -0.508682 -0.560353 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 11.0 864.0

5 rows × 548 columns

In [22]:
%%time
valid_gdf['case_id'].nunique().compute(), valid_gdf['class_vals'].nunique().compute()
CPU times: user 50.1 ms, sys: 16.5 ms, total: 66.6 ms
Wall time: 144 ms
Out[22]:
(285, 20)
In [23]:
%%time
import dask_cudf
test_gdf = dask_cudf.read_parquet(output_test_dir)
test_gdf.head()
CPU times: user 317 ms, sys: 13.5 ms, total: 330 ms
Wall time: 490 ms
Out[23]:
dim_0_0 dim_0_1 dim_0_2 dim_0_3 dim_0_4 dim_0_5 dim_0_6 dim_0_7 dim_0_8 dim_0_9 ... dim_2_174 dim_2_175 dim_2_176 dim_2_177 dim_2_178 dim_2_179 dim_2_180 dim_2_181 class_vals case_id
0 -0.011902 -0.019550 -0.062037 -0.133200 -0.221028 -0.317045 -0.419692 -0.530973 -0.651434 -0.777891 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.0 444.0
1 -0.121770 -0.119831 -0.116177 -0.111483 -0.108602 -0.111719 -0.123581 -0.143425 -0.167864 -0.193229 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 16.0 1199.0
2 -0.122890 -0.123181 -0.124221 -0.127167 -0.133875 -0.146406 -0.166203 -0.193641 -0.228231 -0.269213 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 14.0 1021.0
3 -0.238751 -0.286259 -0.330241 -0.366631 -0.395599 -0.419454 -0.439938 -0.457389 -0.471751 -0.483653 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 124.0
4 -0.121683 -0.119392 -0.114387 -0.105583 -0.092747 -0.076566 -0.058176 -0.038983 -0.020820 -0.005637 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 16.0 1140.0

5 rows × 548 columns

In [24]:
test_gdf.tail()
Out[24]:
dim_0_0 dim_0_1 dim_0_2 dim_0_3 dim_0_4 dim_0_5 dim_0_6 dim_0_7 dim_0_8 dim_0_9 ... dim_2_174 dim_2_175 dim_2_176 dim_2_177 dim_2_178 dim_2_179 dim_2_180 dim_2_181 class_vals case_id
1431 -0.229912 -0.261109 -0.277499 -0.273361 -0.248630 -0.207103 -0.152437 -0.084686 -0.000471 0.103468 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.0 380.0
1432 -0.080856 -0.130531 -0.227994 -0.364505 -0.520037 -0.674525 -0.817338 -0.948717 -1.073356 -1.192411 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 17.0 1266.0
1433 -0.150089 -0.170816 -0.193861 -0.213184 -0.224774 -0.229679 -0.233106 -0.240481 -0.254193 -0.273944 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 14.0 1033.0
1434 -0.187826 -0.219704 -0.255131 -0.291466 -0.327528 -0.363795 -0.401146 -0.439741 -0.479128 -0.519153 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.0
1435 -0.122832 -0.122929 -0.123384 -0.125062 -0.129972 -0.141473 -0.163379 -0.198216 -0.246522 -0.308311 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 14.0 1048.0

5 rows × 548 columns

In [25]:
%%time
test_gdf['case_id'].nunique().compute(), test_gdf['class_vals'].nunique().compute()
CPU times: user 60.5 ms, sys: 4.62 ms, total: 65.1 ms
Wall time: 145 ms
Out[25]:
(1436, 20)

We reset the kernel!!!

In [26]:
%%time
client.shutdown()
client.close()
Traceback (most recent call last):
  File "/home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/distributed/utils.py", line 778, in wrapper
    return await func(*args, **kwargs)
  File "/home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/distributed/client.py", line 1211, in _reconnect
    await self._ensure_connected(timeout=timeout)
  File "/home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/distributed/client.py", line 1241, in _ensure_connected
    comm = await connect(
  File "/home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/distributed/comm/core.py", line 315, in connect
    await asyncio.sleep(backoff)
  File "/home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/asyncio/tasks.py", line 659, in sleep
    return await future
asyncio.exceptions.CancelledError

Traceback (most recent call last):
  File "/home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/distributed/utils.py", line 778, in wrapper
    return await func(*args, **kwargs)
  File "/home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/distributed/client.py", line 1400, in _handle_report
    await self._reconnect()
  File "/home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/distributed/utils.py", line 778, in wrapper
    return await func(*args, **kwargs)
  File "/home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/distributed/client.py", line 1211, in _reconnect
    await self._ensure_connected(timeout=timeout)
  File "/home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/distributed/client.py", line 1241, in _ensure_connected
    comm = await connect(
  File "/home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/distributed/comm/core.py", line 315, in connect
    await asyncio.sleep(backoff)
  File "/home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/asyncio/tasks.py", line 659, in sleep
    return await future
asyncio.exceptions.CancelledError
CPU times: user 38.7 ms, sys: 7.77 ms, total: 46.5 ms
Wall time: 617 ms
In [27]:
from nbdev import nbdev_export
nbdev_export()