In [1]:

    #| default_exp feature_preprocessing.face_detection.target_encoding
    %load_ext autoreload
    %autoreload 2

In [2]:

    # declare a list tasks whose products you want to use as inputs
    upstream = ['parquet_conversion_face_detection']

In [3]:

    # Parameters
    upstream = {"parquet_conversion_face_detection": {"nb": "/home/ubuntu/vitmtsc_nbdev/output/101_data.face_detection.html", "FaceDetection_TRAIN_RAW": "/home/ubuntu/vitmtsc_nbdev/output/FaceDetection/raw/train", "FaceDetection_VALID_RAW": "/home/ubuntu/vitmtsc_nbdev/output/FaceDetection/raw/valid", "FaceDetection_TEST_RAW": "/home/ubuntu/vitmtsc_nbdev/output/FaceDetection/raw/test"}}
    product = {"nb": "/home/ubuntu/vitmtsc_nbdev/output/201_feature_preprocessing.face_detection.target_encoding.html", "FaceDetection_TRAIN_TE": "/home/ubuntu/vitmtsc_nbdev/output/FaceDetection/target_encoding/train", "FaceDetection_VALID_TE": "/home/ubuntu/vitmtsc_nbdev/output/FaceDetection/target_encoding/valid", "FaceDetection_TEST_TE": "/home/ubuntu/vitmtsc_nbdev/output/FaceDetection/target_encoding/test", "FaceDetection_workflow_dir": "/home/ubuntu/vitmtsc_nbdev/output/FaceDetection/target_encoding/nvtabular_workflow"}

In [4]:

    #| hide
    from nbdev.showdoc import *

In [5]:

    #| export
    from vitmtsc import *
    from vitmtsc.core import *
    from vitmtsc.data.face_detection import *
    import os
    import nvtabular as nvt
    import dask_cudf
    from nvtabular import ops

In [6]:

    #| export
    upstream = {
        "parquet_conversion_face_detection": {
            "nb": "/home/ubuntu/vitmtsc_nbdev/output/101_data.face_detection.html",
            "FaceDetection_TRAIN_RAW": "/home/ubuntu/vitmtsc_nbdev/output/FaceDetection/raw/train",
            "FaceDetection_VALID_RAW": "/home/ubuntu/vitmtsc_nbdev/output/FaceDetection/raw/valid",
            "FaceDetection_TEST_RAW": "/home/ubuntu/vitmtsc_nbdev/output/FaceDetection/raw/test",
        }
    }
    product = {
        "nb": "/home/ubuntu/vitmtsc_nbdev/output/201_feature_preprocessing.face_detection.target_encoding.html",
        "FaceDetection_TRAIN_TE": "/home/ubuntu/vitmtsc_nbdev/output/FaceDetection/target_encoding/train",
        "FaceDetection_VALID_TE": "/home/ubuntu/vitmtsc_nbdev/output/FaceDetection/target_encoding/valid",
        "FaceDetection_TEST_TE": "/home/ubuntu/vitmtsc_nbdev/output/FaceDetection/target_encoding/test",
        "FaceDetection_workflow_dir": "/home/ubuntu/vitmtsc_nbdev/output/FaceDetection/target_encoding/nvtabular_workflow",
    }

In [7]:

    !conda list|grep -i nvtabular

## Feature Preprocessing via NVTabular¶

> Fill missing continuous features

> Normalize continuous features

> Categorify categorical features

> Target Encoding of Categorical Variables

In [8]:

    from dask.distributed import Client
    from dask_cuda import LocalCUDACluster

    cluster = LocalCUDACluster(memory_limit='auto', device_memory_limit=0.5, rmm_pool_size='20GB', rmm_managed_memory=True)
    client = Client(cluster)
    client

    2022-09-23 18:55:34,626 - distributed.preloading - INFO - Creating preload: dask_cuda.initialize
    2022-09-23 18:55:34,626 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize
    2022-09-23 18:55:34,628 - distributed.preloading - INFO - Creating preload: dask_cuda.initialize
    2022-09-23 18:55:34,628 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize
    2022-09-23 18:55:34,649 - distributed.preloading - INFO - Creating preload: dask_cuda.initialize
    2022-09-23 18:55:34,649 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize
    2022-09-23 18:55:34,659 - distributed.preloading - INFO - Creating preload: dask_cuda.initialize
    2022-09-23 18:55:34,659 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize

Out[8]:

### Client

Client-4e902290-3b71-11ed-80b5-0a01290f6f4b

Connection method: Cluster object | Cluster type: dask_cuda.LocalCUDACluster
Dashboard: http://127.0.0.1:8787/status |

### Cluster Info

### LocalCUDACluster

8f9d8123

Dashboard: http://127.0.0.1:8787/status | Workers: 4
Total threads: 4 | Total memory: 150.00 GiB
Status: running | Using processes: True

### Scheduler Info

### Scheduler

Scheduler-2e956b6c-5bc3-4804-9ede-2b76fd230a1a

Comm: tcp://127.0.0.1:43109 | Workers: 4
Dashboard: http://127.0.0.1:8787/status | Total threads: 4
Started: Just now | Total memory: 150.00 GiB

### Workers

#### Worker: 0

Comm: tcp://127.0.0.1:33639 | Total threads: 1
Dashboard: http://127.0.0.1:36545/status | Memory: 37.50 GiB
Nanny: tcp://127.0.0.1:42687 |
Local directory: /tmp/dask-worker-space/worker-2nuwguef
GPU: Tesla T4 | GPU memory: 14.76 GiB

#### Worker: 1

Comm: tcp://127.0.0.1:42189 | Total threads: 1
Dashboard: http://127.0.0.1:46569/status | Memory: 37.50 GiB
Nanny: tcp://127.0.0.1:40397 |
Local directory: /tmp/dask-worker-space/worker-grlkfydj
GPU: Tesla T4 | GPU memory: 14.76 GiB

#### Worker: 2

Comm: tcp://127.0.0.1:40765 | Total threads: 1
Dashboard: http://127.0.0.1:39589/status | Memory: 37.50 GiB
Nanny: tcp://127.0.0.1:46299 |
Local directory: /tmp/dask-worker-space/worker-bebdotnw
GPU: Tesla T4 | GPU memory: 14.76 GiB

#### Worker: 3

Comm: tcp://127.0.0.1:40843 | Total threads: 1
Dashboard: http://127.0.0.1:35191/status | Memory: 37.50 GiB
Nanny: tcp://127.0.0.1:36379 |
Local directory: /tmp/dask-worker-space/worker-5x1utzur
GPU: Tesla T4 | GPU memory: 14.76 GiB

COLUMNS: CATEGORICAL, CONTINUOUS and TARGET

In [9]:

    #| export
    import numpy as np
    CATEGORICAL_COLUMNS_DONOT_NEED_ENCODING = ['case_id', 'case_id_seq', 'reading_id']

In [10]:

    #| export
    CATEGORICAL_COLUMNS_NEED_ENCODING = [

    ]

In [11]:

    #| export
    CONTINUOUS_COLUMNS = [
    'dim_0',
    'dim_1',
    'dim_2',
    'dim_3',
    'dim_4',
    'dim_5',
    'dim_6',
    'dim_7',
    'dim_8',
    'dim_9',
    'dim_10',
    'dim_11',
    'dim_12',
    'dim_13',
    'dim_14',
    'dim_15',
    'dim_16',
    'dim_17',
    'dim_18',
    'dim_19',
    'dim_20',
    'dim_21',
    'dim_22',
    'dim_23',
    'dim_24',
    'dim_25',
    'dim_26',
    'dim_27',
    'dim_28',
    'dim_29',
    'dim_30',
    'dim_31',
    'dim_32',
    'dim_33',
    'dim_34',
    'dim_35',
    'dim_36',
    'dim_37',
    'dim_38',
    'dim_39',
    'dim_40',
    'dim_41',
    'dim_42',
    'dim_43',
    'dim_44',
    'dim_45',
    'dim_46',
    'dim_47',
    'dim_48',
    'dim_49',
    'dim_50',
    'dim_51',
    'dim_52',
    'dim_53',
    'dim_54',
    'dim_55',
    'dim_56',
    'dim_57',
    'dim_58',
    'dim_59',
    'dim_60',
    'dim_61',
    'dim_62',
    'dim_63',
    'dim_64',
    'dim_65',
    'dim_66',
    'dim_67',
    'dim_68',
    'dim_69',
    'dim_70',
    'dim_71',
    'dim_72',
    'dim_73',
    'dim_74',
    'dim_75',
    'dim_76',
    'dim_77',
    'dim_78',
    'dim_79',
    'dim_80',
    'dim_81',
    'dim_82',
    'dim_83',
    'dim_84',
    'dim_85',
    'dim_86',
    'dim_87',
    'dim_88',
    'dim_89',
    'dim_90',
    'dim_91',
    'dim_92',
    'dim_93',
    'dim_94',
    'dim_95',
    'dim_96',
    'dim_97',
    'dim_98',
    'dim_99',
    'dim_100',
    'dim_101',
    'dim_102',
    'dim_103',
    'dim_104',
    'dim_105',
    'dim_106',
    'dim_107',
    'dim_108',
    'dim_109',
    'dim_110',
    'dim_111',
    'dim_112',
    'dim_113',
    'dim_114',
    'dim_115',
    'dim_116',
    'dim_117',
    'dim_118',
    'dim_119',
    'dim_120',
    'dim_121',
    'dim_122',
    'dim_123',
    'dim_124',
    'dim_125',
    'dim_126',
    'dim_127',
    'dim_128',
    'dim_129',
    'dim_130',
    'dim_131',
    'dim_132',
    'dim_133',
    'dim_134',
    'dim_135',
    'dim_136',
    'dim_137',
    'dim_138',
    'dim_139',
    'dim_140',
    'dim_141',
    'dim_142',
    'dim_143'
    ]

In [12]:

    #| export
    LABEL_COLUMNS = ['class_vals']

Workflow and Operations

In [13]:

    import cudf
    import numpy as np
    cat_features_no_encoding = nvt.ColumnGroup(CATEGORICAL_COLUMNS_DONOT_NEED_ENCODING)
    #te_features = CATEGORICAL_COLUMNS_NEED_ENCODING >> ops.TargetEncoding(LABEL_COLUMNS, kfold=5, fold_seed=42, p_smooth=20)
    cont_features = CONTINUOUS_COLUMNS >> ops.FillMissing() >> ops.Normalize()
    label_name = LABEL_COLUMNS

    workflow = nvt.Workflow(
        #cat_features_no_encoding + te_features + cont_features + label_name
        #cat_features_no_encoding + te_features + label_name
        cat_features_no_encoding  + cont_features + label_name
    )

Datasets

> Input data

In [14]:

    pre_processed_train_dir = os.path.join("./", upstream['parquet_conversion_face_detection']['FaceDetection_TRAIN_RAW'])
    pre_processed_valid_dir = os.path.join("./", upstream['parquet_conversion_face_detection']['FaceDetection_VALID_RAW'])
    pre_processed_test_dir = os.path.join("./", upstream['parquet_conversion_face_detection']['FaceDetection_TEST_RAW'])

> Training, Validation and Test datasets

In [15]:

    train_dataset = nvt.Dataset(pre_processed_train_dir, engine='parquet')
    valid_dataset = nvt.Dataset(pre_processed_valid_dir, engine='parquet')
    test_dataset = nvt.Dataset(pre_processed_test_dir, engine='parquet')

> Output location

In [16]:

    output_train_dir = os.path.join("./", product['FaceDetection_TRAIN_TE'])
    output_valid_dir = os.path.join("./", product['FaceDetection_VALID_TE'])
    output_test_dir = os.path.join("./", product['FaceDetection_TEST_TE'])

In [17]:

    !mkdir -p $output_train_dir
    !mkdir -p $output_valid_dir
    !mkdir -p $output_test_dir

> Path to save the workflow to

##
Fit: Train Dataset
¶

In [18]:

    %%time
    workflow.fit(train_dataset)

    CPU times: user 418 ms, sys: 54 ms, total: 472 ms
    Wall time: 2.95 s

Out[18]:

    <nvtabular.workflow.workflow.Workflow at 0x7f015045d460>

Save workflow

In [19]:

    %%time
    workflow.save(product['FaceDetection_workflow_dir'])

    CPU times: user 4.58 ms, sys: 667 µs, total: 5.25 ms
    Wall time: 4.78 ms

Clear workflow

In [20]:

    %%time
    workflow = None

    CPU times: user 1 µs, sys: 2 µs, total: 3 µs
    Wall time: 5.72 µs

Load workflow

In [21]:

    %%time
    workflow = nvt.Workflow.load(product['FaceDetection_workflow_dir'], client=client)

    CPU times: user 2.84 ms, sys: 23 µs, total: 2.86 ms
    Wall time: 2.56 ms

##
Transform: Train Dataset
¶

In [22]:

    %%time
    # Write to new "shuffled" and "processed" dataset
    workflow.transform(train_dataset).to_parquet(
        output_train_dir,
        out_files_per_proc=2,
        shuffle=nvt.io.Shuffle.PER_PARTITION,
    )

    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/merlin/io/dataset.py:862: UserWarning: Only created 4 files did not have enough partitions to create 8 files.
      warnings.warn(

    CPU times: user 438 ms, sys: 46.7 ms, total: 485 ms
    Wall time: 3.17 s

##
Transform: Valid Dataset
¶

In [23]:

    %%time
    # Write to new "shuffled" and "processed" dataset
    workflow.transform(valid_dataset).to_parquet(
        output_valid_dir,
        out_files_per_proc=2,
        shuffle=nvt.io.Shuffle.PER_PARTITION,
    )

    CPU times: user 230 ms, sys: 25.9 ms, total: 256 ms
    Wall time: 1.3 s

##
Transform: Test Dataset
¶

In [24]:

    %%time
    # Write to new "shuffled" and "processed" dataset
    workflow.transform(test_dataset).to_parquet(
        output_test_dir,
        out_files_per_proc=2,
        shuffle=nvt.io.Shuffle.PER_PARTITION,
    )

    CPU times: user 217 ms, sys: 21.7 ms, total: 238 ms
    Wall time: 865 ms

Verify Data

In [25]:

    train_gdf = dask_cudf.read_parquet(output_train_dir)

In [26]:

    %%time
    train_gdf.head()

    CPU times: user 27.2 ms, sys: 0 ns, total: 27.2 ms
    Wall time: 65.5 ms

Out[26]:

| dim_0 | dim_1 | dim_2 | dim_3 | dim_4 | dim_5 | dim_6 | dim_7 | dim_8 | dim_9 | ... | dim_138 | dim_139 | dim_140 | dim_141 | dim_142 | dim_143 | case_id | case_id_seq | reading_id | class_vals
--- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | ---
0 | -0.121314 | 0.393771 | 0.079717 | 0.077422 | -0.981932 | 0.357935 | -0.030971 | -0.694245 | 0.078503 | 0.073602 | ... | -1.123991 | 1.145524 | 0.441915 | 0.071174 | -0.414679 | 0.074954 | 661 | 661 | 25 | 0
1 | -0.841474 | -0.797672 | -0.263793 | 1.298514 | -0.739589 | -0.381691 | 0.588058 | 0.608006 | -0.247028 | 0.193751 | ... | -0.440068 | -1.319766 | 0.219988 | 0.081958 | -1.039737 | -0.218774 | 659 | 659 | 58 | 0
2 | -0.234124 | -0.014298 | 0.851526 | -0.699747 | -0.870767 | 0.805499 | -0.786541 | 0.730033 | -0.032784 | -1.821225 | ... | -0.219486 | -1.016099 | 1.122216 | 0.559855 | -1.778112 | 0.560210 | 35 | 35 | 19 | 1
3 | 0.092242 | -1.140328 | -1.318047 | 0.468520 | 1.570720 | -1.111088 | 0.701104 | -0.652357 | -0.760031 | 0.140818 | ... | -0.873041 | -0.757116 | 0.953688 | -0.137620 | -0.252380 | -0.262697 | 145 | 145 | 50 | 0
4 | -1.052806 | -0.138503 | 1.347148 | -2.277875 | 0.361970 | 1.458319 | 0.831894 | -1.265934 | 0.839550 | -0.019017 | ... | -0.863022 | 1.170422 | 0.106141 | 0.650703 | 0.308283 | 0.802099 | 496 | 496 | 51 | 1

5 rows × 148 columns

In [27]:

    %%time
    train_gdf['case_id'].nunique().compute()

    CPU times: user 118 ms, sys: 10.3 ms, total: 128 ms
    Wall time: 159 ms

Out[27]:

    4712

In [28]:

    valid_gdf = dask_cudf.read_parquet(output_valid_dir)

In [29]:

    %%time
    valid_gdf.head()

    CPU times: user 23.7 ms, sys: 3.67 ms, total: 27.3 ms
    Wall time: 54.4 ms

Out[29]:

| dim_0 | dim_1 | dim_2 | dim_3 | dim_4 | dim_5 | dim_6 | dim_7 | dim_8 | dim_9 | ... | dim_138 | dim_139 | dim_140 | dim_141 | dim_142 | dim_143 | case_id | case_id_seq | reading_id | class_vals
--- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | ---
0 | -1.171568 | 0.063259 | 1.015736 | -0.603317 | -0.068818 | 0.628579 | -1.884459 | -0.731820 | 0.468230 | -0.309108 | ... | -0.057336 | -0.179852 | 0.144208 | -0.210516 | 0.664078 | -0.668529 | 3425 | 3425 | 8 | 1
1 | 1.062399 | 0.965218 | 1.212485 | 0.076351 | -0.475522 | 1.061865 | -0.675021 | -2.317005 | 0.837066 | 0.906042 | ... | -0.559026 | 0.347611 | -0.171397 | -0.918838 | -0.932959 | 0.113725 | 3475 | 3475 | 41 | 1
2 | 0.116676 | 0.389882 | 0.880675 | -1.038531 | 1.072086 | 0.739722 | -0.344597 | -0.158225 | -0.273703 | -1.396459 | ... | -0.687636 | 1.054171 | -1.007294 | -0.104828 | 0.093016 | -0.221463 | 3652 | 3652 | 44 | 0
3 | -1.258221 | 0.739932 | 0.593104 | -0.613782 | 0.654134 | 0.956879 | -2.337000 | 0.211169 | 0.336160 | -0.995167 | ... | 0.388883 | 0.482879 | 0.002319 | 0.053174 | 0.364765 | -1.163029 | 3262 | 3262 | 46 | 0
4 | -1.168730 | 1.659656 | 0.043099 | -1.012270 | 0.041338 | 0.602217 | -1.092073 | 0.892507 | 0.115993 | -1.454313 | ... | 0.397347 | -1.141881 | 1.721545 | 0.485349 | 0.405445 | 1.025795 | 3365 | 3365 | 28 | 0

5 rows × 148 columns

In [30]:

    %%time
    valid_gdf['case_id'].nunique().compute()

    CPU times: user 24.6 ms, sys: 0 ns, total: 24.6 ms
    Wall time: 55.1 ms

Out[30]:

    1178

In [31]:

    test_gdf = dask_cudf.read_parquet(output_test_dir)

In [32]:

    %%time
    test_gdf.head()

    CPU times: user 22.7 ms, sys: 2.56 ms, total: 25.2 ms
    Wall time: 61.1 ms

Out[32]:

| dim_0 | dim_1 | dim_2 | dim_3 | dim_4 | dim_5 | dim_6 | dim_7 | dim_8 | dim_9 | ... | dim_138 | dim_139 | dim_140 | dim_141 | dim_142 | dim_143 | case_id | case_id_seq | reading_id | class_vals
--- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | ---
0 | -0.260010 | -0.489144 | 0.656876 | 0.815443 | -1.202993 | 0.525935 | -0.255497 | -1.572738 | 0.652261 | -0.035183 | ... | -0.353590 | -0.996325 | -1.411985 | -1.146100 | -0.456622 | -0.571731 | 73 | 73 | 44 | 1
1 | -0.809563 | 2.231060 | -0.720697 | -0.630477 | 1.837997 | -0.958108 | -1.111681 | 0.387655 | -1.175156 | -1.467910 | ... | -0.291857 | -1.778868 | 0.294018 | -1.296351 | -1.230731 | 0.473268 | 235 | 235 | 8 | 0
2 | 1.141494 | -0.968512 | 0.786951 | 0.102245 | -0.277603 | 0.819223 | -0.489941 | 0.603016 | 0.746006 | -0.624365 | ... | 0.615759 | -0.401089 | -0.092868 | -0.397284 | 0.553770 | -1.266353 | 193 | 193 | 42 | 0
3 | 1.004016 | -1.891377 | 0.658439 | 0.303093 | -0.724539 | 0.268960 | 1.223566 | -1.376977 | 0.542260 | -0.992891 | ... | -0.173092 | 0.488560 | -1.316807 | -1.073103 | -0.584971 | -0.645980 | 350 | 350 | 25 | 0
4 | -0.686177 | -0.800100 | 0.220677 | -0.225374 | 0.300138 | 0.571094 | -2.010938 | -1.444248 | 0.146692 | 0.097607 | ... | -0.182355 | -0.060018 | 0.325988 | 0.027354 | 0.819301 | -1.251991 | 261 | 261 | 5 | 1

5 rows × 148 columns

In [33]:

    %%time
    test_gdf['case_id'].nunique().compute()

    CPU times: user 24.5 ms, sys: 3.56 ms, total: 28.1 ms
    Wall time: 60.1 ms

Out[33]:

    3524

In [34]:

    test_gdf.columns

Out[34]:

    Index(['dim_0', 'dim_1', 'dim_2', 'dim_3', 'dim_4', 'dim_5', 'dim_6', 'dim_7',
           'dim_8', 'dim_9',
           ...
           'dim_138', 'dim_139', 'dim_140', 'dim_141', 'dim_142', 'dim_143',
           'case_id', 'case_id_seq', 'reading_id', 'class_vals'],
          dtype='object', length=148)

In [35]:

    !ls -lrt --block-size=M $output_train_dir

    total 323M
    -rw-r--r-- 1 root root  1M Sep 23 18:55 schema.pbtxt
    -rw-r--r-- 1 root root 41M Sep 23 18:55 part_7.parquet
    -rw-r--r-- 1 root root 41M Sep 23 18:55 part_6.parquet
    -rw-r--r-- 1 root root 41M Sep 23 18:55 part_5.parquet
    -rw-r--r-- 1 root root 41M Sep 23 18:55 part_4.parquet
    -rw-r--r-- 1 root root 41M Sep 23 18:55 part_3.parquet
    -rw-r--r-- 1 root root 41M Sep 23 18:55 part_2.parquet
    -rw-r--r-- 1 root root 41M Sep 23 18:55 part_1.parquet
    -rw-r--r-- 1 root root 41M Sep 23 18:55 part_0.parquet
    -rw-r--r-- 1 root root  1M Sep 23 18:55 _metadata.json
    -rw-r--r-- 1 root root  1M Sep 23 18:55 _file_list.txt
    -rw-r--r-- 1 root root  1M Sep 23 18:55 _metadata

In [36]:

    !ls -lrt --block-size=M $output_valid_dir

    total 82M
    -rw-r--r-- 1 root root  1M Sep 23 18:55 schema.pbtxt
    -rw-r--r-- 1 root root 11M Sep 23 18:55 part_7.parquet
    -rw-r--r-- 1 root root 11M Sep 23 18:55 part_6.parquet
    -rw-r--r-- 1 root root 11M Sep 23 18:55 part_3.parquet
    -rw-r--r-- 1 root root 11M Sep 23 18:55 part_2.parquet
    -rw-r--r-- 1 root root 11M Sep 23 18:55 part_5.parquet
    -rw-r--r-- 1 root root 11M Sep 23 18:55 part_4.parquet
    -rw-r--r-- 1 root root 11M Sep 23 18:55 part_1.parquet
    -rw-r--r-- 1 root root 11M Sep 23 18:55 part_0.parquet
    -rw-r--r-- 1 root root  1M Sep 23 18:55 _metadata.json
    -rw-r--r-- 1 root root  1M Sep 23 18:55 _file_list.txt
    -rw-r--r-- 1 root root  1M Sep 23 18:55 _metadata

In [37]:

    !ls -lrt --block-size=M $output_test_dir

    total 242M
    -rw-r--r-- 1 root root  1M Sep 23 18:55 schema.pbtxt
    -rw-r--r-- 1 root root 31M Sep 23 18:55 part_5.parquet
    -rw-r--r-- 1 root root 31M Sep 23 18:55 part_4.parquet
    -rw-r--r-- 1 root root 31M Sep 23 18:55 part_7.parquet
    -rw-r--r-- 1 root root 31M Sep 23 18:55 part_6.parquet
    -rw-r--r-- 1 root root 31M Sep 23 18:55 part_3.parquet
    -rw-r--r-- 1 root root 30M Sep 23 18:55 part_2.parquet
    -rw-r--r-- 1 root root 31M Sep 23 18:55 part_1.parquet
    -rw-r--r-- 1 root root 31M Sep 23 18:55 part_0.parquet
    -rw-r--r-- 1 root root  1M Sep 23 18:55 _metadata.json
    -rw-r--r-- 1 root root  1M Sep 23 18:55 _file_list.txt
    -rw-r--r-- 1 root root  1M Sep 23 18:55 _metadata

We reset the kernel!!!

In [38]:

    %%time
    client.shutdown()
    client.close()

    Traceback (most recent call last):
      File "/home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/distributed/utils.py", line 778, in wrapper
        return await func(*args, **kwargs)
      File "/home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/distributed/client.py", line 1211, in _reconnect
        await self._ensure_connected(timeout=timeout)
      File "/home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/distributed/client.py", line 1241, in _ensure_connected
        comm = await connect(
      File "/home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/distributed/comm/core.py", line 315, in connect
        await asyncio.sleep(backoff)
      File "/home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/asyncio/tasks.py", line 659, in sleep
        return await future
    asyncio.exceptions.CancelledError

    Traceback (most recent call last):
      File "/home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/distributed/utils.py", line 778, in wrapper
        return await func(*args, **kwargs)
      File "/home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/distributed/client.py", line 1400, in _handle_report
        await self._reconnect()
      File "/home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/distributed/utils.py", line 778, in wrapper
        return await func(*args, **kwargs)
      File "/home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/distributed/client.py", line 1211, in _reconnect
        await self._ensure_connected(timeout=timeout)
      File "/home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/distributed/client.py", line 1241, in _ensure_connected
        comm = await connect(
      File "/home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/distributed/comm/core.py", line 315, in connect
        await asyncio.sleep(backoff)
      File "/home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/asyncio/tasks.py", line 659, in sleep
        return await future
    asyncio.exceptions.CancelledError

    CPU times: user 26.4 ms, sys: 22.3 ms, total: 48.7 ms
    Wall time: 669 ms

In [39]:

    from nbdev import nbdev_export
    nbdev_export()