In [1]:

    #| default_exp model.optimization.nn.tsc.vittsc.character_trajectories_evaluation_mask
    %load_ext autoreload
    %autoreload 2

In [2]:

    # declare a list tasks whose products you want to use as inputs
    upstream = ['tabular_to_timeseries_character_trajectories', 'model_training_character_trajectories']

In [3]:

    # Parameters
    upstream = {
        "tabular_to_timeseries_character_trajectories": {
            "nb": "/home/ubuntu/vitmtsc_nbdev/output/305_feature_preprocessing.character_trajectories.tabular_to_timeseries.html",
            "CharacterTrajectories_TRAIN_MODEL_INPUT": "/home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/target_encoding-nn/train",
            "CharacterTrajectories_VALID_MODEL_INPUT": "/home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/target_encoding-nn/valid",
            "CharacterTrajectories_TEST_MODEL_INPUT": "/home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/target_encoding-nn/test",
        },
        "model_training_character_trajectories": {
            "nb": "/home/ubuntu/vitmtsc_nbdev/output/405_model.optimization.nn.tsc.vittsc.character_trajectories_training_mask_tune.html",
            "CharacterTrajectories_MODEL_TUNE_OUTPUT": "/home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/ray_results",
            "CharacterTrajectories_MODEL_TRAINING_OUTPUT": "/home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/experiments_result",
            "CharacterTrajectories_MODEL_TRAINING_CHECKPOINT_OUTPUT": "/home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/experiments_result/checkpoint",
            "CharacterTrajectories_BEST_MODEL": "/home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/experiments_result/best_model.ckpt",
            "CharacterTrajectories_BEST_MODEL_CONFIG": "/home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/experiments_result/best_model_config.json",
        },
    }
    product = {
        "nb": "/home/ubuntu/vitmtsc_nbdev/output/505_model.optimization.nn.tsc.vittsc.character_trajectories_evaluation_mask.html",
        "CharacterTrajectories_MODEL_VALID_EVAL_OUTPUT": "/home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/experiments_result/evaluation/valid",
        "CharacterTrajectories_MODEL_TEST_EVAL_OUTPUT": "/home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/experiments_result/evaluation/test",
    }

In [4]:

    #| hide
    from nbdev.showdoc import *

In [5]:

    #| export
    import sys
    import pathlib as p

    def is_running_from_ipython():
        from IPython import get_ipython
        return get_ipython() is not None

    if not is_running_from_ipython() and __package__ is None:
        DIR = p.Path(__file__).resolve().parent
        sys.path.insert(0, str(DIR.parent))
        __package__ = DIR.name

In [6]:

    #| export
    from vitmtsc.model.optimization.nn.tsc.vittsc.character_trajectories_training_mask_tune import *

    class_weight: [0.42425373 0.46598361 0.49868421 0.47375    0.40035211 0.51681818
     0.5685     0.41195652 0.55735294 0.53632075 0.52638889 0.52638889
     0.60478723 0.51681818 0.5685     0.58010204 0.48177966 0.5685
     0.45119048 0.45846774]

In [7]:

    # |export
    upstream = {
        "tabular_to_timeseries_character_trajectories": {
            "nb": "/home/ubuntu/vitmtsc_nbdev/output/305_feature_preprocessing.character_trajectories.tabular_to_timeseries.html",
            "CharacterTrajectories_TRAIN_MODEL_INPUT": "/home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/target_encoding-nn/train",
            "CharacterTrajectories_VALID_MODEL_INPUT": "/home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/target_encoding-nn/valid",
            "CharacterTrajectories_TEST_MODEL_INPUT": "/home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/target_encoding-nn/test",
        },
        "model_training_character_trajectories": {
            "nb": "/home/ubuntu/vitmtsc_nbdev/output/405_model.optimization.nn.tsc.vittsc.character_trajectories_training_mask_tune.html",
            "CharacterTrajectories_MODEL_TUNE_OUTPUT": "/home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/ray_results",
            "CharacterTrajectories_MODEL_TRAINING_OUTPUT": "/home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/experiments_result",
            "CharacterTrajectories_MODEL_TRAINING_CHECKPOINT_OUTPUT": "/home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/experiments_result/checkpoint",
            "CharacterTrajectories_BEST_MODEL": "/home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/experiments_result/best_model.ckpt",
            "CharacterTrajectories_BEST_MODEL_CONFIG": "/home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/experiments_result/best_model_config.json",
        },
    }
    product = {
        "nb": "/home/ubuntu/vitmtsc_nbdev/output/505_model.optimization.nn.tsc.vittsc.character_trajectories_evaluation_mask.html",
        "CharacterTrajectories_MODEL_VALID_EVAL_OUTPUT": "/home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/experiments_result/evaluation/valid",
        "CharacterTrajectories_MODEL_TEST_EVAL_OUTPUT": "/home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/experiments_result/evaluation/test",
    }

In [8]:

    # |export
    import json
    def get_best_model_config():
        with open(upstream['model_training_character_trajectories']['CharacterTrajectories_BEST_MODEL_CONFIG'], 'r') as json_file:
            return json.load(json_file)

In [9]:

    #| export
    import pandas as pd
    import os
    import torch
    import math
    import glob

    import pytorch_lightning as pl
    from torch.nn import functional as F
    import matplotlib.pyplot as plt
    import scikitplot as skplt
    from pytorch_lightning import LightningModule
    from pytorch_lightning import Trainer
    from petastorm import make_batch_reader
    from petastorm.pytorch import DataLoader

## Vision Transformer for Multivariate Time-Series Classification (VitMTSC) Model with Masking - Evaluation¶

> Load Model

> Model Evaluation: Evaluate Model on test and validation dataset using PR-AUC

In [10]:

    #| export
    DATASET_NAME = 'CharacterTrajectories'
    VALID_DATA_DIR = f"file://{upstream['tabular_to_timeseries_character_trajectories']['CharacterTrajectories_VALID_MODEL_INPUT']}"
    TEST_DATA_DIR = f"file://{upstream['tabular_to_timeseries_character_trajectories']['CharacterTrajectories_TEST_MODEL_INPUT']}"
    VALID_EVAL_OUTPUT_DIR = product['CharacterTrajectories_MODEL_VALID_EVAL_OUTPUT']
    TEST_EVAL_OUTPUT_DIR = product['CharacterTrajectories_MODEL_TEST_EVAL_OUTPUT']
    BEST_MODEL_CHECKPOINT = upstream['model_training_character_trajectories']['CharacterTrajectories_BEST_MODEL']
    NUM_WORKERS=1
    SHARD_COUNT=1
    BATCH_SIZE = 64
    TOTAL_VALID_BATCHES = math.ceil(get_valid_dataset_size()/BATCH_SIZE)
    TOTAL_TEST_BATCHES = math.ceil(get_test_dataset_size()/BATCH_SIZE)
    BEST_MODEL_CHECKPOINT, TOTAL_VALID_BATCHES, TOTAL_TEST_BATCHES, VALID_DATA_DIR, TEST_DATA_DIR, VALID_EVAL_OUTPUT_DIR, TEST_EVAL_OUTPUT_DIR

Out[10]:

    ('/home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/experiments_result/best_model.ckpt',
     5,
     23,
     'file:///home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/target_encoding-nn/valid',
     'file:///home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/target_encoding-nn/test',
     '/home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/experiments_result/evaluation/valid',
     '/home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/experiments_result/evaluation/test')

In [11]:

    !mkdir -p $VALID_EVAL_OUTPUT_DIR
    !mkdir -p $TEST_EVAL_OUTPUT_DIR

## 1. VitMTSC Classification Prediction Task¶

In [12]:

    #| export
    class VitMTSCClassificationPredictionTask(LightningModule):
        def __init__(self,
                     model,
                     output_pred_dir,
                     input_data_dir,
                     batch_size=BATCH_SIZE,
                     num_workers=NUM_WORKERS,
                     shard_count = SHARD_COUNT):
            super().__init__()
            pl.seed_everything(42, workers=True)
            self.model = model
            self.case_id = []
            self.probability_0 = []
            self.probability_1 = []
            self.probability_2 = []
            self.probability_3 = []
            self.probability_4 = []
            self.probability_5 = []
            self.probability_6 = []
            self.probability_7 = []
            self.probability_8 = []
            self.probability_9 = []
            self.probability_10 = []
            self.probability_11 = []
            self.probability_12 = []
            self.probability_13 = []
            self.probability_14 = []
            self.probability_15 = []
            self.probability_16 = []
            self.probability_17 = []
            self.probability_18 = []
            self.probability_19 = []
            self.prediction = []
            self.target = []
            self.output_pred_dir = output_pred_dir
            self.input_data_dir = input_data_dir
            self.prediction_files = input_data_dir
            self.batch_size = batch_size
            self.num_workers = num_workers
            self.shard_count = shard_count

        def test_step(self, batch, batch_idx):
            x, y, case_id_1, mask = batch
            y_hat = self.model(x, mask)
            self.case_id.extend(case_id_1.to('cpu').numpy())
            self.probability_0.extend(F.softmax(y_hat, dim=1)[:,0].to('cpu').numpy())
            self.probability_1.extend(F.softmax(y_hat, dim=1)[:,1].to('cpu').numpy())
            self.probability_2.extend(F.softmax(y_hat, dim=1)[:,2].to('cpu').numpy())
            self.probability_3.extend(F.softmax(y_hat, dim=1)[:,3].to('cpu').numpy())
            self.probability_4.extend(F.softmax(y_hat, dim=1)[:,4].to('cpu').numpy())
            self.probability_5.extend(F.softmax(y_hat, dim=1)[:,5].to('cpu').numpy())
            self.probability_6.extend(F.softmax(y_hat, dim=1)[:,6].to('cpu').numpy())
            self.probability_7.extend(F.softmax(y_hat, dim=1)[:,7].to('cpu').numpy())
            self.probability_8.extend(F.softmax(y_hat, dim=1)[:,8].to('cpu').numpy())
            self.probability_9.extend(F.softmax(y_hat, dim=1)[:,9].to('cpu').numpy())
            self.probability_10.extend(F.softmax(y_hat, dim=1)[:,10].to('cpu').numpy())
            self.probability_11.extend(F.softmax(y_hat, dim=1)[:,11].to('cpu').numpy())
            self.probability_12.extend(F.softmax(y_hat, dim=1)[:,12].to('cpu').numpy())
            self.probability_13.extend(F.softmax(y_hat, dim=1)[:,13].to('cpu').numpy())
            self.probability_14.extend(F.softmax(y_hat, dim=1)[:,14].to('cpu').numpy())
            self.probability_15.extend(F.softmax(y_hat, dim=1)[:,15].to('cpu').numpy())
            self.probability_16.extend(F.softmax(y_hat, dim=1)[:,16].to('cpu').numpy())
            self.probability_17.extend(F.softmax(y_hat, dim=1)[:,17].to('cpu').numpy())
            self.probability_18.extend(F.softmax(y_hat, dim=1)[:,18].to('cpu').numpy())
            self.probability_19.extend(F.softmax(y_hat, dim=1)[:,19].to('cpu').numpy())
            self.prediction.extend(torch.max(y_hat.data, 1)[1].to('cpu').numpy())
            self.target.extend(y.to('cpu').numpy())

        def test_dataloader(self):
            print('test_dataloader: local rank :', int(os.environ['LOCAL_RANK']), 'shard count: ', self.shard_count)
            self.test_ds = make_batch_reader(self.prediction_files, workers_count=self.num_workers,
                                             cur_shard = int(os.environ['LOCAL_RANK']),
                                             shard_count = self.shard_count, num_epochs = 2)
            return DataLoader(self.test_ds, batch_size = self.batch_size, collate_fn= petastorm_collate_fn)

        def test_epoch_end(self, outputs):
            print('Consolidating predictions on GPU:', os.environ['LOCAL_RANK'])
            df_text_predictions = pd.DataFrame({'case_id': self.case_id,
                                                'probability_0': self.probability_0,
                                                'probability_1': self.probability_1,
                                                'probability_2': self.probability_2,
                                                'probability_3': self.probability_3,
                                                'probability_4': self.probability_4,
                                                'probability_5': self.probability_5,
                                                'probability_6': self.probability_6,
                                                'probability_7': self.probability_7,
                                                'probability_8': self.probability_8,
                                                'probability_9': self.probability_9,
                                                'probability_10': self.probability_10,
                                                'probability_11': self.probability_11,
                                                'probability_12': self.probability_12,
                                                'probability_13': self.probability_13,
                                                'probability_14': self.probability_14,
                                                'probability_15': self.probability_15,
                                                'probability_16': self.probability_16,
                                                'probability_17': self.probability_17,
                                                'probability_18': self.probability_18,
                                                'probability_19': self.probability_19,
                                                'prediction': self.prediction,
                                                'target': self.target
                                                })
            print('Writing predictions on GPU:', os.environ['LOCAL_RANK'])
            df_text_predictions.to_csv(self.output_pred_dir + "/" + os.environ['LOCAL_RANK'] + '_predictions.csv', index=False)
            print('Finished Writing predictions on GPU:', os.environ['LOCAL_RANK'])

In [13]:

    #| export
    def get_model_for_prediction(BEST_MODEL_CHECKPOINT, config, output_pred_dir, input_data_dir, shard_count = SHARD_COUNT):
        # load the best model
        pl.seed_everything(42, workers=True)
        model = VitTimeSeriesTransformer.load_from_checkpoint(BEST_MODEL_CHECKPOINT, config = config)
        model.eval()
        return VitMTSCClassificationPredictionTask(model = model, shard_count = shard_count, output_pred_dir = output_pred_dir, input_data_dir = input_data_dir)

In [14]:

    #| export
    def write_prediction_for_valid_dataset(BEST_MODEL_CHECKPOINT,
                                          config,
                                          shard_count,
                                          output_pred_dir = VALID_EVAL_OUTPUT_DIR,
                                          input_data_dir=VALID_DATA_DIR):
        pl.seed_everything(42, workers=True)
        model = get_model_for_prediction(BEST_MODEL_CHECKPOINT = BEST_MODEL_CHECKPOINT,
                                         config = config,
                                         shard_count = shard_count,
                                         output_pred_dir = output_pred_dir,
                                         input_data_dir = input_data_dir)
        trainer = Trainer(gpus = [0],
                          accelerator='dp',
                          progress_bar_refresh_rate=1,
                          limit_test_batches = TOTAL_VALID_BATCHES)
        trainer.test(model)

    def write_prediction_for_test_dataset(BEST_MODEL_CHECKPOINT,
                                          config,
                                          shard_count,
                                          output_pred_dir = TEST_EVAL_OUTPUT_DIR,
                                          input_data_dir=TEST_DATA_DIR):
        pl.seed_everything(42, workers=True)
        model = get_model_for_prediction(BEST_MODEL_CHECKPOINT = BEST_MODEL_CHECKPOINT,
                                         config = config,
                                         shard_count = shard_count,
                                         output_pred_dir = output_pred_dir,
                                         input_data_dir = input_data_dir)
        trainer = Trainer(gpus = [0],
                          accelerator='dp',
                          progress_bar_refresh_rate=1,
                          limit_test_batches = TOTAL_TEST_BATCHES)
        trainer.test(model)

In [15]:

    %env LOCAL_RANK=0

    env: LOCAL_RANK=0

In [16]:

    #| export
    if __name__ == "__main__":
        print('Processing valid dataset...\n')
        write_prediction_for_valid_dataset(BEST_MODEL_CHECKPOINT = BEST_MODEL_CHECKPOINT,
                                          config = get_best_model_config(),
                                          shard_count = SHARD_COUNT)
        print('Finished Processing valid dataset!!!\n')

        print('Processing test dataset...\n')
        write_prediction_for_test_dataset(BEST_MODEL_CHECKPOINT = BEST_MODEL_CHECKPOINT,
                                          config = get_best_model_config(),
                                          shard_count = SHARD_COUNT)
        print('Finished Processing test dataset!!!\n')

    Global seed set to 42
    Global seed set to 42
    Global seed set to 42
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/pytorch_lightning/trainer/connectors/accelerator_connector.py:297: LightningDeprecationWarning: Passing `Trainer(accelerator='dp')` has been deprecated in v1.5 and will be removed in v1.7. Use `Trainer(strategy='dp')` instead.
      rank_zero_deprecation(
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/pytorch_lightning/loops/utilities.py:91: PossibleUserWarning: `max_epochs` was not set. Setting it to 1000 epochs. To train without an epoch limit, set `max_epochs=-1`.
      rank_zero_warn(
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/pytorch_lightning/trainer/connectors/callback_connector.py:96: LightningDeprecationWarning: Setting `Trainer(progress_bar_refresh_rate=1)` is deprecated in v1.5 and will be removed in v1.7. Please pass `pytorch_lightning.callbacks.progress.TQDMProgressBar` with `refresh_rate` directly to the Trainer's `callbacks` argument instead. Or, to disable the progress bar pass `enable_progress_bar = False` to the Trainer.
      rank_zero_deprecation(
    GPU available: True, used: True
    TPU available: False, using: 0 TPU cores
    IPU available: False, using: 0 IPUs
    HPU available: False, using: 0 HPUs

    Processing valid dataset...

    LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7]

    test_dataloader: local rank : 0 shard count:  1

    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/fs_utils.py:88: FutureWarning: pyarrow.localfs is deprecated as of 2.0.0, please use pyarrow.fs.LocalFileSystem instead.
      self._filesystem = pyarrow.localfs
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/etl/dataset_metadata.py:402: FutureWarning: Specifying the 'metadata_nthreads' argument is deprecated as of pyarrow 8.0.0, and the argument will be removed in a future version
      dataset = pq.ParquetDataset(path_or_paths, filesystem=fs, validate_schema=False, metadata_nthreads=10)
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/etl/dataset_metadata.py:362: FutureWarning: 'ParquetDataset.common_metadata' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version.
      if not dataset.common_metadata:
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/reader.py:405: FutureWarning: Specifying the 'metadata_nthreads' argument is deprecated as of pyarrow 8.0.0, and the argument will be removed in a future version
      self.dataset = pq.ParquetDataset(dataset_path, filesystem=pyarrow_filesystem,
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/unischema.py:317: FutureWarning: 'ParquetDataset.pieces' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version. Specify 'use_legacy_dataset=False' while constructing the ParquetDataset, and then use the '.fragments' attribute instead.
      meta = parquet_dataset.pieces[0].get_metadata()
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/unischema.py:321: FutureWarning: 'ParquetDataset.partitions' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version. Specify 'use_legacy_dataset=False' while constructing the ParquetDataset, and then use the '.partitioning' attribute instead.
      for partition in (parquet_dataset.partitions or []):
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/etl/dataset_metadata.py:253: FutureWarning: 'ParquetDataset.metadata' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version.
      metadata = dataset.metadata
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/etl/dataset_metadata.py:254: FutureWarning: 'ParquetDataset.common_metadata' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version.
      common_metadata = dataset.common_metadata
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/etl/dataset_metadata.py:350: FutureWarning: 'ParquetDataset.pieces' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version. Specify 'use_legacy_dataset=False' while constructing the ParquetDataset, and then use the '.fragments' attribute instead.
      futures_list = [thread_pool.submit(_split_piece, piece, dataset.fs.open) for piece in dataset.pieces]
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/etl/dataset_metadata.py:350: FutureWarning: 'ParquetDataset.fs' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version. Specify 'use_legacy_dataset=False' while constructing the ParquetDataset, and then use the '.filesystem' attribute instead.
      futures_list = [thread_pool.submit(_split_piece, piece, dataset.fs.open) for piece in dataset.pieces]
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/etl/dataset_metadata.py:334: FutureWarning: ParquetDatasetPiece is deprecated as of pyarrow 5.0.0 and will be removed in a future version.
      return [pq.ParquetDatasetPiece(piece.path, open_file_func=fs_open,
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/arrow_reader_worker.py:138: FutureWarning: 'ParquetDataset.fs' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version. Specify 'use_legacy_dataset=False' while constructing the ParquetDataset, and then use the '.filesystem' attribute instead.
      parquet_file = ParquetFile(self._dataset.fs.open(piece.path))
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/arrow_reader_worker.py:286: FutureWarning: 'ParquetDataset.partitions' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version. Specify 'use_legacy_dataset=False' while constructing the ParquetDataset, and then use the '.partitioning' attribute instead.
      partition_names = self._dataset.partitions.partition_names if self._dataset.partitions else set()
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/arrow_reader_worker.py:289: FutureWarning: 'ParquetDataset.partitions' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version. Specify 'use_legacy_dataset=False' while constructing the ParquetDataset, and then use the '.partitioning' attribute instead.
      table = piece.read(columns=column_names - partition_names, partitions=self._dataset.partitions)

    Testing: 0it [00:00, ?it/s]

    Global seed set to 42
    Global seed set to 42
    Global seed set to 42
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/pytorch_lightning/trainer/connectors/accelerator_connector.py:297: LightningDeprecationWarning: Passing `Trainer(accelerator='dp')` has been deprecated in v1.5 and will be removed in v1.7. Use `Trainer(strategy='dp')` instead.
      rank_zero_deprecation(
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/pytorch_lightning/loops/utilities.py:91: PossibleUserWarning: `max_epochs` was not set. Setting it to 1000 epochs. To train without an epoch limit, set `max_epochs=-1`.
      rank_zero_warn(
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/pytorch_lightning/trainer/connectors/callback_connector.py:96: LightningDeprecationWarning: Setting `Trainer(progress_bar_refresh_rate=1)` is deprecated in v1.5 and will be removed in v1.7. Please pass `pytorch_lightning.callbacks.progress.TQDMProgressBar` with `refresh_rate` directly to the Trainer's `callbacks` argument instead. Or, to disable the progress bar pass `enable_progress_bar = False` to the Trainer.
      rank_zero_deprecation(
    GPU available: True, used: True
    TPU available: False, using: 0 TPU cores
    IPU available: False, using: 0 IPUs
    HPU available: False, using: 0 HPUs
    LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7]

    Consolidating predictions on GPU: 0
    Writing predictions on GPU: 0
    Finished Writing predictions on GPU: 0
    Finished Processing valid dataset!!!

    Processing test dataset...

    test_dataloader: local rank : 0 shard count:  1

    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/fs_utils.py:88: FutureWarning: pyarrow.localfs is deprecated as of 2.0.0, please use pyarrow.fs.LocalFileSystem instead.
      self._filesystem = pyarrow.localfs
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/etl/dataset_metadata.py:402: FutureWarning: Specifying the 'metadata_nthreads' argument is deprecated as of pyarrow 8.0.0, and the argument will be removed in a future version
      dataset = pq.ParquetDataset(path_or_paths, filesystem=fs, validate_schema=False, metadata_nthreads=10)
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/etl/dataset_metadata.py:362: FutureWarning: 'ParquetDataset.common_metadata' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version.
      if not dataset.common_metadata:
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/reader.py:405: FutureWarning: Specifying the 'metadata_nthreads' argument is deprecated as of pyarrow 8.0.0, and the argument will be removed in a future version
      self.dataset = pq.ParquetDataset(dataset_path, filesystem=pyarrow_filesystem,
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/unischema.py:317: FutureWarning: 'ParquetDataset.pieces' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version. Specify 'use_legacy_dataset=False' while constructing the ParquetDataset, and then use the '.fragments' attribute instead.
      meta = parquet_dataset.pieces[0].get_metadata()
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/unischema.py:321: FutureWarning: 'ParquetDataset.partitions' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version. Specify 'use_legacy_dataset=False' while constructing the ParquetDataset, and then use the '.partitioning' attribute instead.
      for partition in (parquet_dataset.partitions or []):
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/etl/dataset_metadata.py:253: FutureWarning: 'ParquetDataset.metadata' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version.
      metadata = dataset.metadata
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/etl/dataset_metadata.py:254: FutureWarning: 'ParquetDataset.common_metadata' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version.
      common_metadata = dataset.common_metadata
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/etl/dataset_metadata.py:350: FutureWarning: 'ParquetDataset.pieces' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version. Specify 'use_legacy_dataset=False' while constructing the ParquetDataset, and then use the '.fragments' attribute instead.
      futures_list = [thread_pool.submit(_split_piece, piece, dataset.fs.open) for piece in dataset.pieces]
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/etl/dataset_metadata.py:350: FutureWarning: 'ParquetDataset.fs' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version. Specify 'use_legacy_dataset=False' while constructing the ParquetDataset, and then use the '.filesystem' attribute instead.
      futures_list = [thread_pool.submit(_split_piece, piece, dataset.fs.open) for piece in dataset.pieces]
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/etl/dataset_metadata.py:334: FutureWarning: ParquetDatasetPiece is deprecated as of pyarrow 5.0.0 and will be removed in a future version.
      return [pq.ParquetDatasetPiece(piece.path, open_file_func=fs_open,
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/arrow_reader_worker.py:138: FutureWarning: 'ParquetDataset.fs' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version. Specify 'use_legacy_dataset=False' while constructing the ParquetDataset, and then use the '.filesystem' attribute instead.
      parquet_file = ParquetFile(self._dataset.fs.open(piece.path))
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/arrow_reader_worker.py:286: FutureWarning: 'ParquetDataset.partitions' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version. Specify 'use_legacy_dataset=False' while constructing the ParquetDataset, and then use the '.partitioning' attribute instead.
      partition_names = self._dataset.partitions.partition_names if self._dataset.partitions else set()
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/arrow_reader_worker.py:289: FutureWarning: 'ParquetDataset.partitions' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version. Specify 'use_legacy_dataset=False' while constructing the ParquetDataset, and then use the '.partitioning' attribute instead.
      table = piece.read(columns=column_names - partition_names, partitions=self._dataset.partitions)

    Testing: 0it [00:00, ?it/s]

    Consolidating predictions on GPU: 0
    Writing predictions on GPU: 0
    Finished Writing predictions on GPU: 0
    Finished Processing test dataset!!!

## 2. Valid Dataset Prediction Evaluation¶

In [17]:

    import scikitplot as skplt
    import matplotlib.pyplot as plt
    from sklearn.metrics import f1_score
    from sklearn.metrics import f1_score

    valid_gdf = pd.concat(map(pd.read_csv, glob.glob(f'{VALID_EVAL_OUTPUT_DIR}/*.csv')))
    valid_gdf['target'] = valid_gdf['target'].astype('int64')
    valid_gdf['case_id'] = valid_gdf['case_id'].astype('int64')
    valid_gdf = valid_gdf.drop_duplicates()
    valid_gdf

Out[17]:

| case_id | probability_0 | probability_1 | probability_2 | probability_3 | probability_4 | probability_5 | probability_6 | probability_7 | probability_8 | ... | probability_12 | probability_13 | probability_14 | probability_15 | probability_16 | probability_17 | probability_18 | probability_19 | prediction | target
--- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | ---
0 | 1247 | 0.000005 | 1.800549e-08 | 0.000003 | 4.978003e-08 | 0.000064 | 2.203251e-06 | 0.005333 | 2.795083e-05 | 1.925722e-04 | ... | 0.000002 | 0.002031 | 4.821588e-07 | 1.409102e-03 | 3.449435e-04 | 0.988479 | 1.574322e-06 | 1.628327e-05 | 17 | 17
1 | 1110 | 0.000119 | 7.097371e-07 | 0.000004 | 3.528850e-05 | 0.000015 | 9.364832e-05 | 0.009002 | 8.786424e-08 | 4.860365e-04 | ... | 0.000002 | 0.000006 | 1.400727e-08 | 9.884116e-01 | 6.448830e-08 | 0.000773 | 1.757000e-08 | 1.161904e-05 | 15 | 15
2 | 351 | 0.000169 | 7.811192e-05 | 0.000792 | 7.466219e-05 | 0.997487 | 5.877511e-06 | 0.000118 | 2.070772e-04 | 2.598430e-04 | ... | 0.000052 | 0.000087 | 5.438850e-05 | 6.434592e-06 | 4.706672e-07 | 0.000023 | 1.553872e-04 | 3.189330e-04 | 4 | 4
3 | 1395 | 0.000016 | 3.417864e-04 | 0.000757 | 3.401216e-07 | 0.000239 | 5.614206e-07 | 0.000307 | 1.133085e-05 | 9.270803e-07 | ... | 0.000057 | 0.001487 | 1.930174e-06 | 3.425110e-06 | 2.159835e-06 | 0.000021 | 1.211828e-04 | 9.958110e-01 | 19 | 19
4 | 381 | 0.000108 | 3.093974e-05 | 0.000460 | 8.585231e-05 | 0.997562 | 5.313093e-06 | 0.000141 | 2.703119e-04 | 5.565743e-04 | ... | 0.000048 | 0.000054 | 4.188575e-05 | 1.346345e-05 | 1.392741e-06 | 0.000141 | 3.198070e-04 | 6.577518e-05 | 4 | 4
... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ...
280 | 1073 | 0.000169 | 9.672062e-07 | 0.000003 | 1.123490e-04 | 0.000011 | 2.457128e-04 | 0.002427 | 6.079590e-08 | 1.181867e-03 | ... | 0.000004 | 0.000008 | 3.296374e-08 | 9.927385e-01 | 1.267222e-07 | 0.000573 | 2.726948e-08 | 1.277382e-05 | 15 | 15
281 | 365 | 0.000080 | 1.564584e-04 | 0.000504 | 4.725363e-05 | 0.996125 | 1.561526e-05 | 0.000101 | 8.384814e-04 | 3.465322e-04 | ... | 0.000068 | 0.000054 | 8.335247e-05 | 8.603818e-06 | 4.055886e-06 | 0.000059 | 1.224610e-03 | 1.735724e-04 | 4 | 4
282 | 506 | 0.000291 | 1.009728e-04 | 0.000156 | 1.160212e-08 | 0.000188 | 2.938335e-06 | 0.995613 | 8.970255e-04 | 2.384480e-06 | ... | 0.000001 | 0.000278 | 3.443917e-07 | 7.626945e-04 | 9.737359e-08 | 0.000789 | 8.984470e-07 | 5.664811e-04 | 6 | 6
283 | 1034 | 0.002187 | 1.162090e-04 | 0.000252 | 1.119654e-06 | 0.000431 | 3.867204e-03 | 0.000003 | 5.774761e-04 | 1.784824e-06 | ... | 0.000007 | 0.000812 | 9.912626e-01 | 1.386680e-07 | 2.834754e-05 | 0.000002 | 1.121123e-04 | 2.895197e-07 | 14 | 14
284 | 864 | 0.000095 | 1.690539e-03 | 0.000039 | 8.365983e-10 | 0.000019 | 8.274544e-06 | 0.000080 | 5.788982e-04 | 6.960399e-06 | ... | 0.000375 | 0.000426 | 5.692011e-05 | 5.004014e-05 | 4.330669e-04 | 0.000098 | 1.535336e-04 | 1.878347e-06 | 11 | 11

285 rows × 23 columns

In [18]:

    valid_gdf[valid_gdf.prediction == valid_gdf.target].count()

Out[18]:

    case_id           279
    probability_0     279
    probability_1     279
    probability_2     279
    probability_3     279
    probability_4     279
    probability_5     279
    probability_6     279
    probability_7     279
    probability_8     279
    probability_9     279
    probability_10    279
    probability_11    279
    probability_12    279
    probability_13    279
    probability_14    279
    probability_15    279
    probability_16    279
    probability_17    279
    probability_18    279
    probability_19    279
    prediction        279
    target            279
    dtype: int64

In [19]:

    valid_gdf['target'].min(), valid_gdf['prediction'].min(), valid_gdf['target'].max(), valid_gdf['prediction'].max()

Out[19]:

    (0, 0, 19, 19)

In [20]:

    skplt.metrics.plot_precision_recall(valid_gdf['target'].to_numpy(),
                                        valid_gdf[['probability_0', 'probability_1', 'probability_2', 'probability_3', 'probability_4',
                                                 'probability_5', 'probability_6', 'probability_7', 'probability_8', 'probability_9',
                                                 'probability_10', 'probability_11', 'probability_12', 'probability_13', 'probability_14',
                                                 'probability_15', 'probability_16', 'probability_17', 'probability_18', 'probability_19']].to_numpy(),
                                        cmap='nipy_spectral')
    plt.show()

In [21]:

    skplt.metrics.plot_roc(valid_gdf['target'].to_numpy(),
                           valid_gdf[['probability_0', 'probability_1', 'probability_2', 'probability_3', 'probability_4',
                                     'probability_5', 'probability_6', 'probability_7', 'probability_8', 'probability_9',
                                     'probability_10', 'probability_11', 'probability_12', 'probability_13', 'probability_14',
                                     'probability_15', 'probability_16', 'probability_17', 'probability_18', 'probability_19']].to_numpy(),
                           cmap='nipy_spectral')
    plt.show()

In [22]:

    f1_score(valid_gdf['target'], valid_gdf['prediction'], average='macro')

Out[22]:

    0.9714021164021164

In [23]:

    f1_score(valid_gdf['target'], valid_gdf['prediction'], average='weighted')

Out[23]:

    0.9790290541167734

## 3. Test Dataset Prediction Evaluation¶

In [24]:

    test_gdf = pd.concat(map(pd.read_csv, glob.glob(f'{TEST_EVAL_OUTPUT_DIR}/*.csv')))
    test_gdf['target'] = test_gdf['target'].astype('int64')
    test_gdf['case_id'] = test_gdf['case_id'].astype('int64')
    test_gdf = test_gdf.drop_duplicates()
    test_gdf

Out[24]:

| case_id | probability_0 | probability_1 | probability_2 | probability_3 | probability_4 | probability_5 | probability_6 | probability_7 | probability_8 | ... | probability_12 | probability_13 | probability_14 | probability_15 | probability_16 | probability_17 | probability_18 | probability_19 | prediction | target
--- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | ---
0 | 444 | 0.000172 | 1.037028e-03 | 3.117014e-07 | 2.910569e-05 | 1.233196e-04 | 0.993130 | 4.753550e-05 | 0.000491 | 6.611674e-06 | ... | 8.183048e-04 | 0.000013 | 6.615310e-04 | 3.307198e-04 | 7.027909e-04 | 1.605262e-05 | 1.643743e-03 | 1.048297e-06 | 5 | 5
1 | 1199 | 0.000001 | 2.401770e-06 | 1.267826e-07 | 2.715637e-08 | 1.103245e-06 | 0.000600 | 4.922082e-07 | 0.000297 | 2.269512e-05 | ... | 1.763938e-05 | 0.000535 | 3.357174e-04 | 2.024172e-06 | 9.950300e-01 | 1.306225e-03 | 3.319567e-04 | 1.265936e-06 | 16 | 16
2 | 1021 | 0.002151 | 1.027874e-04 | 3.858115e-04 | 7.276445e-07 | 3.449778e-04 | 0.001274 | 1.343417e-06 | 0.000218 | 1.322393e-06 | ... | 1.126214e-05 | 0.000730 | 9.941750e-01 | 1.073941e-07 | 2.081826e-05 | 1.054842e-06 | 6.369646e-05 | 1.030365e-07 | 14 | 14
3 | 124 | 0.001457 | 9.963020e-01 | 2.952667e-05 | 1.483841e-07 | 2.379206e-04 | 0.000143 | 1.144316e-04 | 0.000236 | 2.536245e-07 | ... | 4.243579e-04 | 0.000045 | 4.940667e-05 | 4.915080e-06 | 5.566135e-07 | 8.225066e-08 | 1.356605e-04 | 1.172750e-04 | 1 | 1
4 | 1140 | 0.000002 | 1.176415e-06 | 5.768387e-08 | 2.464314e-08 | 8.210303e-07 | 0.000179 | 5.727268e-07 | 0.000107 | 3.064495e-05 | ... | 1.448508e-05 | 0.000711 | 1.203556e-04 | 1.696867e-06 | 9.951472e-01 | 2.335096e-03 | 6.481477e-05 | 1.925988e-06 | 16 | 16
... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ...
1431 | 380 | 0.000063 | 2.033213e-04 | 5.279934e-04 | 3.418998e-05 | 9.952267e-01 | 0.000018 | 1.702243e-04 | 0.001261 | 2.150317e-04 | ... | 1.173532e-04 | 0.000084 | 6.206902e-05 | 9.797611e-06 | 3.771024e-06 | 4.759164e-05 | 1.340888e-03 | 3.791930e-04 | 4 | 4
1432 | 1266 | 0.000002 | 9.874934e-09 | 2.779127e-06 | 6.762707e-09 | 1.474600e-05 | 0.000001 | 4.113225e-03 | 0.000058 | 2.678453e-04 | ... | 2.632184e-07 | 0.001154 | 2.848131e-07 | 1.215115e-03 | 6.137536e-04 | 9.911638e-01 | 9.573644e-07 | 1.139139e-05 | 17 | 17
1433 | 1033 | 0.001047 | 4.295251e-05 | 2.014314e-04 | 8.089319e-07 | 1.393150e-04 | 0.001960 | 6.300509e-07 | 0.000104 | 6.722193e-07 | ... | 1.108413e-05 | 0.000680 | 9.952625e-01 | 8.005173e-08 | 1.938103e-05 | 6.571366e-07 | 6.498972e-05 | 7.892843e-08 | 14 | 14
1434 | 3 | 0.996562 | 8.724795e-04 | 1.061585e-05 | 2.483608e-04 | 2.525395e-04 | 0.000221 | 3.146714e-04 | 0.000005 | 1.422494e-04 | ... | 3.577650e-04 | 0.000582 | 3.027739e-04 | 5.406051e-05 | 4.300754e-07 | 4.417862e-06 | 1.964553e-08 | 1.859201e-06 | 0 | 0
1435 | 1048 | 0.001243 | 7.601092e-05 | 1.461502e-04 | 9.618224e-07 | 1.337476e-04 | 0.006343 | 1.056817e-06 | 0.000173 | 7.327389e-07 | ... | 1.600992e-05 | 0.000524 | 9.906502e-01 | 1.940328e-07 | 2.845857e-05 | 8.360539e-07 | 1.089045e-04 | 7.802863e-08 | 14 | 14

1436 rows × 23 columns

In [25]:

    test_gdf[test_gdf.prediction == test_gdf.target].count()

Out[25]:

    case_id           1416
    probability_0     1416
    probability_1     1416
    probability_2     1416
    probability_3     1416
    probability_4     1416
    probability_5     1416
    probability_6     1416
    probability_7     1416
    probability_8     1416
    probability_9     1416
    probability_10    1416
    probability_11    1416
    probability_12    1416
    probability_13    1416
    probability_14    1416
    probability_15    1416
    probability_16    1416
    probability_17    1416
    probability_18    1416
    probability_19    1416
    prediction        1416
    target            1416
    dtype: int64

In [26]:

    test_gdf['target'].min(), test_gdf['prediction'].min(), test_gdf['target'].max(), test_gdf['prediction'].max()

Out[26]:

    (0, 0, 19, 19)

In [27]:

    skplt.metrics.plot_precision_recall(test_gdf['target'].to_numpy(),
                                        test_gdf[['probability_0', 'probability_1', 'probability_2', 'probability_3', 'probability_4',
                                                 'probability_5', 'probability_6', 'probability_7', 'probability_8', 'probability_9',
                                                 'probability_10', 'probability_11', 'probability_12', 'probability_13', 'probability_14',
                                                 'probability_15', 'probability_16', 'probability_17', 'probability_18', 'probability_19']].to_numpy(),
                                        cmap='nipy_spectral')
    plt.show()

In [28]:

    skplt.metrics.plot_roc(test_gdf['target'].to_numpy(),
                           test_gdf[['probability_0', 'probability_1', 'probability_2', 'probability_3', 'probability_4',
                                     'probability_5', 'probability_6', 'probability_7', 'probability_8', 'probability_9',
                                     'probability_10', 'probability_11', 'probability_12', 'probability_13', 'probability_14',
                                     'probability_15', 'probability_16', 'probability_17', 'probability_18', 'probability_19']].to_numpy(),
                           cmap='nipy_spectral')
    plt.show()

In [29]:

    f1_score(test_gdf['target'], test_gdf['prediction'], average='macro')

Out[29]:

    0.9850644844697183

In [30]:

    f1_score(test_gdf['target'], test_gdf['prediction'], average='weighted')

Out[30]:

    0.9861184541805629

We shutdown the kernel!!!

In [31]:

    from nbdev import nbdev_export
    nbdev_export()