In [1]:

    #| default_exp model.optimization.nn.tsc.vittsc.insect_wingbeat_evaluation_mask
    %load_ext autoreload
    %autoreload 2

In [2]:

    # declare a list tasks whose products you want to use as inputs
    upstream = ['tabular_to_timeseries_insect_wingbeat', 'model_training_insect_wingbeat']

In [3]:

    # Parameters
    upstream = {"model_training_insect_wingbeat": {"nb": "/home/ubuntu/vitmtsc_nbdev/output/402_model.optimization.nn.tsc.vittsc.insect_wingbeat_training_mask_tune.html", "InsectWingbeat_MODEL_TUNE_OUTPUT": "/home/ubuntu/vitmtsc_nbdev/output/InsectWingbeat/ray_results", "InsectWingbeat_MODEL_TRAINING_OUTPUT": "/home/ubuntu/vitmtsc_nbdev/output/InsectWingbeat/experiments_result", "InsectWingbeat_MODEL_TRAINING_CHECKPOINT_OUTPUT": "/home/ubuntu/vitmtsc_nbdev/output/InsectWingbeat/experiments_result/checkpoint", "InsectWingbeat_BEST_MODEL": "/home/ubuntu/vitmtsc_nbdev/output/InsectWingbeat/experiments_result/best_model.ckpt", "InsectWingbeat_BEST_MODEL_CONFIG": "/home/ubuntu/vitmtsc_nbdev/output/InsectWingbeat/experiments_result/best_model_config.json"}, "tabular_to_timeseries_insect_wingbeat": {"nb": "/home/ubuntu/vitmtsc_nbdev/output/302_feature_preprocessing.insect_wingbeat.tabular_to_timeseries.html", "InsectWingbeat_TRAIN_MODEL_INPUT": "/home/ubuntu/vitmtsc_nbdev/output/InsectWingbeat/target_encoding-nn/train", "InsectWingbeat_VALID_MODEL_INPUT": "/home/ubuntu/vitmtsc_nbdev/output/InsectWingbeat/target_encoding-nn/valid", "InsectWingbeat_TEST_MODEL_INPUT": "/home/ubuntu/vitmtsc_nbdev/output/InsectWingbeat/target_encoding-nn/test"}}
    product = {"nb": "/home/ubuntu/vitmtsc_nbdev/output/502_model.optimization.nn.tsc.vittsc.insect_wingbeat_evaluation_mask.html", "InsectWingbeat_MODEL_VALID_EVAL_OUTPUT": "/home/ubuntu/vitmtsc_nbdev/output/InsectWingbeat/experiments_result/evaluation/valid", "InsectWingbeat_MODEL_TEST_EVAL_OUTPUT": "/home/ubuntu/vitmtsc_nbdev/output/InsectWingbeat/experiments_result/evaluation/test"}

In [4]:

    #| hide
    from nbdev.showdoc import *

In [5]:

    #| export
    import sys
    import pathlib as p

    def is_running_from_ipython():
        from IPython import get_ipython
        return get_ipython() is not None

    if not is_running_from_ipython() and __package__ is None:
        DIR = p.Path(__file__).resolve().parent
        sys.path.insert(0, str(DIR.parent))
        __package__ = DIR.name

In [6]:

    #| export
    from vitmtsc.model.optimization.nn.tsc.vittsc.insect_wingbeat_training_mask_tune import *

    class_weight: [0.4982561  0.50709939 0.4982561  0.50175615 0.49850449 0.49776008
     0.49627792 0.501002   0.50075113 0.5005005 ]

In [7]:

    #| export
    upstream = {
        "tabular_to_timeseries_insect_wingbeat": {
            "nb": "/home/ubuntu/vitmtsc_nbdev/output/302_feature_preprocessing.insect_wingbeat.tabular_to_timeseries.html",
            "InsectWingbeat_TRAIN_MODEL_INPUT": "/home/ubuntu/vitmtsc_nbdev/output/InsectWingbeat/target_encoding-nn/train",
            "InsectWingbeat_VALID_MODEL_INPUT": "/home/ubuntu/vitmtsc_nbdev/output/InsectWingbeat/target_encoding-nn/valid",
            "InsectWingbeat_TEST_MODEL_INPUT": "/home/ubuntu/vitmtsc_nbdev/output/InsectWingbeat/target_encoding-nn/test",
        },
        "model_training_insect_wingbeat": {
            "nb": "/home/ubuntu/vitmtsc_nbdev/output/402_model.optimization.nn.tsc.vittsc.insect_wingbeat_training_mask_tune.html",
            "InsectWingbeat_MODEL_TUNE_OUTPUT": "/home/ubuntu/vitmtsc_nbdev/output/InsectWingbeat/ray_results",
            "InsectWingbeat_MODEL_TRAINING_OUTPUT": "/home/ubuntu/vitmtsc_nbdev/output/InsectWingbeat/experiments_result",
            "InsectWingbeat_MODEL_TRAINING_CHECKPOINT_OUTPUT": "/home/ubuntu/vitmtsc_nbdev/output/InsectWingbeat/experiments_result/checkpoint",
            "InsectWingbeat_BEST_MODEL": "/home/ubuntu/vitmtsc_nbdev/output/InsectWingbeat/experiments_result/best_model.ckpt",
            "InsectWingbeat_BEST_MODEL_CONFIG": "/home/ubuntu/vitmtsc_nbdev/output/InsectWingbeat/experiments_result/best_model_config.json",
        },
    }
    product = {
        "nb": "/home/ubuntu/vitmtsc_nbdev/output/502_model.optimization.nn.tsc.vittsc.insect_wingbeat_evaluation_mask.html",
        "InsectWingbeat_MODEL_VALID_EVAL_OUTPUT": "/home/ubuntu/vitmtsc_nbdev/output/InsectWingbeat/experiments_result/evaluation/valid",
        "InsectWingbeat_MODEL_TEST_EVAL_OUTPUT": "/home/ubuntu/vitmtsc_nbdev/output/InsectWingbeat/experiments_result/evaluation/test",
    }

In [8]:

    # |export
    import json
    def get_best_model_config():
        with open(upstream['model_training_insect_wingbeat']['InsectWingbeat_BEST_MODEL_CONFIG'], 'r') as json_file:
            return json.load(json_file)

In [9]:

    #| export
    import pandas as pd
    import os
    import torch
    import math
    import glob

    import pytorch_lightning as pl
    from torch.nn import functional as F
    import matplotlib.pyplot as plt
    import scikitplot as skplt
    from pytorch_lightning import LightningModule
    from pytorch_lightning import Trainer
    from petastorm import make_batch_reader
    from petastorm.pytorch import DataLoader

## Vision Transformer for Multivariate Time-Series Classification (VitMTSC) Model with Masking - Evaluation¶

> Load Model

> Model Evaluation: Evaluate Model on test and validation dataset using PR-AUC

In [10]:

    #| export
    DATASET_NAME = 'InsectWingbeat'
    VALID_DATA_DIR = f"file://{upstream['tabular_to_timeseries_insect_wingbeat']['InsectWingbeat_VALID_MODEL_INPUT']}"
    TEST_DATA_DIR = f"file://{upstream['tabular_to_timeseries_insect_wingbeat']['InsectWingbeat_TEST_MODEL_INPUT']}"
    VALID_EVAL_OUTPUT_DIR = product['InsectWingbeat_MODEL_VALID_EVAL_OUTPUT']
    TEST_EVAL_OUTPUT_DIR = product['InsectWingbeat_MODEL_TEST_EVAL_OUTPUT']
    BEST_MODEL_CHECKPOINT = upstream['model_training_insect_wingbeat']['InsectWingbeat_BEST_MODEL']
    NUM_WORKERS=1
    SHARD_COUNT=1
    BATCH_SIZE = 64
    TOTAL_VALID_BATCHES = math.ceil(get_valid_dataset_size()/BATCH_SIZE)
    TOTAL_TEST_BATCHES = math.ceil(get_test_dataset_size()/BATCH_SIZE)
    BEST_MODEL_CHECKPOINT, TOTAL_VALID_BATCHES, TOTAL_TEST_BATCHES, VALID_DATA_DIR, TEST_DATA_DIR, VALID_EVAL_OUTPUT_DIR, TEST_EVAL_OUTPUT_DIR

Out[10]:

    ('/home/ubuntu/vitmtsc_nbdev/output/InsectWingbeat/experiments_result/best_model.ckpt',
     79,
     391,
     'file:///home/ubuntu/vitmtsc_nbdev/output/InsectWingbeat/target_encoding-nn/valid',
     'file:///home/ubuntu/vitmtsc_nbdev/output/InsectWingbeat/target_encoding-nn/test',
     '/home/ubuntu/vitmtsc_nbdev/output/InsectWingbeat/experiments_result/evaluation/valid',
     '/home/ubuntu/vitmtsc_nbdev/output/InsectWingbeat/experiments_result/evaluation/test')

In [11]:

    !mkdir -p $VALID_EVAL_OUTPUT_DIR
    !mkdir -p $TEST_EVAL_OUTPUT_DIR

## 1. VitMTSC Classification Prediction Task¶

In [12]:

    #| export
    class VitMTSCClassificationPredictionTask(LightningModule):
        def __init__(self,
                     model,
                     output_pred_dir,
                     input_data_dir,
                     batch_size=BATCH_SIZE,
                     num_workers=NUM_WORKERS,
                     shard_count = SHARD_COUNT):
            super().__init__()
            pl.seed_everything(42, workers=True)
            self.model = model
            self.case_id = []
            self.probability_0 = []
            self.probability_1 = []
            self.probability_2 = []
            self.probability_3 = []
            self.probability_4 = []
            self.probability_5 = []
            self.probability_6 = []
            self.probability_7 = []
            self.probability_8 = []
            self.probability_9 = []
            self.prediction = []
            self.target = []
            self.output_pred_dir = output_pred_dir
            self.input_data_dir = input_data_dir
            self.prediction_files = input_data_dir
            self.batch_size = batch_size
            self.num_workers = num_workers
            self.shard_count = shard_count

        def test_step(self, batch, batch_idx):
            x, y, case_id_1, mask = batch
            y_hat = self.model(x, mask)
            self.case_id.extend(case_id_1.to('cpu').numpy())
            self.probability_0.extend(F.softmax(y_hat, dim=1)[:,0].to('cpu').numpy())
            self.probability_1.extend(F.softmax(y_hat, dim=1)[:,1].to('cpu').numpy())
            self.probability_2.extend(F.softmax(y_hat, dim=1)[:,2].to('cpu').numpy())
            self.probability_3.extend(F.softmax(y_hat, dim=1)[:,3].to('cpu').numpy())
            self.probability_4.extend(F.softmax(y_hat, dim=1)[:,4].to('cpu').numpy())
            self.probability_5.extend(F.softmax(y_hat, dim=1)[:,5].to('cpu').numpy())
            self.probability_6.extend(F.softmax(y_hat, dim=1)[:,6].to('cpu').numpy())
            self.probability_7.extend(F.softmax(y_hat, dim=1)[:,7].to('cpu').numpy())
            self.probability_8.extend(F.softmax(y_hat, dim=1)[:,8].to('cpu').numpy())
            self.probability_9.extend(F.softmax(y_hat, dim=1)[:,9].to('cpu').numpy())
            self.prediction.extend(torch.max(y_hat.data, 1)[1].to('cpu').numpy())
            self.target.extend(y.to('cpu').numpy())

        def test_dataloader(self):
            print('test_dataloader: local rank :', int(os.environ['LOCAL_RANK']), 'shard count: ', self.shard_count)
            self.test_ds = make_batch_reader(self.prediction_files, workers_count=self.num_workers,
                                             cur_shard = int(os.environ['LOCAL_RANK']),
                                             shard_count = self.shard_count, num_epochs = 2)
            return DataLoader(self.test_ds, batch_size = self.batch_size, collate_fn= petastorm_collate_fn)

        def test_epoch_end(self, outputs):
            print('Consolidating predictions on GPU:', os.environ['LOCAL_RANK'])
            df_text_predictions = pd.DataFrame({'case_id': self.case_id,
                                                'probability_0': self.probability_0,
                                                'probability_1': self.probability_1,
                                                'probability_2': self.probability_2,
                                                'probability_3': self.probability_3,
                                                'probability_4': self.probability_4,
                                                'probability_5': self.probability_5,
                                                'probability_6': self.probability_6,
                                                'probability_7': self.probability_7,
                                                'probability_8': self.probability_8,
                                                'probability_9': self.probability_9,
                                                'prediction': self.prediction,
                                                'target': self.target
                                                })
            print('Writing predictions on GPU:', os.environ['LOCAL_RANK'])
            df_text_predictions.to_csv(self.output_pred_dir + "/" + os.environ['LOCAL_RANK'] + '_predictions.csv', index=False)
            print('Finished Writing predictions on GPU:', os.environ['LOCAL_RANK'])

In [13]:

    #| export
    def get_model_for_prediction(BEST_MODEL_CHECKPOINT, config, output_pred_dir, input_data_dir, shard_count = SHARD_COUNT):
        # load the best model
        pl.seed_everything(42, workers=True)
        model = VitTimeSeriesTransformer.load_from_checkpoint(BEST_MODEL_CHECKPOINT, config = config)
        model.eval()
        return VitMTSCClassificationPredictionTask(model = model, shard_count = shard_count, output_pred_dir = output_pred_dir, input_data_dir = input_data_dir)

In [14]:

    #| export
    def write_prediction_for_valid_dataset(BEST_MODEL_CHECKPOINT,
                                          config,
                                          shard_count,
                                          output_pred_dir = VALID_EVAL_OUTPUT_DIR,
                                          input_data_dir=VALID_DATA_DIR):
        pl.seed_everything(42, workers=True)
        model = get_model_for_prediction(BEST_MODEL_CHECKPOINT = BEST_MODEL_CHECKPOINT,
                                         config = config,
                                         shard_count = shard_count,
                                         output_pred_dir = output_pred_dir,
                                         input_data_dir = input_data_dir)
        trainer = Trainer(gpus = [0],
                          accelerator='dp',
                          progress_bar_refresh_rate=1,
                          limit_test_batches = TOTAL_VALID_BATCHES)
        trainer.test(model)

    def write_prediction_for_test_dataset(BEST_MODEL_CHECKPOINT,
                                          config,
                                          shard_count,
                                          output_pred_dir = TEST_EVAL_OUTPUT_DIR,
                                          input_data_dir=TEST_DATA_DIR):
        pl.seed_everything(42, workers=True)
        model = get_model_for_prediction(BEST_MODEL_CHECKPOINT = BEST_MODEL_CHECKPOINT,
                                         config = config,
                                         shard_count = shard_count,
                                         output_pred_dir = output_pred_dir,
                                         input_data_dir = input_data_dir)
        trainer = Trainer(gpus = [0],
                          accelerator='dp',
                          progress_bar_refresh_rate=1,
                          limit_test_batches = TOTAL_TEST_BATCHES)
        trainer.test(model)

In [15]:

    %env LOCAL_RANK=0

    env: LOCAL_RANK=0

In [16]:

    #| export
    if __name__ == "__main__":
        print('Processing valid dataset...\n')
        write_prediction_for_valid_dataset(BEST_MODEL_CHECKPOINT = BEST_MODEL_CHECKPOINT,
                                          config = get_best_model_config(),
                                          shard_count = SHARD_COUNT)
        print('Finished Processing valid dataset!!!\n')

        print('Processing test dataset...\n')
        write_prediction_for_test_dataset(BEST_MODEL_CHECKPOINT = BEST_MODEL_CHECKPOINT,
                                          config = get_best_model_config(),
                                          shard_count = SHARD_COUNT)
        print('Finished Processing test dataset!!!\n')

    Global seed set to 42
    Global seed set to 42
    Global seed set to 42
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/pytorch_lightning/trainer/connectors/accelerator_connector.py:297: LightningDeprecationWarning: Passing `Trainer(accelerator='dp')` has been deprecated in v1.5 and will be removed in v1.7. Use `Trainer(strategy='dp')` instead.
      rank_zero_deprecation(
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/pytorch_lightning/loops/utilities.py:91: PossibleUserWarning: `max_epochs` was not set. Setting it to 1000 epochs. To train without an epoch limit, set `max_epochs=-1`.
      rank_zero_warn(
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/pytorch_lightning/trainer/connectors/callback_connector.py:96: LightningDeprecationWarning: Setting `Trainer(progress_bar_refresh_rate=1)` is deprecated in v1.5 and will be removed in v1.7. Please pass `pytorch_lightning.callbacks.progress.TQDMProgressBar` with `refresh_rate` directly to the Trainer's `callbacks` argument instead. Or, to disable the progress bar pass `enable_progress_bar = False` to the Trainer.
      rank_zero_deprecation(
    GPU available: True, used: True
    TPU available: False, using: 0 TPU cores
    IPU available: False, using: 0 IPUs
    HPU available: False, using: 0 HPUs
    Missing logger folder: /home/ubuntu/vitmtsc_nbdev/lightning_logs

    Processing valid dataset...

    LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3]
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/fs_utils.py:88: FutureWarning: pyarrow.localfs is deprecated as of 2.0.0, please use pyarrow.fs.LocalFileSystem instead.
      self._filesystem = pyarrow.localfs
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/etl/dataset_metadata.py:402: FutureWarning: Specifying the 'metadata_nthreads' argument is deprecated as of pyarrow 8.0.0, and the argument will be removed in a future version
      dataset = pq.ParquetDataset(path_or_paths, filesystem=fs, validate_schema=False, metadata_nthreads=10)
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/etl/dataset_metadata.py:362: FutureWarning: 'ParquetDataset.common_metadata' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version.
      if not dataset.common_metadata:
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/reader.py:405: FutureWarning: Specifying the 'metadata_nthreads' argument is deprecated as of pyarrow 8.0.0, and the argument will be removed in a future version
      self.dataset = pq.ParquetDataset(dataset_path, filesystem=pyarrow_filesystem,
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/unischema.py:317: FutureWarning: 'ParquetDataset.pieces' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version. Specify 'use_legacy_dataset=False' while constructing the ParquetDataset, and then use the '.fragments' attribute instead.
      meta = parquet_dataset.pieces[0].get_metadata()
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/unischema.py:321: FutureWarning: 'ParquetDataset.partitions' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version. Specify 'use_legacy_dataset=False' while constructing the ParquetDataset, and then use the '.partitioning' attribute instead.
      for partition in (parquet_dataset.partitions or []):

    test_dataloader: local rank : 0 shard count:  1

    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/etl/dataset_metadata.py:253: FutureWarning: 'ParquetDataset.metadata' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version.
      metadata = dataset.metadata
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/etl/dataset_metadata.py:254: FutureWarning: 'ParquetDataset.common_metadata' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version.
      common_metadata = dataset.common_metadata
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/etl/dataset_metadata.py:350: FutureWarning: 'ParquetDataset.pieces' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version. Specify 'use_legacy_dataset=False' while constructing the ParquetDataset, and then use the '.fragments' attribute instead.
      futures_list = [thread_pool.submit(_split_piece, piece, dataset.fs.open) for piece in dataset.pieces]
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/etl/dataset_metadata.py:350: FutureWarning: 'ParquetDataset.fs' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version. Specify 'use_legacy_dataset=False' while constructing the ParquetDataset, and then use the '.filesystem' attribute instead.
      futures_list = [thread_pool.submit(_split_piece, piece, dataset.fs.open) for piece in dataset.pieces]
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/etl/dataset_metadata.py:334: FutureWarning: ParquetDatasetPiece is deprecated as of pyarrow 5.0.0 and will be removed in a future version.
      return [pq.ParquetDatasetPiece(piece.path, open_file_func=fs_open,
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/arrow_reader_worker.py:138: FutureWarning: 'ParquetDataset.fs' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version. Specify 'use_legacy_dataset=False' while constructing the ParquetDataset, and then use the '.filesystem' attribute instead.
      parquet_file = ParquetFile(self._dataset.fs.open(piece.path))

    Testing: 0it [00:00, ?it/s]

    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/arrow_reader_worker.py:286: FutureWarning: 'ParquetDataset.partitions' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version. Specify 'use_legacy_dataset=False' while constructing the ParquetDataset, and then use the '.partitioning' attribute instead.
      partition_names = self._dataset.partitions.partition_names if self._dataset.partitions else set()
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/arrow_reader_worker.py:289: FutureWarning: 'ParquetDataset.partitions' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version. Specify 'use_legacy_dataset=False' while constructing the ParquetDataset, and then use the '.partitioning' attribute instead.
      table = piece.read(columns=column_names - partition_names, partitions=self._dataset.partitions)
    Global seed set to 42
    Global seed set to 42
    Global seed set to 42
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/pytorch_lightning/trainer/connectors/accelerator_connector.py:297: LightningDeprecationWarning: Passing `Trainer(accelerator='dp')` has been deprecated in v1.5 and will be removed in v1.7. Use `Trainer(strategy='dp')` instead.
      rank_zero_deprecation(
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/pytorch_lightning/loops/utilities.py:91: PossibleUserWarning: `max_epochs` was not set. Setting it to 1000 epochs. To train without an epoch limit, set `max_epochs=-1`.
      rank_zero_warn(
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/pytorch_lightning/trainer/connectors/callback_connector.py:96: LightningDeprecationWarning: Setting `Trainer(progress_bar_refresh_rate=1)` is deprecated in v1.5 and will be removed in v1.7. Please pass `pytorch_lightning.callbacks.progress.TQDMProgressBar` with `refresh_rate` directly to the Trainer's `callbacks` argument instead. Or, to disable the progress bar pass `enable_progress_bar = False` to the Trainer.
      rank_zero_deprecation(
    GPU available: True, used: True
    TPU available: False, using: 0 TPU cores
    IPU available: False, using: 0 IPUs
    HPU available: False, using: 0 HPUs
    LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3]

    Consolidating predictions on GPU: 0
    Writing predictions on GPU: 0
    Finished Writing predictions on GPU: 0
    Finished Processing valid dataset!!!

    Processing test dataset...

    test_dataloader: local rank : 0 shard count:  1

    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/fs_utils.py:88: FutureWarning: pyarrow.localfs is deprecated as of 2.0.0, please use pyarrow.fs.LocalFileSystem instead.
      self._filesystem = pyarrow.localfs
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/etl/dataset_metadata.py:402: FutureWarning: Specifying the 'metadata_nthreads' argument is deprecated as of pyarrow 8.0.0, and the argument will be removed in a future version
      dataset = pq.ParquetDataset(path_or_paths, filesystem=fs, validate_schema=False, metadata_nthreads=10)
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/etl/dataset_metadata.py:362: FutureWarning: 'ParquetDataset.common_metadata' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version.
      if not dataset.common_metadata:
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/reader.py:405: FutureWarning: Specifying the 'metadata_nthreads' argument is deprecated as of pyarrow 8.0.0, and the argument will be removed in a future version
      self.dataset = pq.ParquetDataset(dataset_path, filesystem=pyarrow_filesystem,
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/unischema.py:317: FutureWarning: 'ParquetDataset.pieces' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version. Specify 'use_legacy_dataset=False' while constructing the ParquetDataset, and then use the '.fragments' attribute instead.
      meta = parquet_dataset.pieces[0].get_metadata()
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/unischema.py:321: FutureWarning: 'ParquetDataset.partitions' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version. Specify 'use_legacy_dataset=False' while constructing the ParquetDataset, and then use the '.partitioning' attribute instead.
      for partition in (parquet_dataset.partitions or []):
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/etl/dataset_metadata.py:253: FutureWarning: 'ParquetDataset.metadata' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version.
      metadata = dataset.metadata
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/etl/dataset_metadata.py:254: FutureWarning: 'ParquetDataset.common_metadata' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version.
      common_metadata = dataset.common_metadata
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/etl/dataset_metadata.py:350: FutureWarning: 'ParquetDataset.pieces' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version. Specify 'use_legacy_dataset=False' while constructing the ParquetDataset, and then use the '.fragments' attribute instead.
      futures_list = [thread_pool.submit(_split_piece, piece, dataset.fs.open) for piece in dataset.pieces]
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/etl/dataset_metadata.py:350: FutureWarning: 'ParquetDataset.fs' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version. Specify 'use_legacy_dataset=False' while constructing the ParquetDataset, and then use the '.filesystem' attribute instead.
      futures_list = [thread_pool.submit(_split_piece, piece, dataset.fs.open) for piece in dataset.pieces]
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/etl/dataset_metadata.py:334: FutureWarning: ParquetDatasetPiece is deprecated as of pyarrow 5.0.0 and will be removed in a future version.
      return [pq.ParquetDatasetPiece(piece.path, open_file_func=fs_open,
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/arrow_reader_worker.py:138: FutureWarning: 'ParquetDataset.fs' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version. Specify 'use_legacy_dataset=False' while constructing the ParquetDataset, and then use the '.filesystem' attribute instead.
      parquet_file = ParquetFile(self._dataset.fs.open(piece.path))

    Testing: 0it [00:00, ?it/s]

    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/arrow_reader_worker.py:286: FutureWarning: 'ParquetDataset.partitions' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version. Specify 'use_legacy_dataset=False' while constructing the ParquetDataset, and then use the '.partitioning' attribute instead.
      partition_names = self._dataset.partitions.partition_names if self._dataset.partitions else set()
    /home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/arrow_reader_worker.py:289: FutureWarning: 'ParquetDataset.partitions' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version. Specify 'use_legacy_dataset=False' while constructing the ParquetDataset, and then use the '.partitioning' attribute instead.
      table = piece.read(columns=column_names - partition_names, partitions=self._dataset.partitions)

    Consolidating predictions on GPU: 0
    Writing predictions on GPU: 0
    Finished Writing predictions on GPU: 0
    Finished Processing test dataset!!!

## 2. Valid Dataset Prediction Evaluation¶

In [17]:

    import scikitplot as skplt
    import matplotlib.pyplot as plt
    from sklearn.metrics import f1_score
    from sklearn.metrics import f1_score

    valid_gdf = pd.concat(map(pd.read_csv, glob.glob(f'{VALID_EVAL_OUTPUT_DIR}/*.csv')))
    valid_gdf['target'] = valid_gdf['target'].astype('int64')
    valid_gdf['case_id'] = valid_gdf['case_id'].astype('int64')
    valid_gdf = valid_gdf.drop_duplicates()
    valid_gdf

Out[17]:

| case_id | probability_0 | probability_1 | probability_2 | probability_3 | probability_4 | probability_5 | probability_6 | probability_7 | probability_8 | probability_9 | prediction | target
--- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | ---
0 | 17775 | 0.000719 | 0.000081 | 0.001260 | 0.000038 | 0.000173 | 0.007430 | 0.000055 | 0.964737 | 0.000136 | 0.025372 | 7 | 7
1 | 360 | 0.468183 | 0.000600 | 0.000996 | 0.000088 | 0.480209 | 0.000031 | 0.043181 | 0.000244 | 0.005694 | 0.000774 | 4 | 0
2 | 24655 | 0.000167 | 0.000821 | 0.000199 | 0.000097 | 0.000012 | 0.053536 | 0.000023 | 0.003885 | 0.001921 | 0.939340 | 9 | 9
3 | 13884 | 0.001054 | 0.003093 | 0.000853 | 0.000442 | 0.000078 | 0.322287 | 0.000234 | 0.042530 | 0.003628 | 0.625801 | 9 | 5
4 | 24342 | 0.000215 | 0.002035 | 0.001367 | 0.001076 | 0.000126 | 0.097914 | 0.000137 | 0.011127 | 0.001714 | 0.884288 | 9 | 9
... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ...
4995 | 4139 | 0.000015 | 0.985024 | 0.000271 | 0.000049 | 0.001111 | 0.007339 | 0.001797 | 0.000005 | 0.001028 | 0.003362 | 1 | 1
4996 | 7212 | 0.000070 | 0.000030 | 0.324991 | 0.659194 | 0.002170 | 0.001399 | 0.001001 | 0.000560 | 0.009179 | 0.001405 | 3 | 2
4997 | 367 | 0.853697 | 0.000081 | 0.001168 | 0.000137 | 0.138451 | 0.000033 | 0.003394 | 0.000576 | 0.001804 | 0.000660 | 0 | 0
4998 | 22976 | 0.000302 | 0.000394 | 0.004926 | 0.022799 | 0.000454 | 0.012415 | 0.000163 | 0.014769 | 0.006282 | 0.937496 | 9 | 9
4999 | 11262 | 0.181651 | 0.000111 | 0.159829 | 0.003066 | 0.569693 | 0.000253 | 0.079842 | 0.002178 | 0.001055 | 0.002321 | 4 | 4

5000 rows × 13 columns

In [18]:

    valid_gdf[valid_gdf.prediction == valid_gdf.target].count()

Out[18]:

    case_id          3069
    probability_0    3069
    probability_1    3069
    probability_2    3069
    probability_3    3069
    probability_4    3069
    probability_5    3069
    probability_6    3069
    probability_7    3069
    probability_8    3069
    probability_9    3069
    prediction       3069
    target           3069
    dtype: int64

In [19]:

    valid_gdf['target'].min(), valid_gdf['prediction'].min(), valid_gdf['target'].max(), valid_gdf['prediction'].max()

Out[19]:

    (0, 0, 9, 9)

In [20]:

    skplt.metrics.plot_precision_recall(valid_gdf['target'].to_numpy(),
                                        valid_gdf[['probability_0', 'probability_1', 'probability_2', 'probability_3', 'probability_4',
                                                 'probability_5', 'probability_6', 'probability_7', 'probability_8', 'probability_9']].to_numpy(),
                                        cmap='nipy_spectral')
    plt.show()

In [21]:

    skplt.metrics.plot_roc(valid_gdf['target'].to_numpy(),
                           valid_gdf[['probability_0', 'probability_1', 'probability_2', 'probability_3', 'probability_4',
                                     'probability_5', 'probability_6', 'probability_7', 'probability_8', 'probability_9']].to_numpy(),
                           cmap='nipy_spectral')
    plt.show()

In [22]:

    f1_score(valid_gdf['target'], valid_gdf['prediction'], average='macro')

Out[22]:

    0.6046931935637698

In [23]:

    f1_score(valid_gdf['target'], valid_gdf['prediction'], average='weighted')

Out[23]:

    0.6070342741372103

## 2. Test Dataset Prediction Evaluation¶

In [24]:

    test_gdf = pd.concat(map(pd.read_csv, glob.glob(f'{TEST_EVAL_OUTPUT_DIR}/*.csv')))
    test_gdf['target'] = test_gdf['target'].astype('int64')
    test_gdf['case_id'] = test_gdf['case_id'].astype('int64')
    test_gdf = test_gdf.drop_duplicates()
    test_gdf

Out[24]:

| case_id | probability_0 | probability_1 | probability_2 | probability_3 | probability_4 | probability_5 | probability_6 | probability_7 | probability_8 | probability_9 | prediction | target
--- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | ---
0 | 20752 | 0.001732 | 0.022110 | 0.001565 | 0.001264 | 0.004064 | 0.001369 | 0.000663 | 0.000510 | 0.951560 | 0.015163 | 8 | 8
1 | 23346 | 0.000964 | 0.000905 | 0.001626 | 0.000432 | 0.000118 | 0.105983 | 0.000124 | 0.168956 | 0.001822 | 0.719070 | 9 | 9
2 | 7044 | 0.002176 | 0.000140 | 0.959897 | 0.007124 | 0.003592 | 0.003555 | 0.017872 | 0.000406 | 0.000597 | 0.004641 | 2 | 2
3 | 21564 | 0.000743 | 0.000060 | 0.001555 | 0.004610 | 0.000130 | 0.000109 | 0.001190 | 0.000099 | 0.990819 | 0.000684 | 8 | 8
4 | 2522 | 0.000411 | 0.524795 | 0.001104 | 0.000343 | 0.005327 | 0.179365 | 0.002506 | 0.003562 | 0.010552 | 0.272036 | 1 | 1
... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ... | ...
24995 | 13941 | 0.002211 | 0.009192 | 0.021890 | 0.031116 | 0.001418 | 0.435406 | 0.005469 | 0.001047 | 0.013095 | 0.479155 | 9 | 5
24996 | 8571 | 0.001059 | 0.000021 | 0.297408 | 0.667079 | 0.001738 | 0.006222 | 0.001574 | 0.005202 | 0.007937 | 0.011760 | 3 | 3
24997 | 5055 | 0.001345 | 0.000084 | 0.899550 | 0.011500 | 0.034766 | 0.001306 | 0.049383 | 0.000604 | 0.000312 | 0.001150 | 2 | 2
24998 | 15300 | 0.023531 | 0.005317 | 0.137862 | 0.511946 | 0.112035 | 0.072465 | 0.057051 | 0.030369 | 0.008904 | 0.040517 | 3 | 6
24999 | 10680 | 0.169720 | 0.000415 | 0.011426 | 0.000279 | 0.509616 | 0.000158 | 0.302143 | 0.000715 | 0.004338 | 0.001191 | 4 | 4

25000 rows × 13 columns

In [25]:

    test_gdf[test_gdf.prediction == test_gdf.target].count()

Out[25]:

    case_id          15239
    probability_0    15239
    probability_1    15239
    probability_2    15239
    probability_3    15239
    probability_4    15239
    probability_5    15239
    probability_6    15239
    probability_7    15239
    probability_8    15239
    probability_9    15239
    prediction       15239
    target           15239
    dtype: int64

In [26]:

    test_gdf['target'].min(), test_gdf['prediction'].min(), test_gdf['target'].max(), test_gdf['prediction'].max()

Out[26]:

    (0, 0, 9, 9)

In [27]:

    skplt.metrics.plot_precision_recall(test_gdf['target'].to_numpy(),
                                        test_gdf[['probability_0', 'probability_1', 'probability_2', 'probability_3', 'probability_4',
                                                 'probability_5', 'probability_6', 'probability_7', 'probability_8', 'probability_9']].to_numpy(),
                                        cmap='nipy_spectral')
    plt.show()

In [28]:

    skplt.metrics.plot_roc(test_gdf['target'].to_numpy(),
                           test_gdf[['probability_0', 'probability_1', 'probability_2', 'probability_3', 'probability_4',
                                     'probability_5', 'probability_6', 'probability_7', 'probability_8', 'probability_9']].to_numpy(),
                           cmap='nipy_spectral')
    plt.show()

In [29]:

    f1_score(test_gdf['target'], test_gdf['prediction'], average='macro')

Out[29]:

    0.6031890154587815

In [30]:

    f1_score(test_gdf['target'], test_gdf['prediction'], average='weighted')

Out[30]:

    0.6031890154587815

In [31]:

    from nbdev import nbdev_export
    nbdev_export()