In [1]:
#| default_exp model.optimization.nn.tsc.vittsc.character_trajectories_evaluation_mask
%load_ext autoreload
%autoreload 2
In [2]:
# declare a list tasks whose products you want to use as inputs
upstream = ['tabular_to_timeseries_character_trajectories', 'model_training_character_trajectories']
In [3]:
# Parameters
upstream = {
    "tabular_to_timeseries_character_trajectories": {
        "nb": "/home/ubuntu/vitmtsc_nbdev/output/305_feature_preprocessing.character_trajectories.tabular_to_timeseries.html",
        "CharacterTrajectories_TRAIN_MODEL_INPUT": "/home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/target_encoding-nn/train",
        "CharacterTrajectories_VALID_MODEL_INPUT": "/home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/target_encoding-nn/valid",
        "CharacterTrajectories_TEST_MODEL_INPUT": "/home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/target_encoding-nn/test",
    },
    "model_training_character_trajectories": {
        "nb": "/home/ubuntu/vitmtsc_nbdev/output/405_model.optimization.nn.tsc.vittsc.character_trajectories_training_mask_tune.html",
        "CharacterTrajectories_MODEL_TUNE_OUTPUT": "/home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/ray_results",
        "CharacterTrajectories_MODEL_TRAINING_OUTPUT": "/home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/experiments_result",
        "CharacterTrajectories_MODEL_TRAINING_CHECKPOINT_OUTPUT": "/home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/experiments_result/checkpoint",
        "CharacterTrajectories_BEST_MODEL": "/home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/experiments_result/best_model.ckpt",
        "CharacterTrajectories_BEST_MODEL_CONFIG": "/home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/experiments_result/best_model_config.json",
    },
}
product = {
    "nb": "/home/ubuntu/vitmtsc_nbdev/output/505_model.optimization.nn.tsc.vittsc.character_trajectories_evaluation_mask.html",
    "CharacterTrajectories_MODEL_VALID_EVAL_OUTPUT": "/home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/experiments_result/evaluation/valid",
    "CharacterTrajectories_MODEL_TEST_EVAL_OUTPUT": "/home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/experiments_result/evaluation/test",
}
In [4]:
#| hide
from nbdev.showdoc import *
In [5]:
#| export
import sys
import pathlib as p

def is_running_from_ipython():
    from IPython import get_ipython
    return get_ipython() is not None

if not is_running_from_ipython() and __package__ is None:
    DIR = p.Path(__file__).resolve().parent
    sys.path.insert(0, str(DIR.parent))
    __package__ = DIR.name
In [6]:
#| export
from vitmtsc.model.optimization.nn.tsc.vittsc.character_trajectories_training_mask_tune import *

class_weight: [0.42425373 0.46598361 0.49868421 0.47375    0.40035211 0.51681818
 0.5685     0.41195652 0.55735294 0.53632075 0.52638889 0.52638889
 0.60478723 0.51681818 0.5685     0.58010204 0.48177966 0.5685
 0.45119048 0.45846774]
In [7]:
# |export
upstream = {
    "tabular_to_timeseries_character_trajectories": {
        "nb": "/home/ubuntu/vitmtsc_nbdev/output/305_feature_preprocessing.character_trajectories.tabular_to_timeseries.html",
        "CharacterTrajectories_TRAIN_MODEL_INPUT": "/home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/target_encoding-nn/train",
        "CharacterTrajectories_VALID_MODEL_INPUT": "/home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/target_encoding-nn/valid",
        "CharacterTrajectories_TEST_MODEL_INPUT": "/home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/target_encoding-nn/test",
    },
    "model_training_character_trajectories": {
        "nb": "/home/ubuntu/vitmtsc_nbdev/output/405_model.optimization.nn.tsc.vittsc.character_trajectories_training_mask_tune.html",
        "CharacterTrajectories_MODEL_TUNE_OUTPUT": "/home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/ray_results",
        "CharacterTrajectories_MODEL_TRAINING_OUTPUT": "/home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/experiments_result",
        "CharacterTrajectories_MODEL_TRAINING_CHECKPOINT_OUTPUT": "/home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/experiments_result/checkpoint",
        "CharacterTrajectories_BEST_MODEL": "/home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/experiments_result/best_model.ckpt",
        "CharacterTrajectories_BEST_MODEL_CONFIG": "/home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/experiments_result/best_model_config.json",
    },
}
product = {
    "nb": "/home/ubuntu/vitmtsc_nbdev/output/505_model.optimization.nn.tsc.vittsc.character_trajectories_evaluation_mask.html",
    "CharacterTrajectories_MODEL_VALID_EVAL_OUTPUT": "/home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/experiments_result/evaluation/valid",
    "CharacterTrajectories_MODEL_TEST_EVAL_OUTPUT": "/home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/experiments_result/evaluation/test",
}
In [8]:
# |export
import json
def get_best_model_config():
    with open(upstream['model_training_character_trajectories']['CharacterTrajectories_BEST_MODEL_CONFIG'], 'r') as json_file:
        return json.load(json_file)
In [9]:
#| export
import pandas as pd
import os
import torch
import math
import glob

import pytorch_lightning as pl
from torch.nn import functional as F
import matplotlib.pyplot as plt
import scikitplot as skplt
from pytorch_lightning import LightningModule
from pytorch_lightning import Trainer
from petastorm import make_batch_reader
from petastorm.pytorch import DataLoader

Vision Transformer for Multivariate Time-Series Classification (VitMTSC) Model with Masking - Evaluation¶

Load Model

Model Evaluation: Evaluate Model on test and validation dataset using PR-AUC

In [10]:
#| export
DATASET_NAME = 'CharacterTrajectories'
VALID_DATA_DIR = f"file://{upstream['tabular_to_timeseries_character_trajectories']['CharacterTrajectories_VALID_MODEL_INPUT']}"
TEST_DATA_DIR = f"file://{upstream['tabular_to_timeseries_character_trajectories']['CharacterTrajectories_TEST_MODEL_INPUT']}"
VALID_EVAL_OUTPUT_DIR = product['CharacterTrajectories_MODEL_VALID_EVAL_OUTPUT']
TEST_EVAL_OUTPUT_DIR = product['CharacterTrajectories_MODEL_TEST_EVAL_OUTPUT']
BEST_MODEL_CHECKPOINT = upstream['model_training_character_trajectories']['CharacterTrajectories_BEST_MODEL']
NUM_WORKERS=1
SHARD_COUNT=1
BATCH_SIZE = 64
TOTAL_VALID_BATCHES = math.ceil(get_valid_dataset_size()/BATCH_SIZE)
TOTAL_TEST_BATCHES = math.ceil(get_test_dataset_size()/BATCH_SIZE)
BEST_MODEL_CHECKPOINT, TOTAL_VALID_BATCHES, TOTAL_TEST_BATCHES, VALID_DATA_DIR, TEST_DATA_DIR, VALID_EVAL_OUTPUT_DIR, TEST_EVAL_OUTPUT_DIR
Out[10]:
('/home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/experiments_result/best_model.ckpt',
 5,
 23,
 'file:///home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/target_encoding-nn/valid',
 'file:///home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/target_encoding-nn/test',
 '/home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/experiments_result/evaluation/valid',
 '/home/ubuntu/vitmtsc_nbdev/output/CharacterTrajectories/experiments_result/evaluation/test')
In [11]:
!mkdir -p $VALID_EVAL_OUTPUT_DIR
!mkdir -p $TEST_EVAL_OUTPUT_DIR

1. VitMTSC Classification Prediction Task¶

In [12]:
#| export
class VitMTSCClassificationPredictionTask(LightningModule):
    def __init__(self, 
                 model,
                 output_pred_dir,
                 input_data_dir,
                 batch_size=BATCH_SIZE,
                 num_workers=NUM_WORKERS,
                 shard_count = SHARD_COUNT):
        super().__init__()
        pl.seed_everything(42, workers=True)
        self.model = model
        self.case_id = []
        self.probability_0 = []
        self.probability_1 = []
        self.probability_2 = []
        self.probability_3 = []
        self.probability_4 = []
        self.probability_5 = []
        self.probability_6 = []
        self.probability_7 = []
        self.probability_8 = []
        self.probability_9 = []
        self.probability_10 = []
        self.probability_11 = []
        self.probability_12 = []
        self.probability_13 = []
        self.probability_14 = []
        self.probability_15 = []
        self.probability_16 = []
        self.probability_17 = []
        self.probability_18 = []
        self.probability_19 = []
        self.prediction = []
        self.target = []
        self.output_pred_dir = output_pred_dir
        self.input_data_dir = input_data_dir
        self.prediction_files = input_data_dir
        self.batch_size = batch_size
        self.num_workers = num_workers
        self.shard_count = shard_count
       
    def test_step(self, batch, batch_idx):
        x, y, case_id_1, mask = batch
        y_hat = self.model(x, mask)
        self.case_id.extend(case_id_1.to('cpu').numpy())
        self.probability_0.extend(F.softmax(y_hat, dim=1)[:,0].to('cpu').numpy())
        self.probability_1.extend(F.softmax(y_hat, dim=1)[:,1].to('cpu').numpy())
        self.probability_2.extend(F.softmax(y_hat, dim=1)[:,2].to('cpu').numpy())
        self.probability_3.extend(F.softmax(y_hat, dim=1)[:,3].to('cpu').numpy())
        self.probability_4.extend(F.softmax(y_hat, dim=1)[:,4].to('cpu').numpy())
        self.probability_5.extend(F.softmax(y_hat, dim=1)[:,5].to('cpu').numpy())
        self.probability_6.extend(F.softmax(y_hat, dim=1)[:,6].to('cpu').numpy())
        self.probability_7.extend(F.softmax(y_hat, dim=1)[:,7].to('cpu').numpy())
        self.probability_8.extend(F.softmax(y_hat, dim=1)[:,8].to('cpu').numpy())
        self.probability_9.extend(F.softmax(y_hat, dim=1)[:,9].to('cpu').numpy())
        self.probability_10.extend(F.softmax(y_hat, dim=1)[:,10].to('cpu').numpy())
        self.probability_11.extend(F.softmax(y_hat, dim=1)[:,11].to('cpu').numpy())
        self.probability_12.extend(F.softmax(y_hat, dim=1)[:,12].to('cpu').numpy())
        self.probability_13.extend(F.softmax(y_hat, dim=1)[:,13].to('cpu').numpy())
        self.probability_14.extend(F.softmax(y_hat, dim=1)[:,14].to('cpu').numpy())
        self.probability_15.extend(F.softmax(y_hat, dim=1)[:,15].to('cpu').numpy())
        self.probability_16.extend(F.softmax(y_hat, dim=1)[:,16].to('cpu').numpy())
        self.probability_17.extend(F.softmax(y_hat, dim=1)[:,17].to('cpu').numpy())
        self.probability_18.extend(F.softmax(y_hat, dim=1)[:,18].to('cpu').numpy())
        self.probability_19.extend(F.softmax(y_hat, dim=1)[:,19].to('cpu').numpy())
        self.prediction.extend(torch.max(y_hat.data, 1)[1].to('cpu').numpy())
        self.target.extend(y.to('cpu').numpy())
        
    def test_dataloader(self):
        print('test_dataloader: local rank :', int(os.environ['LOCAL_RANK']), 'shard count: ', self.shard_count)
        self.test_ds = make_batch_reader(self.prediction_files, workers_count=self.num_workers, 
                                         cur_shard = int(os.environ['LOCAL_RANK']), 
                                         shard_count = self.shard_count, num_epochs = 2)
        return DataLoader(self.test_ds, batch_size = self.batch_size, collate_fn= petastorm_collate_fn) 
    
    def test_epoch_end(self, outputs):
        print('Consolidating predictions on GPU:', os.environ['LOCAL_RANK'])
        df_text_predictions = pd.DataFrame({'case_id': self.case_id, 
                                            'probability_0': self.probability_0, 
                                            'probability_1': self.probability_1, 
                                            'probability_2': self.probability_2, 
                                            'probability_3': self.probability_3, 
                                            'probability_4': self.probability_4, 
                                            'probability_5': self.probability_5, 
                                            'probability_6': self.probability_6, 
                                            'probability_7': self.probability_7, 
                                            'probability_8': self.probability_8, 
                                            'probability_9': self.probability_9,
                                            'probability_10': self.probability_10, 
                                            'probability_11': self.probability_11, 
                                            'probability_12': self.probability_12, 
                                            'probability_13': self.probability_13, 
                                            'probability_14': self.probability_14, 
                                            'probability_15': self.probability_15, 
                                            'probability_16': self.probability_16, 
                                            'probability_17': self.probability_17, 
                                            'probability_18': self.probability_18, 
                                            'probability_19': self.probability_19,
                                            'prediction': self.prediction,
                                            'target': self.target
                                            })
        print('Writing predictions on GPU:', os.environ['LOCAL_RANK'])
        df_text_predictions.to_csv(self.output_pred_dir + "/" + os.environ['LOCAL_RANK'] + '_predictions.csv', index=False)
        print('Finished Writing predictions on GPU:', os.environ['LOCAL_RANK'])
In [13]:
#| export
def get_model_for_prediction(BEST_MODEL_CHECKPOINT, config, output_pred_dir, input_data_dir, shard_count = SHARD_COUNT):
    # load the best model
    pl.seed_everything(42, workers=True)
    model = VitTimeSeriesTransformer.load_from_checkpoint(BEST_MODEL_CHECKPOINT, config = config)
    model.eval()
    return VitMTSCClassificationPredictionTask(model = model, shard_count = shard_count, output_pred_dir = output_pred_dir, input_data_dir = input_data_dir)
In [14]:
#| export   
def write_prediction_for_valid_dataset(BEST_MODEL_CHECKPOINT,
                                      config,
                                      shard_count,
                                      output_pred_dir = VALID_EVAL_OUTPUT_DIR,
                                      input_data_dir=VALID_DATA_DIR):
    pl.seed_everything(42, workers=True)
    model = get_model_for_prediction(BEST_MODEL_CHECKPOINT = BEST_MODEL_CHECKPOINT, 
                                     config = config,
                                     shard_count = shard_count, 
                                     output_pred_dir = output_pred_dir, 
                                     input_data_dir = input_data_dir)
    trainer = Trainer(gpus = [0], 
                      accelerator='dp', 
                      progress_bar_refresh_rate=1, 
                      limit_test_batches = TOTAL_VALID_BATCHES)
    trainer.test(model)
    
def write_prediction_for_test_dataset(BEST_MODEL_CHECKPOINT,
                                      config,
                                      shard_count,
                                      output_pred_dir = TEST_EVAL_OUTPUT_DIR,
                                      input_data_dir=TEST_DATA_DIR):
    pl.seed_everything(42, workers=True)
    model = get_model_for_prediction(BEST_MODEL_CHECKPOINT = BEST_MODEL_CHECKPOINT, 
                                     config = config,
                                     shard_count = shard_count, 
                                     output_pred_dir = output_pred_dir, 
                                     input_data_dir = input_data_dir)
    trainer = Trainer(gpus = [0], 
                      accelerator='dp', 
                      progress_bar_refresh_rate=1, 
                      limit_test_batches = TOTAL_TEST_BATCHES)
    trainer.test(model)
In [15]:
%env LOCAL_RANK=0

env: LOCAL_RANK=0
In [16]:
#| export
if __name__ == "__main__":
    print('Processing valid dataset...\n')
    write_prediction_for_valid_dataset(BEST_MODEL_CHECKPOINT = BEST_MODEL_CHECKPOINT, 
                                      config = get_best_model_config(),
                                      shard_count = SHARD_COUNT)
    print('Finished Processing valid dataset!!!\n')
    
    print('Processing test dataset...\n')
    write_prediction_for_test_dataset(BEST_MODEL_CHECKPOINT = BEST_MODEL_CHECKPOINT, 
                                      config = get_best_model_config(),
                                      shard_count = SHARD_COUNT)
    print('Finished Processing test dataset!!!\n')

Global seed set to 42
Global seed set to 42
Global seed set to 42
/home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/pytorch_lightning/trainer/connectors/accelerator_connector.py:297: LightningDeprecationWarning: Passing `Trainer(accelerator='dp')` has been deprecated in v1.5 and will be removed in v1.7. Use `Trainer(strategy='dp')` instead.
  rank_zero_deprecation(
/home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/pytorch_lightning/loops/utilities.py:91: PossibleUserWarning: `max_epochs` was not set. Setting it to 1000 epochs. To train without an epoch limit, set `max_epochs=-1`.
  rank_zero_warn(
/home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/pytorch_lightning/trainer/connectors/callback_connector.py:96: LightningDeprecationWarning: Setting `Trainer(progress_bar_refresh_rate=1)` is deprecated in v1.5 and will be removed in v1.7. Please pass `pytorch_lightning.callbacks.progress.TQDMProgressBar` with `refresh_rate` directly to the Trainer's `callbacks` argument instead. Or, to disable the progress bar pass `enable_progress_bar = False` to the Trainer.
  rank_zero_deprecation(
GPU available: True, used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs

Processing valid dataset...


LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7]

test_dataloader: local rank : 0 shard count:  1

/home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/fs_utils.py:88: FutureWarning: pyarrow.localfs is deprecated as of 2.0.0, please use pyarrow.fs.LocalFileSystem instead.
  self._filesystem = pyarrow.localfs
/home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/etl/dataset_metadata.py:402: FutureWarning: Specifying the 'metadata_nthreads' argument is deprecated as of pyarrow 8.0.0, and the argument will be removed in a future version
  dataset = pq.ParquetDataset(path_or_paths, filesystem=fs, validate_schema=False, metadata_nthreads=10)
/home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/etl/dataset_metadata.py:362: FutureWarning: 'ParquetDataset.common_metadata' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version.
  if not dataset.common_metadata:
/home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/reader.py:405: FutureWarning: Specifying the 'metadata_nthreads' argument is deprecated as of pyarrow 8.0.0, and the argument will be removed in a future version
  self.dataset = pq.ParquetDataset(dataset_path, filesystem=pyarrow_filesystem,
/home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/unischema.py:317: FutureWarning: 'ParquetDataset.pieces' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version. Specify 'use_legacy_dataset=False' while constructing the ParquetDataset, and then use the '.fragments' attribute instead.
  meta = parquet_dataset.pieces[0].get_metadata()
/home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/unischema.py:321: FutureWarning: 'ParquetDataset.partitions' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version. Specify 'use_legacy_dataset=False' while constructing the ParquetDataset, and then use the '.partitioning' attribute instead.
  for partition in (parquet_dataset.partitions or []):
/home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/etl/dataset_metadata.py:253: FutureWarning: 'ParquetDataset.metadata' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version.
  metadata = dataset.metadata
/home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/etl/dataset_metadata.py:254: FutureWarning: 'ParquetDataset.common_metadata' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version.
  common_metadata = dataset.common_metadata
/home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/etl/dataset_metadata.py:350: FutureWarning: 'ParquetDataset.pieces' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version. Specify 'use_legacy_dataset=False' while constructing the ParquetDataset, and then use the '.fragments' attribute instead.
  futures_list = [thread_pool.submit(_split_piece, piece, dataset.fs.open) for piece in dataset.pieces]
/home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/etl/dataset_metadata.py:350: FutureWarning: 'ParquetDataset.fs' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version. Specify 'use_legacy_dataset=False' while constructing the ParquetDataset, and then use the '.filesystem' attribute instead.
  futures_list = [thread_pool.submit(_split_piece, piece, dataset.fs.open) for piece in dataset.pieces]
/home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/etl/dataset_metadata.py:334: FutureWarning: ParquetDatasetPiece is deprecated as of pyarrow 5.0.0 and will be removed in a future version.
  return [pq.ParquetDatasetPiece(piece.path, open_file_func=fs_open,
/home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/arrow_reader_worker.py:138: FutureWarning: 'ParquetDataset.fs' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version. Specify 'use_legacy_dataset=False' while constructing the ParquetDataset, and then use the '.filesystem' attribute instead.
  parquet_file = ParquetFile(self._dataset.fs.open(piece.path))
/home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/arrow_reader_worker.py:286: FutureWarning: 'ParquetDataset.partitions' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version. Specify 'use_legacy_dataset=False' while constructing the ParquetDataset, and then use the '.partitioning' attribute instead.
  partition_names = self._dataset.partitions.partition_names if self._dataset.partitions else set()
/home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/arrow_reader_worker.py:289: FutureWarning: 'ParquetDataset.partitions' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version. Specify 'use_legacy_dataset=False' while constructing the ParquetDataset, and then use the '.partitioning' attribute instead.
  table = piece.read(columns=column_names - partition_names, partitions=self._dataset.partitions)

Testing: 0it [00:00, ?it/s]
Global seed set to 42
Global seed set to 42
Global seed set to 42
/home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/pytorch_lightning/trainer/connectors/accelerator_connector.py:297: LightningDeprecationWarning: Passing `Trainer(accelerator='dp')` has been deprecated in v1.5 and will be removed in v1.7. Use `Trainer(strategy='dp')` instead.
  rank_zero_deprecation(
/home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/pytorch_lightning/loops/utilities.py:91: PossibleUserWarning: `max_epochs` was not set. Setting it to 1000 epochs. To train without an epoch limit, set `max_epochs=-1`.
  rank_zero_warn(
/home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/pytorch_lightning/trainer/connectors/callback_connector.py:96: LightningDeprecationWarning: Setting `Trainer(progress_bar_refresh_rate=1)` is deprecated in v1.5 and will be removed in v1.7. Please pass `pytorch_lightning.callbacks.progress.TQDMProgressBar` with `refresh_rate` directly to the Trainer's `callbacks` argument instead. Or, to disable the progress bar pass `enable_progress_bar = False` to the Trainer.
  rank_zero_deprecation(
GPU available: True, used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs
LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7]

Consolidating predictions on GPU: 0
Writing predictions on GPU: 0
Finished Writing predictions on GPU: 0
Finished Processing valid dataset!!!

Processing test dataset...

test_dataloader: local rank : 0 shard count:  1

/home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/fs_utils.py:88: FutureWarning: pyarrow.localfs is deprecated as of 2.0.0, please use pyarrow.fs.LocalFileSystem instead.
  self._filesystem = pyarrow.localfs
/home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/etl/dataset_metadata.py:402: FutureWarning: Specifying the 'metadata_nthreads' argument is deprecated as of pyarrow 8.0.0, and the argument will be removed in a future version
  dataset = pq.ParquetDataset(path_or_paths, filesystem=fs, validate_schema=False, metadata_nthreads=10)
/home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/etl/dataset_metadata.py:362: FutureWarning: 'ParquetDataset.common_metadata' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version.
  if not dataset.common_metadata:
/home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/reader.py:405: FutureWarning: Specifying the 'metadata_nthreads' argument is deprecated as of pyarrow 8.0.0, and the argument will be removed in a future version
  self.dataset = pq.ParquetDataset(dataset_path, filesystem=pyarrow_filesystem,
/home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/unischema.py:317: FutureWarning: 'ParquetDataset.pieces' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version. Specify 'use_legacy_dataset=False' while constructing the ParquetDataset, and then use the '.fragments' attribute instead.
  meta = parquet_dataset.pieces[0].get_metadata()
/home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/unischema.py:321: FutureWarning: 'ParquetDataset.partitions' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version. Specify 'use_legacy_dataset=False' while constructing the ParquetDataset, and then use the '.partitioning' attribute instead.
  for partition in (parquet_dataset.partitions or []):
/home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/etl/dataset_metadata.py:253: FutureWarning: 'ParquetDataset.metadata' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version.
  metadata = dataset.metadata
/home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/etl/dataset_metadata.py:254: FutureWarning: 'ParquetDataset.common_metadata' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version.
  common_metadata = dataset.common_metadata
/home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/etl/dataset_metadata.py:350: FutureWarning: 'ParquetDataset.pieces' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version. Specify 'use_legacy_dataset=False' while constructing the ParquetDataset, and then use the '.fragments' attribute instead.
  futures_list = [thread_pool.submit(_split_piece, piece, dataset.fs.open) for piece in dataset.pieces]
/home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/etl/dataset_metadata.py:350: FutureWarning: 'ParquetDataset.fs' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version. Specify 'use_legacy_dataset=False' while constructing the ParquetDataset, and then use the '.filesystem' attribute instead.
  futures_list = [thread_pool.submit(_split_piece, piece, dataset.fs.open) for piece in dataset.pieces]
/home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/etl/dataset_metadata.py:334: FutureWarning: ParquetDatasetPiece is deprecated as of pyarrow 5.0.0 and will be removed in a future version.
  return [pq.ParquetDatasetPiece(piece.path, open_file_func=fs_open,
/home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/arrow_reader_worker.py:138: FutureWarning: 'ParquetDataset.fs' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version. Specify 'use_legacy_dataset=False' while constructing the ParquetDataset, and then use the '.filesystem' attribute instead.
  parquet_file = ParquetFile(self._dataset.fs.open(piece.path))
/home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/arrow_reader_worker.py:286: FutureWarning: 'ParquetDataset.partitions' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version. Specify 'use_legacy_dataset=False' while constructing the ParquetDataset, and then use the '.partitioning' attribute instead.
  partition_names = self._dataset.partitions.partition_names if self._dataset.partitions else set()
/home/ubuntu/anaconda3/envs/rapids-22.08_ploomber/lib/python3.8/site-packages/petastorm/arrow_reader_worker.py:289: FutureWarning: 'ParquetDataset.partitions' attribute is deprecated as of pyarrow 5.0.0 and will be removed in a future version. Specify 'use_legacy_dataset=False' while constructing the ParquetDataset, and then use the '.partitioning' attribute instead.
  table = piece.read(columns=column_names - partition_names, partitions=self._dataset.partitions)

Testing: 0it [00:00, ?it/s]
Consolidating predictions on GPU: 0
Writing predictions on GPU: 0
Finished Writing predictions on GPU: 0
Finished Processing test dataset!!!

2. Valid Dataset Prediction Evaluation¶

In [17]:
import scikitplot as skplt
import matplotlib.pyplot as plt
from sklearn.metrics import f1_score
from sklearn.metrics import f1_score

valid_gdf = pd.concat(map(pd.read_csv, glob.glob(f'{VALID_EVAL_OUTPUT_DIR}/*.csv')))
valid_gdf['target'] = valid_gdf['target'].astype('int64')
valid_gdf['case_id'] = valid_gdf['case_id'].astype('int64')
valid_gdf = valid_gdf.drop_duplicates()
valid_gdf
Out[17]:
case_id probability_0 probability_1 probability_2 probability_3 probability_4 probability_5 probability_6 probability_7 probability_8 ... probability_12 probability_13 probability_14 probability_15 probability_16 probability_17 probability_18 probability_19 prediction target
0 1247 0.000005 1.800549e-08 0.000003 4.978003e-08 0.000064 2.203251e-06 0.005333 2.795083e-05 1.925722e-04 ... 0.000002 0.002031 4.821588e-07 1.409102e-03 3.449435e-04 0.988479 1.574322e-06 1.628327e-05 17 17
1 1110 0.000119 7.097371e-07 0.000004 3.528850e-05 0.000015 9.364832e-05 0.009002 8.786424e-08 4.860365e-04 ... 0.000002 0.000006 1.400727e-08 9.884116e-01 6.448830e-08 0.000773 1.757000e-08 1.161904e-05 15 15
2 351 0.000169 7.811192e-05 0.000792 7.466219e-05 0.997487 5.877511e-06 0.000118 2.070772e-04 2.598430e-04 ... 0.000052 0.000087 5.438850e-05 6.434592e-06 4.706672e-07 0.000023 1.553872e-04 3.189330e-04 4 4
3 1395 0.000016 3.417864e-04 0.000757 3.401216e-07 0.000239 5.614206e-07 0.000307 1.133085e-05 9.270803e-07 ... 0.000057 0.001487 1.930174e-06 3.425110e-06 2.159835e-06 0.000021 1.211828e-04 9.958110e-01 19 19
4 381 0.000108 3.093974e-05 0.000460 8.585231e-05 0.997562 5.313093e-06 0.000141 2.703119e-04 5.565743e-04 ... 0.000048 0.000054 4.188575e-05 1.346345e-05 1.392741e-06 0.000141 3.198070e-04 6.577518e-05 4 4
... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ...
280 1073 0.000169 9.672062e-07 0.000003 1.123490e-04 0.000011 2.457128e-04 0.002427 6.079590e-08 1.181867e-03 ... 0.000004 0.000008 3.296374e-08 9.927385e-01 1.267222e-07 0.000573 2.726948e-08 1.277382e-05 15 15
281 365 0.000080 1.564584e-04 0.000504 4.725363e-05 0.996125 1.561526e-05 0.000101 8.384814e-04 3.465322e-04 ... 0.000068 0.000054 8.335247e-05 8.603818e-06 4.055886e-06 0.000059 1.224610e-03 1.735724e-04 4 4
282 506 0.000291 1.009728e-04 0.000156 1.160212e-08 0.000188 2.938335e-06 0.995613 8.970255e-04 2.384480e-06 ... 0.000001 0.000278 3.443917e-07 7.626945e-04 9.737359e-08 0.000789 8.984470e-07 5.664811e-04 6 6
283 1034 0.002187 1.162090e-04 0.000252 1.119654e-06 0.000431 3.867204e-03 0.000003 5.774761e-04 1.784824e-06 ... 0.000007 0.000812 9.912626e-01 1.386680e-07 2.834754e-05 0.000002 1.121123e-04 2.895197e-07 14 14
284 864 0.000095 1.690539e-03 0.000039 8.365983e-10 0.000019 8.274544e-06 0.000080 5.788982e-04 6.960399e-06 ... 0.000375 0.000426 5.692011e-05 5.004014e-05 4.330669e-04 0.000098 1.535336e-04 1.878347e-06 11 11

285 rows × 23 columns

In [18]:
valid_gdf[valid_gdf.prediction == valid_gdf.target].count()
Out[18]:
case_id           279
probability_0     279
probability_1     279
probability_2     279
probability_3     279
probability_4     279
probability_5     279
probability_6     279
probability_7     279
probability_8     279
probability_9     279
probability_10    279
probability_11    279
probability_12    279
probability_13    279
probability_14    279
probability_15    279
probability_16    279
probability_17    279
probability_18    279
probability_19    279
prediction        279
target            279
dtype: int64
In [19]:
valid_gdf['target'].min(), valid_gdf['prediction'].min(), valid_gdf['target'].max(), valid_gdf['prediction'].max()
Out[19]:
(0, 0, 19, 19)
In [20]:
skplt.metrics.plot_precision_recall(valid_gdf['target'].to_numpy(), 
                                    valid_gdf[['probability_0', 'probability_1', 'probability_2', 'probability_3', 'probability_4',
                                             'probability_5', 'probability_6', 'probability_7', 'probability_8', 'probability_9',
                                             'probability_10', 'probability_11', 'probability_12', 'probability_13', 'probability_14',
                                             'probability_15', 'probability_16', 'probability_17', 'probability_18', 'probability_19']].to_numpy(), 
                                    cmap='nipy_spectral')
plt.show()
In [21]:
skplt.metrics.plot_roc(valid_gdf['target'].to_numpy(), 
                       valid_gdf[['probability_0', 'probability_1', 'probability_2', 'probability_3', 'probability_4',
                                 'probability_5', 'probability_6', 'probability_7', 'probability_8', 'probability_9',
                                 'probability_10', 'probability_11', 'probability_12', 'probability_13', 'probability_14',
                                 'probability_15', 'probability_16', 'probability_17', 'probability_18', 'probability_19']].to_numpy(), 
                       cmap='nipy_spectral')
plt.show()
In [22]:
f1_score(valid_gdf['target'], valid_gdf['prediction'], average='macro')
Out[22]:
0.9714021164021164
In [23]:
f1_score(valid_gdf['target'], valid_gdf['prediction'], average='weighted')
Out[23]:
0.9790290541167734

3. Test Dataset Prediction Evaluation¶

In [24]:
test_gdf = pd.concat(map(pd.read_csv, glob.glob(f'{TEST_EVAL_OUTPUT_DIR}/*.csv')))
test_gdf['target'] = test_gdf['target'].astype('int64')
test_gdf['case_id'] = test_gdf['case_id'].astype('int64')
test_gdf = test_gdf.drop_duplicates()
test_gdf
Out[24]:
case_id probability_0 probability_1 probability_2 probability_3 probability_4 probability_5 probability_6 probability_7 probability_8 ... probability_12 probability_13 probability_14 probability_15 probability_16 probability_17 probability_18 probability_19 prediction target
0 444 0.000172 1.037028e-03 3.117014e-07 2.910569e-05 1.233196e-04 0.993130 4.753550e-05 0.000491 6.611674e-06 ... 8.183048e-04 0.000013 6.615310e-04 3.307198e-04 7.027909e-04 1.605262e-05 1.643743e-03 1.048297e-06 5 5
1 1199 0.000001 2.401770e-06 1.267826e-07 2.715637e-08 1.103245e-06 0.000600 4.922082e-07 0.000297 2.269512e-05 ... 1.763938e-05 0.000535 3.357174e-04 2.024172e-06 9.950300e-01 1.306225e-03 3.319567e-04 1.265936e-06 16 16
2 1021 0.002151 1.027874e-04 3.858115e-04 7.276445e-07 3.449778e-04 0.001274 1.343417e-06 0.000218 1.322393e-06 ... 1.126214e-05 0.000730 9.941750e-01 1.073941e-07 2.081826e-05 1.054842e-06 6.369646e-05 1.030365e-07 14 14
3 124 0.001457 9.963020e-01 2.952667e-05 1.483841e-07 2.379206e-04 0.000143 1.144316e-04 0.000236 2.536245e-07 ... 4.243579e-04 0.000045 4.940667e-05 4.915080e-06 5.566135e-07 8.225066e-08 1.356605e-04 1.172750e-04 1 1
4 1140 0.000002 1.176415e-06 5.768387e-08 2.464314e-08 8.210303e-07 0.000179 5.727268e-07 0.000107 3.064495e-05 ... 1.448508e-05 0.000711 1.203556e-04 1.696867e-06 9.951472e-01 2.335096e-03 6.481477e-05 1.925988e-06 16 16
... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ...
1431 380 0.000063 2.033213e-04 5.279934e-04 3.418998e-05 9.952267e-01 0.000018 1.702243e-04 0.001261 2.150317e-04 ... 1.173532e-04 0.000084 6.206902e-05 9.797611e-06 3.771024e-06 4.759164e-05 1.340888e-03 3.791930e-04 4 4
1432 1266 0.000002 9.874934e-09 2.779127e-06 6.762707e-09 1.474600e-05 0.000001 4.113225e-03 0.000058 2.678453e-04 ... 2.632184e-07 0.001154 2.848131e-07 1.215115e-03 6.137536e-04 9.911638e-01 9.573644e-07 1.139139e-05 17 17
1433 1033 0.001047 4.295251e-05 2.014314e-04 8.089319e-07 1.393150e-04 0.001960 6.300509e-07 0.000104 6.722193e-07 ... 1.108413e-05 0.000680 9.952625e-01 8.005173e-08 1.938103e-05 6.571366e-07 6.498972e-05 7.892843e-08 14 14
1434 3 0.996562 8.724795e-04 1.061585e-05 2.483608e-04 2.525395e-04 0.000221 3.146714e-04 0.000005 1.422494e-04 ... 3.577650e-04 0.000582 3.027739e-04 5.406051e-05 4.300754e-07 4.417862e-06 1.964553e-08 1.859201e-06 0 0
1435 1048 0.001243 7.601092e-05 1.461502e-04 9.618224e-07 1.337476e-04 0.006343 1.056817e-06 0.000173 7.327389e-07 ... 1.600992e-05 0.000524 9.906502e-01 1.940328e-07 2.845857e-05 8.360539e-07 1.089045e-04 7.802863e-08 14 14

1436 rows × 23 columns

In [25]:
test_gdf[test_gdf.prediction == test_gdf.target].count()
Out[25]:
case_id           1416
probability_0     1416
probability_1     1416
probability_2     1416
probability_3     1416
probability_4     1416
probability_5     1416
probability_6     1416
probability_7     1416
probability_8     1416
probability_9     1416
probability_10    1416
probability_11    1416
probability_12    1416
probability_13    1416
probability_14    1416
probability_15    1416
probability_16    1416
probability_17    1416
probability_18    1416
probability_19    1416
prediction        1416
target            1416
dtype: int64
In [26]:
test_gdf['target'].min(), test_gdf['prediction'].min(), test_gdf['target'].max(), test_gdf['prediction'].max()
Out[26]:
(0, 0, 19, 19)
In [27]:
skplt.metrics.plot_precision_recall(test_gdf['target'].to_numpy(), 
                                    test_gdf[['probability_0', 'probability_1', 'probability_2', 'probability_3', 'probability_4',
                                             'probability_5', 'probability_6', 'probability_7', 'probability_8', 'probability_9',
                                             'probability_10', 'probability_11', 'probability_12', 'probability_13', 'probability_14',
                                             'probability_15', 'probability_16', 'probability_17', 'probability_18', 'probability_19']].to_numpy(), 
                                    cmap='nipy_spectral')
plt.show()
In [28]:
skplt.metrics.plot_roc(test_gdf['target'].to_numpy(), 
                       test_gdf[['probability_0', 'probability_1', 'probability_2', 'probability_3', 'probability_4',
                                 'probability_5', 'probability_6', 'probability_7', 'probability_8', 'probability_9',
                                 'probability_10', 'probability_11', 'probability_12', 'probability_13', 'probability_14',
                                 'probability_15', 'probability_16', 'probability_17', 'probability_18', 'probability_19']].to_numpy(), 
                       cmap='nipy_spectral')
plt.show()
In [29]:
f1_score(test_gdf['target'], test_gdf['prediction'], average='macro')
Out[29]:
0.9850644844697183
In [30]:
f1_score(test_gdf['target'], test_gdf['prediction'], average='weighted')
Out[30]:
0.9861184541805629

We shutdown the kernel!!!

In [31]:
from nbdev import nbdev_export
nbdev_export()