{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/taeuk/anaconda3/envs/aif/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/home/taeuk/anaconda3/envs/aif/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/home/taeuk/anaconda3/envs/aif/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/home/taeuk/anaconda3/envs/aif/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/home/taeuk/anaconda3/envs/aif/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/home/taeuk/anaconda3/envs/aif/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "/home/taeuk/anaconda3/envs/aif/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/home/taeuk/anaconda3/envs/aif/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/home/taeuk/anaconda3/envs/aif/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/home/taeuk/anaconda3/envs/aif/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/home/taeuk/anaconda3/envs/aif/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/home/taeuk/anaconda3/envs/aif/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "Using TensorFlow backend.\n",
      "/home/taeuk/anaconda3/envs/aif/lib/python3.6/site-packages/sklearn/utils/linear_assignment_.py:22: FutureWarning: The linear_assignment_ module is deprecated in 0.21 and will be removed from 0.23. Use scipy.optimize.linear_sum_assignment instead.\n",
      "  FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "#!/usr/bin/env python\n",
    "# coding: utf-8\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "#------ Load necessary packages ------#\n",
    "import sys\n",
    "import os\n",
    "import numpy as np\n",
    "# import pandas as pd\n",
    "#from common_utils import compute_metrics\n",
    "sys.path.append(\"../\")\n",
    "\n",
    "# load datasets\n",
    "from aif360.datasets import AdultDataset, GermanDataset, BankDataset, CompasDataset, BinaryLabelDataset, CelebADataset\n",
    "\n",
    "# load metrics\n",
    "from aif360.metrics import BinaryLabelDatasetMetric\n",
    "from aif360.metrics import ClassificationMetric\n",
    "# from aif360.metrics.utils import compute_boolean_conditioning_vector\n",
    "\n",
    "# load preprocessing algorithm\n",
    "from aif360.algorithms.preprocessing.optim_preproc_helpers.data_preproc_functions import load_preproc_data_compas\n",
    "# from aif360.algorithms.preprocessing.optim_preproc_helpers.data_preproc_functions import load_preproc_data_adult\n",
    "# from aif360.algorithms.preprocessing.optim_preproc_helpers.data_preproc_functions import load_preproc_data_german\n",
    "# from aif360.algorithms.preprocessing.optim_preproc_helpers.opt_tools import OptTools\n",
    "# from aif360.algorithms.preprocessing.optim_preproc_helpers.distortion_functions import get_distortion_adult\n",
    "\n",
    "# load algorithms\n",
    "from aif360.algorithms.preprocessing import DisparateImpactRemover\n",
    "from aif360.algorithms.preprocessing.lfr import LFR\n",
    "from aif360.algorithms.preprocessing.optim_preproc import OptimPreproc\n",
    "from aif360.algorithms.preprocessing.reweighing import Reweighing\n",
    "from aif360.algorithms.inprocessing.prejudice_remover import PrejudiceRemover\n",
    "from aif360.algorithms.inprocessing.adversarial_debiasing import AdversarialDebiasing\n",
    "from aif360.algorithms.postprocessing.calibrated_eq_odds_postprocessing import CalibratedEqOddsPostprocessing\n",
    "from aif360.algorithms.postprocessing.reject_option_classification import RejectOptionClassification\n",
    "import keras\n",
    "# load other packages\n",
    "import tensorflow as tf\n",
    "from tqdm import tqdm\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.preprocessing import scale, StandardScaler, MaxAbsScaler\n",
    "from sklearn.metrics import accuracy_score\n",
    "from function_1 import *\n",
    "from fair_main_update1 import *\n",
    "#import fair_bak\n",
    "import fair_ablation\n",
    "import easydict \n",
    "import argparse\n",
    "from plot import plot\n",
    "\n",
    "import pandas as pd\n",
    "# from sklearn.metrics import classification_report\n",
    "# from IPython.display import Markdown, display"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "args = easydict.EasyDict({ \"dataset\": 'celebA', \"senstive_feature\": 0, \"batch_size\": 64, \"repeat\": 1, \"gpu_id\": '3', \"epochs\": 5000, \"network\" : 'vgg16' })\n",
    "\n",
    "dataset_used = args.dataset\n",
    "protected_attribute_used = args.senstive_feature\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_id\n",
    "\n",
    "if dataset_used == \"adult\":\n",
    "    dataset_orig = AdultDataset()\n",
    "    # dataset_orig = load_preproc_data_adult()\n",
    "    if protected_attribute_used == 1:\n",
    "        privileged_groups = [{'sex': 1}]\n",
    "        unprivileged_groups = [{'sex': 0}]\n",
    "        sens_attr = 'sex'\n",
    "    else:\n",
    "        privileged_groups = [{'race': 1}]\n",
    "        unprivileged_groups = [{'race': 0}]\n",
    "        sens_attr = 'race'\n",
    "\n",
    "elif dataset_used == \"german\":\n",
    "    dataset_orig = GermanDataset()\n",
    "    # dataset_orig.labels = dataset_orig.labels-1\n",
    "    if protected_attribute_used == 1:\n",
    "        privileged_groups = [{'sex': 1}]\n",
    "        unprivileged_groups = [{'sex': 0}]\n",
    "        sens_attr = 'sex'\n",
    "    else:\n",
    "        privileged_groups = [{'age': 1}]\n",
    "        unprivileged_groups = [{'age': 0}]\n",
    "        sens_attr = 'age'\n",
    "\n",
    "elif dataset_used == \"compas\":\n",
    "    dataset_orig = CompasDataset()\n",
    "    # dataset_orig = load_preproc_data_compas()\n",
    "    if protected_attribute_used == 1:\n",
    "        privileged_groups = [{'sex': 1}]\n",
    "        unprivileged_groups = [{'sex': 0}]\n",
    "        sens_attr = 'sex'\n",
    "    else:\n",
    "        privileged_groups = [{'race': 1}]\n",
    "        unprivileged_groups = [{'race': 0}]\n",
    "        sens_attr = 'race'\n",
    "\n",
    "elif dataset_used == \"bank\":\n",
    "    dataset_orig = BankDataset()\n",
    "    if protected_attribute_used == 1:\n",
    "        privileged_groups = [{'age': 1}]\n",
    "        unprivileged_groups = [{'age': 0}]\n",
    "        sens_attr = 'age'\n",
    "    else:\n",
    "        privileged_groups = [{'age': 1}]\n",
    "        unprivileged_groups = [{'age': 0}]\n",
    "        sens_attr = 'age'\n",
    "        \n",
    "elif dataset_used == 'celebA':\n",
    "    dataset_orig = CelebADataset(args.network)\n",
    "    privileged_groups = [{'gender': 0}]\n",
    "    unprivileged_groups = [{'gender': 1}]\n",
    "    sens_attr = 'gender'\n",
    "    dataset_orig.features[:,0:2048] += 1e-8 # male = 2, female = 1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_used = 'celebA'\n",
    "dataset_orig = CelebADataset('vgg16')\n",
    "privileged_groups = [{'gender': 0}]\n",
    "unprivileged_groups = [{'gender': 1}]\n",
    "sens_attr = 'gender'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "        \n",
    "sens_idx = (dataset_orig.feature_names).index(sens_attr)\n",
    "\n",
    "min_max_scaler = MaxAbsScaler()\n",
    "dataset_orig.features[:,:-1] = min_max_scaler.fit_transform(dataset_orig.features)[:,:-1]\n",
    "\n",
    "dataset_orig_train, dataset_orig_vt = dataset_orig.split([0.7], shuffle=True)\n",
    "dataset_orig_valid, dataset_orig_test = dataset_orig_vt.split([0.5], shuffle=True)\n",
    "\n",
    "repeat = args.repeat\n",
    "\n",
    "total_acc = np.zeros((11, repeat))\n",
    "total_balanced_acc = np.zeros((11, repeat))\n",
    "total_disimpact = np.zeros((11, repeat))\n",
    "total_eqopp_diff = np.zeros((11, repeat))\n",
    "\n",
    "total_aveodds_diff = np.zeros((11, repeat))\n",
    "total_theil_idx = np.zeros((11, repeat))\n",
    "stat_parity_diff = np.zeros((11, repeat))\n",
    "\n",
    "total_tpr = np.zeros((11, repeat))\n",
    "total_tpr_priv = np.zeros((11, repeat))\n",
    "total_tpr_unpriv = np.zeros((11, repeat))\n",
    "\n",
    "total_fpr = np.zeros((11, repeat))\n",
    "total_fpr_priv = np.zeros((11, repeat))\n",
    "total_fpr_unpriv = np.zeros((11, repeat))\n",
    "total_fpr_diff = np.zeros((11, repeat))\n",
    "\n",
    "total_acc_priv = np.zeros((11, repeat))\n",
    "total_acc_unpriv = np.zeros((11, repeat))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.01\n",
      "0.001\n",
      "WARNING:tensorflow:From /home/taeuk/anaconda3/envs/aif/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py:3313: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "WARNING:tensorflow:From /home/taeuk/anaconda3/envs/aif/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py:422: The name tf.global_variables is deprecated. Please use tf.compat.v1.global_variables instead.\n",
      "\n",
      "\n",
      "\n",
      " best model saved \n",
      "\n",
      " \n",
      "Epoch: 1900, tot_acc: 0.734375, pred_acc: 0.71875, diff_acc: 0.796875, pred_loss: 1.0769, diff_loss: 1.0142, acc: 0.4688, TPR female: 0.9627, TPR male: 0.8208, Eq_opp: 0.1419\n",
      "\n",
      "\n",
      " best model saved \n",
      "\n",
      " \n",
      "Epoch: 2700, tot_acc: 0.734375, pred_acc: 0.78125, diff_acc: 0.46875, pred_loss: 0.6549, diff_loss: 1.3092, acc: 0.5, TPR female: 0.9742, TPR male: 0.8758, Eq_opp: 0.0984\n",
      "FAIAS\n",
      "[0.9440553]\n",
      "[0.84977487]\n",
      "[0.97145236]\n",
      "[-0.12167749]\n",
      "\n",
      "[0.36198791]\n",
      "[0.2214074]\n",
      "[0.58644743]\n",
      "[-0.36504004]\n",
      "\n",
      "[0.7941052]\n",
      "[0.79823769]\n",
      "[0.79116013]\n",
      "[0.00707755]\n",
      "\n",
      "\n",
      "[0.46612813]\n",
      "[-0.12167749]\n",
      "[-0.24335877]\n",
      "[0.07528065]\n",
      "[-0.45220722]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "config = tf.ConfigProto()\n",
    "config.gpu_options.allow_growth = True\n",
    "sess = tf.Session(config = config)\n",
    "sens_idx = (dataset_orig_train.feature_names).index(sens_attr)\n",
    "\n",
    "\n",
    "tmp = dataset_orig_train.labels.astype(int)\n",
    "if dataset_used == \"german\":\n",
    "    tmp = tmp - 1\n",
    "targets = tmp.reshape(-1).tolist()\n",
    "y_train = np.eye(np.unique(tmp).shape[0])[targets]\n",
    "\n",
    "tmp = dataset_orig_valid.labels.astype(int)\n",
    "if dataset_used == \"german\":\n",
    "    tmp = tmp - 1\n",
    "targets = tmp.reshape(-1).tolist()\n",
    "y_valid = np.eye(np.unique(tmp).shape[0])[targets]\n",
    "\n",
    "tmp = dataset_orig_test.labels.astype(int)\n",
    "if dataset_used == \"german\":\n",
    "    tmp = tmp - 1\n",
    "targets = tmp.reshape(-1).tolist()\n",
    "y_test = np.eye(np.unique(tmp).shape[0])[targets]\n",
    "\n",
    "\n",
    "# for lr in [1e-2, 1e-3, 1e-4, 1e-5]:\n",
    "#     for lr_t in [1e-2, 1e-3, 1e-4, 1e-5]:\n",
    "        \n",
    "lr = 1e-2\n",
    "lr_t = 1e-3\n",
    "print(lr)\n",
    "print(lr_t)\n",
    "\n",
    "dataset_orig_test.features[:,-1]+=1\n",
    "dataset_orig_train.features[:,-1]+=1\n",
    "dataset_orig_valid.features[:,-1]+=1\n",
    "\n",
    "outpred = fair_shapley(dataset_orig_train.features, y_train, dataset_orig_valid.features, y_valid, dataset_orig_test.features, \\\n",
    "                       sens_idx, sess, args.batch_size, args.epochs, lr, lr_t)\n",
    "\n",
    "dataset_orig_test.features[:,-1]-=1\n",
    "dataset_orig_train.features[:,-1]-=1\n",
    "dataset_orig_valid.features[:,-1]-=1\n",
    "\n",
    "\n",
    "dataset_pred = dataset_orig_test.copy()\n",
    "\n",
    "dataset_pred.labels = np.argmax(outpred, axis=1)\n",
    "\n",
    "\n",
    "classified_metric = ClassificationMetric(dataset_orig_test,\n",
    "                                     dataset_pred,\n",
    "                                     unprivileged_groups=unprivileged_groups,\n",
    "                                     privileged_groups=privileged_groups)\n",
    "metric_pred = BinaryLabelDatasetMetric(dataset_pred,\n",
    "                                   unprivileged_groups=unprivileged_groups,\n",
    "                                   privileged_groups=privileged_groups)\n",
    "\n",
    "total_acc[9, 0] = classified_metric.accuracy()\n",
    "total_balanced_acc[9, 0] = 0.5 * (classified_metric.true_positive_rate() +\n",
    "                                     classified_metric.true_negative_rate())\n",
    "total_disimpact[9, 0] = metric_pred.disparate_impact()\n",
    "total_eqopp_diff[9, 0] = classified_metric.equal_opportunity_difference()\n",
    "total_aveodds_diff[9, 0] = classified_metric.average_odds_difference()\n",
    "total_theil_idx[9, 0] = classified_metric.theil_index()\n",
    "stat_parity_diff[9, 0] = classified_metric.statistical_parity_difference()\n",
    "total_tpr[9, 0] = classified_metric.recall()\n",
    "\n",
    "\n",
    "#total_tpr[9, 0] = classified_metric_orig_test.recall()\n",
    "total_fpr_diff[9, 0] = classified_metric.false_positive_rate_difference()\n",
    "total_fpr[9, 0] = classified_metric.false_positive_rate()\n",
    "\n",
    "total_tpr_unpriv[9, 0] = classified_metric.performance_measures(True)['TPR']\n",
    "total_tpr_priv[9, 0] = classified_metric.performance_measures(False)['TPR']\n",
    "total_fpr_unpriv[9, 0] = classified_metric.performance_measures(True)['FPR']\n",
    "total_fpr_priv[9, 0] = classified_metric.performance_measures(False)['FPR']\n",
    "\n",
    "\n",
    "total_acc_priv[9, 0] = classified_metric.performance_measures(False)['ACC']\n",
    "total_acc_unpriv[9, 0] = classified_metric.performance_measures(True)['ACC']\n",
    "\n",
    "print('FAIAS')\n",
    "print('Overall TPR : {:.3f}'.format(total_tpr[9].item()))\n",
    "print('Priv TPR : {:.3f}'.format(total_tpr_priv[9].item()))\n",
    "print('Unpriv TPR : {:.3f}'.format(total_tpr_unpriv[9].item()))\n",
    "print('Eq.Opp diff: {:.3f}'.format(total_eqopp_diff[9].item()))\n",
    "print()\n",
    "print('Overall FPR : {:.3f}'.format(total_fpr[9].item()))\n",
    "print('Priv FPR : {:.3f}'.format(total_fpr_priv[9].item()))\n",
    "print('Unpriv FPR : {:.3f}'.format(total_fpr_unpriv[9].item()))\n",
    "print('FPR diff : {:.3f}'.format(total_fpr_diff[9].item()))\n",
    "print()\n",
    "print('Overall ACC : {:.3f}'.format(total_acc[9].item()))\n",
    "print('Priv ACC : {:.3f}'.format(total_acc_priv[9].item()))\n",
    "print('Unpriv ACC : {:.3f}'.format(total_acc_unpriv[9].item()))\n",
    "print('ACC diff : {:.3f}'.format(abs(total_acc_unpriv[9].item() - total_acc_priv[9].item())))\n",
    "print()\n",
    "print()\n",
    "print('Disparate Impact : {:.3f}'.format(total_disimpact[9].item()))\n",
    "print('Eq.Opp diff : {:.3f}'.format(total_eqopp_diff[9].item()))\n",
    "print('Average Odds Diff : {:.3f}'.format(total_aveodds_diff[9].item()))\n",
    "print('Theil Index : {:.3f}'.format(total_theil_idx[9].item()))\n",
    "print('Stat parity diff : {:.3f}'.format(stat_parity_diff[9].item()))\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "FAIAS\n",
      "Overall TPR : 0.944\n",
      "Priv TPR : 0.850\n",
      "Unpriv TPR : 0.971\n",
      "Eq.Opp diff: -0.122\n",
      "\n",
      "Overall FPR : 0.362\n",
      "Priv FPR : 0.221\n",
      "Unpriv FPR : 0.586\n",
      "FPR diff : -0.365\n",
      "\n",
      "Overall ACC : 0.794\n",
      "Priv ACC : 0.798\n",
      "Unpriv ACC : 0.791\n",
      "ACC diff : 0.007\n",
      "\n",
      "\n",
      "Disparate Impact : 0.466\n",
      "Eq.Opp diff : -0.122\n",
      "Average Odds Diff : -0.243\n",
      "Theil Index : 0.075\n",
      "Stat parity diff : -0.452\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print('FAIAS')\n",
    "print('Overall TPR : {:.3f}'.format(total_tpr[9].item()))\n",
    "print('Priv TPR : {:.3f}'.format(total_tpr_priv[9].item()))\n",
    "print('Unpriv TPR : {:.3f}'.format(total_tpr_unpriv[9].item()))\n",
    "print('Eq.Opp diff: {:.3f}'.format(total_eqopp_diff[9].item()))\n",
    "print()\n",
    "print('Overall FPR : {:.3f}'.format(total_fpr[9].item()))\n",
    "print('Priv FPR : {:.3f}'.format(total_fpr_priv[9].item()))\n",
    "print('Unpriv FPR : {:.3f}'.format(total_fpr_unpriv[9].item()))\n",
    "print('FPR diff : {:.3f}'.format(total_fpr_diff[9].item()))\n",
    "print()\n",
    "print('Overall ACC : {:.3f}'.format(total_acc[9].item()))\n",
    "print('Priv ACC : {:.3f}'.format(total_acc_priv[9].item()))\n",
    "print('Unpriv ACC : {:.3f}'.format(total_acc_unpriv[9].item()))\n",
    "print('ACC diff : {:.3f}'.format(abs(total_acc_unpriv[9].item() - total_acc_priv[9].item())))\n",
    "print()\n",
    "print()\n",
    "print('Disparate Impact : {:.3f}'.format(total_disimpact[9].item()))\n",
    "print('Eq.Opp diff : {:.3f}'.format(total_eqopp_diff[9].item()))\n",
    "print('Average Odds Diff : {:.3f}'.format(total_aveodds_diff[9].item()))\n",
    "print('Theil Index : {:.3f}'.format(total_theil_idx[9].item()))\n",
    "print('Stat parity diff : {:.3f}'.format(stat_parity_diff[9].item()))\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:aif]",
   "language": "python",
   "name": "conda-env-aif-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
