"""
This file is part of Automunge which is released under GNU General Public License v3.0.
See file LICENSE or go to (anonymized) for full license details.

contact available via (anonymized)

Copyright (C) 2018, 2019, 2020 (anonymized) - All Rights Reserved

patent pending, applications (anonymized)
"""

#global imports
import numpy as np
import pandas as pd
from copy import deepcopy

#imports for process_time_class, postprocess_time_class
import datetime as dt

#imports for process_bxcx_class
from scipy import stats

#imports for process_hldy_class
from pandas.tseries.holiday import USFederalHolidayCalendar

#imports for evalcategory, getNArows
from collections import Counter
import datetime as dt
from scipy.stats import shapiro
from scipy.stats import skew

#imports for predictinfill, predictpostinfill, trainFSmodel
from sklearn.ensemble import RandomForestRegressor
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import RandomizedSearchCV
from sklearn.metrics import mean_squared_error
# from sklearn.metrics import roc_auc_score
from sklearn.metrics import accuracy_score
#stats may be used for cases where user elects RandomSearchCV hyperparameter tuning
# from scipy import stats

#imports for shuffleaccuracy
from sklearn.metrics import accuracy_score
from sklearn.metrics import mean_squared_log_error

#imports for PCA dimensionality reduction
from sklearn.decomposition import PCA
from sklearn.decomposition import SparsePCA
from sklearn.decomposition import KernelPCA

#imports for automunge
import random
#import datetime as dt
import types

class AutoMunge:
  
  def __init__(self):
    pass

  def assembletransformdict(self, binstransform, NArw_marker):
    '''
    #assembles the range of transformations to be applied based on the evaluated \
    #category of data
    #the primitives are intented as follows:
    #_greatgrandparents_: supplemental column derived from source column, only applied
    #to first generation, with downstream transforms included
    #_grandparents_: supplemental column derived from source column, only applied
    #to first generation
    #_parents_: replace source column, with downstream trasnforms performed
    #_siblings_: supplemental column derived from source column, \
    #with downstream transforms performed
    #_auntsuncles_: replace source column, without downstream transforms performed
    #_cousins_: supplemental column derived from source column, \
    #without downstream transforms performed
    #downstream transform primitives are:
    #_children_: becomes downstream parent
    #_niecenephews_: treated like a downstream sibling
    #_coworkers_: becomes a downstream auntsuncles
    #_friends_: become downstream cousins    
    
    #for example, if we set 'bxcx' entry to have both 'bxcx' as parents and \
    #'nmbr' as cousin, then the output would be column_nmbr, column_bxcx_nmbr
    #(because 'bxcx' has a downstream primitive entry of 'nmbr' as well 
    
    #note a future extension will allow automunge class to run experiments
    #on different configurations of trasnform_dict to improve the feature selection
    '''

    transform_dict = {}

    #initialize bins based on what was passed through application of automunge(.)
    if binstransform is True:
      bint = 'bint'
    else:
      bint = None
        
    if NArw_marker is True:
      NArw = 'NArw'
    else:
      NArw = None

    #initialize trasnform_dict. Note in a future extension the range of categories
    #is intended to be built out
    transform_dict.update({'nmbr' : {'parents'       : ['nmbr'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : [bint]}})
    
    transform_dict.update({'dxdt' : {'parents'       : ['dxdt'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['retn'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['retn'], \
                                     'friends'       : []}})
    
    transform_dict.update({'d2dt' : {'parents'       : ['d2dt'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['retn'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : ['dxdt'], \
                                     'coworkers'     : ['retn'], \
                                     'friends'       : []}})
    
    transform_dict.update({'d3dt' : {'parents'       : ['d3dt'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['retn'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : ['d2dt'], \
                                     'coworkers'     : ['retn'], \
                                     'friends'       : []}})

    transform_dict.update({'d4dt' : {'parents'       : ['d4dt'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['retn'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : ['d3dt'], \
                                     'coworkers'     : ['retn'], \
                                     'friends'       : []}})

    transform_dict.update({'d5dt' : {'parents'       : ['d5dt'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['retn'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : ['d4dt'], \
                                     'coworkers'     : ['retn'], \
                                     'friends'       : []}})

    transform_dict.update({'d6dt' : {'parents'       : ['d6dt'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['retn'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : ['d5dt'], \
                                     'coworkers'     : ['retn'], \
                                     'friends'       : []}})
    
    transform_dict.update({'dxd2' : {'parents'       : ['dxd2'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['retn'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['retn'], \
                                     'friends'       : []}})
    
    transform_dict.update({'d2d2' : {'parents'       : ['d2d2'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['retn'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : ['dxd2'], \
                                     'coworkers'     : ['retn'], \
                                     'friends'       : []}})
    
    transform_dict.update({'d3d2' : {'parents'       : ['d3d2'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['retn'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : ['d2d2'], \
                                     'coworkers'     : ['retn'], \
                                     'friends'       : []}})

    transform_dict.update({'d4d2' : {'parents'       : ['d4d2'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['retn'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : ['d3d2'], \
                                     'coworkers'     : ['retn'], \
                                     'friends'       : []}})

    transform_dict.update({'d5d2' : {'parents'       : ['d5d2'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['retn'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : ['d4d2'], \
                                     'coworkers'     : ['retn'], \
                                     'friends'       : []}})

    transform_dict.update({'d6d2' : {'parents'       : ['d6d2'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['retn'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : ['d5d2'], \
                                     'coworkers'     : ['retn'], \
                                     'friends'       : []}})
    
    transform_dict.update({'nmdx' : {'parents'       : ['nmdx'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : ['dxdt'], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['retn'], \
                                     'friends'       : []}})
    
    transform_dict.update({'nmd2' : {'parents'       : ['nmd2'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : ['d2dt'], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['retn'], \
                                     'friends'       : []}})
    
    transform_dict.update({'nmd3' : {'parents'       : ['nmd3'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : ['d3dt'], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['retn'], \
                                     'friends'       : []}})

    transform_dict.update({'nmd4' : {'parents'       : ['nmd4'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : ['d4dt'], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['retn'], \
                                     'friends'       : []}})

    transform_dict.update({'nmd5' : {'parents'       : ['nmd5'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : ['d5dt'], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['retn'], \
                                     'friends'       : []}})

    transform_dict.update({'nmd6' : {'parents'       : ['nmd6'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : ['d6dt'], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['retn'], \
                                     'friends'       : []}})
    
    transform_dict.update({'mmdx' : {'parents'       : ['mmdx'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['nbr2'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['nbr2'], \
                                     'friends'       : []}})
    
    transform_dict.update({'mmd2' : {'parents'       : ['mmd2'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['nbr2'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : ['mmdx'], \
                                     'coworkers'     : ['nbr2'], \
                                     'friends'       : []}})
    
    transform_dict.update({'mmd3' : {'parents'       : ['mmd3'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['nbr2'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : ['mmd2'], \
                                     'coworkers'     : ['nbr2'], \
                                     'friends'       : []}})

    transform_dict.update({'mmd4' : {'parents'       : ['mmd4'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['nbr2'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : ['mmd3'], \
                                     'coworkers'     : ['nbr2'], \
                                     'friends'       : []}})

    transform_dict.update({'mmd5' : {'parents'       : ['mmd5'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['nbr2'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : ['mmd4'], \
                                     'coworkers'     : ['nbr2'], \
                                     'friends'       : []}})

    transform_dict.update({'mmd6' : {'parents'       : ['mmd6'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['nbr2'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : ['mmd5'], \
                                     'coworkers'     : ['nbr2'], \
                                     'friends'       : []}})
    
    transform_dict.update({'dddt' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['dddt', 'exc2'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'ddd2' : {'parents'       : ['ddd2'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['exc2'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : ['dddt'], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'ddd3' : {'parents'       : ['ddd3'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['exc2'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : ['ddd2'], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})

    transform_dict.update({'ddd4' : {'parents'       : ['ddd4'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['exc2'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : ['ddd3'], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})

    transform_dict.update({'ddd5' : {'parents'       : ['ddd5'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['exc2'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : ['ddd4'], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})

    transform_dict.update({'ddd6' : {'parents'       : ['ddd6'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['exc2'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : ['ddd5'], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'dedt' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['dedt', 'exc2'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'ded2' : {'parents'       : ['ded2'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['exc2'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : ['dedt'], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'ded3' : {'parents'       : ['ded3'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['exc2'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : ['ded2'], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})

    transform_dict.update({'ded4' : {'parents'       : ['ded4'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['exc2'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : ['ded3'], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})

    transform_dict.update({'ded5' : {'parents'       : ['ded5'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['exc2'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : ['ded4'], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})

    transform_dict.update({'ded6' : {'parents'       : ['ded6'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['exc2'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : ['ded5'], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})

    transform_dict.update({'shft' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['shft'], \
                                     'cousins'       : [], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
  
    transform_dict.update({'shf2' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['shf2'], \
                                     'cousins'       : [], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'shf3' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['shf3'], \
                                     'cousins'       : [], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})

    transform_dict.update({'shf4' : {'parents'       : ['shf4'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['retn'], \
                                     'cousins'       : [], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['retn'], \
                                     'friends'       : []}})
  
    transform_dict.update({'shf5' : {'parents'       : ['shf5'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['retn'], \
                                     'cousins'       : [], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['retn'], \
                                     'friends'       : []}})
    
    transform_dict.update({'shf6' : {'parents'       : ['shf6'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['retn'], \
                                     'cousins'       : [], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['retn'], \
                                     'friends'       : []}})

    transform_dict.update({'shf7' : {'parents'       : ['shf4', 'shf5'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['retn'], \
                                     'cousins'       : [], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['retn'], \
                                     'friends'       : []}})
    
    transform_dict.update({'shf8' : {'parents'       : ['shf4', 'shf5', 'shf6'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['retn'], \
                                     'cousins'       : [], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['retn'], \
                                     'friends'       : []}})

    transform_dict.update({'bnry' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['bnry'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'bnr2' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['bnr2'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})

    transform_dict.update({'onht' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['onht'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})

    transform_dict.update({'onh2' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['onht'], \
                                     'cousins'       : ['NArw'], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'text' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['text'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'txt2' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['text'], \
                                     'cousins'       : [NArw, 'splt'], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'txt3' : {'parents'       : ['txt3'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['text'], \
                                     'friends'       : []}})

    transform_dict.update({'lngt' : {'parents'       : ['lngt'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['mnmx'], \
                                     'friends'       : []}})
  
    transform_dict.update({'lnlg' : {'parents'       : ['lnlg'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['log0'], \
                                     'friends'       : []}})

    transform_dict.update({'UPCS' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['UPCS'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'Unht' : {'parents'       : ['Unht'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['onht'], \
                                     'friends'       : []}})
  
    transform_dict.update({'Utxt' : {'parents'       : ['Utxt'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['text'], \
                                     'friends'       : []}})
    
    transform_dict.update({'Utx2' : {'parents'       : ['Utx2'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['text'], \
                                     'friends'       : ['splt']}})

    transform_dict.update({'Utx3' : {'parents'       : ['Utx3'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : ['txt3'], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'Ucct' : {'parents'       : ['Ucct'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['ucct', 'ord3'], \
                                     'friends'       : []}})
    
    transform_dict.update({'Uord' : {'parents'       : ['Uord'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['ordl'], \
                                     'friends'       : []}})
        
    transform_dict.update({'Uor2' : {'parents'       : ['Uor2'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : ['ord2'], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'Uor3' : {'parents'       : ['Uor3'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['ord3'], \
                                     'friends'       : []}})
    
    transform_dict.update({'Uor6' : {'parents'       : ['Uor6'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : ['spl6'], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['ord3'], \
                                     'friends'       : []}})
    
    transform_dict.update({'U101' : {'parents'       : ['U101'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['1010'], \
                                     'friends'       : []}})
    
    transform_dict.update({'splt' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['splt'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'spl2' : {'parents'       : ['spl2'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['ord3'], \
                                     'friends'       : []}})
    
    transform_dict.update({'spl5' : {'parents'       : ['spl5'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['ord3'], \
                                     'friends'       : []}})
    
    transform_dict.update({'spl6' : {'parents'       : ['spl6'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : ['splt'], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : ['ord3']}})
    
    transform_dict.update({'spl7' : {'parents'       : ['spl7'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['ord3'], \
                                     'friends'       : []}})

    transform_dict.update({'spl8' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['spl8'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})

    transform_dict.update({'spl9' : {'parents'       : ['spl9'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['ord3'], \
                                     'friends'       : []}})

    transform_dict.update({'sp10' : {'parents'       : ['sp10'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['ord3'], \
                                     'friends'       : []}})
    
    
    transform_dict.update({'sp11' : {'parents'       : ['sp11'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : ['spl5'], \
                                     'coworkers'     : ['ord3'], \
                                     'friends'       : []}})
    
    transform_dict.update({'sp12' : {'parents'       : ['sp12'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : ['sp11'], \
                                     'coworkers'     : ['ord3'], \
                                     'friends'       : []}})
    
    transform_dict.update({'sp13' : {'parents'       : ['sp13'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : ['sp10'], \
                                     'coworkers'     : ['ord3'], \
                                     'friends'       : []}})
    
    transform_dict.update({'sp14' : {'parents'       : ['sp14'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : ['sp13'], \
                                     'coworkers'     : ['ord3'], \
                                     'friends'       : []}})
    
    transform_dict.update({'sp15' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['sp15'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
  
    transform_dict.update({'sp16' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['sp16'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'sp17' : {'parents'       : ['sp17'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : ['spl5'], \
                                     'coworkers'     : ['ord3'], \
                                     'friends'       : []}})
    
    transform_dict.update({'sp18' : {'parents'       : ['sp18'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : ['sp17'], \
                                     'coworkers'     : ['ord3'], \
                                     'friends'       : []}})

    transform_dict.update({'sp19' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['sp19'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})

    transform_dict.update({'sp20' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['sp20'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})

    transform_dict.update({'sbst' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['sbst'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})

    transform_dict.update({'sbs2' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['sbs2'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})

    transform_dict.update({'sbs3' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['sbs3'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})

    transform_dict.update({'sbs4' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['sbs4'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'srch' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['srch'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
  
    transform_dict.update({'src2' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['src2'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'src3' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['src3'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'src4' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['src4'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'aggt' : {'parents'       : ['aggt'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['ord3'], \
                                     'friends'       : []}})
    
    transform_dict.update({'strn' : {'parents'       : ['strn'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['ord3'], \
                                     'friends'       : []}})

  
    transform_dict.update({'strg' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['strg'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'nmrc' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['nmrc'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
  
    transform_dict.update({'nmr2' : {'parents'       : ['nmr2'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : ['nmbr'], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'nmr3' : {'parents'       : ['nmr3'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : ['mnmx'], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})

    transform_dict.update({'nmr4' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['nmr4'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
  
    transform_dict.update({'nmr5' : {'parents'       : ['nmr5'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : ['nmbr'], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'nmr6' : {'parents'       : ['nmr6'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : ['mnmx'], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'nmr7' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['nmr7'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
  
    transform_dict.update({'nmr8' : {'parents'       : ['nmr8'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : ['nmbr'], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'nmr9' : {'parents'       : ['nmr9'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : ['mnmx'], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'nmcm' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['nmcm'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
  
    transform_dict.update({'nmc2' : {'parents'       : ['nmc2'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : ['nmbr'], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'nmc3' : {'parents'       : ['nmc3'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : ['mnmx'], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})

    transform_dict.update({'nmc4' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['nmc4'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
  
    transform_dict.update({'nmc5' : {'parents'       : ['nmc5'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : ['nmbr'], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'nmc6' : {'parents'       : ['nmc6'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : ['mnmx'], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})

    transform_dict.update({'nmc7' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['nmc7'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
  
    transform_dict.update({'nmc8' : {'parents'       : ['nmc8'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : ['nmbr'], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'nmc9' : {'parents'       : ['nmc9'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : ['mnmx'], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'nmEU' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['nmEU'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
  
    transform_dict.update({'nmE2' : {'parents'       : ['nmE2'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : ['nmbr'], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'nmE3' : {'parents'       : ['nmE3'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : ['mnmx'], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'nmE4' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['nmE4'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
  
    transform_dict.update({'nmE5' : {'parents'       : ['nmE5'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : ['nmbr'], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'nmE6' : {'parents'       : ['nmE6'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : ['mnmx'], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'nmE7' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['nmE7'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
  
    transform_dict.update({'nmE8' : {'parents'       : ['nmE8'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : ['nmbr'], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'nmE9' : {'parents'       : ['nmE9'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : ['mnmx'], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'ors7' : {'parents'       : ['spl6', 'nmr2'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['ord3'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'ors5' : {'parents'       : ['spl5'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['ord3'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'ors6' : {'parents'       : ['spl6'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['ord3'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'ordl' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['ordl'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
        
    transform_dict.update({'ord2' : {'parents'       : ['ord2'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['mnmx'], \
                                     'friends'       : []}})
    
    transform_dict.update({'ord3' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['ord3'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})

    transform_dict.update({'ord5' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['ord5'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'ucct' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['ucct'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
        
    transform_dict.update({'ord4' : {'parents'       : ['ord4'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['mnmx'], \
                                     'friends'       : []}})
    
    transform_dict.update({'ors2' : {'parents'       : ['spl2'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['ord3'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'or10' : {'parents'       : ['ord4'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['1010'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['mnmx'], \
                                     'friends'       : []}})
    
    transform_dict.update({'or11' : {'parents'       : ['sp11'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['1010'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
  
    transform_dict.update({'or12' : {'parents'       : ['nmr2'], \
                                     'siblings'      : ['sp11'], \
                                     'auntsuncles'   : ['1010'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'or13' : {'parents'       : ['sp12'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['1010'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'or14' : {'parents'       : ['nmr2'], \
                                     'siblings'      : ['sp12'], \
                                     'auntsuncles'   : ['1010'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})

    transform_dict.update({'or15' : {'parents'       : ['or15'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : ['sp13'], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['1010'], \
                                     'friends'       : []}})
  
    transform_dict.update({'or16' : {'parents'       : ['or16'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : ['nmr2'], \
                                     'niecesnephews' : ['sp13'], \
                                     'coworkers'     : ['1010'], \
                                     'friends'       : []}})
    
    transform_dict.update({'or17' : {'parents'       : ['or17'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : ['sp14'], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['1010'], \
                                     'friends'       : []}})
    
    transform_dict.update({'or18' : {'parents'       : ['or18'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : ['nmr2'], \
                                     'niecesnephews' : ['sp14'], \
                                     'coworkers'     : ['1010'], \
                                     'friends'       : []}})

    transform_dict.update({'or19' : {'parents'       : ['or19'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : ['nmc8'], \
                                     'niecesnephews' : ['sp13'], \
                                     'coworkers'     : ['1010'], \
                                     'friends'       : []}})
    
    transform_dict.update({'or20' : {'parents'       : ['or20'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : ['nmc8'], \
                                     'niecesnephews' : ['sp14'], \
                                     'coworkers'     : ['1010'], \
                                     'friends'       : []}})
    
    transform_dict.update({'or21' : {'parents'       : ['or21'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : ['nmc8'], \
                                     'niecesnephews' : ['sp17'], \
                                     'coworkers'     : ['1010'], \
                                     'friends'       : []}})
    
    transform_dict.update({'or22' : {'parents'       : ['or22'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : ['nmc8'], \
                                     'niecesnephews' : ['sp18'], \
                                     'coworkers'     : ['1010'], \
                                     'friends'       : []}})
    
    transform_dict.update({'om10' : {'parents'       : ['ord4'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['1010', 'mnmx'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['mnmx'], \
                                     'friends'       : []}})

    transform_dict.update({'mmor' : {'parents'       : ['ord4'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['mnmx'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'1010' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['1010'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})

    transform_dict.update({'null' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['null'], \
                                     'cousins'       : [], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'NArw' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['NArw'], \
                                     'cousins'       : [], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'NAr2' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['NAr2'], \
                                     'cousins'       : [], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'NAr3' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['NAr3'], \
                                     'cousins'       : [], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'NAr4' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['NAr4'], \
                                     'cousins'       : [], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'NAr5' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['NAr5'], \
                                     'cousins'       : [], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'nbr2' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['nmbr'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'nbr3' : {'parents'       : ['nbr3'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : ['bint']}})
    
    transform_dict.update({'MADn' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['MADn'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'MAD2' : {'parents'       : ['MAD2'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : ['nmbr'], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'MAD3' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['MAD3'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'mnmx' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['mnmx'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'mnm2' : {'parents'       : ['nmbr'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['mnmx'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'mnm3' : {'parents'       : ['nmbr'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['mnm3'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'mnm4' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['mnm3'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'mnm5' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['mnmx'], \
                                     'cousins'       : ['nmbr', NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'mnm6' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['mnm6'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'mnm7' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['mnmx', 'bins'], \
                                     'cousins'       : [], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'retn' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['retn'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})

    transform_dict.update({'rtbn' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['retn', 'bsor'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})

    transform_dict.update({'rtb2' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['retn', 'bins'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})

    transform_dict.update({'mean' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['mean'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'mea2' : {'parents'       : ['nmbr'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['mean'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'mea3' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['mean', 'bins'], \
                                     'cousins'       : [], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})

    transform_dict.update({'date' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['year', 'mnth', 'days', 'hour', 'mint', 'scnd'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
  
    transform_dict.update({'dat2' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['bshr', 'wkdy', 'hldy'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'dat3' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['year', 'mnsn', 'mncs', 'dysn', 'dycs', 'hrsn', 'hrcs', 'misn', 'mics', 'scsn', 'sccs'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'dat4' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['year', 'mdsn', 'mdcs', 'hmss', 'hmsc'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'dat5' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['year', 'mdsn', 'mdcs', 'dysn', 'dycs', 'hmss', 'hmsc'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'dat6' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['year', 'mdsn', 'mdcs', 'hmss', 'hmsc', 'bshr', 'wkdy', 'hldy'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'year' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['year'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
  
    transform_dict.update({'yea2' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['year', 'yrsn', 'yrcs', 'mdsn', 'mdcs'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})

    transform_dict.update({'yrcs' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['yrcs'], \
                                     'cousins'       : [], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'yrsn' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['yrsn'], \
                                     'cousins'       : [], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'mnth' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['mnth'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
  
    transform_dict.update({'mnt2' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['mnsn', 'mncs'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'mnt3' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['mnsn', 'mncs', 'dysn', 'dycs', 'hrsn', 'hrcs', 'misn', 'mics', 'scsn', 'sccs'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'mnt4' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['mdsn', 'mdcs'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'mnt5' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['mdsn', 'mdcs', 'hmss', 'hmsc'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'mnt6' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['mdsn', 'mdcs', 'dysn', 'dycs', 'hmss', 'hmsc'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'mnsn' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['mnsn'], \
                                     'cousins'       : [], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'mncs' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['mncs'], \
                                     'cousins'       : [], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'mdsn' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['mdsn'], \
                                     'cousins'       : [], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'mdcs' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['mdcs'], \
                                     'cousins'       : [], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})

    transform_dict.update({'days' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['days'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
  
    transform_dict.update({'day2' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['dysn', 'dycs'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'day3' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['dysn', 'dycs', 'hrsn', 'hrcs', 'misn', 'mics', 'scsn', 'sccs'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'day4' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['dhms', 'dhmc'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'day5' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['dhms', 'dhmc', 'hmss', 'hmsc'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'dysn' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['dysn'], \
                                     'cousins'       : [], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'dycs' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['dycs'], \
                                     'cousins'       : [], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'dhms' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['dhms'], \
                                     'cousins'       : [], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'dhmc' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['dhmc'], \
                                     'cousins'       : [], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'hour' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['hour'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
  
    transform_dict.update({'hrs2' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['hrsn', 'hrcs'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'hrs3' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['hrsn', 'hrcs', 'misn', 'mics', 'scsn', 'sccs'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'hrs4' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['hmss', 'hmsc'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'hrsn' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['hrsn'], \
                                     'cousins'       : [], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'hrcs' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['hrcs'], \
                                     'cousins'       : [], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'hmss' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['hmss'], \
                                     'cousins'       : [], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'hmsc' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['hmsc'], \
                                     'cousins'       : [], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'mint' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['mint'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
  
    transform_dict.update({'min2' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['misn', 'mics'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'min3' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['misn', 'mics', 'scsn', 'sccs'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'min4' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['mssn', 'mscs'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'misn' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['misn'], \
                                     'cousins'       : [], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'mics' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['mics'], \
                                     'cousins'       : [], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'mssn' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['mssn'], \
                                     'cousins'       : [], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'mscs' : {'parents'       : [], \
                                     'siblings': [], \
                                     'auntsuncles'   : ['mscs'], \
                                     'cousins'       : [], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})

    transform_dict.update({'scnd' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['scnd'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
  
    transform_dict.update({'scn2' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['scsn', 'sccs'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'scsn' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['scsn'], \
                                     'cousins'       : [], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'sccs' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['sccs'], \
                                     'cousins'       : [], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'bxcx' : {'parents'       : ['bxcx'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : ['nmbr'], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'bxc2' : {'parents'       : ['bxc2'], \
                                     'siblings'      : ['nmbr'], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : ['nmbr'], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'bxc3' : {'parents'       : ['bxc3'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : ['nmbr'], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'bxc4' : {'parents'       : ['bxc4'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['nbr2'], \
                                     'friends'       : []}})

    transform_dict.update({'bxc5' : {'parents'       : ['bxc5'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['mnmx'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['nbr2', 'bins'], \
                                     'friends'       : []}})

    transform_dict.update({'ntgr' : {'parents'       : ['ntgr'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['retn', '1010', 'ordl'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['mnmx'], \
                                     'friends'       : []}})
    
    transform_dict.update({'ntg2' : {'parents'       : ['ntg2'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['retn', '1010', 'ordl', 'pwr2'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['mnmx'], \
                                     'friends'       : []}})
    
    transform_dict.update({'ntg3' : {'parents'       : ['ntg3'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['retn', 'ordl', 'por2'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['mnmx'], \
                                     'friends'       : []}})
    
    transform_dict.update({'pwrs' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['pwrs'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'pwr2' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['pwr2'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'log0' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['log0'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'log1' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['log0', 'pwr2'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'logn' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['logn'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
  
    transform_dict.update({'lgnm' : {'parents'       : ['lgnm'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['nmbr'], \
                                     'friends'       : []}})
    
    transform_dict.update({'sqrt' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['sqrt'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'addd' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['addd'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
  
    transform_dict.update({'sbtr' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['sbtr'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'mltp' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['mltp'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'divd' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['divd'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'rais' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['rais'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'absl' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['absl'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'bkt1' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['bkt1'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'bkt2' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['bkt2'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'bkt3' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['bkt3'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'bkt4' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['bkt4'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'wkdy' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['wkdy'], \
                                     'cousins'       : [], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'bshr' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['bshr'], \
                                     'cousins'       : [], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'hldy' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['hldy'], \
                                     'cousins'       : [], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'wkds' : {'parents'       : ['wkds'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['text'], \
                                     'friends'       : []}})
  
    transform_dict.update({'wkdo' : {'parents'       : ['wkdo'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['ordl'], \
                                     'friends'       : []}})
    
    transform_dict.update({'mnts' : {'parents'       : ['mnts'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['text'], \
                                     'friends'       : []}})
  
    transform_dict.update({'mnto' : {'parents'       : ['mnto'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['ordl'], \
                                     'friends'       : []}})
    
    transform_dict.update({'bins' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['bins'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})

    transform_dict.update({'bint' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['bint'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'bsor' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['bsor'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})

    transform_dict.update({'btor' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['btor'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'bnwd' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['bnwd'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'bnwK' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['bnwK'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
  
    transform_dict.update({'bnwM' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['bnwM'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'bnwo' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['bnwo'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
  
    transform_dict.update({'bnKo' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['bnKo'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'bnMo' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['bnMo'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})    
    
    transform_dict.update({'bnep' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['bnep'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'bne7' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['bne7'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'bne9' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['bne9'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'bneo' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['bneo'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'bn7o' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['bn7o'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})

    transform_dict.update({'bn9o' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['bn9o'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'tlbn' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['tlbn'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'pwor' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['pwor'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'por2' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['por2'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})

    transform_dict.update({'por3' : {'parents'       : ['por3'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['1010'], \
                                     'friends'       : []}})

    transform_dict.update({'bkb3' : {'parents'       : ['bkb3'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['1010'], \
                                     'friends'       : []}})
  
    transform_dict.update({'bkb4' : {'parents'       : ['bkb4'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['1010'], \
                                     'friends'       : []}})
    
    transform_dict.update({'bsbn' : {'parents'       : ['bsbn'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['1010'], \
                                     'friends'       : []}})
    
    transform_dict.update({'bnwb' : {'parents'       : ['bnwb'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['1010'], \
                                     'friends'       : []}})
    
    transform_dict.update({'bnKb' : {'parents'       : ['bnKb'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['1010'], \
                                     'friends'       : []}})

    transform_dict.update({'bnMb' : {'parents'       : ['bnMb'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['1010'], \
                                     'friends'       : []}})
    
    transform_dict.update({'bneb' : {'parents'       : ['bneb'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['1010'], \
                                     'friends'       : []}})

    transform_dict.update({'bn7b' : {'parents'       : ['bn7b'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['1010'], \
                                     'friends'       : []}})
    
    transform_dict.update({'bn9b' : {'parents'       : ['bn9b'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['1010'], \
                                     'friends'       : []}})
    
    transform_dict.update({'pwbn' : {'parents'       : ['pwbn'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['1010'], \
                                     'friends'       : []}})

    transform_dict.update({'DPnb' : {'parents'       : ['DPn3'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'DPn3' : {'parents'       : ['DPn3'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['DPnb'], \
                                     'friends'       : []}})

    transform_dict.update({'DPmm' : {'parents'       : ['DPm2'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
  
    transform_dict.update({'DPm2' : {'parents'       : ['DPm2'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['DPmm'], \
                                     'friends'       : []}})

    transform_dict.update({'DPrt' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['DPrt'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})

    transform_dict.update({'DLnb' : {'parents'       : ['DLn3'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'DLn3' : {'parents'       : ['DLn3'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['DLnb'], \
                                     'friends'       : []}})

    transform_dict.update({'DLmm' : {'parents'       : ['DLm2'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
  
    transform_dict.update({'DLm2' : {'parents'       : ['DLm2'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['DLmm'], \
                                     'friends'       : []}})

    transform_dict.update({'DLrt' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['DLrt'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
  
    transform_dict.update({'DPbn' : {'parents'       : ['DPb2'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'DPb2' : {'parents'       : ['DPb2'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['DPbn'], \
                                     'friends'       : []}})
    
    transform_dict.update({'DPod' : {'parents'       : ['DPo4'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'DPo4' : {'parents'       : ['DPo4'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['DPod'], \
                                     'friends'       : []}})
    
    transform_dict.update({'DPoh' : {'parents'       : ['DPo5'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : ['onht'], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'DPo5' : {'parents'       : ['DPo5'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : ['DPo2'], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'DPo2' : {'parents'       : ['DPo2'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['onht'], \
                                     'friends'       : []}})
    
    transform_dict.update({'DP10' : {'parents'       : ['DPo6'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : ['1010'], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'DPo6' : {'parents'       : ['DPo6'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : ['DPo3'], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'DPo3' : {'parents'       : ['DPo3'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['1010'], \
                                     'friends'       : []}})
    
    transform_dict.update({'copy' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['copy'], \
                                     'cousins'       : [], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'excl' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['excl'], \
                                     'cousins'       : [], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'exc2' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['exc2'], \
                                     'cousins'       : [], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'exc3' : {'parents'       : ['exc3'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : ['bins']}})
    
    transform_dict.update({'exc4' : {'parents'       : ['exc4'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : ['pwr2']}})
    
    transform_dict.update({'exc5' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['exc5'], \
                                     'cousins'       : [], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})

    #exc6 was removed from library, is now same as excl, just including here so no printout for prior code demonstrations 
    transform_dict.update({'exc6' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['excl'], \
                                     'cousins'       : [], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'shfl' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['shfl'], \
                                     'cousins'       : [], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})

    transform_dict.update({'nmbd' : {'parents'       : ['nmbr'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : [bint]}})

    transform_dict.update({'101d' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['1010'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'ordd' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['ord3'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'texd' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['text'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'bnrd' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['bnry'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'datd' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['year', 'mdsn', 'mdcs', 'hmss', 'hmsc', 'bshr', 'wkdy', 'hldy'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'nuld' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['null'], \
                                     'cousins'       : [], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'lbnm' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['exc2'], \
                                     'cousins'       : [], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})

    transform_dict.update({'lb10' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['text'], \
                                     'cousins'       : [], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'lbor' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['ord3'], \
                                     'cousins'       : [], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})

    transform_dict.update({'lbo5' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['ordl'], \
                                     'cousins'       : [NArw], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})

    transform_dict.update({'lbos' : {'parents'       : ['lbos'], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : [], \
                                     'cousins'       : [], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : ['strg'], \
                                     'friends'       : []}})
    
    transform_dict.update({'lbte' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['text'], \
                                     'cousins'       : [], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'lbbn' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['text'], \
                                     'cousins'       : [], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})
    
    transform_dict.update({'lbda' : {'parents'       : [], \
                                     'siblings'      : [], \
                                     'auntsuncles'   : ['year', 'mdsn', 'mdcs', 'hmss', 'hmsc', 'bshr', 'wkdy', 'hldy'], \
                                     'cousins'       : [], \
                                     'children'      : [], \
                                     'niecesnephews' : [], \
                                     'coworkers'     : [], \
                                     'friends'       : []}})

    return transform_dict
  
  def assembleprocessdict(self):
    '''
    #creates a dictionary storing all of the processing functions for each
    #category. Note that the convention is that every dualprocess entry 
    #(to process both train and text set in automunge) is meant
    #to have a coresponding postprocess entry (to process the test set in 
    #postmunge). If the dualprocess/postprocess pair aren't included a 
    #singleprocess funciton will be instead which processes a single column
    #at a time and is neutral to whether that set is from train or test data.
    
    #note that the functionpointer entry is currenlty only available for user passed processdict
    #this internal library process_dict does not accept functionpointer entries
    
    #NArowtype entries are:
    # - 'numeric' for source columns with expected numeric entries
    # - 'integer' for source column with expected integer entries
    # - 'justNaN' for source columns that may have expected entries other than numeric
    # - 'exclude' for source columns that aren't needing NArow columns derived
    # - 'positivenumeric' for source columns with expected positive numeric entries
    # - 'nonnegativenumeric' for source columns with expected non-nbegative numeric (zero allowed)
    # - 'nonzeronumeric' for source columns with allowed postiive and negative but no zero
    # - 'parsenumeric' marks for infill strings that don't contain any numeric character entries
    # - 'parsenumeric_commas' marks for infill strings that don't contain any numeric character entries, recognizes commas
    # - 'parsenumeric_EU' marks for infill strings that don't contain any numeric character entries, 
    #            recognizes international standard of period deliminator and comma decimal
    # - 'datetime' marks for infill cells that arent' recognized as datetime objects
    
    #MLinfilltype entries are:
    # - 'numeric' for single columns with numeric entries (such as could be floats)
    # - 'singlct' for single column sets with ordinal entries (integers)
    # - 'binary' for single column sets with boolean entries (0/1)
    # - 'multirt' for categorical multicolumn sets with boolean entries (0/1)
    # - 'concurrent_act' for multicolumn sets with boolean entries as may have 
    #multiple entries in the same row
    # - 'concurrent_nmbr' for multicolumn sets with numerical entries
    # - 'exclude' for columns which will be excluded from ML infill
    # - '1010' for binary encoded columns, will be converted to onehot for ML
    # - 'boolexclude' boolean set suitable for Binary transform but exluded from MLinfill
    # - 'totalexclude' sets excluded from all methods that inspect MLinfill, such as for excl category

    #at least one of sets of ('dualprocess' and 'postprocess') or ('singleprocess') needs to be specified
    #'inverseprocess' is optional and supports postmunge inversion
    #'info_retention' is optional boolean required with inversion to prioritize transforms with more information retention
    #'inplace_option' is optional boolean to signal when a transfomration function accepts inplace operations
    #'labelctgy' is associated with feature importance and signals which transform is target for predictive model
    #for cases when a family tree returns multiple configurations and category isapplied to a label set

    #note to self that any future updates such as additional supported entries should be carried through to functionpointer functions
    '''
    
    process_dict = {}
    
    #categories are nmbr, bnry, text, date, bxcx, bins, bint, NArw, null
    #note a future extension will allow the definition of new categories 
    #to automunge

    #dual column functions
    process_dict.update({'nmbr' : {'dualprocess' : self.process_numerical_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_numerical_class, \
                                  'inverseprocess' : self.inverseprocess_nmbr, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'nmbr'}})
    process_dict.update({'dxdt' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_dxdt_class, \
                                  'postprocess' : None, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'retn'}})
    process_dict.update({'d2dt' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_dxdt_class, \
                                  'postprocess' : None, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'retn'}})
    process_dict.update({'d3dt' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_dxdt_class, \
                                  'postprocess' : None, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'retn'}})
    process_dict.update({'d4dt' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_dxdt_class, \
                                  'postprocess' : None, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'retn'}})
    process_dict.update({'d5dt' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_dxdt_class, \
                                  'postprocess' : None, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'retn'}})
    process_dict.update({'d6dt' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_dxdt_class, \
                                  'postprocess' : None, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'retn'}})
    process_dict.update({'dxd2' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_dxd2_class, \
                                  'postprocess' : None, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'retn'}})
    process_dict.update({'d2d2' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_dxd2_class, \
                                  'postprocess' : None, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'retn'}})
    process_dict.update({'d3d2' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_dxd2_class, \
                                  'postprocess' : None, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'retn'}})
    process_dict.update({'d4d2' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_dxd2_class, \
                                  'postprocess' : None, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'retn'}})
    process_dict.update({'d5d2' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_dxd2_class, \
                                  'postprocess' : None, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'retn'}})
    process_dict.update({'d6d2' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_dxd2_class, \
                                  'postprocess' : None, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'retn'}})
    process_dict.update({'nmdx' : {'dualprocess' : self.process_numerical_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_numerical_class, \
                                  'inverseprocess' : self.inverseprocess_nmbr, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'retn'}})
    process_dict.update({'nmd2' : {'dualprocess' : self.process_numerical_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_numerical_class, \
                                  'inverseprocess' : self.inverseprocess_nmbr, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'retn'}})
    process_dict.update({'nmd3' : {'dualprocess' : self.process_numerical_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_numerical_class, \
                                  'inverseprocess' : self.inverseprocess_nmbr, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'retn'}})
    process_dict.update({'nmd4' : {'dualprocess' : self.process_numerical_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_numerical_class, \
                                  'inverseprocess' : self.inverseprocess_nmbr, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'retn'}})
    process_dict.update({'nmd5' : {'dualprocess' : self.process_numerical_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_numerical_class, \
                                  'inverseprocess' : self.inverseprocess_nmbr, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'retn'}})
    process_dict.update({'nmd6' : {'dualprocess' : self.process_numerical_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_numerical_class, \
                                  'inverseprocess' : self.inverseprocess_nmbr, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'retn'}})
    process_dict.update({'mmdx' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_dxdt_class, \
                                  'postprocess' : None, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'nmbr'}})
    process_dict.update({'mmd2' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_dxdt_class, \
                                  'postprocess' : None, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'nmbr'}})
    process_dict.update({'mmd3' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_dxdt_class, \
                                  'postprocess' : None, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'nmbr'}})
    process_dict.update({'mmd4' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_dxdt_class, \
                                  'postprocess' : None, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'nmbr'}})
    process_dict.update({'mmd5' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_dxdt_class, \
                                  'postprocess' : None, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'nmbr'}})
    process_dict.update({'mmd6' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_dxdt_class, \
                                  'postprocess' : None, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'nmbr'}})
    process_dict.update({'dddt' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_dxdt_class, \
                                  'postprocess' : None, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'dxdt'}})
    process_dict.update({'ddd2' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_dxdt_class, \
                                  'postprocess' : None, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'dxdt'}})
    process_dict.update({'ddd3' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_dxdt_class, \
                                  'postprocess' : None, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'dxdt'}})
    process_dict.update({'ddd4' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_dxdt_class, \
                                  'postprocess' : None, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'dxdt'}})
    process_dict.update({'ddd5' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_dxdt_class, \
                                  'postprocess' : None, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'dxdt'}})
    process_dict.update({'ddd6' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_dxdt_class, \
                                  'postprocess' : None, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'dxdt'}})
    process_dict.update({'dedt' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_dxd2_class, \
                                  'postprocess' : None, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'dxd2'}})
    process_dict.update({'ded2' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_dxd2_class, \
                                  'postprocess' : None, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'dxd2'}})
    process_dict.update({'ded3' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_dxd2_class, \
                                  'postprocess' : None, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'dxd2'}})
    process_dict.update({'ded4' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_dxd2_class, \
                                  'postprocess' : None, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'dxd2'}})
    process_dict.update({'ded5' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_dxd2_class, \
                                  'postprocess' : None, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'dxd2'}})
    process_dict.update({'ded6' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_dxd2_class, \
                                  'postprocess' : None, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'dxd2'}})
    process_dict.update({'shft' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_shft_class, \
                                  'postprocess' : None, \
                                  'inverseprocess' : self.inverseprocess_shft, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'exclude', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'shft'}})
    process_dict.update({'shf2' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_shf2_class, \
                                  'postprocess' : None, \
                                  'inverseprocess' : self.inverseprocess_shft, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'exclude', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'shf2'}})
    process_dict.update({'shf3' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_shf3_class, \
                                  'postprocess' : None, \
                                  'inverseprocess' : self.inverseprocess_shft, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'exclude', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'shf3'}})
    process_dict.update({'shf4' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_shft_class, \
                                  'postprocess' : None, \
                                  'inverseprocess' : self.inverseprocess_shft, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'exclude', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'retn'}})
    process_dict.update({'shf5' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_shf2_class, \
                                  'postprocess' : None, \
                                  'inverseprocess' : self.inverseprocess_shft, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'exclude', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'retn'}})
    process_dict.update({'shf6' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_shf3_class, \
                                  'postprocess' : None, \
                                  'inverseprocess' : self.inverseprocess_shft, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'exclude', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'retn'}})
    process_dict.update({'shf7' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_shft_class, \
                                  'postprocess' : None, \
                                  'inverseprocess' : self.inverseprocess_shft, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'exclude', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'retn'}})
    process_dict.update({'shf8' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_shft_class, \
                                  'postprocess' : None, \
                                  'inverseprocess' : self.inverseprocess_shft, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'exclude', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'retn'}})
    process_dict.update({'nbr2' : {'dualprocess' : self.process_numerical_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_numerical_class, \
                                  'inverseprocess' : self.inverseprocess_nmbr, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'nmbr'}})
    process_dict.update({'nbr3' : {'dualprocess' : self.process_numerical_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_numerical_class, \
                                  'inverseprocess' : self.inverseprocess_nmbr, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'nmbr'}})
    process_dict.update({'MADn' : {'dualprocess' : self.process_MADn_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_MADn_class, \
                                  'inverseprocess' : self.inverseprocess_MADn, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'MADn'}})
    process_dict.update({'MAD2' : {'dualprocess' : self.process_MADn_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_MADn_class, \
                                  'inverseprocess' : self.inverseprocess_MADn, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'MADn'}})
    process_dict.update({'MAD3' : {'dualprocess' : self.process_MAD3_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_MAD3_class, \
                                  'inverseprocess' : self.inverseprocess_MAD3, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'MAD3'}})
    process_dict.update({'mnmx' : {'dualprocess' : self.process_mnmx_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_mnmx_class, \
                                  'inverseprocess' : self.inverseprocess_mnmx, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'mnmx'}})
    process_dict.update({'mnm2' : {'dualprocess' : self.process_mnmx_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_mnmx_class, \
                                  'inverseprocess' : self.inverseprocess_mnmx, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'mnmx'}})
    process_dict.update({'mnm3' : {'dualprocess' : self.process_mnm3_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_mnm3_class, \
                                  'inverseprocess' : self.inverseprocess_mnm3, \
                                  'info_retention' : False, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'mnm3'}})
    process_dict.update({'mnm4' : {'dualprocess' : self.process_mnm3_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_mnm3_class, \
                                  'inverseprocess' : self.inverseprocess_mnm3, \
                                  'info_retention' : False, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'mnm3'}})
    process_dict.update({'mnm5' : {'dualprocess' : self.process_mnmx_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_mnmx_class, \
                                  'inverseprocess' : self.inverseprocess_mnmx, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'mnmx'}})
    process_dict.update({'mnm6' : {'dualprocess' : self.process_mnmx_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_mnmx_class, \
                                  'inverseprocess' : self.inverseprocess_mnmx, \
                                  'defaultparams' : {'floor' : True}, \
                                  'info_retention' : False, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'mnmx'}})
    process_dict.update({'mnm7' : {'dualprocess' : None, \
                                  'singleprocess' : None, \
                                  'postprocess' : None, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'mnm7'}})
    process_dict.update({'retn' : {'dualprocess' : self.process_retn_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_retn_class, \
                                  'inverseprocess' : self.inverseprocess_retn, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'retn'}})
    process_dict.update({'rtbn' : {'dualprocess' : self.process_retn_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_retn_class, \
                                  'inverseprocess' : self.inverseprocess_retn, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'retn'}})
    process_dict.update({'rtb2' : {'dualprocess' : self.process_retn_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_retn_class, \
                                  'inverseprocess' : self.inverseprocess_retn, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'retn'}})
    process_dict.update({'mean' : {'dualprocess' : self.process_mean_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_mean_class, \
                                  'inverseprocess' : self.inverseprocess_mean, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'mean'}})
    process_dict.update({'mea2' : {'dualprocess' : self.process_mean_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_mean_class, \
                                  'inverseprocess' : self.inverseprocess_mean, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'mean'}})
    process_dict.update({'mea3' : {'dualprocess' : self.process_mean_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_mean_class, \
                                  'inverseprocess' : self.inverseprocess_mean, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'mean'}})
    process_dict.update({'bnry' : {'dualprocess' : self.process_binary_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_binary_class, \
                                  'inverseprocess' : self.inverseprocess_bnry, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'binary', \
                                  'labelctgy' : 'bnry'}})
    process_dict.update({'bnr2' : {'dualprocess' : self.process_binary2_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_binary2_class, \
                                  'inverseprocess' : self.inverseprocess_bnry, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'binary', \
                                  'labelctgy' : 'bnr2'}})
    process_dict.update({'onht' : {'dualprocess' : self.process_onht_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_onht_class, \
                                  'inverseprocess' : self.inverseprocess_onht, \
                                  'info_retention' : True, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'multirt', \
                                  'labelctgy' : 'onht'}})
    process_dict.update({'onh2' : {'dualprocess' : self.process_onht_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_onht_class, \
                                  'inverseprocess' : self.inverseprocess_onht, \
                                  'info_retention' : True, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'multirt', \
                                  'labelctgy' : 'onht'}})
    process_dict.update({'text' : {'dualprocess' : self.process_text_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_text_class, \
                                  'inverseprocess' : self.inverseprocess_text, \
                                  'info_retention' : True, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'multirt', \
                                  'labelctgy' : 'text'}})
    process_dict.update({'txt2' : {'dualprocess' : self.process_text_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_text_class, \
                                  'inverseprocess' : self.inverseprocess_text, \
                                  'info_retention' : True, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'multirt', \
                                  'labelctgy' : 'text'}})
    process_dict.update({'txt3' : {'dualprocess' : self.process_spl2_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_spl2_class, \
                                  'inverseprocess' : self.inverseprocess_spl2, \
                                  'info_retention' : False, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'singlct', \
                                  'labelctgy' : 'text'}})
    process_dict.update({'lngt' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_lngt_class, \
                                  'postprocess' : None, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'singlct', \
                                  'labelctgy' : 'mnmx'}})
    process_dict.update({'lnlg' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_lngt_class, \
                                  'postprocess' : None, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'singlct', \
                                  'labelctgy' : 'log0'}})
    process_dict.update({'UPCS' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_UPCS_class, \
                                  'postprocess' : None, \
                                  'inverseprocess' : self.inverseprocess_UPCS, \
                                  'info_retention' : False, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'exclude', \
                                  'labelctgy' : 'exclude'}})
    process_dict.update({'Unht' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_UPCS_class, \
                                  'postprocess' : None, \
                                  'inverseprocess' : self.inverseprocess_UPCS, \
                                  'info_retention' : False, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'exclude', \
                                  'labelctgy' : 'onht'}})
    process_dict.update({'Utxt' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_UPCS_class, \
                                  'postprocess' : None, \
                                  'inverseprocess' : self.inverseprocess_UPCS, \
                                  'info_retention' : False, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'exclude', \
                                  'labelctgy' : 'text'}})
    process_dict.update({'Utx2' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_UPCS_class, \
                                  'postprocess' : None, \
                                  'inverseprocess' : self.inverseprocess_UPCS, \
                                  'info_retention' : False, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'exclude', \
                                  'labelctgy' : 'text'}})
    process_dict.update({'Utx3' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_UPCS_class, \
                                  'postprocess' : None, \
                                  'inverseprocess' : self.inverseprocess_UPCS, \
                                  'info_retention' : False, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'exclude', \
                                  'labelctgy' : 'text'}})
    process_dict.update({'Ucct' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_UPCS_class, \
                                  'postprocess' : None, \
                                  'inverseprocess' : self.inverseprocess_UPCS, \
                                  'info_retention' : False, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'exclude', \
                                  'labelctgy' : 'ucct'}})
    process_dict.update({'Uord' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_UPCS_class, \
                                  'postprocess' : None, \
                                  'inverseprocess' : self.inverseprocess_UPCS, \
                                  'info_retention' : False, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'exclude', \
                                  'labelctgy' : 'ordl'}})
    process_dict.update({'Uor2' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_UPCS_class, \
                                  'postprocess' : None, \
                                  'inverseprocess' : self.inverseprocess_UPCS, \
                                  'info_retention' : False, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'exclude', \
                                  'labelctgy' : 'mnmx'}})
    process_dict.update({'Uor3' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_UPCS_class, \
                                  'postprocess' : None, \
                                  'inverseprocess' : self.inverseprocess_UPCS, \
                                  'info_retention' : False, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'exclude', \
                                  'labelctgy' : 'ord3'}})
    process_dict.update({'Uor6' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_UPCS_class, \
                                  'postprocess' : None, \
                                  'inverseprocess' : self.inverseprocess_UPCS, \
                                  'info_retention' : False, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'exclude', \
                                  'labelctgy' : 'ord3'}})
    process_dict.update({'U101' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_UPCS_class, \
                                  'postprocess' : None, \
                                  'inverseprocess' : self.inverseprocess_UPCS, \
                                  'info_retention' : False, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'exclude', \
                                  'labelctgy' : '1010'}})
    process_dict.update({'splt' : {'dualprocess' : self.process_splt_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_splt_class, \
                                  'inverseprocess' : self.inverseprocess_splt, \
                                  'info_retention' : False, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'multirt', \
                                  'labelctgy' : 'splt'}})

    process_dict.update({'spl2' : {'dualprocess' : self.process_spl2_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_spl2_class, \
                                  'inverseprocess' : self.inverseprocess_spl2, \
                                  'info_retention' : False, \
                                  'defaultparams' : {'suffix' : '_spl2', \
                                                     'test_same_as_train' : False, \
                                                     'consolidate_nonoverlaps' : False}, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'exclude', \
                                  'labelctgy' : 'ord3'}})
    process_dict.update({'spl5' : {'dualprocess' : self.process_spl2_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_spl2_class, \
                                  'inverseprocess' : self.inverseprocess_spl2, \
                                  'info_retention' : False, \
                                  'defaultparams' : {'suffix' : '_spl5', \
                                                     'test_same_as_train' : False, \
                                                     'consolidate_nonoverlaps' : True}, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'exclude', \
                                  'labelctgy' : 'ord3'}})
    process_dict.update({'spl6' : {'dualprocess' : self.process_spl2_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_spl2_class, \
                                  'inverseprocess' : self.inverseprocess_spl2, \
                                  'info_retention' : False, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'singlct', \
                                  'labelctgy' : 'ord3'}})
    process_dict.update({'spl7' : {'dualprocess' : self.process_spl2_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_spl2_class, \
                                  'inverseprocess' : self.inverseprocess_spl2, \
                                  'info_retention' : False, \
                                  'defaultparams' : {'suffix' : '_spl7', \
                                                     'test_same_as_train' : False, \
                                                     'consolidate_nonoverlaps' : True, \
                                                     'minsplit' : 1}, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'exclude', \
                                  'labelctgy' : 'ord3'}})
    process_dict.update({'spl8' : {'dualprocess' : self.process_splt_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_splt_class, \
                                  'inverseprocess' : self.inverseprocess_splt, \
                                  'info_retention' : False, \
                                  'defaultparams' : {'suffix' : '_spl8', 'test_same_as_train' : True}, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'multirt', \
                                  'labelctgy' : 'splt'}})
    process_dict.update({'spl9' : {'dualprocess' : self.process_spl2_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_spl2_class, \
                                  'inverseprocess' : self.inverseprocess_spl2, \
                                  'info_retention' : False, \
                                  'defaultparams' : {'suffix' : '_spl9', \
                                                     'test_same_as_train' : True, \
                                                     'consolidate_nonoverlaps' : False}, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'exclude', \
                                  'labelctgy' : 'ord3'}})
    process_dict.update({'sp10' : {'dualprocess' : self.process_spl2_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_spl2_class, \
                                  'inverseprocess' : self.inverseprocess_spl2, \
                                  'info_retention' : False, \
                                  'defaultparams' : {'suffix' : '_sp10', \
                                                     'test_same_as_train' : True, \
                                                     'consolidate_nonoverlaps' : True}, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'exclude', \
                                  'labelctgy' : 'ord3'}})
    process_dict.update({'sp11' : {'dualprocess' : self.process_spl2_class, \
                                   'singleprocess' : None, \
                                   'postprocess' : self.postprocess_spl2_class, \
                                   'inverseprocess' : self.inverseprocess_spl2, \
                                   'info_retention' : False, \
                                  'defaultparams' : {'suffix' : '_spl2', \
                                                     'test_same_as_train' : False, \
                                                     'consolidate_nonoverlaps' : False}, \
                                   'NArowtype' : 'justNaN', \
                                   'MLinfilltype' : 'exclude', \
                                   'labelctgy' : 'ord3'}})
    process_dict.update({'sp12' : {'dualprocess' : self.process_spl2_class, \
                                   'singleprocess' : None, \
                                   'postprocess' : self.postprocess_spl2_class, \
                                   'inverseprocess' : self.inverseprocess_spl2, \
                                   'info_retention' : False, \
                                  'defaultparams' : {'suffix' : '_spl2', \
                                                     'test_same_as_train' : False, \
                                                     'consolidate_nonoverlaps' : False}, \
                                   'NArowtype' : 'justNaN', \
                                   'MLinfilltype' : 'exclude', \
                                   'labelctgy' : 'ord3'}})
    process_dict.update({'sp13' : {'dualprocess' : self.process_spl2_class, \
                                   'singleprocess' : None, \
                                   'postprocess' : self.postprocess_spl2_class, \
                                   'inverseprocess' : self.inverseprocess_spl2, \
                                   'info_retention' : False, \
                                  'defaultparams' : {'suffix' : '_spl9', \
                                                     'test_same_as_train' : True, \
                                                     'consolidate_nonoverlaps' : False}, \
                                   'NArowtype' : 'justNaN', \
                                   'MLinfilltype' : 'exclude', \
                                   'labelctgy' : 'ord3'}})
    process_dict.update({'sp14' : {'dualprocess' : self.process_spl2_class, \
                                   'singleprocess' : None, \
                                   'postprocess' : self.postprocess_spl2_class, \
                                   'inverseprocess' : self.inverseprocess_spl2, \
                                   'info_retention' : False, \
                                  'defaultparams' : {'suffix' : '_spl9', \
                                                     'test_same_as_train' : True, \
                                                     'consolidate_nonoverlaps' : False}, \
                                   'NArowtype' : 'justNaN', \
                                   'MLinfilltype' : 'exclude', \
                                   'labelctgy' : 'ord3'}})
    process_dict.update({'sp15' : {'dualprocess' : self.process_splt_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_splt_class, \
                                  'inverseprocess' : self.inverseprocess_splt, \
                                  'info_retention' : False, \
                                  'defaultparams' : {'suffix' : '_sp15', \
                                                     'concurrent_activations': True, \
                                                     'test_same_as_train' : False}, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'concurrent_act', \
                                  'labelctgy' : 'splt'}})
    process_dict.update({'sp16' : {'dualprocess' : self.process_splt_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_splt_class, \
                                  'inverseprocess' : self.inverseprocess_splt, \
                                  'info_retention' : False, \
                                  'defaultparams' : {'suffix' : '_sp16', \
                                                     'concurrent_activations': True, \
                                                     'test_same_as_train' : True}, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'concurrent_act', \
                                  'labelctgy' : 'splt'}})
    process_dict.update({'sp17' : {'dualprocess' : self.process_spl2_class, \
                                   'singleprocess' : None, \
                                   'postprocess' : self.postprocess_spl2_class, \
                                   'inverseprocess' : self.inverseprocess_spl2, \
                                   'info_retention' : False, \
                                  'defaultparams' : {'suffix' : '_spl2', \
                                                     'test_same_as_train' : False, \
                                                     'consolidate_nonoverlaps' : False}, \
                                   'NArowtype' : 'justNaN', \
                                   'MLinfilltype' : 'exclude', \
                                   'labelctgy' : 'ord3'}})
    process_dict.update({'sp18' : {'dualprocess' : self.process_spl2_class, \
                                   'singleprocess' : None, \
                                   'postprocess' : self.postprocess_spl2_class, \
                                   'inverseprocess' : self.inverseprocess_spl2, \
                                   'info_retention' : False, \
                                  'defaultparams' : {'suffix' : '_spl2', \
                                                     'test_same_as_train' : False, \
                                                     'consolidate_nonoverlaps' : False}, \
                                   'NArowtype' : 'justNaN', \
                                   'MLinfilltype' : 'exclude', \
                                   'labelctgy' : 'ord3'}})
    process_dict.update({'sp19' : {'dualprocess' : self.process_sp19_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_sp19_class, \
                                  'inverseprocess' : self.inverseprocess_sp19, \
                                  'info_retention' : False, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : '1010', \
                                  'labelctgy' : 'sp19'}})
    process_dict.update({'sp20' : {'dualprocess' : self.process_sp19_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_sp19_class, \
                                  'inverseprocess' : self.inverseprocess_sp19, \
                                  'info_retention' : False, \
                                  'defaultparams' : {'suffix' : '_sp20', \
                                                     'test_same_as_train' : True}, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : '1010', \
                                  'labelctgy' : 'sp19'}})
    process_dict.update({'sbst' : {'dualprocess' : self.process_sbst_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_sbst_class, \
                                  'inverseprocess' : self.inverseprocess_sbst, \
                                  'info_retention' : False, \
                                  'defaultparams' : {'suffix' : '_sbst', \
                                                     'test_same_as_train' : False}, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'concurrent_act', \
                                  'labelctgy' : 'sbst'}})
    process_dict.update({'sbs2' : {'dualprocess' : self.process_sbst_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_sbst_class, \
                                  'inverseprocess' : self.inverseprocess_sbst, \
                                  'info_retention' : False, \
                                  'defaultparams' : {'suffix' : '_sbs2', \
                                                     'test_same_as_train' : True}, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'concurrent_act', \
                                  'labelctgy' : 'sbst'}})
    process_dict.update({'sbs3' : {'dualprocess' : self.process_sbs3_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_sbs3_class, \
                                  'inverseprocess' : self.inverseprocess_sbs3, \
                                  'info_retention' : False, \
                                  'defaultparams' : {'suffix' : '_sbs3', \
                                                     'test_same_as_train' : False}, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : '1010', \
                                  'labelctgy' : 'sbs3'}})
    process_dict.update({'sbs4' : {'dualprocess' : self.process_sbs3_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_sbs3_class, \
                                  'inverseprocess' : self.inverseprocess_sbs3, \
                                  'info_retention' : False, \
                                  'defaultparams' : {'suffix' : '_sbs4', \
                                                     'test_same_as_train' : True}, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : '1010', \
                                  'labelctgy' : 'sbs3'}})
    process_dict.update({'srch' : {'dualprocess' : self.process_srch_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_srch_class, \
                                   'inverseprocess' : self.inverseprocess_srch, \
                                   'info_retention' : False, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'concurrent_act', \
                                  'labelctgy' : 'srch'}})
    process_dict.update({'src2' : {'dualprocess' : self.process_src2_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_src2_class, \
                                   'inverseprocess' : self.inverseprocess_src2, \
                                   'info_retention' : False, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'concurrent_act', \
                                  'labelctgy' : 'src2'}})
    process_dict.update({'src3' : {'dualprocess' : self.process_src3_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_src3_class, \
                                   'inverseprocess' : self.inverseprocess_src3, \
                                   'info_retention' : False, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'concurrent_act', \
                                  'labelctgy' : 'src3'}})
    process_dict.update({'src4' : {'dualprocess' : self.process_src4_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_src4_class, \
                                   'inverseprocess' : self.inverseprocess_src4, \
                                   'info_retention' : False, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'singlct', \
                                  'labelctgy' : 'src4'}})
    process_dict.update({'aggt' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_aggt_class, \
                                  'postprocess' : None, \
                                  'inverseprocess' : self.inverseprocess_UPCS, \
                                  'info_retention' : False, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'exclude', \
                                  'labelctgy' : 'ord3'}})
    process_dict.update({'strn' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_strn_class, \
                                  'postprocess' : None, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'exclude', \
                                  'labelctgy' : 'ord3'}})
    process_dict.update({'strg' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_strg_class, \
                                  'postprocess' : None, \
                                  'inverseprocess' : self.inverseprocess_strg, \
                                  'info_retention' : True, \
                                  'NArowtype' : 'integer', \
                                  'MLinfilltype' : 'exclude', \
                                  'labelctgy' : 'ord3'}})
    process_dict.update({'nmrc' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_nmrc_class, \
                                  'postprocess' : None, \
                                  'inverseprocess' : self.inverseprocess_nmrc, \
                                  'info_retention' : True, \
                                  'defaultparams' : {'convention' : 'numbers', \
                                                     'suffix' : '_nmrc'}, \
                                  'NArowtype' : 'parsenumeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'nmrc'}})
    process_dict.update({'nmr2' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_nmrc_class, \
                                  'postprocess' : None, \
                                  'inverseprocess' : self.inverseprocess_nmrc, \
                                  'info_retention' : False, \
                                  'defaultparams' : {'convention' : 'numbers', \
                                                     'suffix' : '_nmrc'}, \
                                  'NArowtype' : 'parsenumeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'nmbr'}})
    process_dict.update({'nmr3' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_nmrc_class, \
                                  'postprocess' : None, \
                                  'inverseprocess' : self.inverseprocess_nmrc, \
                                  'info_retention' : False, \
                                  'defaultparams' : {'convention' : 'numbers', \
                                                     'suffix' : '_nmrc'}, \
                                  'NArowtype' : 'parsenumeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'mnmx'}})
    process_dict.update({'nmr4' : {'dualprocess' : self.process_nmr4_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_nmr4_class, \
                                  'inverseprocess' : self.inverseprocess_nmrc, \
                                  'info_retention' : True, \
                                  'defaultparams' : {'convention' : 'numbers', \
                                                     'suffix' : '_nmr4', \
                                                     'test_same_as_train' : True}, \
                                  'NArowtype' : 'parsenumeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'nmrc'}})
    process_dict.update({'nmr5' : {'dualprocess' : self.process_nmr4_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_nmr4_class, \
                                  'inverseprocess' : self.inverseprocess_nmrc, \
                                  'info_retention' : False, \
                                  'defaultparams' : {'convention' : 'numbers', \
                                                     'suffix' : '_nmr4', \
                                                     'test_same_as_train' : True}, \
                                  'NArowtype' : 'parsenumeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'nmbr'}})
    process_dict.update({'nmr6' : {'dualprocess' : self.process_nmr4_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_nmr4_class, \
                                  'inverseprocess' : self.inverseprocess_nmrc, \
                                  'info_retention' : False, \
                                  'defaultparams' : {'convention' : 'numbers', \
                                                     'suffix' : '_nmr4', \
                                                     'test_same_as_train' : True}, \
                                  'NArowtype' : 'parsenumeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'mnmx'}})
    process_dict.update({'nmr7' : {'dualprocess' : self.process_nmr4_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_nmr4_class, \
                                  'inverseprocess' : self.inverseprocess_nmrc, \
                                  'info_retention' : False, \
                                  'defaultparams' : {'convention' : 'numbers', \
                                                     'suffix' : '_nmr7', \
                                                     'test_same_as_train' : False}, \
                                  'NArowtype' : 'parsenumeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'nmrc'}})
    process_dict.update({'nmr8' : {'dualprocess' : self.process_nmr4_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_nmr4_class, \
                                  'inverseprocess' : self.inverseprocess_nmrc, \
                                  'info_retention' : False, \
                                  'defaultparams' : {'convention' : 'numbers', \
                                                     'suffix' : '_nmr7', \
                                                     'test_same_as_train' : False}, \
                                  'NArowtype' : 'parsenumeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'nmbr'}})
    process_dict.update({'nmr9' : {'dualprocess' : self.process_nmr4_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_nmr4_class, \
                                  'inverseprocess' : self.inverseprocess_nmrc, \
                                  'info_retention' : False, \
                                  'defaultparams' : {'convention' : 'numbers', \
                                                     'suffix' : '_nmr7', \
                                                     'test_same_as_train' : False}, \
                                  'NArowtype' : 'parsenumeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'mnmx'}})
    process_dict.update({'nmcm' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_nmrc_class, \
                                  'postprocess' : None, \
                                  'inverseprocess' : self.inverseprocess_nmrc, \
                                  'info_retention' : True, \
                                  'defaultparams' : {'convention' : 'commas', \
                                                     'suffix' : '_nmcm'}, \
                                  'NArowtype' : 'parsenumeric_commas', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'nmcm'}})
    process_dict.update({'nmc2' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_nmrc_class, \
                                  'postprocess' : None, \
                                  'inverseprocess' : self.inverseprocess_nmrc, \
                                  'info_retention' : False, \
                                  'defaultparams' : {'convention' : 'commas', \
                                                     'suffix' : '_nmcm'}, \
                                  'NArowtype' : 'parsenumeric_commas', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'nmbr'}})
    process_dict.update({'nmc3' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_nmrc_class, \
                                  'postprocess' : None, \
                                  'inverseprocess' : self.inverseprocess_nmrc, \
                                  'info_retention' : False, \
                                  'defaultparams' : {'convention' : 'commas', \
                                                     'suffix' : '_nmcm'}, \
                                  'NArowtype' : 'parsenumeric_commas', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'mnmx'}})
    process_dict.update({'nmc4' : {'dualprocess' : self.process_nmr4_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_nmr4_class, \
                                  'inverseprocess' : self.inverseprocess_nmrc, \
                                  'info_retention' : True, \
                                  'defaultparams' : {'convention' : 'commas', \
                                                     'suffix' : '_nmc4', \
                                                     'test_same_as_train' : True}, \
                                  'NArowtype' : 'parsenumeric_commas', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'nmrc'}})
    process_dict.update({'nmc5' : {'dualprocess' : self.process_nmr4_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_nmr4_class, \
                                  'inverseprocess' : self.inverseprocess_nmrc, \
                                  'info_retention' : False, \
                                  'defaultparams' : {'convention' : 'commas', \
                                                     'suffix' : '_nmc4', \
                                                     'test_same_as_train' : True}, \
                                  'NArowtype' : 'parsenumeric_commas', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'nmbr'}})
    process_dict.update({'nmc6' : {'dualprocess' : self.process_nmr4_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_nmr4_class, \
                                  'inverseprocess' : self.inverseprocess_nmrc, \
                                  'info_retention' : False, \
                                  'defaultparams' : {'convention' : 'commas', \
                                                     'suffix' : '_nmc4', \
                                                     'test_same_as_train' : True}, \
                                  'NArowtype' : 'parsenumeric_commas', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'mnmx'}})
    process_dict.update({'nmc7' : {'dualprocess' : self.process_nmr4_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_nmr4_class, \
                                  'inverseprocess' : self.inverseprocess_nmrc, \
                                  'info_retention' : False, \
                                  'defaultparams' : {'convention' : 'commas', \
                                                     'suffix' : '_nmc7', \
                                                     'test_same_as_train' : False}, \
                                  'NArowtype' : 'parsenumeric_commas', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'nmrc'}})
    process_dict.update({'nmc8' : {'dualprocess' : self.process_nmr4_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_nmr4_class, \
                                  'inverseprocess' : self.inverseprocess_nmrc, \
                                  'info_retention' : False, \
                                  'defaultparams' : {'convention' : 'commas', \
                                                     'suffix' : '_nmc7', \
                                                     'test_same_as_train' : False}, \
                                  'NArowtype' : 'parsenumeric_commas', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'nmbr'}})
    process_dict.update({'nmc9' : {'dualprocess' : self.process_nmr4_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_nmr4_class, \
                                  'inverseprocess' : self.inverseprocess_nmrc, \
                                  'info_retention' : False, \
                                  'defaultparams' : {'convention' : 'commas', \
                                                     'suffix' : '_nmc7', \
                                                     'test_same_as_train' : False}, \
                                  'NArowtype' : 'parsenumeric_commas', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'mnmx'}})
    process_dict.update({'nmEU' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_nmrc_class, \
                                  'postprocess' : None, \
                                  'inverseprocess' : self.inverseprocess_nmrc, \
                                  'info_retention' : True, \
                                  'defaultparams' : {'convention' : 'spaces', \
                                                     'suffix' : '_nmEU'}, \
                                  'NArowtype' : 'parsenumeric_EU', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'nmrc'}})
    process_dict.update({'nmE2' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_nmrc_class, \
                                  'postprocess' : None, \
                                  'inverseprocess' : self.inverseprocess_nmrc, \
                                  'info_retention' : False, \
                                  'defaultparams' : {'convention' : 'spaces', \
                                                     'suffix' : '_nmEU'}, \
                                  'NArowtype' : 'parsenumeric_EU', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'nmbr'}})
    process_dict.update({'nmE3' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_nmrc_class, \
                                  'postprocess' : None, \
                                  'inverseprocess' : self.inverseprocess_nmrc, \
                                  'info_retention' : False, \
                                  'defaultparams' : {'convention' : 'spaces', \
                                                     'suffix' : '_nmEU'}, \
                                  'NArowtype' : 'parsenumeric_EU', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'mnmx'}})
    process_dict.update({'nmE4' : {'dualprocess' : self.process_nmr4_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_nmr4_class, \
                                  'inverseprocess' : self.inverseprocess_nmrc, \
                                  'info_retention' : True, \
                                  'defaultparams' : {'convention' : 'spaces', \
                                                     'suffix' : '_nmE4', \
                                                     'test_same_as_train' : True}, \
                                  'NArowtype' : 'parsenumeric_EU', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'nmrc'}})
    process_dict.update({'nmE5' : {'dualprocess' : self.process_nmr4_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_nmr4_class, \
                                  'inverseprocess' : self.inverseprocess_nmrc, \
                                  'info_retention' : False, \
                                  'defaultparams' : {'convention' : 'spaces', \
                                                     'suffix' : '_nmE4', \
                                                     'test_same_as_train' : True}, \
                                  'NArowtype' : 'parsenumeric_EU', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'nmbr'}})
    process_dict.update({'nmE6' : {'dualprocess' : self.process_nmr4_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_nmr4_class, \
                                  'inverseprocess' : self.inverseprocess_nmrc, \
                                  'info_retention' : False, \
                                  'defaultparams' : {'convention' : 'spaces', \
                                                     'suffix' : '_nmE4', \
                                                     'test_same_as_train' : True}, \
                                  'NArowtype' : 'parsenumeric_EU', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'mnmx'}})
    process_dict.update({'nmE7' : {'dualprocess' : self.process_nmr4_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_nmr4_class, \
                                  'inverseprocess' : self.inverseprocess_nmrc, \
                                  'info_retention' : False, \
                                  'defaultparams' : {'convention' : 'spaces', \
                                                     'suffix' : '_nmE7', \
                                                     'test_same_as_train' : False}, \
                                  'NArowtype' : 'parsenumeric_EU', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'nmrc'}})
    process_dict.update({'nmE8' : {'dualprocess' : self.process_nmr4_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_nmr4_class, \
                                  'inverseprocess' : self.inverseprocess_nmrc, \
                                  'info_retention' : False, \
                                  'defaultparams' : {'convention' : 'spaces', \
                                                     'suffix' : '_nmE7', \
                                                     'test_same_as_train' : False}, \
                                  'NArowtype' : 'parsenumeric_EU', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'nmbr'}})
    process_dict.update({'nmE9' : {'dualprocess' : self.process_nmr4_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_nmr4_class, \
                                  'inverseprocess' : self.inverseprocess_nmrc, \
                                  'info_retention' : False, \
                                  'defaultparams' : {'convention' : 'spaces', \
                                                     'suffix' : '_nmE7', \
                                                     'test_same_as_train' : False}, \
                                  'NArowtype' : 'parsenumeric_EU', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'mnmx'}})
    process_dict.update({'ors7' : {'dualprocess' : self.process_spl2_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_spl2_class, \
                                  'inverseprocess' : self.inverseprocess_spl2, \
                                  'info_retention' : False, \
                                  'defaultparams' : {'suffix' : '_spl5', \
                                                     'test_same_as_train' : False, \
                                                     'consolidate_nonoverlaps' : True}, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'exclude', \
                                  'labelctgy' : 'ord3'}})
    process_dict.update({'ors5' : {'dualprocess' : self.process_spl2_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_spl2_class, \
                                  'inverseprocess' : self.inverseprocess_spl2, \
                                  'info_retention' : False, \
                                  'defaultparams' : {'suffix' : '_spl5', \
                                                     'test_same_as_train' : False, \
                                                     'consolidate_nonoverlaps' : True}, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'exclude', \
                                  'labelctgy' : 'ord3'}})
    process_dict.update({'ors6' : {'dualprocess' : self.process_spl2_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_spl2_class, \
                                  'inverseprocess' : self.inverseprocess_spl2, \
                                  'info_retention' : False, \
                                  'defaultparams' : {'suffix' : '_spl5', \
                                                     'test_same_as_train' : False, \
                                                     'consolidate_nonoverlaps' : True}, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'exclude', \
                                  'labelctgy' : 'ord3'}})
    process_dict.update({'ordl' : {'dualprocess' : self.process_ordl_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_ordl_class, \
                                  'inverseprocess' : self.inverseprocess_ordl, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'singlct', \
                                  'labelctgy' : 'ordl'}})
    process_dict.update({'ord2' : {'dualprocess' : self.process_ordl_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_ordl_class, \
                                  'inverseprocess' : self.inverseprocess_ordl, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'singlct', \
                                  'labelctgy' : 'mnmx'}})
    process_dict.update({'ord3' : {'dualprocess' : self.process_ord3_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_ord3_class, \
                                  'inverseprocess' : self.inverseprocess_ord3, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'singlct', \
                                  'labelctgy' : 'ord3'}})
    process_dict.update({'ord5' : {'dualprocess' : self.process_ordl_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_ordl_class, \
                                  'inverseprocess' : self.inverseprocess_ordl, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'exclude', \
                                  'labelctgy' : 'ordl'}})
    process_dict.update({'ucct' : {'dualprocess' : self.process_ucct_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_ucct_class, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'ucct'}})
    process_dict.update({'ord4' : {'dualprocess' : self.process_ord3_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_ord3_class, \
                                  'inverseprocess' : self.inverseprocess_ord3, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'singlct', \
                                  'labelctgy' : 'mnmx'}})
    process_dict.update({'ors2' : {'dualprocess' : self.process_spl2_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_spl2_class, \
                                  'inverseprocess' : self.inverseprocess_spl2, \
                                  'info_retention' : False, \
                                  'defaultparams' : {'suffix' : '_spl2', \
                                                     'test_same_as_train' : False, \
                                                     'consolidate_nonoverlaps' : False}, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'singlct', \
                                  'labelctgy' : 'ord3'}})
    process_dict.update({'or10' : {'dualprocess' : None, \
                                  'singleprocess' : None, \
                                  'postprocess' : None, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'exclude', \
                                  'labelctgy' : 'mnmx'}})
    process_dict.update({'or11' : {'dualprocess' : self.process_1010_class, \
                                   'singleprocess' : None, \
                                   'postprocess' : self.postprocess_1010_class, \
                                   'inverseprocess' : self.inverseprocess_1010, \
                                   'info_retention' : True, \
                                   'NArowtype' : 'justNaN', \
                                   'MLinfilltype' : '1010', \
                                   'labelctgy' : 'ord3'}})
    process_dict.update({'or12' : {'dualprocess' : self.process_1010_class, \
                                   'singleprocess' : None, \
                                   'postprocess' : self.postprocess_1010_class, \
                                   'inverseprocess' : self.inverseprocess_1010, \
                                   'info_retention' : True, \
                                   'NArowtype' : 'justNaN', \
                                   'MLinfilltype' : '1010', \
                                   'labelctgy' : 'ord3'}})
    process_dict.update({'or13' : {'dualprocess' : self.process_1010_class, \
                                   'singleprocess' : None, \
                                   'postprocess' : self.postprocess_1010_class, \
                                   'inverseprocess' : self.inverseprocess_1010, \
                                   'info_retention' : True, \
                                   'NArowtype' : 'justNaN', \
                                   'MLinfilltype' : '1010', \
                                   'labelctgy' : 'ord3'}})
    process_dict.update({'or14' : {'dualprocess' : self.process_1010_class, \
                                   'singleprocess' : None, \
                                   'postprocess' : self.postprocess_1010_class, \
                                   'inverseprocess' : self.inverseprocess_1010, \
                                   'info_retention' : True, \
                                   'NArowtype' : 'justNaN', \
                                   'MLinfilltype' : '1010', \
                                   'labelctgy' : 'ord3'}})
    process_dict.update({'or15' : {'dualprocess' : None, \
                                   'singleprocess' : self.process_UPCS_class, \
                                   'postprocess' : None, \
                                   'inverseprocess' : self.inverseprocess_UPCS, \
                                   'info_retention' : False, \
                                   'NArowtype' : 'justNaN', \
                                   'MLinfilltype' : 'exclude', \
                                   'labelctgy' : 'ord3'}})
    process_dict.update({'or16' : {'dualprocess' : None, \
                                   'singleprocess' : self.process_UPCS_class, \
                                   'postprocess' : None, \
                                   'inverseprocess' : self.inverseprocess_UPCS, \
                                   'info_retention' : False, \
                                   'NArowtype' : 'justNaN', \
                                   'MLinfilltype' : 'exclude', \
                                   'labelctgy' : 'ord3'}})
    process_dict.update({'or17' : {'dualprocess' : None, \
                                   'singleprocess' : self.process_UPCS_class, \
                                   'postprocess' : None, \
                                   'inverseprocess' : self.inverseprocess_UPCS, \
                                   'info_retention' : False, \
                                   'NArowtype' : 'justNaN', \
                                   'MLinfilltype' : 'exclude', \
                                   'labelctgy' : 'ord3'}})
    process_dict.update({'or18' : {'dualprocess' : None, \
                                   'singleprocess' : self.process_UPCS_class, \
                                   'postprocess' : None, \
                                   'inverseprocess' : self.inverseprocess_UPCS, \
                                   'info_retention' : False, \
                                   'NArowtype' : 'justNaN', \
                                   'MLinfilltype' : 'exclude', \
                                   'labelctgy' : 'ord3'}})
    process_dict.update({'or19' : {'dualprocess' : None, \
                                   'singleprocess' : self.process_UPCS_class, \
                                   'postprocess' : None, \
                                   'inverseprocess' : self.inverseprocess_UPCS, \
                                   'info_retention' : False, \
                                   'NArowtype' : 'justNaN', \
                                   'MLinfilltype' : 'exclude', \
                                   'labelctgy' : 'ord3'}})
    process_dict.update({'or20' : {'dualprocess' : None, \
                                   'singleprocess' : self.process_UPCS_class, \
                                   'postprocess' : None, \
                                   'inverseprocess' : self.inverseprocess_UPCS, \
                                   'info_retention' : False, \
                                   'NArowtype' : 'justNaN', \
                                   'MLinfilltype' : 'exclude', \
                                   'labelctgy' : 'ord3'}})
    process_dict.update({'or21' : {'dualprocess' : None, \
                                   'singleprocess' : self.process_UPCS_class, \
                                   'postprocess' : None, \
                                   'inverseprocess' : self.inverseprocess_UPCS, \
                                   'info_retention' : False, \
                                   'NArowtype' : 'justNaN', \
                                   'MLinfilltype' : 'exclude', \
                                   'labelctgy' : 'ord3'}})
    process_dict.update({'or22' : {'dualprocess' : None, \
                                   'singleprocess' : self.process_UPCS_class, \
                                   'postprocess' : None, \
                                   'inverseprocess' : self.inverseprocess_UPCS, \
                                   'info_retention' : False, \
                                   'NArowtype' : 'justNaN', \
                                   'MLinfilltype' : 'exclude', \
                                   'labelctgy' : 'ord3'}})
    process_dict.update({'om10' : {'dualprocess' : None, \
                                  'singleprocess' : None, \
                                  'postprocess' : None, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'exclude', \
                                  'labelctgy' : 'mnmx'}})
    process_dict.update({'mmor' : {'dualprocess' : None, \
                                  'singleprocess' : None, \
                                  'postprocess' : None, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'mnmx'}})
    process_dict.update({'1010' : {'dualprocess' : self.process_1010_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_1010_class, \
                                  'inverseprocess' : self.inverseprocess_1010, \
                                  'info_retention' : True, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : '1010', \
                                  'labelctgy' : '1010'}})
    process_dict.update({'bxcx' : {'dualprocess' : self.process_bxcx_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_bxcx_class, \
                                  'NArowtype' : 'positivenumeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'nmbr'}})
    process_dict.update({'tmsc' : {'dualprocess' : None, \
                                  'singleprocess' : None, \
                                  'postprocess' : None, \
                                  'NArowtype' : 'datetime', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'tmsc'}})
    process_dict.update({'time' : {'dualprocess' : None, \
                                  'singleprocess' : None, \
                                  'postprocess' : None, \
                                  'NArowtype' : 'datetime', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'time'}})
    process_dict.update({'date' : {'dualprocess' : None, \
                                  'singleprocess' : None, \
                                  'postprocess' : None, \
                                  'NArowtype' : 'datetime', \
                                  'MLinfilltype' : 'exclude', \
                                  'labelctgy' : 'time'}})
    process_dict.update({'dat2' : {'dualprocess' : None, \
                                  'singleprocess' : None, \
                                  'postprocess' : None, \
                                  'NArowtype' : 'datetime', \
                                  'MLinfilltype' : 'exclude', \
                                  'labelctgy' : 'hldy'}})
    process_dict.update({'dat3' : {'dualprocess' : None, \
                                  'singleprocess' : None, \
                                  'postprocess' : None, \
                                  'NArowtype' : 'datetime', \
                                  'MLinfilltype' : 'exclude', \
                                  'labelctgy' : 'tmsc'}})
    process_dict.update({'dat4' : {'dualprocess' : None, \
                                  'singleprocess' : None, \
                                  'postprocess' : None, \
                                  'NArowtype' : 'datetime', \
                                  'MLinfilltype' : 'exclude', \
                                  'labelctgy' : 'tmsc'}})
    process_dict.update({'dat5' : {'dualprocess' : None, \
                                  'singleprocess' : None, \
                                  'postprocess' : None, \
                                  'NArowtype' : 'datetime', \
                                  'MLinfilltype' : 'exclude', \
                                  'labelctgy' : 'tmsc'}})
    process_dict.update({'dat6' : {'dualprocess' : None, \
                                  'singleprocess' : None, \
                                  'postprocess' : None, \
                                  'NArowtype' : 'datetime', \
                                  'MLinfilltype' : 'exclude', \
                                  'labelctgy' : 'tmsc'}})
    process_dict.update({'year' : {'dualprocess' : self.process_time_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_time_class, \
                                  'inverseprocess' : self.inverseprocess_year, \
                                  'info_retention' : False, \
                                  'defaultparams' : {'scale' : 'year', \
                                                     'suffix' : '_year', \
                                                     'normalization' : 'zscore'}, \
                                  'NArowtype' : 'datetime', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'time'}})
    process_dict.update({'yea2' : {'dualprocess' : None, \
                                  'singleprocess' : None, \
                                  'postprocess' : None, \
                                  'NArowtype' : 'datetime', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'time'}})
    process_dict.update({'yrsn' : {'dualprocess' : self.process_tmsc_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_tmsc_class, \
                                  'defaultparams' : {'scale' : 'year', \
                                                     'suffix' : '_yrsn', \
                                                     'function' : 'sin'}, \
                                  'NArowtype' : 'datetime', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'tmsc'}})
    process_dict.update({'yrcs' : {'dualprocess' : self.process_tmsc_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_tmsc_class, \
                                  'defaultparams' : {'scale' : 'year', \
                                                     'suffix' : '_yrcs', \
                                                     'function' : 'cos'}, \
                                  'NArowtype' : 'datetime', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'tmsc'}})
    process_dict.update({'mnth' : {'dualprocess' : self.process_time_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_time_class, \
                                  'defaultparams' : {'scale' : 'month', \
                                                     'suffix' : '_mnth', \
                                                     'normalization' : 'zscore'}, \
                                  'NArowtype' : 'datetime', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'time'}})
    process_dict.update({'mnt2' : {'dualprocess' : None, \
                                  'singleprocess' : None, \
                                  'postprocess' : None, \
                                  'NArowtype' : 'datetime', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'tmsc'}})
    process_dict.update({'mnt3' : {'dualprocess' : None, \
                                  'singleprocess' : None, \
                                  'postprocess' : None, \
                                  'NArowtype' : 'datetime', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'tmsc'}})
    process_dict.update({'mnt4' : {'dualprocess' : None, \
                                  'singleprocess' : None, \
                                  'postprocess' : None, \
                                  'NArowtype' : 'datetime', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'tmsc'}})
    process_dict.update({'mnt5' : {'dualprocess' : None, \
                                  'singleprocess' : None, \
                                  'postprocess' : None, \
                                  'NArowtype' : 'datetime', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'tmsc'}})
    process_dict.update({'mnt6' : {'dualprocess' : None, \
                                  'singleprocess' : None, \
                                  'postprocess' : None, \
                                  'NArowtype' : 'datetime', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'tmsc'}})
    process_dict.update({'mnsn' : {'dualprocess' : self.process_tmsc_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_tmsc_class, \
                                  'defaultparams' : {'scale' : 'month', \
                                                     'suffix' : '_mnsn', \
                                                     'function' : 'sin'}, \
                                  'NArowtype' : 'datetime', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'tmsc'}})
    process_dict.update({'mncs' : {'dualprocess' : self.process_tmsc_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_tmsc_class, \
                                  'defaultparams' : {'scale' : 'month', \
                                                     'suffix' : '_mncs', \
                                                     'function' : 'cos'}, \
                                  'NArowtype' : 'datetime', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'tmsc'}})
    process_dict.update({'mdsn' : {'dualprocess' : self.process_tmsc_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_tmsc_class, \
                                  'defaultparams' : {'scale' : 'monthday', \
                                                     'suffix' : '_mdsn', \
                                                     'function' : 'sin'}, \
                                  'NArowtype' : 'datetime', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'tmsc'}})
    process_dict.update({'mdcs' : {'dualprocess' : self.process_tmsc_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_tmsc_class, \
                                  'defaultparams' : {'scale' : 'monthday', \
                                                     'suffix' : '_mdcs', \
                                                     'function' : 'cos'}, \
                                  'NArowtype' : 'datetime', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'tmsc'}})
    process_dict.update({'days' : {'dualprocess' : self.process_time_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_time_class, \
                                  'defaultparams' : {'scale' : 'day', \
                                                     'suffix' : '_days', \
                                                     'normalization' : 'zscore'}, \
                                  'NArowtype' : 'datetime', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'time'}})
    process_dict.update({'day2' : {'dualprocess' : None, \
                                  'singleprocess' : None, \
                                  'postprocess' : None, \
                                  'NArowtype' : 'datetime', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'tmsc'}})
    process_dict.update({'day3' : {'dualprocess' : None, \
                                  'singleprocess' : None, \
                                  'postprocess' : None, \
                                  'NArowtype' : 'datetime', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'tmsc'}})
    process_dict.update({'day4' : {'dualprocess' : None, \
                                  'singleprocess' : None, \
                                  'postprocess' : None, \
                                  'NArowtype' : 'datetime', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'tmsc'}})
    process_dict.update({'day5' : {'dualprocess' : None, \
                                  'singleprocess' : None, \
                                  'postprocess' : None, \
                                  'NArowtype' : 'datetime', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'tmsc'}})
    process_dict.update({'dysn' : {'dualprocess' : self.process_tmsc_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_tmsc_class, \
                                  'defaultparams' : {'scale' : 'day', \
                                                     'suffix' : '_dysn', \
                                                     'function' : 'sin'}, \
                                  'NArowtype' : 'datetime', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'tmsc'}})
    process_dict.update({'dycs' : {'dualprocess' : self.process_tmsc_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_tmsc_class, \
                                  'defaultparams' : {'scale' : 'day', \
                                                     'suffix' : '_dycs', \
                                                     'function' : 'cos'}, \
                                  'NArowtype' : 'datetime', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'tmsc'}})
    process_dict.update({'dhms' : {'dualprocess' : self.process_tmsc_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_tmsc_class, \
                                  'defaultparams' : {'scale' : 'dayhourminute', \
                                                     'suffix' : '_dhms', \
                                                     'function' : 'sin'}, \
                                  'NArowtype' : 'datetime', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'tmsc'}})
    process_dict.update({'dhmc' : {'dualprocess' : self.process_tmsc_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_tmsc_class, \
                                  'defaultparams' : {'scale' : 'dayhourminute', \
                                                     'suffix' : '_dhmc', \
                                                     'function' : 'cos'}, \
                                  'NArowtype' : 'datetime', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'tmsc'}})
    process_dict.update({'hour' : {'dualprocess' : self.process_time_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_time_class, \
                                  'defaultparams' : {'scale' : 'hour', \
                                                     'suffix' : '_hour', \
                                                     'normalization' : 'zscore'}, \
                                  'NArowtype' : 'datetime', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'time'}})
    process_dict.update({'hrs2' : {'dualprocess' : None, \
                                  'singleprocess' : None, \
                                  'postprocess' : None, \
                                  'NArowtype' : 'datetime', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'tmsc'}})
    process_dict.update({'hrs3' : {'dualprocess' : None, \
                                  'singleprocess' : None, \
                                  'postprocess' : None, \
                                  'NArowtype' : 'datetime', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'tmsc'}})
    process_dict.update({'hrs4' : {'dualprocess' : None, \
                                  'singleprocess' : None, \
                                  'postprocess' : None, \
                                  'NArowtype' : 'datetime', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'tmsc'}})
    process_dict.update({'hrsn' : {'dualprocess' : self.process_tmsc_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_tmsc_class, \
                                  'defaultparams' : {'scale' : 'hour', \
                                                     'suffix' : '_hrsn', \
                                                     'function' : 'sin'}, \
                                  'NArowtype' : 'datetime', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'tmsc'}})
    process_dict.update({'hrcs' : {'dualprocess' : self.process_tmsc_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_tmsc_class, \
                                  'defaultparams' : {'scale' : 'hour', \
                                                     'suffix' : '_hrcs', \
                                                     'function' : 'cos'}, \
                                  'NArowtype' : 'datetime', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'tmsc'}})
    process_dict.update({'hmss' : {'dualprocess' : self.process_tmsc_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_tmsc_class, \
                                  'defaultparams' : {'scale' : 'hourminutesecond', \
                                                     'suffix' : '_hmss', \
                                                     'function' : 'sin'}, \
                                  'NArowtype' : 'datetime', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'tmsc'}})
    process_dict.update({'hmsc' : {'dualprocess' : self.process_tmsc_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_tmsc_class, \
                                  'defaultparams' : {'scale' : 'hourminutesecond', \
                                                     'suffix' : '_hmsc', \
                                                     'function' : 'cos'}, \
                                  'NArowtype' : 'datetime', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'tmsc'}})
    process_dict.update({'mint' : {'dualprocess' : self.process_time_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_time_class, \
                                  'defaultparams' : {'scale' : 'minute', \
                                                     'suffix' : '_mint', \
                                                     'normalization' : 'zscore'}, \
                                  'NArowtype' : 'datetime', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'time'}})
    process_dict.update({'min2' : {'dualprocess' : None, \
                                  'singleprocess' : None, \
                                  'postprocess' : None, \
                                  'NArowtype' : 'datetime', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'tmsc'}})
    process_dict.update({'min3' : {'dualprocess' : None, \
                                  'singleprocess' : None, \
                                  'postprocess' : None, \
                                  'NArowtype' : 'datetime', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'tmsc'}})
    process_dict.update({'min4' : {'dualprocess' : None, \
                                  'singleprocess' : None, \
                                  'postprocess' : None, \
                                  'NArowtype' : 'datetime', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'misn'}})
    process_dict.update({'misn' : {'dualprocess' : self.process_tmsc_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_tmsc_class, \
                                  'defaultparams' : {'scale' : 'minute', \
                                                     'suffix' : '_misn', \
                                                     'function' : 'sin'}, \
                                  'NArowtype' : 'datetime', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'tmsc'}})
    process_dict.update({'mics' : {'dualprocess' : self.process_tmsc_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_tmsc_class, \
                                  'defaultparams' : {'scale' : 'minute', \
                                                     'suffix' : '_mics', \
                                                     'function' : 'cos'}, \
                                  'NArowtype' : 'datetime', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'mics'}})
    process_dict.update({'mssn' : {'dualprocess' : self.process_tmsc_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_tmsc_class, \
                                  'defaultparams' : {'scale' : 'minutesecond', \
                                                     'suffix' : '_mssn', \
                                                     'function' : 'sin'}, \
                                  'NArowtype' : 'datetime', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'tmsc'}})
    process_dict.update({'mscs' : {'dualprocess' : self.process_tmsc_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_tmsc_class, \
                                  'defaultparams' : {'scale' : 'minutesecond', \
                                                     'suffix' : '_mscs', \
                                                     'function' : 'cos'}, \
                                  'NArowtype' : 'datetime', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'tmsc'}})
    process_dict.update({'scnd' : {'dualprocess' : self.process_time_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_time_class, \
                                  'defaultparams' : {'scale' : 'second', \
                                                     'suffix' : '_scnd', \
                                                     'normalization' : 'zscore'}, \
                                  'NArowtype' : 'datetime', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'time'}})
    process_dict.update({'scn2' : {'dualprocess' : None, \
                                  'singleprocess' : None, \
                                  'postprocess' : None, \
                                  'NArowtype' : 'datetime', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'tmsc'}})
    process_dict.update({'scsn' : {'dualprocess' : self.process_tmsc_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_tmsc_class, \
                                  'defaultparams' : {'scale' : 'second', \
                                                     'suffix' : '_scsn', \
                                                     'function' : 'sin'}, \
                                  'NArowtype' : 'datetime', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'tmsc'}})
    process_dict.update({'sccs' : {'dualprocess' : self.process_tmsc_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_tmsc_class, \
                                  'defaultparams' : {'scale' : 'second', \
                                                     'suffix' : '_sccs', \
                                                     'function' : 'cos'}, \
                                  'NArowtype' : 'datetime', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'tmsc'}})
    process_dict.update({'bxc2' : {'dualprocess' : self.process_bxcx_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_bxcx_class, \
                                  'NArowtype' : 'positivenumeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'nmbr'}})
    process_dict.update({'bxc3' : {'dualprocess' : self.process_bxcx_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_bxcx_class, \
                                  'NArowtype' : 'positivenumeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'nmbr'}})
    process_dict.update({'bxc4' : {'dualprocess' : self.process_bxcx_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_bxcx_class, \
                                  'NArowtype' : 'positivenumeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'nmbr'}})
    process_dict.update({'bxc5' : {'dualprocess' : self.process_bxcx_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_bxcx_class, \
                                  'NArowtype' : 'positivenumeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'nmbr'}})
    process_dict.update({'ntgr' : {'dualprocess' : self.process_ord3_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_ord3_class, \
                                  'inverseprocess' : self.inverseprocess_ord3, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'integer', \
                                  'MLinfilltype' : 'singlct', \
                                  'labelctgy' : 'mnmx'}})
    process_dict.update({'ntg2' : {'dualprocess' : self.process_ord3_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_ord3_class, \
                                  'inverseprocess' : self.inverseprocess_ord3, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'integer', \
                                  'MLinfilltype' : 'singlct', \
                                  'labelctgy' : 'mnmx'}})
    process_dict.update({'ntg3' : {'dualprocess' : self.process_ord3_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_ord3_class, \
                                  'inverseprocess' : self.inverseprocess_ord3, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'integer', \
                                  'MLinfilltype' : 'singlct', \
                                  'labelctgy' : 'mnmx'}})
    process_dict.update({'pwrs' : {'dualprocess' : self.process_pwrs_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_pwrs_class, \
                                  'inverseprocess' : self.inverseprocess_pwr2, \
                                  'info_retention' : False, \
                                  'NArowtype' : 'positivenumeric', \
                                  'MLinfilltype' : 'multirt', \
                                  'labelctgy' : 'pwrs'}})
    process_dict.update({'pwr2' : {'dualprocess' : self.process_pwrs_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_pwrs_class, \
                                  'inverseprocess' : self.inverseprocess_pwr2, \
                                  'info_retention' : False, \
                                  'defaultparams' : {'negvalues' : True}, \
                                  'NArowtype' : 'nonzeronumeric', \
                                  'MLinfilltype' : 'multirt', \
                                  'labelctgy' : 'pwrs'}})
    process_dict.update({'log0' : {'dualprocess' : self.process_log0_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_log0_class, \
                                  'inverseprocess' : self.inverseprocess_log0, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'positivenumeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'log0'}})
    process_dict.update({'log1' : {'dualprocess' : self.process_log0_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_log0_class, \
                                  'inverseprocess' : self.inverseprocess_log0, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'positivenumeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'log0'}})
    process_dict.update({'logn' : {'dualprocess' : self.process_logn_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_logn_class, \
                                  'inverseprocess' : self.inverseprocess_logn, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'positivenumeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'logn'}})
    process_dict.update({'lgnm' : {'dualprocess' : self.process_logn_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_logn_class, \
                                  'inverseprocess' : self.inverseprocess_logn, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'positivenumeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'nmbr'}})
    process_dict.update({'sqrt' : {'dualprocess' : self.process_sqrt_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_sqrt_class, \
                                  'inverseprocess' : self.inverseprocess_sqrt, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'nonnegativenumeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'sqrt'}})
    process_dict.update({'addd' : {'dualprocess' : self.process_addd_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_addd_class, \
                                  'inverseprocess' : self.inverseprocess_addd, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'addd'}})
    process_dict.update({'sbtr' : {'dualprocess' : self.process_sbtr_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_sbtr_class, \
                                  'inverseprocess' : self.inverseprocess_sbtr, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'sbtr'}})
    process_dict.update({'mltp' : {'dualprocess' : self.process_mltp_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_mltp_class, \
                                  'inverseprocess' : self.inverseprocess_mltp, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'mltp'}})
    process_dict.update({'divd' : {'dualprocess' : self.process_divd_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_divd_class, \
                                  'inverseprocess' : self.inverseprocess_divd, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'divd'}})
    process_dict.update({'rais' : {'dualprocess' : self.process_rais_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_rais_class, \
                                  'inverseprocess' : self.inverseprocess_rais, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'rais'}})
    process_dict.update({'absl' : {'dualprocess' : self.process_absl_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_absl_class, \
                                  'inverseprocess' : self.inverseprocess_absl, \
                                  'info_retention' : False, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'absl'}})
    process_dict.update({'bkt1' : {'dualprocess' : self.process_bkt1_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_bkt1_class, \
                                  'inverseprocess' : self.inverseprocess_bkt1, \
                                  'info_retention' : False, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'multirt', \
                                  'labelctgy' : 'bkt1'}})
    process_dict.update({'bkt2' : {'dualprocess' : self.process_bkt2_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_bkt2_class, \
                                  'inverseprocess' : self.inverseprocess_bkt2, \
                                  'info_retention' : False, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'multirt', \
                                  'labelctgy' : 'bkt2'}})
    process_dict.update({'bkt3' : {'dualprocess' : self.process_bkt3_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_bkt3_class, \
                                  'inverseprocess' : self.inverseprocess_bkt3, \
                                  'info_retention' : False, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'singlct', \
                                  'labelctgy' : 'bkt3'}})
    process_dict.update({'bkt4' : {'dualprocess' : self.process_bkt4_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_bkt4_class, \
                                  'inverseprocess' : self.inverseprocess_bkt4, \
                                  'info_retention' : False, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'singlct', \
                                  'labelctgy' : 'bkt4'}})
    process_dict.update({'wkdy' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_wkdy_class, \
                                  'postprocess' : None, \
                                  'NArowtype' : 'datetime', \
                                  'MLinfilltype' : 'binary', \
                                  'labelctgy' : 'wkdy'}})
    process_dict.update({'bshr' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_bshr_class, \
                                  'postprocess' : None, \
                                  'NArowtype' : 'datetime', \
                                  'MLinfilltype' : 'binary', \
                                  'labelctgy' : 'bshr'}})
    process_dict.update({'hldy' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_hldy_class, \
                                  'postprocess' : None, \
                                  'NArowtype' : 'datetime', \
                                  'MLinfilltype' : 'binary', \
                                  'labelctgy' : 'hldy'}})
    process_dict.update({'wkds' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_wkds_class, \
                                  'postprocess' : None, \
                                  'NArowtype' : 'datetime', \
                                  'MLinfilltype' : 'singlct', \
                                  'labelctgy' : 'text'}})
    process_dict.update({'wkdo' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_wkds_class, \
                                  'postprocess' : None, \
                                  'NArowtype' : 'datetime', \
                                  'MLinfilltype' : 'singlct', \
                                  'labelctgy' : 'ord3'}})
    process_dict.update({'mnts' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_mnts_class, \
                                  'postprocess' : None, \
                                  'NArowtype' : 'datetime', \
                                  'MLinfilltype' : 'singlct', \
                                  'labelctgy' : 'text'}})
    process_dict.update({'mnto' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_mnts_class, \
                                  'postprocess' : None, \
                                  'NArowtype' : 'datetime', \
                                  'MLinfilltype' : 'singlct', \
                                  'labelctgy' : 'ord3'}})
    process_dict.update({'bins' : {'dualprocess' : self.process_bins_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_bins_class, \
                                  'inverseprocess' : self.inverseprocess_bins, \
                                  'info_retention' : False, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'multirt', \
                                  'labelctgy' : 'bins'}})
    process_dict.update({'bint' : {'dualprocess' : self.process_bins_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_bins_class, \
                                  'inverseprocess' : self.inverseprocess_bins, \
                                  'info_retention' : False, \
                                  'defaultparams' : {'normalizedinput' : True}, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'multirt', \
                                  'labelctgy' : 'bins'}})
    process_dict.update({'bsor' : {'dualprocess' : self.process_bsor_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_bsor_class, \
                                  'inverseprocess' : self.inverseprocess_bsor, \
                                  'info_retention' : False, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'singlct', \
                                  'labelctgy' : 'bsor'}})
    process_dict.update({'btor' : {'dualprocess' : self.process_bsor_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_bsor_class, \
                                  'inverseprocess' : self.inverseprocess_bsor, \
                                  'info_retention' : False, \
                                  'defaultparams' : {'normalizedinput' : True}, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'singlct', \
                                  'labelctgy' : 'bsor'}})
    process_dict.update({'bnwd' : {'dualprocess' : self.process_bnwd_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_bnwd_class, \
                                  'inverseprocess' : self.inverseprocess_bnwd, \
                                  'info_retention' : False, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'multirt', \
                                  'labelctgy' : 'bnwd'}})
    process_dict.update({'bnwK' : {'dualprocess' : self.process_bnwd_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_bnwd_class, \
                                  'inverseprocess' : self.inverseprocess_bnwd, \
                                  'info_retention' : False, \
                                  'defaultparams' : {'suffix':'_bnwK', 'width':1000}, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'multirt', \
                                  'labelctgy' : 'bnwd'}})
    process_dict.update({'bnwM' : {'dualprocess' : self.process_bnwd_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_bnwd_class, \
                                  'inverseprocess' : self.inverseprocess_bnwd, \
                                  'info_retention' : False, \
                                  'defaultparams' : {'suffix':'_bnwM', 'width':1000000}, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'multirt', \
                                  'labelctgy' : 'bnwd'}})
    process_dict.update({'bnwo' : {'dualprocess' : self.process_bnwo_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_bnwo_class, \
                                  'inverseprocess' : self.inverseprocess_bnwo, \
                                  'info_retention' : False, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'singlct', \
                                  'labelctgy' : 'bnwo'}})
    process_dict.update({'bnKo' : {'dualprocess' : self.process_bnwo_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_bnwo_class, \
                                  'inverseprocess' : self.inverseprocess_bnwo, \
                                  'info_retention' : False, \
                                  'inplace_option' : True, \
                                  'defaultparams' : {'suffix':'_bnKo', 'width':1000}, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'singlct', \
                                  'labelctgy' : 'bnwo'}})
    process_dict.update({'bnMo' : {'dualprocess' : self.process_bnwo_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_bnwo_class, \
                                  'inverseprocess' : self.inverseprocess_bnwo, \
                                  'info_retention' : False, \
                                  'inplace_option' : True, \
                                  'defaultparams' : {'suffix':'_bnMo', 'width':1000000}, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'singlct', \
                                  'labelctgy' : 'bnwo'}})
    process_dict.update({'bnep' : {'dualprocess' : self.process_bnep_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_bnep_class, \
                                  'inverseprocess' : self.inverseprocess_bnep, \
                                  'info_retention' : False, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'multirt', \
                                  'labelctgy' : 'bnep'}})
    process_dict.update({'bne7' : {'dualprocess' : self.process_bnep_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_bnep_class, \
                                  'inverseprocess' : self.inverseprocess_bnep, \
                                  'info_retention' : False, \
                                  'defaultparams' : {'suffix':'_bne7', 'bincount':7}, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'multirt', \
                                  'labelctgy' : 'bnep'}})
    process_dict.update({'bne9' : {'dualprocess' : self.process_bnep_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_bnep_class, \
                                  'inverseprocess' : self.inverseprocess_bnep, \
                                  'info_retention' : False, \
                                  'defaultparams' : {'suffix':'_bne9', 'bincount':9}, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'multirt', \
                                  'labelctgy' : 'bnep'}})
    process_dict.update({'bneo' : {'dualprocess' : self.process_bneo_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_bneo_class, \
                                  'inverseprocess' : self.inverseprocess_bneo, \
                                  'info_retention' : False, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'singlct', \
                                  'labelctgy' : 'bneo'}})
    process_dict.update({'bn7o' : {'dualprocess' : self.process_bneo_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_bneo_class, \
                                  'inverseprocess' : self.inverseprocess_bneo, \
                                  'info_retention' : False, \
                                  'defaultparams' : {'suffix':'_bn7o', 'bincount':7}, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'singlct', \
                                  'labelctgy' : 'bneo'}})
    process_dict.update({'bn9o' : {'dualprocess' : self.process_bneo_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_bneo_class, \
                                  'inverseprocess' : self.inverseprocess_bneo, \
                                  'info_retention' : False, \
                                  'defaultparams' : {'suffix':'_bn9o', 'bincount':9}, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'singlct', \
                                  'labelctgy' : 'bneo'}})
    process_dict.update({'tlbn' : {'dualprocess' : self.process_tlbn_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_tlbn_class, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'concurrent_nmbr', \
                                  'labelctgy' : 'tlbn'}})
    process_dict.update({'pwor' : {'dualprocess' : self.process_pwor_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_pwor_class, \
                                  'inverseprocess' : self.inverseprocess_por2, \
                                  'info_retention' : False, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'positivenumeric', \
                                  'MLinfilltype' : 'singlct', \
                                  'labelctgy' : 'pwor'}})
    process_dict.update({'por2' : {'dualprocess' : self.process_pwor_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_pwor_class, \
                                  'inverseprocess' : self.inverseprocess_por2, \
                                  'info_retention' : False, \
                                  'inplace_option' : True, \
                                  'defaultparams' : {'negvalues' : True}, \
                                  'NArowtype' : 'nonzeronumeric', \
                                  'MLinfilltype' : 'singlct', \
                                  'labelctgy' : 'pwor'}})
    process_dict.update({'por3' : {'dualprocess' : self.process_pwor_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_pwor_class, \
                                  'inverseprocess' : self.inverseprocess_por2, \
                                  'info_retention' : False, \
                                  'inplace_option' : True, \
                                  'defaultparams' : {'negvalues' : True}, \
                                  'NArowtype' : 'nonzeronumeric', \
                                  'MLinfilltype' : 'singlct', \
                                  'labelctgy' : '1010'}})
    process_dict.update({'bkb3' : {'dualprocess' : self.process_bkt3_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_bkt3_class, \
                                  'inverseprocess' : self.inverseprocess_bkt3, \
                                  'info_retention' : False, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'singlct', \
                                  'labelctgy' : '1010'}})
    process_dict.update({'bkb4' : {'dualprocess' : self.process_bkt4_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_bkt4_class, \
                                  'inverseprocess' : self.inverseprocess_bkt4, \
                                  'info_retention' : False, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'singlct', \
                                  'labelctgy' : '1010'}})
    process_dict.update({'bsbn' : {'dualprocess' : self.process_bsor_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_bsor_class, \
                                  'inverseprocess' : self.inverseprocess_bsor, \
                                  'info_retention' : False, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'singlct', \
                                  'labelctgy' : '1010'}})
    process_dict.update({'bnwb' : {'dualprocess' : self.process_bnwo_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_bnwo_class, \
                                  'inverseprocess' : self.inverseprocess_bnwo, \
                                  'info_retention' : False, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'singlct', \
                                  'labelctgy' : '1010'}})
    process_dict.update({'bnKb' : {'dualprocess' : self.process_bnwo_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_bnwo_class, \
                                  'inverseprocess' : self.inverseprocess_bnwo, \
                                  'info_retention' : False, \
                                  'defaultparams' : {'suffix':'_bnKo', 'width':1000}, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'singlct', \
                                  'labelctgy' : '1010'}})
    process_dict.update({'bnMb' : {'dualprocess' : self.process_bnwo_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_bnwo_class, \
                                  'inverseprocess' : self.inverseprocess_bnwo, \
                                  'info_retention' : False, \
                                  'defaultparams' : {'suffix':'_bnMo', 'width':1000000}, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'singlct', \
                                  'labelctgy' : '1010'}})
    process_dict.update({'bneb' : {'dualprocess' : self.process_bneo_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_bneo_class, \
                                  'inverseprocess' : self.inverseprocess_bneo, \
                                  'info_retention' : False, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'singlct', \
                                  'labelctgy' : '1010'}})
    process_dict.update({'bn7b' : {'dualprocess' : self.process_bneo_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_bneo_class, \
                                  'inverseprocess' : self.inverseprocess_bneo, \
                                  'info_retention' : False, \
                                  'defaultparams' : {'suffix':'_bn7o', 'bincount':7}, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'singlct', \
                                  'labelctgy' : '1010'}})
    process_dict.update({'bn9b' : {'dualprocess' : self.process_bneo_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_bneo_class, \
                                  'inverseprocess' : self.inverseprocess_bneo, \
                                  'info_retention' : False, \
                                  'defaultparams' : {'suffix':'_bn9o', 'bincount':9}, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'singlct', \
                                  'labelctgy' : '1010'}})
    process_dict.update({'pwbn' : {'dualprocess' : self.process_pwor_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_pwor_class, \
                                  'inverseprocess' : self.inverseprocess_por2, \
                                  'info_retention' : False, \
                                  'NArowtype' : 'positivenumeric', \
                                  'MLinfilltype' : 'singlct', \
                                  'labelctgy' : '1010'}})
    process_dict.update({'DPn3' : {'dualprocess' : self.process_numerical_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_numerical_class, \
                                  'inverseprocess' : self.inverseprocess_nmbr, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'DPnb'}})
    process_dict.update({'DPnb' : {'dualprocess' : self.process_DPnb_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_DPnb_class, \
                                  'inverseprocess' : self.inverseprocess_UPCS, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'DPnb'}})
    process_dict.update({'DPm2' : {'dualprocess' : self.process_mnmx_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_mnmx_class, \
                                  'inverseprocess' : self.inverseprocess_mnmx, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'DPmm'}})
    process_dict.update({'DPmm' : {'dualprocess' : self.process_DPmm_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_DPmm_class, \
                                  'inverseprocess' : self.inverseprocess_UPCS, \
                                  'info_retention' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'DPmm'}})
    process_dict.update({'DPrt' : {'dualprocess' : self.process_DPrt_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_DPrt_class, \
                                  'inverseprocess' : self.inverseprocess_retn, \
                                  'info_retention' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'DPrt'}})
    process_dict.update({'DLn3' : {'dualprocess' : self.process_numerical_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_numerical_class, \
                                  'inverseprocess' : self.inverseprocess_nmbr, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'DLnb'}})
    process_dict.update({'DLnb' : {'dualprocess' : self.process_DPnb_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_DPnb_class, \
                                  'inverseprocess' : self.inverseprocess_UPCS, \
                                  'info_retention' : True, \
                                  'defaultparams' : {'noisedistribution' : 'laplace'}, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'DLnb'}})
    process_dict.update({'DLm2' : {'dualprocess' : self.process_mnmx_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_mnmx_class, \
                                  'inverseprocess' : self.inverseprocess_mnmx, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'DLmm'}})
    process_dict.update({'DLmm' : {'dualprocess' : self.process_DPmm_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_DPmm_class, \
                                  'inverseprocess' : self.inverseprocess_UPCS, \
                                  'info_retention' : True, \
                                  'defaultparams' : {'noisedistribution' : 'laplace'}, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'DLmm'}})
    process_dict.update({'DLrt' : {'dualprocess' : self.process_DPrt_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_DPrt_class, \
                                  'inverseprocess' : self.inverseprocess_retn, \
                                  'info_retention' : True, \
                                  'defaultparams' : {'noisedistribution' : 'laplace'}, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'DLrt'}})
    process_dict.update({'DPb2' : {'dualprocess' : self.process_binary_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_binary_class, \
                                  'inverseprocess' : self.inverseprocess_bnry, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'binary', \
                                  'labelctgy' : 'DPbn'}})
    process_dict.update({'DPbn' : {'dualprocess' : self.process_DPbn_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_DPbn_class, \
                                  'inverseprocess' : self.inverseprocess_UPCS, \
                                  'info_retention' : True, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'binary', \
                                  'labelctgy' : 'DPbn'}})
    process_dict.update({'DPo4' : {'dualprocess' : self.process_ord3_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_ord3_class, \
                                  'inverseprocess' : self.inverseprocess_ord3, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'singlct', \
                                  'labelctgy' : 'DPod'}})
    process_dict.update({'DPod' : {'dualprocess' : self.process_DPod_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_DPod_class, \
                                  'inverseprocess' : self.inverseprocess_UPCS, \
                                  'info_retention' : True, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'singlct', \
                                  'labelctgy' : 'DPod'}})
    process_dict.update({'DPo5' : {'dualprocess' : self.process_ord3_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_ord3_class, \
                                  'inverseprocess' : self.inverseprocess_ord3, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'singlct', \
                                  'labelctgy' : 'onht'}})
    process_dict.update({'DPo2' : {'dualprocess' : self.process_DPod_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_DPod_class, \
                                  'inverseprocess' : self.inverseprocess_UPCS, \
                                  'info_retention' : True, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'singlct', \
                                  'labelctgy' : 'onht'}})
    process_dict.update({'DPoh' : {'dualprocess' : self.process_DPod_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_DPod_class, \
                                  'inverseprocess' : self.inverseprocess_UPCS, \
                                  'info_retention' : True, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'singlct', \
                                  'labelctgy' : 'onht'}})
    process_dict.update({'DP10' : {'dualprocess' : self.process_DPod_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_DPod_class, \
                                  'inverseprocess' : self.inverseprocess_UPCS, \
                                  'info_retention' : True, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'singlct', \
                                  'labelctgy' : '1010'}})
    process_dict.update({'DPo6' : {'dualprocess' : self.process_ord3_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_ord3_class, \
                                  'inverseprocess' : self.inverseprocess_ord3, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'singlct', \
                                  'labelctgy' : '1010'}})
    process_dict.update({'DPo3' : {'dualprocess' : self.process_DPod_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_DPod_class, \
                                  'inverseprocess' : self.inverseprocess_UPCS, \
                                  'info_retention' : True, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'singlct', \
                                  'labelctgy' : '1010'}})
    process_dict.update({'NArw' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_NArw_class, \
                                  'postprocess' : None, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'boolexclude', \
                                  'labelctgy' : 'NArw'}})
    process_dict.update({'NAr2' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_NArw_class, \
                                  'postprocess' : None, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'boolexclude', \
                                  'labelctgy' : 'NArw'}})
    process_dict.update({'NAr3' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_NArw_class, \
                                  'postprocess' : None, \
                                  'NArowtype' : 'positivenumeric', \
                                  'MLinfilltype' : 'boolexclude', \
                                  'labelctgy' : 'NArw'}})
    process_dict.update({'NAr4' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_NArw_class, \
                                  'postprocess' : None, \
                                  'NArowtype' : 'nonnegativenumeric', \
                                  'MLinfilltype' : 'boolexclude', \
                                  'labelctgy' : 'NArw'}})
    process_dict.update({'NAr5' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_NArw_class, \
                                  'postprocess' : None, \
                                  'NArowtype' : 'integer', \
                                  'MLinfilltype' : 'boolexclude', \
                                  'labelctgy' : 'NArw'}})
    process_dict.update({'null' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_null_class, \
                                  'postprocess' : None, \
                                  'NArowtype' : 'exclude', \
                                  'MLinfilltype' : 'exclude', \
                                  'labelctgy' : None}})
    process_dict.update({'copy' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_copy_class, \
                                  'postprocess' : None, \
                                  'inverseprocess' : self.inverseprocess_excl, \
                                  'info_retention' : True, \
                                  'NArowtype' : 'exclude', \
                                  'MLinfilltype' : 'exclude', \
                                  'labelctgy' : 'copy'}})
    process_dict.update({'excl' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_excl_class, \
                                  'postprocess' : None, \
                                  'inverseprocess' : self.inverseprocess_excl, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'exclude', \
                                  'MLinfilltype' : 'totalexclude', \
                                  'labelctgy' : 'excl'}})
    process_dict.update({'exc2' : {'dualprocess' : self.process_exc2_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_exc2_class, \
                                  'inverseprocess' : self.inverseprocess_UPCS, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'exc2'}})
    process_dict.update({'exc3' : {'dualprocess' : self.process_exc2_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_exc2_class, \
                                  'inverseprocess' : self.inverseprocess_UPCS, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'exc2'}})
    process_dict.update({'exc4' : {'dualprocess' : self.process_exc2_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_exc2_class, \
                                  'inverseprocess' : self.inverseprocess_UPCS, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'exc2'}})
    process_dict.update({'exc5' : {'dualprocess' : self.process_exc5_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_exc5_class, \
                                  'inverseprocess' : self.inverseprocess_UPCS, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'integer', \
                                  'MLinfilltype' : 'singlct', \
                                  'labelctgy' : 'exc5'}})
    process_dict.update({'exc6' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_excl_class, \
                                  'postprocess' : None, \
                                  'inverseprocess' : self.inverseprocess_excl, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'exclude', \
                                  'MLinfilltype' : 'totalexclude', \
                                  'labelctgy' : 'excl'}})
    process_dict.update({'shfl' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_shfl_class, \
                                  'postprocess' : None, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'exclude', \
                                  'MLinfilltype' : 'exclude', \
                                  'labelctgy' : 'shfl'}})
    process_dict.update({'nmbd' : {'dualprocess' : self.process_numerical_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_numerical_class, \
                                  'inverseprocess' : self.inverseprocess_nmbr, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'nmbr'}})
    process_dict.update({'101d' : {'dualprocess' : self.process_1010_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_1010_class, \
                                  'inverseprocess' : self.inverseprocess_1010, \
                                  'info_retention' : True, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : '1010', \
                                  'labelctgy' : '1010'}})
    process_dict.update({'ordd' : {'dualprocess' : self.process_ord3_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_ord3_class, \
                                  'inverseprocess' : self.inverseprocess_ord3, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'singlct', \
                                  'labelctgy' : 'ord3'}})
    process_dict.update({'texd' : {'dualprocess' : self.process_text_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_text_class, \
                                  'inverseprocess' : self.inverseprocess_text, \
                                  'info_retention' : True, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'multirt', \
                                  'labelctgy' : 'text'}})
    process_dict.update({'bnrd' : {'dualprocess' : self.process_binary_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_binary_class, \
                                  'inverseprocess' : self.inverseprocess_bnry, \
                                  'info_retention' : True, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'binary', \
                                  'labelctgy' : 'bnry'}})
    process_dict.update({'datd' : {'dualprocess' : None, \
                                  'singleprocess' : None, \
                                  'postprocess' : None, \
                                  'NArowtype' : 'datetime', \
                                  'MLinfilltype' : 'exclude', \
                                  'labelctgy' : 'mdsn'}})
    process_dict.update({'nuld' : {'dualprocess' : None, \
                                  'singleprocess' : self.process_null_class, \
                                  'postprocess' : None, \
                                  'NArowtype' : 'exclude', \
                                  'MLinfilltype' : 'exclude', \
                                  'labelctgy' : None}})
    process_dict.update({'lbnm' : {'dualprocess' : self.process_exc2_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_exc2_class, \
                                  'inverseprocess' : self.inverseprocess_UPCS, \
                                  'info_retention' : True, \
                                  'NArowtype' : 'numeric', \
                                  'MLinfilltype' : 'numeric', \
                                  'labelctgy' : 'exc2'}})
    process_dict.update({'lb10' : {'dualprocess' : self.process_text_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_text_class, \
                                  'inverseprocess' : self.inverseprocess_text, \
                                  'info_retention' : True, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'multirt', \
                                  'labelctgy' : 'text'}})
    process_dict.update({'lbor' : {'dualprocess' : self.process_ord3_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_ord3_class, \
                                  'inverseprocess' : self.inverseprocess_ord3, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'singlct', \
                                  'labelctgy' : 'ord3'}})
    process_dict.update({'lbo5' : {'dualprocess' : self.process_ordl_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_ordl_class, \
                                  'inverseprocess' : self.inverseprocess_ordl, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'exclude', \
                                  'labelctgy' : 'ordl'}})
    process_dict.update({'lbos' : {'dualprocess' : self.process_ord3_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_ord3_class, \
                                  'inverseprocess' : self.inverseprocess_ord3, \
                                  'info_retention' : True, \
                                  'inplace_option' : True, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'singlct', \
                                  'labelctgy' : 'ord3'}})
    process_dict.update({'lbte' : {'dualprocess' : self.process_text_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_text_class, \
                                  'inverseprocess' : self.inverseprocess_text, \
                                  'info_retention' : True, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'multirt', \
                                  'labelctgy' : 'text'}})
    process_dict.update({'lbbn' : {'dualprocess' : self.process_binary_class, \
                                  'singleprocess' : None, \
                                  'postprocess' : self.postprocess_binary_class, \
                                  'inverseprocess' : self.inverseprocess_bnry, \
                                  'info_retention' : True, \
                                  'NArowtype' : 'justNaN', \
                                  'MLinfilltype' : 'multirt', \
                                  'labelctgy' : 'text'}})
    process_dict.update({'lbda' : {'dualprocess' : None, \
                                  'singleprocess' : None, \
                                  'postprocess' : None, \
                                  'NArowtype' : 'datetime', \
                                  'MLinfilltype' : 'exclude', \
                                  'labelctgy' : 'mdsn'}})

    return process_dict

  def processfamily(self, df_train, df_test, column, category, origcategory, process_dict, \
                    transform_dict, postprocess_dict, assign_param):
    '''
    #as automunge runs a for loop through each column in automunge, this is the master 
    #processing function applied which runs through the different family primitives
    #populated in the transform_dict by assembletransformdict
    
    #we will run in order of
    #siblings, cousins, parents, auntsuncles
    '''
    
    inplaceperformed = False
    
    #final upstream transform from parents or auntsuncles is elligible for inplace
    #as long as no supplement transforms were applied
    final_upstream = False
    if len(transform_dict[category]['auntsuncles']) == 0:
      if len(transform_dict[category]['parents']) > 0:
        final_upstream = transform_dict[category]['parents'][-1]
    else:
      if len(transform_dict[category]['auntsuncles']) > 0:
        final_upstream = transform_dict[category]['auntsuncles'][-1]

    #process the siblings (with downstream, supplemental)
    for sibling in transform_dict[category]['siblings']:

      if sibling != None:
        #note we use the processparent function here
        df_train, df_test, postprocess_dict, inplaceperformed = \
        self.processparent(df_train, df_test, column, sibling, origcategory, final_upstream, \
                          process_dict, transform_dict, postprocess_dict, assign_param)
    
    #process the cousins (no downstream, supplemental)
    for cousin in transform_dict[category]['cousins']:
      
      #this if statement kind of a placeholder such as for validation of primitive entry
      if cousin != None:

        #note we use the processcousin function here
        df_train, df_test, postprocess_dict, inplaceperformed = \
        self.processcousin(df_train, df_test, column, cousin, origcategory, final_upstream, \
                            process_dict, transform_dict, postprocess_dict, assign_param)

    #process the parents (with downstream, with replacement)
    for parent in transform_dict[category]['parents']:

      if parent != None:

        df_train, df_test, postprocess_dict, inplaceperformed = \
        self.processparent(df_train, df_test, column, parent, origcategory, final_upstream, \
                          process_dict, transform_dict, postprocess_dict, assign_param)
        
    #process the auntsuncles (no downstream, with replacement)
    for auntuncle in transform_dict[category]['auntsuncles']:

      if auntuncle != None:

        #note we use the processcousin function here
        df_train, df_test, postprocess_dict, inplaceperformed = \
        self.processcousin(df_train, df_test, column, auntuncle, origcategory, final_upstream, \
                            process_dict, transform_dict, postprocess_dict, assign_param)

    #if we had replacement transformations performed then mark column for deletion
    #(circle of life)
    if len(transform_dict[category]['auntsuncles']) \
    + len(transform_dict[category]['parents']) > 0 \
    and inplaceperformed is False:
      #here we'll only address downstream generaitons
      if column in postprocess_dict['column_dict']:
        postprocess_dict['column_dict'][column]['deletecolumn'] = True
      else:
        if column not in postprocess_dict['orig_noinplace']:
          postprocess_dict['orig_noinplace'].append(column)  
    elif inplaceperformed is True:
      if column in postprocess_dict['column_dict']:
        postprocess_dict['column_dict'][column]['deletecolumn'] = 'inplace'

    return df_train, df_test, postprocess_dict

  def circleoflife(self, df_train, df_test, column, category, origcategory, process_dict, \
                    transform_dict, postprocess_dict, templist1):
    '''
    #This function deletes source column for cases where family primitives 
    #included replacement, with maintenance of the associated data structures.
    
    #templist1 is the list of df_train columns before processfamily
    '''

    #if we had replacement transformations performed on first generation \
    #then delete the original column
    if len(transform_dict[category]['auntsuncles']) \
    + len(transform_dict[category]['parents']) > 0:
      
      if column in postprocess_dict['orig_noinplace']:
        del df_train[column]
        del df_test[column]

    #if we had replacement transformations performed on downstream generation \
    #then delete the associated parent column 
    
    newcolumns = set(df_train) - set(templist1)
    
    #this one is for columns replaced as part of inplace operation
    if len(newcolumns) > 0:
      anewcolumn = list(newcolumns)[0]
      temp_columnslist = postprocess_dict['column_dict'][anewcolumn]['columnslist'].copy()
      for newcolumn in temp_columnslist:
        if postprocess_dict['column_dict'][newcolumn]['deletecolumn'] == 'inplace':
          for newcolumn2 in temp_columnslist:
            if newcolumn in postprocess_dict['column_dict'][newcolumn2]['columnslist']:        
              postprocess_dict['column_dict'][newcolumn2]['columnslist'].remove(newcolumn)
    
    #this one is for columns we manually delete
    for newcolumn in newcolumns:
      if postprocess_dict['column_dict'][newcolumn]['deletecolumn'] is True:
        for newcolumn2 in newcolumns:
          if newcolumn in postprocess_dict['column_dict'][newcolumn2]['columnslist']:
            postprocess_dict['column_dict'][newcolumn2]['columnslist'].remove(newcolumn)
          
        #now we'll delete column
        #note this only worksa on single column  parents, need to incioroprate categorylist
        #for multicolumn parents (future extension)
        if newcolumn in df_train.columns:
          del df_train[newcolumn]
          del df_test[newcolumn]

    return df_train, df_test, postprocess_dict

  def dictupdate(self, column, column_dict, postprocess_dict):
    '''
    #dictupdate function takes as input column_dict, postprocess_dict, then for cases
    #where origcolmn is the same fo rhte two combines the columnslist and the 
    #normalization_dict, then appends the column_dict onto the postprocess_dict
    #returns the column_dict and postprocess_dict. Note that the passed column name
    #"column" is the column name prior to the applicaiton of processing, and the
    #name of the column after the. last processing funciton is saved as a key
    #in the column_dict
    '''

    #(reason for "key2" instead of key1 is some shuffling during editing)
    for key2 in column_dict:

      #first address carry-though of origcolumn and origcategory from parent to child
      if column in postprocess_dict['column_dict']:

        #if column is not origcolumn in postprocess_dict
        if postprocess_dict['column_dict'][column]['origcolumn'] \
        != column:

          #assign origcolumn from postprocess_dict to column_dict
          column_dict[key2]['origcolumn'] = \
          postprocess_dict['column_dict'][column]['origcolumn']

          #assign origcategory from postprocess_dict to column_dict
          column_dict[key2]['origcategory'] = \
          postprocess_dict['column_dict'][column]['origcategory']

      for key1 in postprocess_dict['column_dict']:

        #if origcolumn is the same between column_dict saved in postprocess_dict and
        #the column_dict outputed from our processing, we'll combine a few values
        if postprocess_dict['column_dict'][key1]['origcolumn'] == column_dict[key2]['origcolumn']:
          #first we'll combine the columnslist capturing all columns 
          #originating from same origcolumn for these two sets
          postprocess_dict['column_dict'][key1]['columnslist'] = \
          list(set(postprocess_dict['column_dict'][key1]['columnslist'])|set(column_dict[key2]['columnslist']))
          #apply that value to the column_dict columnslist as well
          column_dict[key2]['columnslist'] = postprocess_dict['column_dict'][key1]['columnslist']

    #now append column_dict onto postprocess_dict
    postprocess_dict['column_dict'].update(column_dict)

    #return column_dict, postprocess_dict
    return postprocess_dict

  def processcousin(self, df_train, df_test, column, cousin, origcategory, final_upstream, \
                     process_dict, transform_dict, postprocess_dict, assign_param):
    '''
    #cousin is one of the primitives for processfamily function, and it involves
    #transformations without downstream derivations without replacement of source column
    #although this same funciton can be used with the auntsuncles primitive
    #by following with a deletion of original column, also this funciton can be
    #used on the niecesnephews primitive downstream of parents or siblings since 
    #they don't have children (they're way to young for that)
    #note the processing funcitons are accessed through the process_dict

    #reminder the format of assign_param is e.g.
    #assignparam = {'splt' : {'column1' : {'minsplit' : 4}}, \
    #               'spl2' : {'column2' : {'minsplit' : 3}}}

    '''

    #for checking type of processdict entries of custom externally defined transformation functions
    def check_function():
      return
    
    inplaceperformed = False
    inplacecandidate = False
    if final_upstream == cousin:
      inplacecandidate = True

    params = self.grab_params(assign_param, cousin, column, process_dict[cousin], postprocess_dict)

    #if this is a dual process function
    if 'dualprocess' in process_dict[cousin] \
    and (isinstance(process_dict[cousin]['dualprocess'], type(self.processcousin)) \
    or isinstance(process_dict[cousin]['dualprocess'], type(check_function))):
      
      if inplacecandidate is True:
        if 'inplace_option' in process_dict[cousin]:
          if process_dict[cousin]['inplace_option'] is True:
            if 'inplace' not in params:
              inplaceperformed = True
              params.update({'inplace' : True})
            elif ('inplace' in params and params['inplace'] != False):
              inplaceperformed = True
              params.update({'inplace' : True})
            else:
              inplaceperformed = False
      else:
        #user cannot manually specify inplace by design
        if ('inplace' in params and params['inplace'] is True):
          inplaceperformed = False
          params.update({'inplace' : False})

      df_train, df_test, column_dict_list = \
      process_dict[cousin]['dualprocess'](df_train, df_test, column, origcategory, \
                                          postprocess_dict, params)

    #else if this is a single process function process train and test seperately
    elif 'singleprocess' in process_dict[cousin] \
    and (isinstance(process_dict[cousin]['singleprocess'], type(self.processcousin)) \
    or isinstance(process_dict[cousin]['singleprocess'], type(check_function))):
      
      if inplacecandidate is True:
        if 'inplace_option' in process_dict[cousin]:
          if process_dict[cousin]['inplace_option'] is True:
            if 'inplace' not in params:
              inplaceperformed = True
              params.update({'inplace' : True})
            elif ('inplace' in params and params['inplace'] != False):
              inplaceperformed = True
              params.update({'inplace' : True})
            else:
              inplaceperformed = False
      else:
        #user cannot manually specify inplace by design
        if ('inplace' in params and params['inplace'] is True):
          inplaceperformed = False
          params.update({'inplace' : False})

      df_train, column_dict_list =  \
      process_dict[cousin]['singleprocess'](df_train, column, origcategory, \
                                            postprocess_dict, params)

      df_test, _1 = \
      process_dict[cousin]['singleprocess'](df_test, column, origcategory, \
                                            postprocess_dict, params)

    #update the columnslist and normalization_dict for both column_dict and postprocess_dict
    for column_dict in column_dict_list:
      postprocess_dict = self.dictupdate(column, column_dict, postprocess_dict)

    return df_train, df_test, postprocess_dict, inplaceperformed

  def processparent(self, df_train, df_test, column, parent, origcategory, final_upstream, \
                    process_dict, transform_dict, postprocess_dict, assign_param):
    '''
    #parent is one of the primitives for processfamily function, and it involves
    #transformations with downstream derivations with replacement of source column
    #although this same funciton can be used with the siblinga primitive
    #by not following with a deletion of original column, also this funciton can be
    #used on the children primitive downstream of parents or siblings, allowing
    #the children to have children of their own, you know, grandchildren and stuff.
    #note the processing functions are accessed through the process_dict
    
    #reminder the format of assign_param is e.g.
    #assignparam = {'splt' : {'column1' : {'minsplit' : 4}}, \
    #               'spl2' : {'column2' : {'minsplit' : 3}}}
    
    #we want to apply in order of
    #upstream process, niecesnephews, friends, children, coworkers
    '''

    #for checking type of processdict entries of custom externally defined transformation functions
    def check_function():
      return

    #upstream process
    
    inplaceperformed = False
    inplacecandidate = False
    if final_upstream == parent:
      inplacecandidate = True
    
    params = self.grab_params(assign_param, parent, column, process_dict[parent], postprocess_dict)
    
    #if this is a dual process function
    if 'dualprocess' in process_dict[parent] \
    and (isinstance(process_dict[parent]['dualprocess'], type(self.processparent)) \
    or isinstance(process_dict[parent]['dualprocess'], type(check_function))):
      
      if inplacecandidate is True:
        if 'inplace_option' in process_dict[parent]:
          if process_dict[parent]['inplace_option'] is True:
            if 'inplace' not in params:
              inplaceperformed = True
              params.update({'inplace' : True})
            elif ('inplace' in params and params['inplace'] != False):
              inplaceperformed = True
              params.update({'inplace' : True})
            else:
              inplaceperformed = False
      else:
        #user cannot manually specify inplace by design
        if ('inplace' in params and params['inplace'] is True):
          inplaceperformed = False
          params.update({'inplace' : False})

      df_train, df_test, column_dict_list = \
      process_dict[parent]['dualprocess'](df_train, df_test, column, origcategory, \
                                          postprocess_dict, params)

    #else if this is a single process function process train and test seperately
    elif 'singleprocess' in process_dict[parent] \
    and (isinstance(process_dict[parent]['singleprocess'], type(self.processparent)) \
    or isinstance(process_dict[parent]['singleprocess'], type(check_function))):
      
      if inplacecandidate is True:
        if 'inplace_option' in process_dict[parent]:
          if process_dict[parent]['inplace_option'] is True:
            if 'inplace' not in params:
              inplaceperformed = True
              params.update({'inplace' : True})
            elif ('inplace' in params and params['inplace'] != False):
              inplaceperformed = True
              params.update({'inplace' : True})
            else:
              inplaceperformed = False
      else:
        #user cannot manually specify inplace by design
        if ('inplace' in params and params['inplace'] is True):
          inplaceperformed = False
          params.update({'inplace' : False})

      df_train, column_dict_list =  \
      process_dict[parent]['singleprocess'](df_train, column, origcategory, \
                                          postprocess_dict, params)

      df_test, _1 = \
      process_dict[parent]['singleprocess'](df_test, column, origcategory, \
                                          postprocess_dict, params)

    #update the columnslist and normalization_dict for both column_dict and postprocess_dict
    for column_dict in column_dict_list:
      postprocess_dict = self.dictupdate(column, column_dict, postprocess_dict)

      #note this only works for single column source, as currently implemented
      #multicolumn transforms (such as text or bins) cannot serve as parents
      #a future extension may check the categorylist from column_dict for 
      #purposes of transforms applied to multicolumn source
      parentcolumn = list(column_dict.keys())[0]

    #if transform_dict[parent] != None:

    #initialize in case no downstream performed
    parent_inplaceperformed = False
    
    #process any children
    
    #final upstream transform from parents or auntsuncles is elligible for inplace
    #as long as no supplement transforms were applied
    final_downstream = False
    if len(transform_dict[parent]['coworkers']) == 0:
      if len(transform_dict[parent]['children']) > 0:
        final_downstream = transform_dict[parent]['children'][-1]
    else:
      if len(transform_dict[parent]['coworkers']) > 0:
        final_downstream = transform_dict[parent]['coworkers'][-1]

    #process any niecesnephews
    #note the function applied is comparable to processsibling, just a different
    #parent column
    for niecenephew in transform_dict[parent]['niecesnephews']:

      if niecenephew != None:

        #process the niecenephew
        #note the function applied is processparent (using recursion)
        #parent column
        df_train, df_test, postprocess_dict, parent_inplaceperformed = \
        self.processparent(df_train, df_test, parentcolumn, niecenephew, origcategory, final_downstream, \
                           process_dict, transform_dict, postprocess_dict, assign_param)

    #process any friends
    for friend in transform_dict[parent]['friends']:

      if friend != None:

        #process the friend
        #note the function applied is processcousin
        df_train, df_test, postprocess_dict, parent_inplaceperformed = \
        self.processcousin(df_train, df_test, parentcolumn, friend, origcategory, final_downstream, \
                           process_dict, transform_dict, postprocess_dict, assign_param)
    
    for child in transform_dict[parent]['children']:

      if child != None:

        #process the child
        #note the function applied is processparent (using recursion)
        #parent column
        df_train, df_test, postprocess_dict, parent_inplaceperformed = \
        self.processparent(df_train, df_test, parentcolumn, child, origcategory, final_downstream, \
                           process_dict, transform_dict, postprocess_dict, assign_param)

    #process any coworkers
    for coworker in transform_dict[parent]['coworkers']:

      if coworker != None:

        #process the coworker
        #note the function applied is processcousin
        df_train, df_test, postprocess_dict, parent_inplaceperformed = \
        self.processcousin(df_train, df_test, parentcolumn, coworker, origcategory, final_downstream, \
                           process_dict, transform_dict, postprocess_dict, assign_param)

    #if we had replacement transformations performed then mark column for deletion
    #(circle of life)
    if len(transform_dict[parent]['children']) \
    + len(transform_dict[parent]['coworkers']) > 0 \
    and parent_inplaceperformed is False:
      #here we'll only address downstream generaitons
      if parentcolumn in postprocess_dict['column_dict']:
        postprocess_dict['column_dict'][parentcolumn]['deletecolumn'] = True
      else:
        if parentcolumn not in postprocess_dict['orig_noinplace']:
          postprocess_dict['orig_noinplace'].append(parentcolumn)
    elif parent_inplaceperformed is True:
      if parentcolumn in postprocess_dict['column_dict']:
        postprocess_dict['column_dict'][parentcolumn]['deletecolumn'] = 'inplace'

    return df_train, df_test, postprocess_dict, inplaceperformed

  def df_copy_train(self, df_train, column, newcolumn, suffixoverlap_results = {}):
    """
    #performs a copy operation to add column to a df_train
    #Before any new columns added to df_train
    #checks that they are not already present in df_train
    #if so returns error message and logs in suffixoverlap_results
    """
    
    #test for overlap error
    if newcolumn in df_train.columns:
      
      print("*****************")
      print("Warning of suffix overlap error")
      print("When creating new column: ", newcolumn)
      print("The column was already found present in df_train headers.")
      print("")
      print("Some potential quick fixes for this error include:")
      print("- rename columns to integers before passing to automunge(.)")
      print("- strip underscores '_' from column header titles.")
      print("(convention is all suffix appenders include an underscore)")
      print("")
      print("Please note any updates to column headers will need to be carried through to assignment parameters.")
      print("*****************")
      print("")
      
      suffixoverlap_results.update({newcolumn : True})
      
    else:
      
      df_train[newcolumn] = df_train[column].copy()
      
      suffixoverlap_results.update({newcolumn : False})
    
    return df_train, suffixoverlap_results

  def df_check_suffixoverlap(self, df_train, newcolumns, suffixoverlap_results = {}):
    """
    #checks that newcolumns list are not already present in df_train
    #logs in suffixoverlap_results
    """
    
    if not isinstance(newcolumns, list):
      newcolumns = [newcolumns]
    
    for newcolumn in newcolumns:
      
      if newcolumn in df_train.columns:
        
        print("*****************")
        print("Warning of suffix overlap error")
        print("When creating new column: ", newcolumn)
        print("The column was already found present in df_train headers.")
        print("")
        print("Some potential quick fixes for this error include:")
        print("- rename columns to integers before passing to automunge(.)")
        print("- strip underscores '_' from column header titles.")
        print("(convention is all suffix appenders include an underscore)")
        print("")
        print("Please note any updates to column headers will need to be carried through to assignment parameters.")
        print("*****************")
        print("")

        suffixoverlap_results.update({newcolumn : True})

      else:

        suffixoverlap_results.update({newcolumn : False})
        
    return suffixoverlap_results

  def suffix_overlap_final_aggregation_and_printouts(self, postprocess_dict):
    """
    #Performs a final round of printouts in case of identified suffix overlap error
    #Also aggregates the validation results stored in column_dict
    #To a those returned in postprocess_dict['miscparameters_results']
    """
    
    #then at completion of automunge(.), aggregate the suffixoverlap results
    #and do an additional printout if any column overlap error to be sure user sees message
    for entry1 in postprocess_dict['column_dict']:
      for entry2 in postprocess_dict['column_dict'][entry1]['suffixoverlap_results']:
        if postprocess_dict['column_dict'][entry1]['suffixoverlap_results'][entry2] is True:
          
          print("*****************")
          print("Warning of suffix overlap error")
          print("When creating new column: ", entry2)
          print("The column was already found present in df_train headers.")
          print("")
          print("Some potential quick fixes for this error include:")
          print("- rename columns to integers before passing to automunge(.)")
          print("- strip underscores '_' from column header titles.")
          print("(convention is all suffix appenders include an underscore)")
          print("")
          print("Please note any updates to column headers will need to be carried through to assignment parameters.")
          print("*****************")
          print("")
      
      postprocess_dict['miscparameters_results']['suffixoverlap_results'].update(
      postprocess_dict['column_dict'][entry1]['suffixoverlap_results'])

    for entry1 in postprocess_dict['miscparameters_results']['PCA_suffixoverlap_results']:
      if postprocess_dict['miscparameters_results']['PCA_suffixoverlap_results'][entry1] is True:

          print("*****************")
          print("Warning of suffix overlap error")
          print("When creating PCA column: ", entry1)
          print("The column was already found present in df_train headers.")
          print("")
          print("Note that PCA returned columns are of form: PCAcol0")
          print("Where # is integer")
          print("This form of column header should be avoided in passed data.")
          print("")

    for entry1 in postprocess_dict['miscparameters_results']['Binary_suffixoverlap_results']:
      if postprocess_dict['miscparameters_results']['Binary_suffixoverlap_results'][entry1] is True:

          print("*****************")
          print("Warning of suffix overlap error")
          print("When creating Binary column: ", entry1)
          print("The column was already found present in df_train headers.")
          print("")
          print("Note that Binary returned columns are of form: Binary_1010_#")
          print("Where # is integer")
          print("This error might have occured if you passed data including column header 'Binary' to '1010' transform")
          print("This form of column header should be avoided in passed data.")
          print("")

    for entry1 in postprocess_dict['miscparameters_results']['excl_suffixoverlap_results']:
      if postprocess_dict['miscparameters_results']['excl_suffixoverlap_results'][entry1] is True:

          print("*****************")
          print("Warning of suffix overlap error")
          print("When removing '_excl' suffix for column: ", entry1)
          print("The column without suffix was already found present in df_train headers.")
          print("")
          
    return postprocess_dict

  def process_NArw_class(self, df, column, category, postprocess_dict, params = {}):
    '''
    #processing funciton that creates a boolean column indicating 1 for rows
    #corresponding to missing or improperly formated data in source column
    #note this uses the NArows function which has a category specific approach
    #returns same dataframe with new column of name column + '_NArw'
    #note this is a "singleprocess" function since is applied to single dataframe
    '''
    
    suffixoverlap_results = {}
    
    suffixoverlap_results = \
    self.df_check_suffixoverlap(df, column + '_NArw', suffixoverlap_results)

    df[column + '_NArw'] = self.getNArows(df, column, category, postprocess_dict)

    #change NArows data type to 8-bit (1 byte) integers for memory savings
    df[column + '_NArw'] = df[column + '_NArw'].astype(np.int8)

    #create list of columns
    nmbrcolumns = [column + '_NArw']
    
    #for drift report
    pct_NArw = df[column + '_NArw'].sum() / df[column + '_NArw'].shape[0]

    #create normalization dictionary
    NArwnormalization_dict = {column + '_NArw' : {'pct_NArw':pct_NArw}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []
    
    for nc in nmbrcolumns:

      column_dict = { nc : {'category' : 'NArw', \
                           'origcategory' : category, \
                           'normalization_dict' : NArwnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : [nc], \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())

    return df, column_dict_list

  def process_numerical_class(self, mdf_train, mdf_test, column, category, \
                              postprocess_dict, params = {}):
    '''
    #process_numerical_class(mdf_train, mdf_test, column, category)
    #function to normalize data to mean of 0 and standard deviation of 1 \
    #z score normalization) 
    #takes as arguement pandas dataframe of training and test data (mdf_train), (mdf_test)\
    #and the name of the column string ('column') and parent category (category)
    #replaces missing or improperly formatted data with mean of remaining values
    #returns same dataframes with new column of name column + '_nmbr'
    #note this is a "dualprocess" function since is applied to both dataframes
    #expect this approach works better when the numerical distribution is thin tailed
    #if only have training but not test data handy, use same training data for both dataframe inputs
    '''
    
    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    #initialize parameters
    #offset is just an added constant applied after multiplier
    if 'offset' in params:
      offset = params['offset']
    else:
      offset = 0
    
    #multiplier scales the set by multiplication prior to offset
    if 'multiplier' in params:
      multiplier = params['multiplier']
    else:
      multiplier = 1
    
    #cap can be passed as True for max of training data or as a specific value prior to normalization, False for no cap
    if 'cap' in params:
      cap = params['cap']
    else:
      cap = False
      
    #floor can be passed as True for min of training data or as a specific value prior to normalization, False for no floor
    if 'floor' in params:
      floor = params['floor']
    else:
      floor = False
      
    #adjinfill accepts True/False to change default infill from mean inputation to adjacent cell
    if 'adjinfill' in params:
      adjinfill = params['adjinfill']
    else:
      adjinfill = False

    if inplace is not True:
      
      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.df_copy_train(mdf_train, column, column + '_nmbr', suffixoverlap_results)

      mdf_test[column + '_nmbr'] = mdf_test[column].copy()
    
    else:
      
      suffixoverlap_results = \
      self.df_check_suffixoverlap(mdf_train, column + '_nmbr', suffixoverlap_results)
      
      mdf_train.rename(columns = {column : column + '_nmbr'}, inplace = True)
      mdf_test.rename(columns = {column : column + '_nmbr'}, inplace = True)

    #convert all values to either numeric or NaN
    mdf_train[column + '_nmbr'] = pd.to_numeric(mdf_train[column + '_nmbr'], errors='coerce')
    mdf_test[column + '_nmbr'] = pd.to_numeric(mdf_test[column + '_nmbr'], errors='coerce')
    
    #a few more metrics collected for driftreport
    #get maximum value of training column
    maximum = mdf_train[column + '_nmbr'].max()
    #get minimum value of training column
    minimum = mdf_train[column + '_nmbr'].min()
    
    #if cap < maximum, maximum = cap
    if cap is not False and cap is not True:
      if cap < maximum:
        maximum = cap
    if floor is not False and floor is not True:
      if floor > minimum:
        minimum = floor
        
    #cap and floor application
    if cap is True:
      cap = maximum
    if floor is True:
      floor = minimum
      
    if cap is not False:
      #replace values in test > cap with cap
      mdf_train.loc[mdf_train[column + '_nmbr'] > cap, (column + '_nmbr')] \
      = cap
      
      mdf_test.loc[mdf_test[column + '_nmbr'] > cap, (column + '_nmbr')] \
      = cap
    
    if floor is not False:
      #replace values in test < floor with floor
      mdf_train.loc[mdf_train[column + '_nmbr'] < floor, (column + '_nmbr')] \
      = floor
      
      mdf_test.loc[mdf_test[column + '_nmbr'] < floor, (column + '_nmbr')] \
      = floor

    #get mean of training data
    mean = mdf_train[column + '_nmbr'].mean()
    if mean != mean:
      mean = 0
      
    if adjinfill is True:
      mdf_train[column + '_nmbr'] = mdf_train[column + '_nmbr'].fillna(method='ffill')
      mdf_test[column + '_nmbr'] = mdf_test[column + '_nmbr'].fillna(method='ffill')
      mdf_train[column + '_nmbr'] = mdf_train[column + '_nmbr'].fillna(method='bfill')
      mdf_test[column + '_nmbr'] = mdf_test[column + '_nmbr'].fillna(method='bfill')

    #replace missing data with training set mean
    mdf_train[column + '_nmbr'] = mdf_train[column + '_nmbr'].fillna(mean)
    mdf_test[column + '_nmbr'] = mdf_test[column + '_nmbr'].fillna(mean)

    #subtract mean from column for both train and test
    mdf_train[column + '_nmbr'] = mdf_train[column + '_nmbr'] - mean
    mdf_test[column + '_nmbr'] = mdf_test[column + '_nmbr'] - mean

    #get standard deviation of training data
    std = mdf_train[column + '_nmbr'].std()
    
    #special case, if standard deviation is 0 we'll set it to 1 to avoid division by 0
    if std == 0:
      std = 1

    #divide column values by std for both training and test data
    #offset, multiplier are parameters that defaults to zero, one
    mdf_train[column + '_nmbr'] = mdf_train[column + '_nmbr'] / std * multiplier + offset
    mdf_test[column + '_nmbr'] = mdf_test[column + '_nmbr'] / std * multiplier + offset
    
#     #change data type for memory savings
#     mdf_train[column + '_nmbr'] = mdf_train[column + '_nmbr'].astype(np.float32)
#     mdf_test[column + '_nmbr'] = mdf_test[column + '_nmbr'].astype(np.float32)

    #create list of columns
    nmbrcolumns = [column + '_nmbr']

    nmbrnormalization_dict = {column + '_nmbr' : {'mean' : mean, 'std' : std, \
                                                  'max' : maximum, 'min' : minimum, \
                                                  'offset' : offset, 'multiplier': multiplier, \
                                                  'cap' : cap, 'floor' : floor, \
                                                  'adjinfill' : adjinfill}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in nmbrcolumns:

      column_dict = { nc : {'category' : 'nmbr', \
                           'origcategory' : category, \
                           'normalization_dict' : nmbrnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())
        
    return mdf_train, mdf_test, column_dict_list
  
  def process_dxdt_class(self, df, column, category, postprocess_dict, params = {}):
    '''
    #process_dxdt_class(df, column, category, postprocess_dict)
    #function to translate a continues variable into a bounded variable
    #by taking delta of row from preceding row
    #assumes the rows are not shuffled and represent a continuous funciton 
    #with consistent time steps
    
    #for missing values, uses adjacent cell infill as default
    '''
    
    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    #initialize parameters
    if 'periods' in params:
      periods = params['periods']
    else:
      periods = 1
    
    if inplace is not True:
      
      #copy source column into new column
      df, suffixoverlap_results = \
      self.df_copy_train(df, column, column + '_dxdt', suffixoverlap_results)
    
    else:
      
      suffixoverlap_results = \
      self.df_check_suffixoverlap(df, column + '_dxdt', suffixoverlap_results)
      
      df.rename(columns = {column : column + '_dxdt'}, inplace = True)
    
    #convert all values to either numeric or NaN
    df[column + '_dxdt'] = pd.to_numeric(df[column + '_dxdt'], errors='coerce')
    
    #apply ffill to replace NArows with value from adjacent cell in pre4ceding row
    df[column + '_dxdt'] = df[column + '_dxdt'].fillna(method='ffill')
    
    #we'll follow with a bfill just in case first row had a nan
    df[column + '_dxdt'] = df[column + '_dxdt'].fillna(method='bfill') 
    
    #subtract preceding row
    df[column + '_dxdt'] = df[column + '_dxdt'] - df[column + '_dxdt'].shift(periods = periods)
    
    #first row will have a nan so just one more backfill
    df[column + '_dxdt'] = df[column + '_dxdt'].fillna(method='bfill')
    
    #then one more infill with to address scenario when data wasn't numeric
    #get arbitrary cell value, if one is nan then all will be
    value = df[column + '_dxdt'].values[0]
    if value != value:
      value = 0

      df[column + '_dxdt'] = df[column + '_dxdt'].fillna(value)
    
    #create list of columns
    nmbrcolumns = [column + '_dxdt']

    #grab some driftreport metrics
    #note that if this function implemented for data streams at scale it may be appropriate
    #to consider creating an alternate to dxdt without the driftreport metrics for postmunge efficiency
    positiveratio = df[df[column + '_dxdt'] >= 0].shape[0] / df[column + '_dxdt'].shape[0]
    negativeratio = df[df[column + '_dxdt'] < 0].shape[0] / df[column + '_dxdt'].shape[0]
    zeroratio = df[df[column + '_dxdt'] == 0].shape[0] / df[column + '_dxdt'].shape[0]
    minimum = df[column + '_dxdt'].min()
    maximum = df[column + '_dxdt'].max()
    mean = df[column + '_dxdt'].mean()
    std = df[column + '_dxdt'].std()

    nmbrnormalization_dict = {column + '_dxdt' : {'positiveratio' : positiveratio, \
                                                  'negativeratio' : negativeratio, \
                                                  'zeroratio' : zeroratio, \
                                                  'minimum' : minimum, \
                                                  'maximum' : maximum, \
                                                  'mean' : mean, \
                                                  'std' : std, \
                                                  'periods' : periods}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in nmbrcolumns:
      
      column_dict = { nc : {'category' : 'dxdt', \
                           'origcategory' : category, \
                           'normalization_dict' : nmbrnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())
        
    return df, column_dict_list

  def process_dxd2_class(self, df, column, category, postprocess_dict, params = {}):
    '''
    #process_dxd2_class(df, column, category, postprocess_dict)
    #function to translate a continues variable into a bounded variable
    #by taking delta of average of last two rows minus 
    #average of preceding two rows before that
    #should take a littel noise out of noisy data
    #assumes the rows are not shuffled and represent a continuous funciton 
    #with consistent time steps
    
    #for missing values, uses adjacent cell infill as default
    '''
    
    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    #initialize parameters
    if 'periods' in params:
      periods = params['periods']
    else:
      periods = 2
    
    if inplace is not True:
      
      #copy source column into new column
      df, suffixoverlap_results = \
      self.df_copy_train(df, column, column + '_dxd2', suffixoverlap_results)
    
    else:
      
      suffixoverlap_results = \
      self.df_check_suffixoverlap(df, column + '_dxd2', suffixoverlap_results)
      
      df.rename(columns = {column : column + '_dxd2'}, inplace = True)
    
    #convert all values to either numeric or NaN
    df[column + '_dxd2'] = pd.to_numeric(df[column + '_dxd2'], errors='coerce')
    
    #apply ffill to replace NArows with value from adjacent cell in pre4ceding row
    df[column + '_dxd2'] = df[column + '_dxd2'].fillna(method='ffill')
    
    #we'll follow with a bfill just in case first row had a nan
    df[column + '_dxd2'] = df[column + '_dxd2'].fillna(method='bfill')  
    
#     #we're going to take difference of average of last two rows with two rows preceding
#     df[column + '_dxd2'] = (df[column + '_dxd2'] + df[column + '_dxd2'].shift()) / 2 \
#                            - ((df[column + '_dxd2'].shift(periods=2) + df[column + '_dxd2'].shift(periods=3)) / 2)

    suffixoverlap_results = \
    self.df_check_suffixoverlap(df, [column + '_temp1'], suffixoverlap_results)

    df[column + '_temp1'] = df[column + '_dxd2'].copy()
    # df_train['number7_temp3'] = df_train['number7'].copy()

    for i in range(periods-1):
      df[column + '_temp1'] = df[column + '_temp1'] + df[column + '_dxd2'].shift(periods = i+1)

    df[column + '_dxd2'] = (df[column + '_temp1'] - df[column + '_temp1'].shift(periods = periods)) / periods
    
    #first row will have a nan so just one more backfill
    df[column + '_dxd2'] = df[column + '_dxd2'].fillna(method='bfill')
    
    #then one more infill with to address scenario when data wasn't numeric
    #get arbitrary cell value, if one is nan then all will be
    value = df[column + '_dxd2'].values[0]
    if value != value:
      value = 0

      df[column + '_dxd2'] = df[column + '_dxd2'].fillna(value)
    
    del df[column + '_temp1']
    
    #create list of columns
    nmbrcolumns = [column + '_dxd2']

    #grab some driftreport metrics
    #note that if this function implemented for data streams at scale it may be appropriate
    #to consider creating an alternate to dxd2 without the driftreport metrics for postmunge efficiency
    positiveratio = df[df[column + '_dxd2'] >= 0].shape[0] / df[column + '_dxd2'].shape[0]
    negativeratio = df[df[column + '_dxd2'] < 0].shape[0] / df[column + '_dxd2'].shape[0]
    zeroratio = df[df[column + '_dxd2'] == 0].shape[0] / df[column + '_dxd2'].shape[0]
    minimum = df[column + '_dxd2'].min()
    maximum = df[column + '_dxd2'].max()
    mean = df[column + '_dxd2'].mean()
    std = df[column + '_dxd2'].std()
  
    nmbrnormalization_dict = {column + '_dxd2' : {'positiveratio' : positiveratio, \
                                                  'negativeratio' : negativeratio, \
                                                  'zeroratio' : zeroratio, \
                                                  'minimum' : minimum, \
                                                  'maximum' : maximum, \
                                                  'mean' : mean, \
                                                  'std' : std, \
                                                  'periods' : periods}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in nmbrcolumns:

      column_dict = { nc : {'category' : 'dxd2', \
                           'origcategory' : category, \
                           'normalization_dict' : nmbrnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())
        
    return df, column_dict_list

  def process_shft_class(self, df, column, category, postprocess_dict, params = {}):
    '''
    #process_shft_class(df, column, category, postprocess_dict)
    #function to shift a sequential set forward by one or more time steps    
    #for missing values, uses adjacent cell infill as default
    #accepts parameter 'periods' for number of time steps, defaults to one
    #accepts parameter 'suffix' for column suffix appender
    #such as may be useful if applying this transform to the same column more than once
    '''
    
    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    #initialize parameters
    if 'periods' in params:
      periods = params['periods']
    else:
      periods = 1
    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = 'shft'
      
    shft_column = column + '_' + suffix
    
    if inplace is not True:
      
      #copy source column into new column
      df, suffixoverlap_results = \
      self.df_copy_train(df, column, shft_column, suffixoverlap_results)
    
    else:
      
      suffixoverlap_results = \
      self.df_check_suffixoverlap(df, shft_column, suffixoverlap_results)
      
      df.rename(columns = {column : shft_column}, inplace = True)
    
    #convert all values to either numeric or NaN
    df[shft_column] = pd.to_numeric(df[shft_column], errors='coerce')
    
    #apply ffill to replace NArows with value from adjacent cell in pre4ceding row
    df[shft_column] = df[shft_column].fillna(method='ffill')
    
    #we'll follow with a bfill just in case first row had a nan
    df[shft_column] = df[shft_column].fillna(method='bfill') 
    
    #shift from preceding row
    df[shft_column] = df[shft_column].shift(periods = periods)
    
    #first row will have a nan so just one more backfill
    df[shft_column] = df[shft_column].fillna(method='bfill')
    
    #then one more infill with to address scenario when data wasn't numeric
    #get arbitrary cell value, if one is nan then all will be
    value = df[shft_column].values[0]
    if value != value:
      value = 0

      df[shft_column] = df[shft_column].fillna(value)
    
    #create list of columns
    nmbrcolumns = [shft_column]

    #grab some driftreport metrics
    #note that if this function implemented for data streams at scale it may be appropriate
    #to consider creating an alternate to dxdt without the driftreport metrics for postmunge efficiency
    positiveratio = df[df[shft_column] >= 0].shape[0] / df[shft_column].shape[0]
    negativeratio = df[df[shft_column] < 0].shape[0] / df[shft_column].shape[0]
    zeroratio = df[df[shft_column] == 0].shape[0] / df[shft_column].shape[0]
    minimum = df[shft_column].min()
    maximum = df[shft_column].max()
    mean = df[shft_column].mean()
    std = df[shft_column].std()

    nmbrnormalization_dict = {shft_column :      {'positiveratio' : positiveratio, \
                                                  'negativeratio' : negativeratio, \
                                                  'zeroratio' : zeroratio, \
                                                  'minimum' : minimum, \
                                                  'maximum' : maximum, \
                                                  'mean' : mean, \
                                                  'std' : std, \
                                                  'periods' : periods, \
                                                  'suffix' : suffix}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in nmbrcolumns:
      
      column_dict = { nc : {'category' : 'shft', \
                           'origcategory' : category, \
                           'normalization_dict' : nmbrnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())
        
    return df, column_dict_list
  
  def process_shf2_class(self, df, column, category, postprocess_dict, params = {}):
    '''
    #process_shft_class(df, column, category, postprocess_dict)
    #function to shift a sequential set forward by one or more time steps    
    #for missing values, uses adjacent cell infill as default
    #accepts parameter 'periods' for number of time steps, defaults to one
    #accepts parameter 'suffix' for column suffix appender
    #such as may be useful if applying this transform to the same column more than once
    '''
    
    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    #initialize parameters
    if 'periods' in params:
      periods = params['periods']
    else:
      periods = 2
    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = 'shf2'
      
    shft_column = column + '_' + suffix
    
    if inplace is not True:
      
      #copy source column into new column
      df, suffixoverlap_results = \
      self.df_copy_train(df, column, shft_column, suffixoverlap_results)
    
    else:
      
      suffixoverlap_results = \
      self.df_check_suffixoverlap(df, shft_column, suffixoverlap_results)
      
      df.rename(columns = {column : shft_column}, inplace = True)
    
    #convert all values to either numeric or NaN
    df[shft_column] = pd.to_numeric(df[shft_column], errors='coerce')
    
    #apply ffill to replace NArows with value from adjacent cell in pre4ceding row
    df[shft_column] = df[shft_column].fillna(method='ffill')
    
    #we'll follow with a bfill just in case first row had a nan
    df[shft_column] = df[shft_column].fillna(method='bfill') 
    
    #shift from preceding row
    df[shft_column] = df[shft_column].shift(periods = periods)
    
    #first row will have a nan so just one more backfill
    df[shft_column] = df[shft_column].fillna(method='bfill')
    
    #then one more infill with to address scenario when data wasn't numeric
    #get arbitrary cell value, if one is nan then all will be
    value = df[shft_column].values[0]
    if value != value:
      value = 0

      df[shft_column] = df[shft_column].fillna(value)
    
    #create list of columns
    nmbrcolumns = [shft_column]

    #grab some driftreport metrics
    #note that if this function implemented for data streams at scale it may be appropriate
    #to consider creating an alternate to dxdt without the driftreport metrics for postmunge efficiency
    positiveratio = df[df[shft_column] >= 0].shape[0] / df[shft_column].shape[0]
    negativeratio = df[df[shft_column] < 0].shape[0] / df[shft_column].shape[0]
    zeroratio = df[df[shft_column] == 0].shape[0] / df[shft_column].shape[0]
    minimum = df[shft_column].min()
    maximum = df[shft_column].max()
    mean = df[shft_column].mean()
    std = df[shft_column].std()

    nmbrnormalization_dict = {shft_column :      {'positiveratio' : positiveratio, \
                                                  'negativeratio' : negativeratio, \
                                                  'zeroratio' : zeroratio, \
                                                  'minimum' : minimum, \
                                                  'maximum' : maximum, \
                                                  'mean' : mean, \
                                                  'std' : std, \
                                                  'periods' : periods, \
                                                  'suffix' : suffix}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in nmbrcolumns:
      
      column_dict = { nc : {'category' : 'shf2', \
                           'origcategory' : category, \
                           'normalization_dict' : nmbrnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())
        
    return df, column_dict_list
  
  def process_shf3_class(self, df, column, category, postprocess_dict, params = {}):
    '''
    #process_shft_class(df, column, category, postprocess_dict)
    #function to shift a sequential set forward by one or more time steps    
    #for missing values, uses adjacent cell infill as default
    #accepts parameter 'periods' for number of time steps, defaults to one
    #accepts parameter 'suffix' for column suffix appender
    #such as may be useful if applying this transform to the same column more than once
    '''
    
    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    #initialize parameters
    if 'periods' in params:
      periods = params['periods']
    else:
      periods = 3
    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = 'shf3'
      
    shft_column = column + '_' + suffix
    
    if inplace is not True:
      
      #copy source column into new column
      df, suffixoverlap_results = \
      self.df_copy_train(df, column, shft_column, suffixoverlap_results)
    
    else:
      
      suffixoverlap_results = \
      self.df_check_suffixoverlap(df, shft_column, suffixoverlap_results)
      
      df.rename(columns = {column : shft_column}, inplace = True)
    
    #convert all values to either numeric or NaN
    df[shft_column] = pd.to_numeric(df[shft_column], errors='coerce')
    
    #apply ffill to replace NArows with value from adjacent cell in pre4ceding row
    df[shft_column] = df[shft_column].fillna(method='ffill')
    
    #we'll follow with a bfill just in case first row had a nan
    df[shft_column] = df[shft_column].fillna(method='bfill') 
    
    #shift from preceding row
    df[shft_column] = df[shft_column].shift(periods = periods)
    
    #first row will have a nan so just one more backfill
    df[shft_column] = df[shft_column].fillna(method='bfill')
    
    #then one more infill with to address scenario when data wasn't numeric
    #get arbitrary cell value, if one is nan then all will be
    value = df[shft_column].values[0]
    if value != value:
      value = 0

      df[shft_column] = df[shft_column].fillna(value)
    
    #create list of columns
    nmbrcolumns = [shft_column]

    #grab some driftreport metrics
    #note that if this function implemented for data streams at scale it may be appropriate
    #to consider creating an alternate to dxdt without the driftreport metrics for postmunge efficiency
    positiveratio = df[df[shft_column] >= 0].shape[0] / df[shft_column].shape[0]
    negativeratio = df[df[shft_column] < 0].shape[0] / df[shft_column].shape[0]
    zeroratio = df[df[shft_column] == 0].shape[0] / df[shft_column].shape[0]
    minimum = df[shft_column].min()
    maximum = df[shft_column].max()
    mean = df[shft_column].mean()
    std = df[shft_column].std()

    nmbrnormalization_dict = {shft_column :      {'positiveratio' : positiveratio, \
                                                  'negativeratio' : negativeratio, \
                                                  'zeroratio' : zeroratio, \
                                                  'minimum' : minimum, \
                                                  'maximum' : maximum, \
                                                  'mean' : mean, \
                                                  'std' : std, \
                                                  'periods' : periods, \
                                                  'suffix' : suffix}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in nmbrcolumns:
      
      column_dict = { nc : {'category' : 'shf3', \
                           'origcategory' : category, \
                           'normalization_dict' : nmbrnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())
        
    return df, column_dict_list

  def process_MADn_class(self, mdf_train, mdf_test, column, category, \
                              postprocess_dict, params = {}):
    '''
    #process_MADn_class(mdf_train, mdf_test, column, category)
    #function to normalize data to mean of 0 and mean absolute deviation of 1
    #takes as arguement pandas dataframe of training and test data (mdf_train), (mdf_test)\
    #and the name of the column string ('column') and parent category (category)
    #replaces missing or improperly formatted data with mean of remaining values
    #returns same dataframes with new column of name column + '_MADn'
    #note this is a "dualprocess" function since is applied to both train and test dataframes
    #expect this approach works better than z-score for when the numerical distribution isn't thin tailed
    #if only have training but not test data handy, use same training data for both dataframe inputs
    '''
    
    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    #adjinfill accepts True/False to change default infill from mean inputation to adjacent cell
    if 'adjinfill' in params:
      adjinfill = params['adjinfill']
    else:
      adjinfill = False
    
    if inplace is not True:
      
      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.df_copy_train(mdf_train, column, column + '_MADn', suffixoverlap_results)

      mdf_test[column + '_MADn'] = mdf_test[column].copy()
    
    else:
      
      suffixoverlap_results = \
      self.df_check_suffixoverlap(mdf_train, column + '_MADn', suffixoverlap_results)
      
      mdf_train.rename(columns = {column : column + '_MADn'}, inplace = True)
      mdf_test.rename(columns = {column : column + '_MADn'}, inplace = True)

    #convert all values to either numeric or NaN
    mdf_train[column + '_MADn'] = pd.to_numeric(mdf_train[column + '_MADn'], errors='coerce')
    mdf_test[column + '_MADn'] = pd.to_numeric(mdf_test[column + '_MADn'], errors='coerce')
    
    #a few more metrics collected for driftreport
    #get maximum value of training column
    maximum = mdf_train[column + '_MADn'].max()
    #get minimum value of training column
    minimum = mdf_train[column + '_MADn'].min()

    #get mean of training data
    mean = mdf_train[column + '_MADn'].mean() 
    if mean != mean:
      mean = 0
      
    if adjinfill is True:
      mdf_train[column + '_MADn'] = mdf_train[column + '_MADn'].fillna(method='ffill')
      mdf_test[column + '_MADn'] = mdf_test[column + '_MADn'].fillna(method='ffill')
      mdf_train[column + '_MADn'] = mdf_train[column + '_MADn'].fillna(method='bfill')
      mdf_test[column + '_MADn'] = mdf_test[column + '_MADn'].fillna(method='bfill')

    #replace missing data with training set mean
    mdf_train[column + '_MADn'] = mdf_train[column + '_MADn'].fillna(mean)
    mdf_test[column + '_MADn'] = mdf_test[column + '_MADn'].fillna(mean)

    #subtract mean from column for both train and test
    mdf_train[column + '_MADn'] = mdf_train[column + '_MADn'] - mean
    mdf_test[column + '_MADn'] = mdf_test[column + '_MADn'] - mean

    #get mean absolute deviation of training data
    MAD = mdf_train[column + '_MADn'].mad()
    
    #special case to avoid div by 0
    if MAD == 0:
      MAD = 1

    #divide column values by mad for both training and test data
    mdf_train[column + '_MADn'] = mdf_train[column + '_MADn'] / MAD
    mdf_test[column + '_MADn'] = mdf_test[column + '_MADn'] / MAD

#     #change data type for memory savings
#     mdf_train[column + '_MADn'] = mdf_train[column + '_MADn'].astype(np.float32)
#     mdf_test[column + '_MADn'] = mdf_test[column + '_MADn'].astype(np.float32)

    #create list of columns
    nmbrcolumns = [column + '_MADn']

    nmbrnormalization_dict = {column + '_MADn' : {'mean' : mean, 'MAD' : MAD, \
                                                  'maximum':maximum, 'minimum':minimum, \
                                                  'adjinfill':adjinfill}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in nmbrcolumns:

      column_dict = { nc : {'category' : 'MADn', \
                           'origcategory' : category, \
                           'normalization_dict' : nmbrnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())
    
    return mdf_train, mdf_test, column_dict_list

  def process_MAD3_class(self, mdf_train, mdf_test, column, category, \
                              postprocess_dict, params = {}):
    '''
    #process_MAD3_class(mdf_train, mdf_test, column, category)
    #function to normalize data by subtracting maximum and dividing by median absolute deviation
    #takes as arguement pandas dataframe of training and test data (mdf_train), (mdf_test)\
    #and the name of the column string ('column') and parent category (category)
    #replaces missing or improperly formatted data with mean of remaining values
    #returns same dataframes with new column of name column + '_MADn'
    #note this is a "dualprocess" function since is applied to both train and test dataframes
    #expect this approach works better than z-score for when the numerical distribution isn't thin tailed
    #if only have training but not test data handy, use same training data for both dataframe inputs
    #the use of maximum instead of mean for normalization based on comment from RWRI lectures 
    #documented in (anonymized)
    '''
    
    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    #adjinfill accepts True/False to change default infill from mean inputation to adjacent cell
    if 'adjinfill' in params:
      adjinfill = params['adjinfill']
    else:
      adjinfill = False
    
    if inplace is not True:
      
      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.df_copy_train(mdf_train, column, column + '_MAD3', suffixoverlap_results)

      mdf_test[column + '_MAD3'] = mdf_test[column].copy()
    
    else:
      
      suffixoverlap_results = \
      self.df_check_suffixoverlap(mdf_train, column + '_MAD3', suffixoverlap_results)
      
      mdf_train.rename(columns = {column : column + '_MAD3'}, inplace = True)
      mdf_test.rename(columns = {column : column + '_MAD3'}, inplace = True)

    #convert all values to either numeric or NaN
    mdf_train[column + '_MAD3'] = pd.to_numeric(mdf_train[column + '_MAD3'], errors='coerce')
    mdf_test[column + '_MAD3'] = pd.to_numeric(mdf_test[column + '_MAD3'], errors='coerce')
    
    #a few more metrics collected for driftreport
    #get maximum value of training column
    maximum = mdf_train[column + '_MAD3'].max()
    #get minimum value of training column
    minimum = mdf_train[column + '_MAD3'].min()

    #get mean of training data
    mean = mdf_train[column + '_MAD3'].mean()
    if mean != mean:
      mean = 0
      
    if adjinfill is True:
      mdf_train[column + '_MAD3'] = mdf_train[column + '_MAD3'].fillna(method='ffill')
      mdf_test[column + '_MAD3'] = mdf_test[column + '_MAD3'].fillna(method='ffill')
      mdf_train[column + '_MAD3'] = mdf_train[column + '_MAD3'].fillna(method='bfill')
      mdf_test[column + '_MAD3'] = mdf_test[column + '_MAD3'].fillna(method='bfill')
    
    #replace missing data with training set mean
    mdf_train[column + '_MAD3'] = mdf_train[column + '_MAD3'].fillna(mean)
    mdf_test[column + '_MAD3'] = mdf_test[column + '_MAD3'].fillna(mean)

    #get max of training data
    datamax = mdf_train[column + '_MAD3'].max()
    
    #get mean absolute deviation of training data
    MAD = mdf_train[column + '_MAD3'].mad()
    
    #special case to avoid div by 0
    if MAD == 0:
      MAD = 1
    
    #subtract max from column for both train and test
    mdf_train[column + '_MAD3'] = mdf_train[column + '_MAD3'] - datamax
    mdf_test[column + '_MAD3'] = mdf_test[column + '_MAD3'] - datamax

    #divide column values by mad for both training and test data
    mdf_train[column + '_MAD3'] = mdf_train[column + '_MAD3'] / MAD
    mdf_test[column + '_MAD3'] = mdf_test[column + '_MAD3'] / MAD

#     #change data type for memory savings
#     mdf_train[column + '_MAD3'] = mdf_train[column + '_MAD3'].astype(np.float32)
#     mdf_test[column + '_MAD3'] = mdf_test[column + '_MAD3'].astype(np.float32)

    #create list of columns
    nmbrcolumns = [column + '_MAD3']

    nmbrnormalization_dict = {column + '_MAD3' : {'mean' : mean, 'MAD' : MAD, 'datamax' : datamax, \
                                                  'maximum':maximum, 'minimum':minimum, 'adjinfill':adjinfill}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in nmbrcolumns:

      column_dict = { nc : {'category' : 'MAD3', \
                           'origcategory' : category, \
                           'normalization_dict' : nmbrnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())
        
    return mdf_train, mdf_test, column_dict_list

  def process_mnmx_class(self, mdf_train, mdf_test, column, category, \
                         postprocess_dict, params = {}):
    '''
    #process_mnmx_class(mdf_train, mdf_test, column, category)
    #function to scale data to minimum of 0 and maximum of 1 \
    #based on min/max values from training set for this column
    #takes as arguement pandas dataframe of training and test data (mdf_train), (mdf_test)\
    #and the name of the column string ('column') and parent category (category)
    #replaces missing or improperly formatted data with mean of remaining values
    #returns same dataframes with new column of name column + '_mnmx'
    #note this is a "dualprocess" function since is applied to both dataframes
    #expect this approach works better when the numerical distribution is thin tailed
    #if only have training but not test data handy, use same training data for both
    #dataframe inputs
    '''
    
    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    #for cap ands floor, False means not applied, True means based on set's found max/min in train set
    
    #initialize parameters
    #cap can be passed as True for max of training data or as a specific value prior to normalizaiton, False for no cap
    if 'cap' in params:
      cap = params['cap']
    else:
      cap = False
      
    #floor can be passed as True for min of training data or as a specific value prior to normalizaiton, False for no floor
    if 'floor' in params:
      floor = params['floor']
    else:
      floor = False
      
    #adjinfill accepts True/False to change default infill from mean inputation to adjacent cell
    if 'adjinfill' in params:
      adjinfill = params['adjinfill']
    else:
      adjinfill = False
    
    if inplace is not True:
      
      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.df_copy_train(mdf_train, column, column + '_mnmx', suffixoverlap_results)

      mdf_test[column + '_mnmx'] = mdf_test[column].copy()
    
    else:
      
      suffixoverlap_results = \
      self.df_check_suffixoverlap(mdf_train, column + '_mnmx', suffixoverlap_results)
      
      mdf_train.rename(columns = {column : column + '_mnmx'}, inplace = True)
      mdf_test.rename(columns = {column : column + '_mnmx'}, inplace = True)

    #convert all values to either numeric or NaN
    mdf_train[column + '_mnmx'] = pd.to_numeric(mdf_train[column + '_mnmx'], errors='coerce')
    mdf_test[column + '_mnmx'] = pd.to_numeric(mdf_test[column + '_mnmx'], errors='coerce')
    
    #a few more metrics collected for driftreport
    #get standard deviation of training data
    std = mdf_train[column + '_mnmx'].std()

    #get mean of training data
    mean = mdf_train[column + '_mnmx'].mean()   
    if mean != mean:
      mean = 0
      
    if adjinfill is True:
      mdf_train[column + '_mnmx'] = mdf_train[column + '_mnmx'].fillna(method='ffill')
      mdf_test[column + '_mnmx'] = mdf_test[column + '_mnmx'].fillna(method='ffill')
      mdf_train[column + '_mnmx'] = mdf_train[column + '_mnmx'].fillna(method='bfill')
      mdf_test[column + '_mnmx'] = mdf_test[column + '_mnmx'].fillna(method='bfill')

    #replace missing data with training set mean
    mdf_train[column + '_mnmx'] = mdf_train[column + '_mnmx'].fillna(mean)
    mdf_test[column + '_mnmx'] = mdf_test[column + '_mnmx'].fillna(mean)
    
    #get maximum value of training column
    maximum = mdf_train[column + '_mnmx'].max()
    
    #get minimum value of training column
    minimum = mdf_train[column + '_mnmx'].min()
    
    #if cap < maximum, maximum = cap
    if cap is not False and cap is not True:
      if cap < maximum:
        maximum = cap
    if floor is not False and floor is not True:
      if floor > minimum:
        minimum = floor
    
    #avoid outlier div by zero when max = min
    maxminusmin = maximum - minimum
    if maxminusmin == 0:
      maxminusmin = 1
    
    #perform min-max scaling to train and test sets using values from train
    mdf_train[column + '_mnmx'] = (mdf_train[column + '_mnmx'] - minimum) / \
                                  (maxminusmin)
    
    mdf_test[column + '_mnmx'] = (mdf_test[column + '_mnmx'] - minimum) / \
                                 (maxminusmin)

    #cap and floor application
    if cap is True:
      cap = maximum
    if floor is True:
      floor = minimum
    
    if cap is not False:
      #replace values in test > cap with cap
      mdf_train.loc[mdf_train[column + '_mnmx'] > (cap - minimum)/maxminusmin, (column + '_mnmx')] \
      = (cap - minimum)/maxminusmin
      
      mdf_test.loc[mdf_test[column + '_mnmx'] > (cap - minimum)/maxminusmin, (column + '_mnmx')] \
      = (cap - minimum)/maxminusmin
    
    if floor is not False:
      #replace values in test < floor with floor
      mdf_train.loc[mdf_train[column + '_mnmx'] < (floor - minimum)/maxminusmin, (column + '_mnmx')] \
      = (floor - minimum)/maxminusmin
      
      mdf_test.loc[mdf_test[column + '_mnmx'] < (floor - minimum)/maxminusmin, (column + '_mnmx')] \
      = (floor - minimum)/maxminusmin
    
    #create list of columns
    nmbrcolumns = [column + '_mnmx']

    nmbrnormalization_dict = {column + '_mnmx' : {'minimum' : minimum, \
                                                  'maximum' : maximum, \
                                                  'maxminusmin' : maxminusmin, \
                                                  'mean' : mean, \
                                                  'std' : std, \
                                                  'cap' : cap, \
                                                  'floor' : floor, \
                                                  'adjinfill' : adjinfill}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in nmbrcolumns:

      column_dict = { nc : {'category' : 'mnmx', \
                           'origcategory' : category, \
                           'normalization_dict' : nmbrnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())
        
    return mdf_train, mdf_test, column_dict_list

  def process_mnm3_class(self, mdf_train, mdf_test, column, category, \
                         postprocess_dict, params = {}):
    '''
    #process_mnmx_class(mdf_train, mdf_test, column, category)
    #function to scale data to minimum of 0 and maximum of 1 \
    #after replacing extreme values above the 0.99 quantile with
    #the value of 0.99 quantile and extreme values below the 0.01
    #quantile with the value of 0.01 quantile
    #(accepts parameters qmax and qmin to customize these 0.99/0.01 values)
    #takes as arguement pandas dataframe of training and test data (mdf_train), (mdf_test)\
    #and the name of the column string ('column') and parent category (category)
    #replaces missing or improperly formatted data with mean of remaining values
    #returns same dataframes with new column of name column + '_mnmx'
    #note this is a "dualprocess" function since is applied to both dataframes
    '''
    
    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False

    #initialize parameters
    if 'qmax' in params:
      qmax = params['qmax']
    else:
      qmax = 0.99
      
    if 'qmin' in params:
      qmin = params['qmin']
    else:
      qmin = 0.01
      
    #adjinfill accepts True/False to change default infill from mean inputation to adjacent cell
    if 'adjinfill' in params:
      adjinfill = params['adjinfill']
    else:
      adjinfill = False
    
    if inplace is not True:
      
      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.df_copy_train(mdf_train, column, column + '_mnm3', suffixoverlap_results)

      mdf_test[column + '_mnm3'] = mdf_test[column].copy()
    
    else:
      
      suffixoverlap_results = \
      self.df_check_suffixoverlap(mdf_train, column + '_mnm3', suffixoverlap_results)
      
      mdf_train.rename(columns = {column : column + '_mnm3'}, inplace = True)
      mdf_test.rename(columns = {column : column + '_mnm3'}, inplace = True)

    #convert all values to either numeric or NaN
    mdf_train[column + '_mnm3'] = pd.to_numeric(mdf_train[column + '_mnm3'], errors='coerce')
    mdf_test[column + '_mnm3'] = pd.to_numeric(mdf_test[column + '_mnm3'], errors='coerce')
    
    #a few more metrics collected for driftreport
    #get standard deviation of training data
    std = mdf_train[column + '_mnm3'].std()

    #get maximum value of training column
    quantilemax = mdf_train[column + '_mnm3'].quantile(qmax)
    
    if quantilemax != quantilemax:
      quantilemax = 0

    #get minimum value of training column
    quantilemin = mdf_train[column + '_mnm3'].quantile(qmin)
    
    if quantilemin != quantilemin:
      quantilemin = 0

    #replace values > quantilemax with quantilemax
    mdf_train.loc[mdf_train[column + '_mnm3'] > quantilemax, (column + '_mnm3')] \
    = quantilemax
    mdf_test.loc[mdf_test[column + '_mnm3'] > quantilemax, (column + '_mnm3')] \
    = quantilemax
    #replace values < quantile10 with quantile10
    mdf_train.loc[mdf_train[column + '_mnm3'] < quantilemin, (column + '_mnm3')] \
    = quantilemin
    mdf_test.loc[mdf_test[column + '_mnm3'] < quantilemin, (column + '_mnm3')] \
    = quantilemin

    #note this step is now performed after the quantile evaluation / replacement

    #get mean of training data
    mean = mdf_train[column + '_mnm3'].mean()    
    if mean != mean:
      mean = 0
      
    if adjinfill is True:
      mdf_train[column + '_mnm3'] = mdf_train[column + '_mnm3'].fillna(method='ffill')
      mdf_test[column + '_mnm3'] = mdf_test[column + '_mnm3'].fillna(method='ffill')
      mdf_train[column + '_mnm3'] = mdf_train[column + '_mnm3'].fillna(method='bfill')
      mdf_test[column + '_mnm3'] = mdf_test[column + '_mnm3'].fillna(method='bfill')
    
    #replace missing data with training set mean
    mdf_train[column + '_mnm3'] = mdf_train[column + '_mnm3'].fillna(mean)
    mdf_test[column + '_mnm3'] = mdf_test[column + '_mnm3'].fillna(mean)
    
    #avoid outlier div by zero when max = min
    maxminusmin = quantilemax - quantilemin
    if maxminusmin == 0:
      maxminusmin = 1

    #perform min-max scaling to train and test sets using values from train
    mdf_train[column + '_mnm3'] = (mdf_train[column + '_mnm3'] - quantilemin) / \
                                  (maxminusmin)

    mdf_test[column + '_mnm3'] = (mdf_test[column + '_mnm3'] - quantilemin) / \
                                 (maxminusmin)

#     #change data type for memory savings
#     mdf_train[column + '_mnm3'] = mdf_train[column + '_mnm3'].astype(np.float32)
#     mdf_test[column + '_mnm3'] = mdf_test[column + '_mnm3'].astype(np.float32)
    
    #create list of columns
    nmbrcolumns = [column + '_mnm3']

    nmbrnormalization_dict = {column + '_mnm3' : {'quantilemin' : quantilemin, \
                                                  'quantilemax' : quantilemax, \
                                                  'maxminusmin' : maxminusmin, \
                                                  'mean' : mean, \
                                                  'std' : std, \
                                                  'qmax' : qmax, \
                                                  'qmin' : qmin, \
                                                  'adjinfill' : adjinfill }}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in nmbrcolumns:

      column_dict = { nc : {'category' : 'mnm3', \
                           'origcategory' : category, \
                           'normalization_dict' : nmbrnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())

    return mdf_train, mdf_test, column_dict_list
  
  def process_retn_class(self, mdf_train, mdf_test, column, category, postprocess_dict, params = {}):
    """
    #process_retn_class(mdf_train, mdf_test, column, category)
    #function to scale data as follows:
    
    # if max >= 0 and min <= 0:
    #   #scaling based on 
    #   x = x / (max - min)

    # elif max >= 0 and min >= 0:
    #   #traditional min/max
    #   x = (x - min) / (max - min)
    
    # elif max <= 0 and min <= 0:
    #   #max/min (retains negative values)
    #   x = (x - max) / (max - min)
    
    #replaces missing or improperly formatted data with mean of remaining values
    
    #returns same dataframes with new column of name column + '_retn'
    #note this is a "dualprocess" function since is applied to both dataframes
    
    #note with parameters divisor can also be set as standard deviation
    #also aprameters accepted for cap/floor/mulitplier/offset
    #where cap/floor based on pretransform values
    #multiplier/offset based on posttransform values, muoltiplier applied betfore offset
    """
    
    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    #initialize parameters
    
    #accepts divisor parameters of 'minmax' or 'std', eg divisor for normalization equation
    #note that standard deviation doesn't have same properties for sign retention when all values > or < 0
    if 'divisor' in params:
      divisor = params['divisor']
    else:
      divisor = 'minmax'
    
    #offset is just an added constant applied after multiplier
    if 'offset' in params:
      offset = params['offset']
    else:
      offset = 0
    
    #multiplier scales the set by multiplication prior to offset
    if 'multiplier' in params:
      multiplier = params['multiplier']
    else:
      multiplier = 1
    
    #cap can be passed as True for max of training data or as a specific value prior to normalization, False for no cap
    if 'cap' in params:
      cap = params['cap']
    else:
      cap = False
    
    #floor can be passed as True for min of training data or as a specific value prior to normalization, False for no floor
    if 'floor' in params:
      floor = params['floor']
    else:
      floor = False
      
    #adjinfill accepts True/False to change default infill from mean inputation to adjacent cell
    if 'adjinfill' in params:
      adjinfill = params['adjinfill']
    else:
      adjinfill = False
    
    if inplace is not True:
      
      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.df_copy_train(mdf_train, column, column + '_retn', suffixoverlap_results)

      mdf_test[column + '_retn'] = mdf_test[column].copy()
    
    else:
      
      suffixoverlap_results = \
      self.df_check_suffixoverlap(mdf_train, column + '_retn', suffixoverlap_results)
      
      mdf_train.rename(columns = {column : column + '_retn'}, inplace = True)
      mdf_test.rename(columns = {column : column + '_retn'}, inplace = True)

    #convert all values to either numeric or NaN
    mdf_train[column + '_retn'] = pd.to_numeric(mdf_train[column + '_retn'], errors='coerce')
    mdf_test[column + '_retn'] = pd.to_numeric(mdf_test[column + '_retn'], errors='coerce')
    
    #a few more metrics collected for driftreport
    #get standard deviation of training data
    std = mdf_train[column + '_retn'].std()
    
    mad = mdf_train[column + '_retn'].mad()
    
    #get maximum value of training column
    maximum = mdf_train[column + '_retn'].max()
    
    #get minimum value of training column
    minimum = mdf_train[column + '_retn'].min()
    
    #avoid outlier div by zero when max = min
    maxminusmin = maximum - minimum
    if maxminusmin == 0 or maxminusmin != maxminusmin:
      maxminusmin = 1
      
    if std != std or std == 0:
      std = 1
      
    if mad != mad or mad == 0:
      mad = 1
      
    #if cap < maximum, maximum = cap
    if cap is not False and cap is not True:
      if cap < maximum:
        maximum = cap
    if floor is not False and floor is not True:
      if floor > minimum:
        minimum = floor
        
    #cap and floor application
    if cap is True:
      cap = maximum
    if floor is True:
      floor = minimum
      
    if cap is not False:
      #replace values in test > cap with cap
      mdf_train.loc[mdf_train[column + '_retn'] > cap, (column + '_retn')] \
      = cap
      
      mdf_test.loc[mdf_test[column + '_retn'] > cap, (column + '_retn')] \
      = cap
    
    if floor is not False:
      #replace values in test < floor with floor
      mdf_train.loc[mdf_train[column + '_retn'] < floor, (column + '_retn')] \
      = floor
      
      mdf_test.loc[mdf_test[column + '_retn'] < floor, (column + '_retn')] \
      = floor
      
    #get mean of training data
    mean = mdf_train[column + '_retn'].mean()
    if mean != mean:
      mean = 0
      
    if adjinfill is True:
      mdf_train[column + '_retn'] = mdf_train[column + '_retn'].fillna(method='ffill')
      mdf_test[column + '_retn'] = mdf_test[column + '_retn'].fillna(method='ffill')
      mdf_train[column + '_retn'] = mdf_train[column + '_retn'].fillna(method='bfill')
      mdf_test[column + '_retn'] = mdf_test[column + '_retn'].fillna(method='bfill')

    #replace missing data with training set mean
    mdf_train[column + '_retn'] = mdf_train[column + '_retn'].fillna(mean)
    mdf_test[column + '_retn'] = mdf_test[column + '_retn'].fillna(mean)
    
    #edge case (only neccesary so scalingapproach is assigned)
    if maximum != maximum:
      maximum = 0
    if minimum != minimum:
      minimum = 0
    
    #divisor
    if divisor not in ['minmax', 'std', 'mad']:
      print("Error: retn transform parameter 'divisor' only accepts entries of 'minmax' 'mad' or 'std'")
    if divisor == 'minmax':
      divisor = maxminusmin
    elif divisor == 'mad':
      divisor = mad
    else:
      divisor = std
      
    if divisor == 0 or divisor != divisor:
      divisor = 1
    
    #driftreport metric scalingapproach returned as 'retn' or 'mnmx' or 'mxmn'
    #where mnmx is for cases where all values in train set are positive
    #mxmn is for cases where all values in train set are negative
    
    if maximum >= 0 and minimum <= 0:
      
      mdf_train[column + '_retn'] = (mdf_train[column + '_retn']) / \
                                    (divisor) * multiplier + offset
      
      mdf_test[column + '_retn'] = (mdf_test[column + '_retn']) / \
                                    (divisor) * multiplier + offset
      
      scalingapproach = 'retn'
      
    elif maximum >= 0 and minimum >= 0:
    
      #perform min-max scaling to train and test sets using values from train
      mdf_train[column + '_retn'] = (mdf_train[column + '_retn'] - minimum) / \
                                    (divisor) * multiplier + offset

      mdf_test[column + '_retn'] = (mdf_test[column + '_retn'] - minimum) / \
                                   (divisor) * multiplier + offset
      
      scalingapproach = 'mnmx'
      
    elif maximum <= 0 and minimum <= 0:
    
      #perform min-max scaling to train and test sets using values from train
      mdf_train[column + '_retn'] = (mdf_train[column + '_retn'] - maximum) / \
                                    (divisor) * multiplier + offset

      mdf_test[column + '_retn'] = (mdf_test[column + '_retn'] - maximum) / \
                                   (divisor) * multiplier + offset
      
      scalingapproach = 'mxmn'
    
    #create list of columns
    nmbrcolumns = [column + '_retn']

    nmbrnormalization_dict = {column + '_retn' : {'minimum' : minimum, \
                                                  'maximum' : maximum, \
                                                  'mean' : mean, \
                                                  'std' : std, \
                                                  'mad' : mad, \
                                                  'scalingapproach' : scalingapproach, \
                                                  'offset' : offset, \
                                                  'multiplier': multiplier, \
                                                  'cap' : cap, \
                                                  'floor' : floor, \
                                                  'divisor' : divisor, \
                                                  'adjinfill' : adjinfill}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in nmbrcolumns:

      column_dict = { nc : {'category' : 'retn', \
                           'origcategory' : category, \
                           'normalization_dict' : nmbrnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())
        
    return mdf_train, mdf_test, column_dict_list

  def process_mean_class(self, mdf_train, mdf_test, column, category, postprocess_dict, params = {}):
    '''
    #process_mean_class(mdf_train, mdf_test, column, category)
    #function to scale data to minimum of 0 and maximum of 1 \
    #based on min/max values from training set for this column
    #takes as arguement pandas dataframe of training and test data (mdf_train), (mdf_test)\
    #and the name of the column string ('column') and parent category (category)
    #replaces missing or improperly formatted data with mean of remaining values
    #returns same dataframes with new column of name column + '_mnmx'
    #note this is a "dualprocess" function since is applied to both dataframes
    #expect this approach works better when the numerical distribution is thin tailed
    #if only have training but not test data handy, use same training data for both
    #dataframe inputs
    '''
    
    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    #initialize parameters
    if 'offset' in params:
      offset = params['offset']
    else:
      offset = 0
      
    if 'multiplier' in params:
      multiplier = params['multiplier']
    else:
      multiplier = 1
    
    if 'cap' in params:
      cap = params['cap']
    else:
      cap = False
      
    if 'floor' in params:
      floor = params['floor']
    else:
      floor = False
      
    #adjinfill accepts True/False to change default infill from mean inputation to adjacent cell
    if 'adjinfill' in params:
      adjinfill = params['adjinfill']
    else:
      adjinfill = False
    
    if inplace is not True:
      
      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.df_copy_train(mdf_train, column, column + '_mean', suffixoverlap_results)

      mdf_test[column + '_mean'] = mdf_test[column].copy()
    
    else:
      
      suffixoverlap_results = \
      self.df_check_suffixoverlap(mdf_train, column + '_mean', suffixoverlap_results)
      
      mdf_train.rename(columns = {column : column + '_mean'}, inplace = True)
      mdf_test.rename(columns = {column : column + '_mean'}, inplace = True)

    #convert all values to either numeric or NaN
    mdf_train[column + '_mean'] = pd.to_numeric(mdf_train[column + '_mean'], errors='coerce')
    mdf_test[column + '_mean'] = pd.to_numeric(mdf_test[column + '_mean'], errors='coerce')
    
    #get maximum value of training column
    maximum = mdf_train[column + '_mean'].max()
    
    #get minimum value of training column
    minimum = mdf_train[column + '_mean'].min()
    
    #if cap < maximum, maximum = cap
    if cap is not False and cap is not True:
      if cap < maximum:
        maximum = cap
    if floor is not False and floor is not True:
      if floor > minimum:
        minimum = floor
        
    #cap and floor application
    if cap is True:
      cap = maximum
    if floor is True:
      floor = minimum
      
    if cap is not False:
      #replace values in test > cap with cap
      mdf_train.loc[mdf_train[column + '_mean'] > cap, (column + '_mean')] \
      = cap
      
      mdf_test.loc[mdf_test[column + '_mean'] > cap, (column + '_mean')] \
      = cap
    
    if floor is not False:
      #replace values in test < floor with floor
      mdf_train.loc[mdf_train[column + '_mean'] < floor, (column + '_mean')] \
      = floor
      
      mdf_test.loc[mdf_test[column + '_mean'] < floor, (column + '_mean')] \
      = floor
    
    #a few more metrics collected for driftreport
    #get standard deviation of training data
    std = mdf_train[column + '_mean'].std()

    #get mean of training data
    mean = mdf_train[column + '_mean'].mean()
    if mean != mean:
      mean = 0

    if adjinfill is True:
      mdf_train[column + '_mean'] = mdf_train[column + '_mean'].fillna(method='ffill')
      mdf_test[column + '_mean'] = mdf_test[column + '_mean'].fillna(method='ffill')
      mdf_train[column + '_mean'] = mdf_train[column + '_mean'].fillna(method='bfill')
      mdf_test[column + '_mean'] = mdf_test[column + '_mean'].fillna(method='bfill')
      
    #replace missing data with training set mean
    mdf_train[column + '_mean'] = mdf_train[column + '_mean'].fillna(mean)
    mdf_test[column + '_mean'] = mdf_test[column + '_mean'].fillna(mean)
    
    #avoid outlier div by zero when max = min
    maxminusmin = maximum - minimum
    if maxminusmin == 0 or maxminusmin != maxminusmin:
      maxminusmin = 1
    
    #perform min-max scaling to train and test sets using values from train
    mdf_train[column + '_mean'] = (mdf_train[column + '_mean'] - mean) / \
                                  (maxminusmin) * multiplier + offset
    
    mdf_test[column + '_mean'] = (mdf_test[column + '_mean'] - mean) / \
                                 (maxminusmin) * multiplier + offset

#     #change data type for memory savings
#     mdf_train[column + '_mnmx'] = mdf_train[column + '_mnmx'].astype(np.float32)
#     mdf_test[column + '_mnmx'] = mdf_test[column + '_mnmx'].astype(np.float32)
    
    #create list of columns
    nmbrcolumns = [column + '_mean']

    nmbrnormalization_dict = {column + '_mean' : {'minimum' : minimum, \
                                                  'maximum' : maximum, \
                                                  'maxminusmin' : maxminusmin, \
                                                  'mean' : mean, \
                                                  'std' : std, \
                                                  'offset' : offset, \
                                                  'multiplier': multiplier, \
                                                  'cap' : cap, \
                                                  'floor' : floor, \
                                                  'adjinfill' : adjinfill}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in nmbrcolumns:

      column_dict = { nc : {'category' : 'mean', \
                           'origcategory' : category, \
                           'normalization_dict' : nmbrnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())
        
    return mdf_train, mdf_test, column_dict_list

  def process_binary_class(self, mdf_train, mdf_test, column, category, \
                           postprocess_dict, params = {}):
    '''
    #process_binary_class(mdf, column, missing)
    #converts binary classification values to 0 or 1
    #takes as arguement a pandas dataframe (mdf_train, mdf_test), \
    #the name of the column string ('column') \
    #and the category from parent columkn (category)
    #fills missing valules with most common value
    #returns same dataframes with new column of name column + '_bnry'
    #note this is a "dualprocess" function since is applied to both dataframes
    '''
    
    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    #adjinfill accepts True/False to change default infill from mean inputation to adjacent cell
    if 'adjinfill' in params:
      adjinfill = params['adjinfill']
    else:
      adjinfill = False
      
    #str_convert provides consistent encodings between numbers and string equivalent, eg 2 == '2'
    if 'str_convert' in params:
      str_convert = params['str_convert']
    else:
      str_convert = False
    
    if inplace is not True:
      
      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.df_copy_train(mdf_train, column, column + '_bnry', suffixoverlap_results)

      mdf_test[column + '_bnry'] = mdf_test[column].copy()
    
    else:
      
      suffixoverlap_results = \
      self.df_check_suffixoverlap(mdf_train, column + '_bnry', suffixoverlap_results)
      
      mdf_train.rename(columns = {column : column + '_bnry'}, inplace = True)
      mdf_test.rename(columns = {column : column + '_bnry'}, inplace = True)
    
    if str_convert is True:
      mdf_train[column + '_bnry'] = mdf_train[column + '_bnry'].astype(str)
      mdf_test[column + '_bnry'] = mdf_test[column + '_bnry'].astype(str)

    #create plug value for missing cells as most common value
    valuecounts = pd.DataFrame(mdf_train[column + '_bnry'].value_counts())
    valuecounts = valuecounts.rename_axis('zzzinfill').sort_values(by = [column + '_bnry', 'zzzinfill'], ascending = [False, True])
    valuecounts = list(valuecounts.index)
    
    if len(valuecounts) > 0:

      if len(valuecounts) > 1:
        binary_missing_plug = valuecounts[0]
      else:
        #making an executive decision here to deviate from standardinfill of most common value
        #for this edge case where a column evaluated as binary has only single value and NaN's
        binary_missing_plug = 'zzzinfill'

      #test for nan
      if binary_missing_plug != binary_missing_plug:
        binary_missing_plug = valuecounts[1]

      #edge case when applying this transform to set with >2 values
      #this only comes up when caluclating driftreport in postmunge
      extravalues = []
      if len(valuecounts) > 2:
        i=0
        for value in valuecounts:
          if i>1:
            extravalues.append(value)
          i+=1

      #replace nan in valuecounts with binary_missing_plug so we can sort
      valuecounts = [x if x == x else binary_missing_plug for x in valuecounts]
  #     #convert everything to string for sort
  #     valuecounts = [str(x) for x in valuecounts]

      #note LabelBinarizer encodes alphabetically, with 1 assigned to first and 0 to second
      #we'll take different approach of going by most common value to 1 unless 0 or 1
      #are already in the set then we'll defer to keeping those designations in place
      #there's some added complexity here to deal with edge case of passing this function
      #to a set with >2 values as we might run into when caluclating drift in postmunge

  #     valuecounts.sort()
  #     valuecounts = sorted(valuecounts)
      #in case this includes both strings and integers for instance we'll sort this way
  #     valuecounts = sorted(valuecounts, key=lambda p: str(p))

      #we'll save these in the normalization dictionary for future reference
      onevalue = valuecounts[0]
      if len(valuecounts) > 1:
        zerovalue = valuecounts[1]
      else:
        zerovalue = 'zzzinfill'

      #special case for when the source column is already encoded as 0/1

      if len(valuecounts) <= 2:

        if 0 in valuecounts:
          zerovalue = 0
          if 1 in valuecounts:
            onevalue = 1
          else:
            if valuecounts[0] == 0:
              if len(valuecounts) > 1:
                onevalue = valuecounts[1]
              else:
                onevalue = 'zzzinfill'

        if 1 in valuecounts:
          if 0 not in valuecounts:
            if valuecounts[0] != 1:
              onevalue = 1
              zerovalue = valuecounts[0]

      #edge case same as above but when values of 0 or 1. are in set and 
      #len(valuecounts) > 2
      if len(valuecounts) > 2:
        valuecounts2 = valuecounts[:2]

        if 0 in valuecounts2:
          zerovalue = 0
          if 1 in valuecounts2:
            onevalue = 1
          else:
            if valuecounts2[0] == 0:
              if len(valuecounts) > 1:
                onevalue = valuecounts2[1]
              else:
                onevalue = 'zzzinfill'

        if 1 in valuecounts2:
          if 0 not in valuecounts2:
            if valuecounts2[0] != 1:
              onevalue = 1
              zerovalue = valuecounts2[0]

      #edge case that might come up in drift report
      if binary_missing_plug not in [onevalue, zerovalue]:
        binary_missing_plug = onevalue

      #edge case when applying this transform to set with >2 values
      #this only comes up when caluclating driftreport in postmunge
      if len(valuecounts) > 2:
        for value in extravalues:
          mdf_train[column + '_bnry'] = \
          np.where(mdf_train[column + '_bnry'] == value, binary_missing_plug, mdf_train[column + '_bnry'])
          mdf_test[column + '_bnry'] = \
          np.where(mdf_test[column + '_bnry'] == value, binary_missing_plug, mdf_test[column + '_bnry'])
          
      if adjinfill is True:
        mdf_train[column + '_bnry'] = mdf_train[column + '_bnry'].fillna(method='ffill')
        mdf_test[column + '_bnry'] = mdf_test[column + '_bnry'].fillna(method='ffill')
        mdf_train[column + '_bnry'] = mdf_train[column + '_bnry'].fillna(method='bfill')
        mdf_test[column + '_bnry'] = mdf_test[column + '_bnry'].fillna(method='bfill')

      #replace missing data with specified classification
      mdf_train[column + '_bnry'] = mdf_train[column + '_bnry'].fillna(binary_missing_plug)
      mdf_test[column + '_bnry'] = mdf_test[column + '_bnry'].fillna(binary_missing_plug)

      #this addressess issue where nunique for mdftest > than that for mdf_train
      #note is currently an oportunity for improvement that NArows won't identify these poinsts as candiadates
      #for user specified infill, and as currently addressed will default to infill with most common value
      #in the mean time a workaround could be for user to manually replace extra values with nan prior to
      #postmunge application such as if they want to apply ML infill
      #this will only be an issue when nunique for df_train == 2, and nunique for df_test > 2
      #if len(mdf_test[column + '_bnry'].unique()) > 2:
      uniqueintest = mdf_test[column + '_bnry'].unique()
      for unique in uniqueintest:
        if unique not in [onevalue, zerovalue]:
          mdf_test[column + '_bnry'] = \
          np.where(mdf_test[column + '_bnry'] == unique, binary_missing_plug, mdf_test[column + '_bnry'])

      #convert column to binary 0/1 classification (replaces scikit LabelBinarizer)
      mdf_train[column + '_bnry'] = np.where(mdf_train[column + '_bnry'] == onevalue, 1, 0)
      mdf_test[column + '_bnry'] = np.where(mdf_test[column + '_bnry'] == onevalue, 1, 0)

      #create list of columns
      bnrycolumns = [column + '_bnry']

      #change data types to 8-bit (1 byte) integers for memory savings
      mdf_train[column + '_bnry'] = mdf_train[column + '_bnry'].astype(np.int8)
      mdf_test[column + '_bnry'] = mdf_test[column + '_bnry'].astype(np.int8)

      #a few more metrics collected for driftreport
      oneratio = mdf_train[column + '_bnry'].sum() / mdf_train[column + '_bnry'].shape[0]
      zeroratio = (mdf_train[column + '_bnry'].shape[0] - mdf_train[column + '_bnry'].sum() )\
                  / mdf_train[column + '_bnry'].shape[0]

      #create list of columns associated with categorical transform (blank for now)
      categorylist = []
    
    else:
      mdf_train[column + '_bnry'] = 0
      mdf_test[column + '_bnry'] = 0
      
      binary_missing_plug = 0
      onevalue = 1
      zerovalue = 0
      extravalues = 0
      oneratio = 0
      zeroratio = 0
      bnrycolumns = [column + '_bnry']

  #     bnrynormalization_dict = {column + '_bnry' : {'missing' : binary_missing_plug, \
  #                                                   'onevalue' : onevalue, \
  #                                                   'zerovalue' : zerovalue}}
    
    bnrynormalization_dict = {column + '_bnry' : {'missing' : binary_missing_plug, \
                                                  1 : onevalue, \
                                                  0 : zerovalue, \
                                                  'extravalues' : extravalues, \
                                                  'oneratio' : oneratio, \
                                                  'zeroratio' : zeroratio, \
                                                  'adjinfill' : adjinfill, \
                                                  'str_convert' : str_convert}}

    #store some values in the column_dict{} for use later in ML infill methods
    column_dict_list = []

    for bc in bnrycolumns:

      column_dict = { bc : {'category' : 'bnry', \
                           'origcategory' : category, \
                           'normalization_dict' : bnrynormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : bnrycolumns, \
                           'categorylist' : bnrycolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())

    return mdf_train, mdf_test, column_dict_list

  def process_binary2_class(self, mdf_train, mdf_test, column, category, postprocess_dict, params = {}):
    '''
    #process_binary2_class(mdf, column, missing)
    #converts binary classification values to 0 or 1
    #takes as arguement a pandas dataframe (mdf_train, mdf_test), \
    #the name of the column string ('column') \
    #and the category from parent columkn (category)
    #fills missing valules with least common value (different than bnry)
    #returns same dataframes with new column of name column + '_bnry'
    #note this is a "dualprocess" function since is applied to both dataframes
    '''
    
    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    #adjinfill accepts True/False to change default infill from mean inputation to adjacent cell
    if 'adjinfill' in params:
      adjinfill = params['adjinfill']
    else:
      adjinfill = False
      
    #str_convert provides consistent encodings between numbers and string equivalent, eg 2 == '2'
    if 'str_convert' in params:
      str_convert = params['str_convert']
    else:
      str_convert = False
    
    if inplace is not True:
      
      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.df_copy_train(mdf_train, column, column + '_bnr2', suffixoverlap_results)

      mdf_test[column + '_bnr2'] = mdf_test[column].copy()
    
    else:
      
      suffixoverlap_results = \
      self.df_check_suffixoverlap(mdf_train, column + '_bnr2', suffixoverlap_results)
      
      mdf_train.rename(columns = {column : column + '_bnr2'}, inplace = True)
      mdf_test.rename(columns = {column : column + '_bnr2'}, inplace = True)
    
    if str_convert is True:
      mdf_train[column + '_bnr2'] = mdf_train[column + '_bnr2'].astype(str)
      mdf_test[column + '_bnr2'] = mdf_test[column + '_bnr2'].astype(str)

    #create plug value for missing cells as most common value
    valuecounts = pd.DataFrame(mdf_train[column + '_bnr2'].value_counts())
    valuecounts = valuecounts.rename_axis('zzzinfill').sort_values(by = [column + '_bnr2', 'zzzinfill'], ascending = [False, True])
    valuecounts = list(valuecounts.index)
    
    if len(valuecounts) > 0:

      if len(valuecounts) > 1:
        #binary_missing_plug = valuecounts[0]
        binary_missing_plug = valuecounts[1]
      else:
        #making an executive decision here to deviate from standardinfill of most common value
        #for this edge case where a column evaluated as binary has only single value and NaN's
        binary_missing_plug = 'zzzinfill'

      #test for nan
      if binary_missing_plug != binary_missing_plug:
        #binary_missing_plug = valuecounts[1]
        binary_missing_plug = valuecounts[0]

      #edge case when applying this transform to set with >2 values
      #this only comes up when caluclating driftreport in postmunge
      extravalues = []
      if len(valuecounts) > 2:
        i=0
        for value in valuecounts:
          if i>1:
            extravalues.append(value)
          i+=1

      #replace nan in valuecounts with binary_missing_plug so we can sort
      valuecounts = [x if x == x else binary_missing_plug for x in valuecounts]
  #     #convert everything to string for sort
  #     valuecounts = [str(x) for x in valuecounts]

      #note LabelBinarizer encodes alphabetically, with 1 assigned to first and 0 to second
      #we'll take different approach of going by most common value to 1 unless 0 or 1
      #are already in the set then we'll defer to keeping those designations in place
      #there's some added complexity here to deal with edge case of passing this function
      #to a set with >2 values as we might run into when caluclating drift in postmunge

  #     valuecounts.sort()
  #     valuecounts = sorted(valuecounts)
      #in case this includes both strings and integers for instance we'll sort this way
  #     valuecounts = sorted(valuecounts, key=lambda p: str(p))

      #we'll save these in the normalization dictionary for future reference
      onevalue = valuecounts[0]
      if len(valuecounts) > 1:
        zerovalue = valuecounts[1]
      else:
        zerovalue = 'zzzinfill'

      #special case for when the source column is already encoded as 0/1

      if len(valuecounts) <= 2:

        if 0 in valuecounts:
          zerovalue = 0
          if 1 in valuecounts:
            onevalue = 1
          else:
            if valuecounts[0] == 0:
              if len(valuecounts) > 1:
                onevalue = valuecounts[1]
              else:
                onevalue = 'zzzinfill'

        if 1 in valuecounts:
          if 0 not in valuecounts:
            if valuecounts[0] != 1:
              onevalue = 1
              zerovalue = valuecounts[0]

      #edge case same as above but when values of 0 or 1. are in set and 
      #len(valuecounts) > 2
      if len(valuecounts) > 2:
        valuecounts2 = valuecounts[:2]

        if 0 in valuecounts2:
          zerovalue = 0
          if 1 in valuecounts2:
            onevalue = 1
          else:
            if valuecounts2[0] == 0:
              if len(valuecounts) > 1:
                onevalue = valuecounts2[1]
              else:
                onevalue = 'zzzinfill'

        if 1 in valuecounts2:
          if 0 not in valuecounts2:
            if valuecounts2[0] != 1:
              onevalue = 1
              zerovalue = valuecounts2[0]

      #edge case that might come up in drift report
      if binary_missing_plug not in [onevalue, zerovalue]:
        #binary_missing_plug = onevalue
        binary_missing_plug = zerovalue

      #edge case when applying this transform to set with >2 values
      #this only comes up when caluclating driftreport in postmunge
      if len(valuecounts) > 2:
        for value in extravalues:
          mdf_train[column + '_bnr2'] = \
          np.where(mdf_train[column + '_bnr2'] == value, binary_missing_plug, mdf_train[column + '_bnr2'])
          mdf_test[column + '_bnr2'] = \
          np.where(mdf_test[column + '_bnr2'] == value, binary_missing_plug, mdf_test[column + '_bnr2'])
          
      if adjinfill is True:
        mdf_train[column + '_bnr2'] = mdf_train[column + '_bnr2'].fillna(method='ffill')
        mdf_test[column + '_bnr2'] = mdf_test[column + '_bnr2'].fillna(method='ffill')
        mdf_train[column + '_bnr2'] = mdf_train[column + '_bnr2'].fillna(method='bfill')
        mdf_test[column + '_bnr2'] = mdf_test[column + '_bnr2'].fillna(method='bfill')

      #replace missing data with specified classification
      mdf_train[column + '_bnr2'] = mdf_train[column + '_bnr2'].fillna(binary_missing_plug)
      mdf_test[column + '_bnr2'] = mdf_test[column + '_bnr2'].fillna(binary_missing_plug)

      #this addressess issue where nunique for mdftest > than that for mdf_train
      #note is currently an oportunity for improvement that NArows won't identify these poinsts as candiadates
      #for user specified infill, and as currently addressed will default to infill with most common value
      #in the mean time a workaround could be for user to manually replace extra values with nan prior to
      #postmunge application such as if they want to apply ML infill
      #this will only be an issue when nunique for df_train == 2, and nunique for df_test > 2
      #if len(mdf_test[column + '_bnry'].unique()) > 2:
      uniqueintest = mdf_test[column + '_bnr2'].unique()
      for unique in uniqueintest:
        if unique not in [onevalue, zerovalue]:
          mdf_test[column + '_bnr2'] = \
          np.where(mdf_test[column + '_bnr2'] == unique, binary_missing_plug, mdf_test[column + '_bnr2'])

      #convert column to binary 0/1 classification (replaces scikit LabelBinarizer)
      mdf_train[column + '_bnr2'] = np.where(mdf_train[column + '_bnr2'] == onevalue, 1, 0)
      mdf_test[column + '_bnr2'] = np.where(mdf_test[column + '_bnr2'] == onevalue, 1, 0)

      #create list of columns
      bnrycolumns = [column + '_bnr2']

      #change data types to 8-bit (1 byte) integers for memory savings
      mdf_train[column + '_bnr2'] = mdf_train[column + '_bnr2'].astype(np.int8)
      mdf_test[column + '_bnr2'] = mdf_test[column + '_bnr2'].astype(np.int8)

      #a few more metrics collected for driftreport
      oneratio = mdf_train[column + '_bnr2'].sum() / mdf_train[column + '_bnr2'].shape[0]
      zeroratio = (mdf_train[column + '_bnr2'].shape[0] - mdf_train[column + '_bnr2'].sum() )\
                  / mdf_train[column + '_bnr2'].shape[0]

      #create list of columns associated with categorical transform (blank for now)
      categorylist = []
    
    else:
      mdf_train[column + '_bnr2'] = 0
      mdf_test[column + '_bnr2'] = 0
      
      binary_missing_plug = 0
      onevalue = 1
      zerovalue = 0
      extravalues = 0
      oneratio = 0
      zeroratio = 0
      bnrycolumns = [column + '_bnr2']

  #     bnrynormalization_dict = {column + '_bnry' : {'missing' : binary_missing_plug, \
  #                                                   'onevalue' : onevalue, \
  #                                                   'zerovalue' : zerovalue}}
    
    bnrynormalization_dict = {column + '_bnr2' : {'missing' : binary_missing_plug, \
                                                  1 : onevalue, \
                                                  0 : zerovalue, \
                                                  'extravalues' : extravalues, \
                                                  'oneratio' : oneratio, \
                                                  'zeroratio' : zeroratio, \
                                                  'adjinfill' : adjinfill, \
                                                  'str_convert' : str_convert}}

    #store some values in the column_dict{} for use later in ML infill methods
    column_dict_list = []

    for bc in bnrycolumns:

      column_dict = { bc : {'category' : 'bnr2', \
                           'origcategory' : category, \
                           'normalization_dict' : bnrynormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : bnrycolumns, \
                           'categorylist' : bnrycolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())

    #return mdf, bnrycolumns, categorylist, column_dict_list
    return mdf_train, mdf_test, column_dict_list
  
  def process_onht_class(self, mdf_train, mdf_test, column, category, postprocess_dict, params = {}):
    '''
    #process_onht_class(mdf_train, mdf_test, column, category, postprocess_dict, params = {})
    #preprocess column with one hot encoding
    #same as 'text' transform except labels returned column with integer instead of entry appender
    '''
    
    suffixoverlap_results = {}
    
    #adjinfill accepts True/False to change default infill from mean inputation to adjacent cell
    if 'adjinfill' in params:
      adjinfill = params['adjinfill']
    else:
      adjinfill = False
      
    #str_convert provides consistent encodings between numbers and string equivalent, eg 2 == '2'
    if 'str_convert' in params:
      str_convert = params['str_convert']
    else:
      str_convert = False
    
    tempcolumn = column + '_onht_'
    
    suffixoverlap_results = \
    self.df_check_suffixoverlap(mdf_train, tempcolumn, suffixoverlap_results)
    
    #store original column for later retrieval
    mdf_train[tempcolumn] = mdf_train[column].copy()
    mdf_test[tempcolumn] = mdf_test[column].copy()

    #convert column to category
    mdf_train[tempcolumn] = mdf_train[tempcolumn].astype('category')
    mdf_test[tempcolumn] = mdf_test[tempcolumn].astype('category')

    #if set is categorical we'll need the plug value for missing values included
    if 'zzzinfill' not in mdf_train[tempcolumn].cat.categories:
      mdf_train[tempcolumn] = mdf_train[tempcolumn].cat.add_categories(['zzzinfill'])
    if 'zzzinfill' not in mdf_test[tempcolumn].cat.categories:
      mdf_test[tempcolumn] = mdf_test[tempcolumn].cat.add_categories(['zzzinfill'])
      
    if adjinfill is True:
      mdf_train[tempcolumn] = mdf_train[tempcolumn].fillna(method='ffill')
      mdf_test[tempcolumn] = mdf_test[tempcolumn].fillna(method='ffill')
      mdf_train[tempcolumn] = mdf_train[tempcolumn].fillna(method='bfill')
      mdf_test[tempcolumn] = mdf_test[tempcolumn].fillna(method='bfill')

    #replace NA with a dummy variable
    mdf_train[tempcolumn] = mdf_train[tempcolumn].fillna('zzzinfill')
    mdf_test[tempcolumn] = mdf_test[tempcolumn].fillna('zzzinfill')

    if str_convert is True:
      #replace numerical with string equivalent
      mdf_train[tempcolumn] = mdf_train[tempcolumn].astype(str)
      mdf_test[tempcolumn] = mdf_test[tempcolumn].astype(str)
    else:
      mdf_train[tempcolumn] = mdf_train[tempcolumn].astype('object')
      mdf_test[tempcolumn] = mdf_test[tempcolumn].astype('object')

    #extract categories for column labels
    #note that .unique() extracts the labels as a numpy array
    labels_train = mdf_train[tempcolumn].unique()
#     labels_train.sort(axis=0)
    labels_train = sorted(labels_train, key=str)
    labels_train = list(labels_train)
    orig_labels_train = list(labels_train.copy())
    labels_test = mdf_test[tempcolumn].unique()
#     labels_test.sort(axis=0)
    labels_test = sorted(labels_test, key=str)
    labels_test = list(labels_test)

    #pandas one hot encoding doesn't sort integers and strings properly so using my own
    df_train_cat = pd.DataFrame(mdf_train[tempcolumn])
    df_test_cat = pd.DataFrame(mdf_test[tempcolumn])
    for entry in labels_train:
      df_train_cat[entry] = np.where(mdf_train[tempcolumn] == entry, 1, 0)
      df_test_cat[entry] = np.where(mdf_test[tempcolumn] == entry, 1, 0)
    del df_train_cat[tempcolumn]
    del df_test_cat[tempcolumn]
    
    labels_dict = {}
    i = 0
    for entry in labels_train:
      labels_dict.update({entry : column + '_onht_' + str(i)})
      i += 1
    
    #convert sparse array to pandas dataframe with column labels
    df_train_cat.columns = labels_train
    df_test_cat.columns = labels_train

    #Get missing columns in test set that are present in training set
    missing_cols = set( df_train_cat.columns ) - set( labels_test )
    
    suffixoverlap_results = \
    self.df_check_suffixoverlap(mdf_train, list(df_train_cat), suffixoverlap_results)

    #concatinate the sparse set with the rest of our training data
    mdf_train = pd.concat([mdf_train, df_train_cat], axis=1)
    mdf_test = pd.concat([mdf_test, df_test_cat], axis=1)

    del mdf_train[tempcolumn]    
    del mdf_test[tempcolumn]
    
    #delete _NArw column, this will be processed seperately in the processfamily function
    #delete support NArw2 column
#     columnNArw = column + '_NArw'
    columnNAr2 = column + '_zzzinfill'
    if columnNAr2 in mdf_train.columns:
      del mdf_train[columnNAr2]
    if columnNAr2 in mdf_test.columns:
      del mdf_test[columnNAr2]
    if 'zzzinfill' in orig_labels_train:
      orig_labels_train.remove('zzzinfill')

#     del mdf_train[column + '_NAr2']    
#     del mdf_test[column + '_NAr2']
    
    #create output of a list of the created column names
    NAcolumn = columnNAr2
    labels_train = list(df_train_cat)
    if NAcolumn in labels_train:
      labels_train.remove(NAcolumn)
    textcolumns = labels_train
    
    #now we'll creaate a dicitonary of the columns : categories for later reference
    #reminder here is list of. unque values from original column
    #labels_train
    
    normalizationdictvalues = labels_train
    normalizationdictkeys = textcolumns
    
    normalizationdictkeys = sorted(normalizationdictkeys, key=str)
    normalizationdictvalues = sorted(normalizationdictvalues, key=str)
    
    #textlabelsdict = dict(zip(normalizationdictkeys, normalizationdictvalues))
    textlabelsdict = dict(zip(normalizationdictvalues, orig_labels_train))
    
    #change data types to 8-bit (1 byte) integers for memory savings
    for textcolumn in textcolumns:
      mdf_train[textcolumn] = mdf_train[textcolumn].astype(np.int8)
      mdf_test[textcolumn] = mdf_test[textcolumn].astype(np.int8)

    #store some values in the text_dict{} for use later in ML infill methods
    column_dict_list = []

    categorylist = textcolumns.copy()
#     categorylist.remove(columnNArw)

    #now convert coloumn headers from text convention to onht convention
    mdf_train = mdf_train.rename(columns=labels_dict)
    mdf_test  = mdf_test.rename(columns=labels_dict)
    
    textcolumns = [labels_dict[entry] for entry in textcolumns]
    
    inverse_labels_dict = {value:key for key,value in labels_dict.items()}

    for tc in textcolumns:
    
      #new parameter collected for driftreport
      tc_ratio = tc + '_ratio'
      tcratio = mdf_train[tc].sum() / mdf_train[tc].shape[0]

      textnormalization_dict = {tc : {'textlabelsdict_onht' : textlabelsdict, \
                                      tc_ratio : tcratio, \
                                      'labels_dict' : labels_dict, \
                                      'inverse_labels_dict' : inverse_labels_dict, \
                                      'text_categorylist' : categorylist, \
                                      'adjinfill' : adjinfill, \
                                      'str_convert' : str_convert}}
      
      column_dict = {tc : {'category' : 'onht', \
                           'origcategory' : category, \
                           'normalization_dict' : textnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : textcolumns, \
                           'categorylist' : textcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())
    
    return mdf_train, mdf_test, column_dict_list

  def process_text_class(self, mdf_train, mdf_test, column, category, postprocess_dict, params = {}):
    '''
    #process_text_class(mdf_train, mdf_test, column, category)
    #preprocess column with text categories
    #takes as arguement two pandas dataframe containing training and test data respectively 
    #(mdf_train, mdf_test), and the name of the column string ('column')
    #and the name of the category from parent column (category)
    #note this trains both training and test data simultaneously due to unique treatment if any category
    #missing from training set but not from test set to ensure consistent formatting 
    #doesn't delete the original column from master dataframe but
    #creates onehot encodings
    #with columns named after column_ + text categories
    #any categories missing from the training set removed from test set
    #any category present in training but missing from test set given a column of zeros for consistent formatting
    #ensures order of all new columns consistent between both sets
    #returns two transformed dataframe (mdf_train, mdf_test) \
    #and a list of the new column names (textcolumns)
    
    #if only have training but not test data handy, use same training data for both dataframe inputs
    '''
    
    suffixoverlap_results = {}
    
    #adjinfill accepts True/False to change default infill from mean inputation to adjacent cell
    if 'adjinfill' in params:
      adjinfill = params['adjinfill']
    else:
      adjinfill = False
    
    tempsuffix = str(mdf_train[column].unique()[0])
    
    tempcolumn = column + '_' + tempsuffix
    
    suffixoverlap_results = \
    self.df_check_suffixoverlap(mdf_train, tempcolumn, suffixoverlap_results)
    
    #store original column for later retrieval
    mdf_train[tempcolumn] = mdf_train[column].copy()
    mdf_test[tempcolumn] = mdf_test[column].copy()

    #convert column to category
    mdf_train[tempcolumn] = mdf_train[tempcolumn].astype('category')
    mdf_test[tempcolumn] = mdf_test[tempcolumn].astype('category')

    #if set is categorical we'll need the plug value for missing values included
    if 'zzzinfill' not in mdf_train[tempcolumn].cat.categories:
      mdf_train[tempcolumn] = mdf_train[tempcolumn].cat.add_categories(['zzzinfill'])
    if 'zzzinfill' not in mdf_test[tempcolumn].cat.categories:
      mdf_test[tempcolumn] = mdf_test[tempcolumn].cat.add_categories(['zzzinfill'])
    
    if adjinfill is True:
      mdf_train[tempcolumn] = mdf_train[tempcolumn].fillna(method='ffill')
      mdf_test[tempcolumn] = mdf_test[tempcolumn].fillna(method='ffill')
      mdf_train[tempcolumn] = mdf_train[tempcolumn].fillna(method='bfill')
      mdf_test[tempcolumn] = mdf_test[tempcolumn].fillna(method='bfill')
      
    #replace NA with a dummy variable
    mdf_train[tempcolumn] = mdf_train[tempcolumn].fillna('zzzinfill')
    mdf_test[tempcolumn] = mdf_test[tempcolumn].fillna('zzzinfill')

    #replace numerical with string equivalent
    mdf_train[tempcolumn] = mdf_train[tempcolumn].astype(str)
    mdf_test[tempcolumn] = mdf_test[tempcolumn].astype(str)

    #extract categories for column labels
    #note that .unique() extracts the labels as a numpy array
    labels_train = mdf_train[tempcolumn].unique()
    labels_train.sort(axis=0)
    labels_train = list(labels_train)
    orig_labels_train = list(labels_train.copy())
    labels_test = mdf_test[tempcolumn].unique()
    labels_test.sort(axis=0)
    labels_test = list(labels_test)

    #pandas one hot encoder
    df_train_cat = pd.get_dummies(mdf_train[tempcolumn])
    df_test_cat = pd.get_dummies(mdf_test[tempcolumn])

    #append column header name to each category listing
    labels_train = [column + '_' + entry for entry in labels_train]
    labels_test = [column + '_' + entry for entry in labels_test]
    
    #convert sparse array to pandas dataframe with column labels
    df_train_cat.columns = labels_train
    df_test_cat.columns = labels_test

    #Get missing columns in test set that are present in training set
    missing_cols = set( df_train_cat.columns ) - set( df_test_cat.columns )

    #Add a missing column in test set with default value equal to 0
    for c in missing_cols:
        df_test_cat[c] = 0
    #Ensure the order of column in the test set is in the same order than in train set
    #Note this also removes categories in test set that aren't present in training set
    df_test_cat = df_test_cat[df_train_cat.columns]

    del mdf_train[tempcolumn]    
    del mdf_test[tempcolumn]
    
    suffixoverlap_results = \
    self.df_check_suffixoverlap(mdf_train, list(df_train_cat), suffixoverlap_results)
    
    #concatinate the sparse set with the rest of our training data
    mdf_train = pd.concat([mdf_train, df_train_cat], axis=1)
    mdf_test = pd.concat([mdf_test, df_test_cat], axis=1)
    
    #delete _NArw column, this will be processed seperately in the processfamily function
    #delete support NArw2 column
#     columnNArw = column + '_NArw'
    columnNAr2 = column + '_zzzinfill'
    if columnNAr2 in mdf_train.columns:
      del mdf_train[columnNAr2]
    if columnNAr2 in mdf_test.columns:
      del mdf_test[columnNAr2]
    if 'zzzinfill' in orig_labels_train:
      orig_labels_train.remove('zzzinfill')

#     del mdf_train[column + '_NAr2']    
#     del mdf_test[column + '_NAr2']
    
    #create output of a list of the created column names
    NAcolumn = columnNAr2
    labels_train = list(df_train_cat)
    if NAcolumn in labels_train:
      labels_train.remove(NAcolumn)
    textcolumns = labels_train
    
    #now we'll creaate a dicitonary of the columns : categories for later reference
    #reminder here is list of. unque values from original column
    #labels_train
    
    normalizationdictvalues = labels_train
    normalizationdictkeys = textcolumns
    
    normalizationdictkeys.sort()
    normalizationdictvalues.sort()
    
    #textlabelsdict = dict(zip(normalizationdictkeys, normalizationdictvalues))
    textlabelsdict = dict(zip(normalizationdictvalues, orig_labels_train))
    
    #change data types to 8-bit (1 byte) integers for memory savings
    for textcolumn in textcolumns:
      mdf_train[textcolumn] = mdf_train[textcolumn].astype(np.int8)
      mdf_test[textcolumn] = mdf_test[textcolumn].astype(np.int8)

    #store some values in the text_dict{} for use later in ML infill methods
    column_dict_list = []

    for tc in textcolumns:
    
      #new parameter collected for driftreport
      tc_ratio = tc + '_ratio'
      tcratio = mdf_train[tc].sum() / mdf_train[tc].shape[0]

      textnormalization_dict = {tc : {'textlabelsdict_text' : textlabelsdict, \
                                      tc_ratio : tcratio, \
                                      'adjinfill' : adjinfill}}
      
      column_dict = {tc : {'category' : 'text', \
                           'origcategory' : category, \
                           'normalization_dict' : textnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : textcolumns, \
                           'categorylist' : textcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())
    
    return mdf_train, mdf_test, column_dict_list

  def process_lngt_class(self, df, column, category, postprocess_dict, params = {}):
    '''
    #processing funciton that length of string for each entry
    #such as a heuristic for information content
    #default infill is len(str(np.nan)) = 3
    #note this is a "singleprocess" function since is applied to single dataframe
    '''
    
    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    if inplace is not True:
      
      #copy source column into new column
      df, suffixoverlap_results = \
      self.df_copy_train(df, column, column + '_lngt', suffixoverlap_results)
    
    else:
      
      suffixoverlap_results = \
      self.df_check_suffixoverlap(df, column + '_lngt', suffixoverlap_results)
      
      df.rename(columns = {column : column + '_lngt'}, inplace = True)
    
    df[column + '_lngt'] = df[column + '_lngt'].astype(str).apply(len)
    
    #grab a fe4w driftreport metrics:
    #get maximum value of training column
    maximum = df[column + '_lngt'].max()
    
    #get minimum value of training column
    minimum = df[column + '_lngt'].min()
    
    #get minimum value of training column
    mean = df[column + '_lngt'].mean()
    
    #get standard deviation of training column
    std = df[column + '_lngt'].std()

    #create list of columns
    columns = [column + '_lngt']

    #create normalization dictionary
    normalization_dict = {column + '_lngt' : {'maximum' : maximum, \
                                              'minimum' : minimum, \
                                              'mean' : mean, \
                                              'std' : std }}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []
    
    for nc in columns:

      column_dict = { nc : {'category' : 'lngt', \
                           'origcategory' : category, \
                           'normalization_dict' : normalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : columns, \
                           'categorylist' : columns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())

    return df, column_dict_list
  
  def process_UPCS_class(self, df, column, category, postprocess_dict, params = {}):
    '''
    #processing funciton that converts columns to uppercase strings
    #such as to allow consistnet encoding if data has upper/lower case discrepencies
    #default infill is a distinct entry as string NAN
    #note that with assigninfill this can be converted to other infill methods
    #returns same dataframe with new column of name column + '_UPCS'
    #note this is a "singleprocess" function since is applied to single dataframe
    '''
    
    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    if 'activate' in params:
      activate = params['activate']
    else:
      activate = True
    
    if inplace is not True:
      
      #copy source column into new column
      df, suffixoverlap_results = \
      self.df_copy_train(df, column, column + '_UPCS', suffixoverlap_results)
    
    else:
      
      suffixoverlap_results = \
      self.df_check_suffixoverlap(df, column + '_UPCS', suffixoverlap_results)
      
      df.rename(columns = {column : column + '_UPCS'}, inplace = True)
    
    #convert to uppercase string based on activate parameter
    if activate is True:
      #convert column to string except for nan infill points
      df[column + '_UPCS'] = \
      np.where(df[column + '_UPCS'] == df[column + '_UPCS'], df[column + '_UPCS'].astype(str), df[column + '_UPCS'])
      #convert to uppercase
      df[column + '_UPCS'] = \
      np.where(df[column + '_UPCS'] == df[column + '_UPCS'], df[column + '_UPCS'].str.upper(), df[column + '_UPCS'])

    #create list of columns
    UPCScolumns = [column + '_UPCS']

    #create normalization dictionary
    normalization_dict = {column + '_UPCS' : {'activate' : activate}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []
    
    for nc in UPCScolumns:

      column_dict = { nc : {'category' : 'UPCS', \
                           'origcategory' : category, \
                           'normalization_dict' : normalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : UPCScolumns, \
                           'categorylist' : UPCScolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())

    return df, column_dict_list

  def process_splt_class(self, mdf_train, mdf_test, column, category, \
                         postprocess_dict, params = {}):
    '''
    #process_splt_class(mdf_train, mdf_test, column, category)
    #preprocess column with categorical entries as strings
    #identifies overlaps of subsets of those strings and records
    #as a new boolan column
    #for example, if a categoical set consisted of unique values 
    #['west', 'north', 'northeast']
    #then a new column would be created idenitifying cells which included 
    #'north' in their entries
    #(here for north and northeast)
    #returns as column titled origcolumn_splt_entry    
    #missing values are ignored by default
    '''
    
    suffixoverlap_results = {}
    
    #overlap_lengths = [20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7 , 6, 5]
    
    if 'minsplit' in params:
      minsplit = params['minsplit'] - 1
    else:
      minsplit = 4
      
    if 'space_and_punctuation' in params:
      space_and_punctuation = params['space_and_punctuation']
    else:
      space_and_punctuation = True
      
    if 'excluded_characters' in params:
      excluded_characters = params['excluded_characters']
    else:
      excluded_characters = [' ', ',', '.', '?', '!', '(', ')']
      
    if 'concurrent_activations' in params:
      concurrent_activations = params['concurrent_activations']
    else:
      concurrent_activations = False
      
    if 'int_headers' in params:
      int_headers = params['int_headers']
    else:
      int_headers = False
      
    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = '_splt'
      
    if 'test_same_as_train' in params:
      test_same_as_train = params['test_same_as_train']
    else:
      test_same_as_train = False
    
    #first we find overlaps from mdf_train
    
    unique_list = list(mdf_train[column].unique())

    unique_list = list(map(str, unique_list))
    
    maxlength = max(len(x) for x in unique_list)
    
    overlap_lengths = list(range(maxlength - 1, minsplit, -1))

    overlap_dict = {}

    #we'll populate overlap_dict as
    #{extract_with_overlap : [list of associate categories with that overlap]}

    #we'll cycle through the overlap lengths and only record an overlap 
    #if it is not a subset of those already recorded
    
    for overlap_length in overlap_lengths:

      for unique in unique_list:

        len_unique = len(unique)

        if len_unique >= overlap_length:

          nbr_iterations = len_unique - overlap_length + 1

          for i in range(nbr_iterations):

            extract = unique[i:(overlap_length+i)]

            extract_already_in_overlap_dict = False

            for key in overlap_dict:

              len_key = len(key)

              if len_key >= overlap_length:

                nbr_iterations3 = len_key - overlap_length + 1

                for k in range(nbr_iterations3):

                  extract3 = key[k:(overlap_length+k)]
                  
                  if concurrent_activations is False:

                    if extract == extract3:

                      extract_already_in_overlap_dict = True
                      
                      break
                      
                  elif concurrent_activations is True:
                    
                    if extract == extract3 and unique in overlap_dict[key]:

                      extract_already_in_overlap_dict = True
                      
                      break
                      
                if extract_already_in_overlap_dict is True:
                  
                  break

            if extract_already_in_overlap_dict is False:

              for unique2 in unique_list:

                if unique2 != unique:

                  len_unique2 = len(unique2)

                  nbr_iterations2 = len_unique2 - overlap_length + 1

                  for j in range(nbr_iterations2):

                    extract2 = unique2[j:(overlap_length+j)]

                    #________
                    
                    if space_and_punctuation is True:

                      if extract2 == extract:

                        if extract in overlap_dict:

                          if unique2 not in overlap_dict[extract]:

                            overlap_dict[extract].append(unique2)
                            
                            if concurrent_activations is False:

                              break

                          if unique not in overlap_dict[extract]:

                            overlap_dict[extract].append(unique)
                            
                            if concurrent_activations is False:

                              break

                        #else if we don't have a key for extract
                        else:

                          overlap_dict.update({extract : [unique, unique2]})
                          
                          if concurrent_activations is False:

                            break
                          
                    elif space_and_punctuation is False:
                      
                      for scrub_punctuation in excluded_characters:
                        
                        extract2 = extract2.replace(scrub_punctuation, '')
                        
                      #if any punctuation was scrubbed these two extracts will be different lengths
                      if extract2 == extract:

                        if extract in overlap_dict:

                          if unique2 not in overlap_dict[extract]:

                            overlap_dict[extract].append(unique2)
                            
                            if concurrent_activations is False:

                              break

                          if unique not in overlap_dict[extract]:

                            overlap_dict[extract].append(unique)
                            
                            if concurrent_activations is False:

                              break

                        #else if we don't have a key for extract
                        else:

                          overlap_dict.update({extract : [unique, unique2]})
                          
                          if concurrent_activations is False:

                            break
     
    #now for mdf_test we'll only consider those overlaps already identified from train set
    
    if test_same_as_train is True:
      test_overlap_dict = overlap_dict
    
    elif test_same_as_train is False:

      unique_list_test = list(mdf_test[column].unique())

      unique_list_test = list(map(str, unique_list_test))

      test_overlap_dict = {}

      train_keys = list(overlap_dict)

      train_keys.sort(key = len, reverse=True)

      for key in train_keys:

        test_overlap_dict.update({key:[]})

      for dict_key in train_keys:

        for unique_test in unique_list_test:

          len_key = len(dict_key)

          if len(unique_test) >= len_key:

            nbr_iterations4 = len(unique_test) - len_key + 1

            for l in range(nbr_iterations4):

              extract4 = unique_test[l:(len_key+l)]

              if extract4 == dict_key:

                test_overlap_dict[dict_key].append(unique_test)

                if concurrent_activations is False:

                  break
    
    newcolumns = []

    for dict_key in overlap_dict:

      newcolumn = column + suffix + '_' + dict_key
      
      mdf_train, suffixoverlap_results = \
      self.df_copy_train(mdf_train, column, newcolumn, suffixoverlap_results)
      
      mdf_test[newcolumn] = mdf_test[column].copy()

      mdf_train[newcolumn] = mdf_train[newcolumn].astype(str)
      mdf_test[newcolumn] = mdf_test[newcolumn].astype(str)

      mdf_train[newcolumn] = mdf_train[newcolumn].isin(overlap_dict[dict_key])
      mdf_train[newcolumn] = mdf_train[newcolumn].astype(np.int8)
      
      mdf_test[newcolumn] = mdf_test[newcolumn].isin(test_overlap_dict[dict_key])
      mdf_test[newcolumn] = mdf_test[newcolumn].astype(np.int8)

      newcolumns.append(newcolumn)
      
    preint_newcolumns = newcolumns.copy()
      
    if int_headers is True:
      
      int_labels_dict = {}
      i = 0
      for entry in newcolumns:
        int_labels_dict.update({entry : column + suffix + '_' + str(i)})
        i += 1
        
      newcolumns = [int_labels_dict[entry] for entry in newcolumns]
        
      #now convert column headers from string to int convention
      suffixoverlap_results = \
      self.df_check_suffixoverlap(mdf_train, newcolumns, suffixoverlap_results)
      
      mdf_train = mdf_train.rename(columns=int_labels_dict)
      mdf_test  = mdf_test.rename(columns=int_labels_dict)

      inverse_int_labels_dict = {value:key for key,value in int_labels_dict.items()}
      for key in inverse_int_labels_dict:
        inverse_int_labels_dict[key] = inverse_int_labels_dict[key][len(column) + 1:]
        
    else:
      int_labels_dict = False
      inverse_int_labels_dict = False
    
    column_dict_list = []

    for tc in newcolumns:

      textnormalization_dict = {tc : {'suffix' : suffix, \
                                      'test_same_as_train' : test_same_as_train, \
                                      'overlap_dict' : overlap_dict, \
                                      'splt_newcolumns_splt'   : newcolumns, \
                                      'minsplit' : minsplit, \
                                      'concurrent_activations' : concurrent_activations, \
                                      'preint_newcolumns' : preint_newcolumns, \
                                      'int_headers' : int_headers, \
                                      'int_labels_dict' : int_labels_dict, \
                                      'inverse_int_labels_dict' : inverse_int_labels_dict}}
      
      column_dict = {tc : {'category' : 'splt', \
                           'origcategory' : category, \
                           'normalization_dict' : textnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : newcolumns, \
                           'categorylist' : newcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())
      
    if len(newcolumns) == 0:
      
      column_dict_list = []
    
    return mdf_train, mdf_test, column_dict_list

  def process_spl2_class(self, mdf_train, mdf_test, column, category, \
                         postprocess_dict, params = {}):
    '''
    #process_spl2_class(mdf_train, mdf_test, column, category)
    #preprocess column with categorical entries as strings
    #identifies overlaps of subsets of those strings and replaces entries 
    #with their redecued overlap
    #replaces entries without overlap to 0 (unique to spl5)
    #for example, if a categorical set consisted of unique values 
    #['west', 'north', 'northeast']
    #then a new column would be created in which the entry 'north' 
    #replaced cells with north in their entries
    #(here for north and northeast)
    #and cells with west would be set to 0
    #returns as column titled origcolumn_spl2
    #missing values are ignored by default
    #this alternative to splt may be benficial for instance if one wanted 
    #to follow with an ordl encoding
    '''
    
    suffixoverlap_results = {}
    
#     overlap_lengths = [20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7 , 6, 5]

    if 'minsplit' in params:
        
      minsplit = params['minsplit'] - 1
    
    else:
      
      minsplit = 4
      
    if 'space_and_punctuation' in params:
      space_and_punctuation = params['space_and_punctuation']
    else:
      space_and_punctuation = True
      
    if 'excluded_characters' in params:
      excluded_characters = params['excluded_characters']
    else:
      excluded_characters = [' ', ',', '.', '?', '!', '(', ')']
      
    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = '_spl2'
      
    if 'test_same_as_train' in params:
      test_same_as_train = params['test_same_as_train']
    else:
      test_same_as_train = False
      
    if 'consolidate_nonoverlaps' in params:
      consolidate_nonoverlaps = params['consolidate_nonoverlaps']
    else:
      consolidate_nonoverlaps = False
    
    #first we find overlaps from mdf_train
    
    unique_list = list(mdf_train[column].unique())

    unique_list = list(map(str, unique_list))
    
    maxlength = max(len(x) for x in unique_list)
    
    overlap_lengths = list(range(maxlength - 1, minsplit, -1))

    overlap_dict = {}

    #we'll populate overlap_dict as
    #{extract_with_overlap : [list of associate categories with that overlap]}

    #we'll cycle through the overlap lengths and only record an overlap 
    #if it is not a subset of those already recorded
    
    for overlap_length in overlap_lengths:

      for unique in unique_list:

        len_unique = len(unique)

        if len_unique >= overlap_length:

          nbr_iterations = len_unique - overlap_length + 1

          for i in range(nbr_iterations):

            extract = unique[i:(overlap_length+i)]

            extract_already_in_overlap_dict = False

            for key in overlap_dict:

              len_key = len(key)

              if len_key >= overlap_length:

                nbr_iterations3 = len_key - overlap_length + 1

                for k in range(nbr_iterations3):

                  extract3 = key[k:(overlap_length+k)]

                  if extract == extract3:

                    extract_already_in_overlap_dict = True
                    
                    break
                    
                if extract_already_in_overlap_dict is True:
                  
                  break

            if extract_already_in_overlap_dict is False:

              for unique2 in unique_list:

                if unique2 != unique:

                  len_unique2 = len(unique2)

                  nbr_iterations2 = len_unique2 - overlap_length + 1

                  for j in range(nbr_iterations2):

                    extract2 = unique2[j:(overlap_length+j)]

                    #________
                    
                    if space_and_punctuation is True:

                      if extract2 == extract:

                        if extract in overlap_dict:

                          if unique2 not in overlap_dict[extract]:

                            overlap_dict[extract].append(unique2)
                            
                            break

                          if unique not in overlap_dict[extract]:

                            overlap_dict[extract].append(unique)
                            
                            break

                        #else if we don't have a key for extract
                        else:

                          overlap_dict.update({extract : [unique, unique2]})
                          
                          break
                          
                    elif space_and_punctuation is False:
                      
                      for scrub_punctuation in excluded_characters:
                        
                        extract2 = extract2.replace(scrub_punctuation, '')
                        
                      #if any punctuation was scrubbed these two extracts will be different lengths
                      if extract2 == extract:

                        if extract in overlap_dict:

                          if unique2 not in overlap_dict[extract]:

                            overlap_dict[extract].append(unique2)
                            
                            break

                          if unique not in overlap_dict[extract]:

                            overlap_dict[extract].append(unique)
                            
                            break

                        #else if we don't have a key for extract
                        else:

                          overlap_dict.update({extract : [unique, unique2]})
                          
                          break
      
    #now for mdf_test we'll only consider those overlaps already 
    #identified from train set
    
    if test_same_as_train is True:
      test_overlap_dict = overlap_dict
    
    elif test_same_as_train is False:
    
      unique_list_test = list(mdf_test[column].unique())

      unique_list_test = list(map(str, unique_list_test))

      test_overlap_dict = {}

      train_keys = list(overlap_dict)

      train_keys.sort(key = len, reverse=True)

      for key in train_keys:

        test_overlap_dict.update({key:[]})

      for dict_key in train_keys:

        for unique_test in unique_list_test:

          len_key = len(dict_key)

          if len(unique_test) >= len_key:

            nbr_iterations4 = len(unique_test) - len_key + 1

            for l in range(nbr_iterations4):

              extract4 = unique_test[l:(len_key+l)]

              if extract4 == dict_key:

                test_overlap_dict[dict_key].append(unique_test)

                break
    
    #so that was all comparable to splt, now for spl2 we'll create a new 
    #dictionary structred as
    #{original unique value : overlap extract for replacement}
    
    #since one original unique value may have entries as multiple overlaps, 
    #we'll prioritize overlaps with
    #longer string lengths and then alphabetical
    
    spl2_overlap_dict = {}
    
    overlap_key_list = list(overlap_dict)
    
    overlap_key_list.sort()
    overlap_key_list.sort(key = len, reverse=True)
    
    for overlap_key in overlap_key_list:
      
      for entry in overlap_dict[overlap_key]:
        
        if entry not in spl2_overlap_dict:
          
          spl2_overlap_dict.update({entry : overlap_key})
    
    #here's where we identify values to set to 0 for spl5
    spl5_zero_dict = {}
    if consolidate_nonoverlaps is True:
      for entry in unique_list:
        if entry not in spl2_overlap_dict:
          spl5_zero_dict.update({entry : 0})
    
    #then we'll do same for test set
    
    spl2_test_overlap_dict = {}
    
    test_overlap_key_list = list(test_overlap_dict)
    
    test_overlap_key_list.sort()
    test_overlap_key_list.sort(key = len, reverse=True)
    
    for overlap_key in test_overlap_key_list:
      
      for entry in test_overlap_dict[overlap_key]:
        
        if entry not in spl2_test_overlap_dict:
          
          spl2_test_overlap_dict.update({entry : overlap_key})
    
    #here's where we identify values to set to 0 for spl5
    spl5_test_zero_dict = {}
    if consolidate_nonoverlaps is True:

      if test_same_as_train is True:
        unique_list_test = list(mdf_test[column].unique())
        unique_list_test = list(map(str, unique_list_test))

      for entry in unique_list_test:
        if entry not in spl2_test_overlap_dict:
          spl5_test_zero_dict.update({entry : 0})
    
    newcolumns = []

#     for dict_key in overlap_dict:

    newcolumn = column + suffix
    
    mdf_train, suffixoverlap_results = \
    self.df_copy_train(mdf_train, column, newcolumn, suffixoverlap_results)
    
    mdf_test[newcolumn] = mdf_test[column].copy()
    
    mdf_train[newcolumn] = mdf_train[newcolumn].astype(str)
    mdf_test[newcolumn] = mdf_test[newcolumn].astype(str)

    mdf_train[newcolumn] = mdf_train[newcolumn].replace(spl2_overlap_dict)
    mdf_train[newcolumn] = mdf_train[newcolumn].replace(spl5_zero_dict)

#       mdf_train[newcolumn] = mdf_train[column].isin(overlap_dict[dict_key])
#       mdf_train[newcolumn] = mdf_train[newcolumn].astype(np.int8)

    mdf_test[newcolumn] = mdf_test[newcolumn].replace(spl2_test_overlap_dict)
    mdf_test[newcolumn] = mdf_test[newcolumn].replace(spl5_test_zero_dict)

#       mdf_test[newcolumn] = mdf_test[column].isin(test_overlap_dict[dict_key])
#       mdf_test[newcolumn] = mdf_test[newcolumn].astype(np.int8)

    newcolumns.append(newcolumn)
    
    column_dict_list = []

    for tc in newcolumns:

      textnormalization_dict = {tc : {'suffix' : suffix, \
                                      'test_same_as_train' : test_same_as_train, \
                                      'consolidate_nonoverlaps' : consolidate_nonoverlaps, \
                                      'overlap_dict' : overlap_dict, \
                                      'spl2_newcolumns'   : newcolumns, 
                                      'spl2_overlap_dict' : spl2_overlap_dict, \
                                      'spl2_test_overlap_dict' : spl2_test_overlap_dict, \
                                      'spl5_zero_dict' : spl5_zero_dict, \
                                      'minsplit' : minsplit}}
      
      column_dict = {tc : {'category' : 'spl2', \
                           'origcategory' : category, \
                           'normalization_dict' : textnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : newcolumns, \
                           'categorylist' : newcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())
      
    if len(newcolumns) == 0:
      
      column_dict_list = []

    return mdf_train, mdf_test, column_dict_list

  def process_sp19_class(self, mdf_train, mdf_test, column, category, \
                         postprocess_dict, params = {}):
    '''
    #process_splt_class(mdf_train, mdf_test, column, category)
    #preprocess column with categorical entries as strings
    #identifies overlaps of subsets of those strings and records
    #as a new boolan column
    #for example, if a categoical set consisted of unique values 
    #['west', 'north', 'northeast']
    #then a new column would be created idenitifying cells which included 
    #'north' in their entries
    #(here for north and northeast)
    #returns as column titled origcolumn_splt_entry    
    #missing values are ignored by default
    
    #sp15 is comparable to splt but multiple concurrent activations allowed
    #so requires a different MLinfilltype in processdict
    
    #sp19 is comparable to sp15 but with a returned binary encoding aggregation
    '''
    
    suffixoverlap_results = {}
    
    #overlap_lengths = [20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7 , 6, 5]
    
    if 'minsplit' in params:
      minsplit = params['minsplit'] - 1
    else:
      minsplit = 4
      
    if 'space_and_punctuation' in params:
      space_and_punctuation = params['space_and_punctuation']
    else:
      space_and_punctuation = True
      
    if 'excluded_characters' in params:
      excluded_characters = params['excluded_characters']
    else:
      excluded_characters = [' ', ',', '.', '?', '!', '(', ')']
      
    if 'int_headers' in params:
      int_headers = params['int_headers']
    else:
      int_headers = False
      
    #note that same MLinfilltype in processdict ('1010')
    #may be used for both configurations but applying concurrent_activations = False
    #with sp11 is less efficient then running splt
    if 'concurrent_activations' in params:
      concurrent_activations = params['concurrent_activations']
    else:
      concurrent_activations = True
      
    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = '_sp19'
      
    if 'test_same_as_train' in params:
      test_same_as_train = params['test_same_as_train']
    else:
      test_same_as_train = False
    
    #first we find overlaps from mdf_train
    
    unique_list = list(mdf_train[column].unique())

    unique_list = list(map(str, unique_list))
    
    maxlength = max(len(x) for x in unique_list)
    
    overlap_lengths = list(range(maxlength - 1, minsplit, -1))

    overlap_dict = {}

    #we'll populate overlap_dict as
    #{extract_with_overlap : [list of associate categories with that overlap]}

    #we'll cycle through the overlap lengths and only record an overlap 
    #if it is not a subset of those already recorded
    
    for overlap_length in overlap_lengths:

      for unique in unique_list:

        len_unique = len(unique)

        if len_unique >= overlap_length:

          nbr_iterations = len_unique - overlap_length + 1

          for i in range(nbr_iterations):

            extract = unique[i:(overlap_length+i)]

            extract_already_in_overlap_dict = False

            for key in overlap_dict:

              len_key = len(key)

              if len_key >= overlap_length:

                nbr_iterations3 = len_key - overlap_length + 1

                for k in range(nbr_iterations3):

                  extract3 = key[k:(overlap_length+k)]
                  
                  if concurrent_activations is False:

                    if extract == extract3:

                      extract_already_in_overlap_dict = True
                      
                      break
                      
                  elif concurrent_activations is True:
                    
                    if extract == extract3 and unique in overlap_dict[key]:

                      extract_already_in_overlap_dict = True
                      
                      break
                      
                if extract_already_in_overlap_dict is True:
                  
                  break

            if extract_already_in_overlap_dict is False:

              for unique2 in unique_list:

                if unique2 != unique:

                  len_unique2 = len(unique2)

                  nbr_iterations2 = len_unique2 - overlap_length + 1

                  for j in range(nbr_iterations2):

                    extract2 = unique2[j:(overlap_length+j)]

                    #________
                    
                    if space_and_punctuation is True:

                      if extract2 == extract:

                        if extract in overlap_dict:

                          if unique2 not in overlap_dict[extract]:

                            overlap_dict[extract].append(unique2)
                            
                            if concurrent_activations is False:

                              break

                          if unique not in overlap_dict[extract]:

                            overlap_dict[extract].append(unique)
                            
                            if concurrent_activations is False:

                              break

                        #else if we don't have a key for extract
                        else:

                          overlap_dict.update({extract : [unique, unique2]})
                          
                          if concurrent_activations is False:

                            break
                          
                    elif space_and_punctuation is False:
                      
                      for scrub_punctuation in excluded_characters:
                        
                        extract2 = extract2.replace(scrub_punctuation, '')
                        
                      #if any punctuation was scrubbed these two extracts will be different lengths
                      if extract2 == extract:

                        if extract in overlap_dict:

                          if unique2 not in overlap_dict[extract]:

                            overlap_dict[extract].append(unique2)
                            
                            if concurrent_activations is False:

                              break

                          if unique not in overlap_dict[extract]:

                            overlap_dict[extract].append(unique)
                            
                            if concurrent_activations is False:

                              break

                        #else if we don't have a key for extract
                        else:

                          overlap_dict.update({extract : [unique, unique2]})
                          
                          if concurrent_activations is False:

                            break
        
    #now for mdf_test we'll only consider those overlaps already identified from train set
    
    if test_same_as_train is True:
      test_overlap_dict = overlap_dict
    
    elif test_same_as_train is False:

      unique_list_test = list(mdf_test[column].unique())

      unique_list_test = list(map(str, unique_list_test))

      test_overlap_dict = {}

      train_keys = list(overlap_dict)

      train_keys.sort(key = len, reverse=True)

      for key in train_keys:

        test_overlap_dict.update({key:[]})

      for dict_key in train_keys:

        for unique_test in unique_list_test:

          len_key = len(dict_key)

          if len(unique_test) >= len_key:

            nbr_iterations4 = len(unique_test) - len_key + 1

            for l in range(nbr_iterations4):

              extract4 = unique_test[l:(len_key+l)]

              if extract4 == dict_key:

                test_overlap_dict[dict_key].append(unique_test)

                if concurrent_activations is False:

                  break
    
    newcolumns = []

    for dict_key in overlap_dict:

      newcolumn = column + '_sp15_' + dict_key
      
      mdf_train, suffixoverlap_results = \
      self.df_copy_train(mdf_train, column, newcolumn, suffixoverlap_results)
#       mdf_train[newcolumn] = mdf_train[column].copy()
      
      mdf_test[newcolumn] = mdf_test[column].copy()

      mdf_train[newcolumn] = mdf_train[newcolumn].astype(str)
      mdf_test[newcolumn] = mdf_test[newcolumn].astype(str)

      mdf_train[newcolumn] = mdf_train[newcolumn].isin(overlap_dict[dict_key])
      mdf_train[newcolumn] = mdf_train[newcolumn].astype(np.int8)
      
      mdf_test[newcolumn] = mdf_test[newcolumn].isin(test_overlap_dict[dict_key])
      mdf_test[newcolumn] = mdf_test[newcolumn].astype(np.int8)

      newcolumns.append(newcolumn)
      
    preint_newcolumns = newcolumns.copy()
      
    if int_headers is True:
      
      int_labels_dict = {}
      i = 0
      for entry in newcolumns:
        int_labels_dict.update({entry : column + '_sp15_' + str(i)})
        i += 1
        
      #now convert column headers from string to int convention
      mdf_train = mdf_train.rename(columns=int_labels_dict)
      mdf_test  = mdf_test.rename(columns=int_labels_dict)

      newcolumns = [int_labels_dict[entry] for entry in newcolumns]

      inverse_int_labels_dict = {value:key for key,value in int_labels_dict.items()}
      for key in inverse_int_labels_dict:
        inverse_int_labels_dict[key] = inverse_int_labels_dict[key][len(column) + 1:]
        
    else:
      int_labels_dict = False
      inverse_int_labels_dict = False
    
    column_dict_list = []
    
    #begin binary encoding of set, leaving the int_headers here out of convenience, not really needed
    
    if len(newcolumns) > 0:
      
      sp19_column = column + suffix
    
      #aggregate collection of activations as string set
      #the suffix 'activations_' is to avoid potential of overlap with binary encoding and aggregated activations
      mdf_train[sp19_column] = 'activations_'
      mdf_test[sp19_column] = 'activations_'

      for entry in newcolumns:
        mdf_train[sp19_column] = mdf_train[sp19_column] + mdf_train[entry].astype(str)
        mdf_test[sp19_column] = mdf_test[sp19_column] + mdf_test[entry].astype(str)

      #extract categories for column labels
      #note that .unique() extracts the labels as a numpy array
      labels_train = list(mdf_train[sp19_column].unique())
      labels_train.sort()
      labels_test = list(mdf_test[sp19_column].unique())
      labels_test.sort()

      #if infill not present in train set, insert
      if 'zzzinfill' not in labels_train:
        labels_train = labels_train + ['zzzinfill']
        labels_train.sort()
      if 'zzzinfill' not in labels_test:
        labels_test = labels_test + ['zzzinfill']
        labels_test.sort()

      #get length of the list
      listlength = len(labels_train)

      #calculate number of columns we'll need
      binary_column_count = int(np.ceil(np.log2(listlength)))

      #initialize dictionaryt to store encodings
      binary_encoding_dict = {}
      encoding_list = []

      for i in range(listlength):

        #this converts the integer i to binary encoding
        #where f is an f string for inserting the column coount into the string to designate length of encoding
        #0 is to pad out the encoding with 0's for the length
        #and b is telling it to convert to binary 
        #note this returns a string
        encoding = format(i, f"0{binary_column_count}b")

        if i < len(labels_train):

          #store the encoding in a dictionary
          binary_encoding_dict.update({labels_train[i] : encoding})

          #store the encoding in a list for checking in next step
          encoding_list.append(encoding)


      #clear up memory
      del encoding_list
  #     del overlap_list

      #new driftreport metric _1010_activations_dict
      _1010_activations_dict = {}
      for key in binary_encoding_dict:
        sumcalc = (mdf_train[sp19_column] == key).sum() 
        ratio = sumcalc / mdf_train[sp19_column].shape[0]
        _1010_activations_dict.update({key:ratio})


      #replace the cateogries in train set via ordinal trasnformation
      mdf_train[sp19_column] = mdf_train[sp19_column].replace(binary_encoding_dict) 

      #in test set, we'll need to strike any categories that weren't present in train
      #first let'/s identify what applies
      testspecificcategories = list(set(labels_test)-set(labels_train))

      #so we'll just replace those items with our plug value
      testplug_dict = dict(zip(testspecificcategories, ['zzzinfill'] * len(testspecificcategories)))
      mdf_test[sp19_column] = mdf_test[sp19_column].replace(testplug_dict)  

      #now we'll apply the 1010 transformation to the test set
      mdf_test[sp19_column] = mdf_test[sp19_column].replace(binary_encoding_dict)    

      #ok let's create a list of columns to store each entry of the binary encoding
      _1010_columnlist = []

      for i in range(binary_column_count):

        _1010_columnlist.append(column + suffix + '_' + str(i))

      suffixoverlap_results = \
      self.df_check_suffixoverlap(mdf_train, _1010_columnlist, suffixoverlap_results)

      #now let's store the encoding
      i=0
      for _1010_column in _1010_columnlist:

        mdf_train[_1010_column] = mdf_train[sp19_column].str.slice(i,i+1).astype(np.int8)

        mdf_test[_1010_column] = mdf_test[sp19_column].str.slice(i,i+1).astype(np.int8)

        i+=1

      #now delete the support column
      del mdf_train[sp19_column]
      del mdf_test[sp19_column]

      for entry in newcolumns:
        del mdf_train[entry]
        del mdf_test[entry]

      #now store the column_dict entries
      categorylist = _1010_columnlist

      column_dict_list = []

      for tc in categorylist:

        #                                   '_1010_overlap_replace' : overlap_replace, \
        normalization_dict = {tc : {'suffix' : suffix, \
                                    'test_same_as_train' : test_same_as_train, \
                                    '_1010_binary_encoding_dict' : binary_encoding_dict, \
                                    '_1010_binary_column_count' : binary_column_count, \
                                    '_1010_activations_dict' : _1010_activations_dict, \
                                    'categorylist' : categorylist, \
                                    'overlap_dict' : overlap_dict, \
                                    'splt_newcolumns_sp19'   : newcolumns, \
                                    'minsplit' : minsplit, \
                                    'concurrent_activations' : concurrent_activations, \
                                    'preint_newcolumns' : preint_newcolumns, \
                                    'int_headers' : int_headers, \
                                    'int_labels_dict' : int_labels_dict, \
                                    'inverse_int_labels_dict' : inverse_int_labels_dict}}

        column_dict = {tc : {'category' : 'sp19', \
                             'origcategory' : category, \
                             'normalization_dict' : normalization_dict, \
                             'origcolumn' : column, \
                             'inputcolumn' : column, \
                             'columnslist' : categorylist, \
                             'categorylist' : categorylist, \
                             'infillmodel' : False, \
                             'infillcomplete' : False, \
                             'suffixoverlap_results' : suffixoverlap_results, \
                             'deletecolumn' : False}}

        column_dict_list.append(column_dict.copy())
      
    else:
      
      column_dict_list = []
    
    return mdf_train, mdf_test, column_dict_list

  def process_sbst_class(self, mdf_train, mdf_test, column, category, \
                         postprocess_dict, params = {}):
    '''
    #process_sbst_class(mdf_train, mdf_test, column, category)
    #preprocess column with categorical entries as strings
    #identifies cases where a full unique value is present 
    #as a subset of a longer length unique value
    #and returns one-hot activations to aggregate those cases
    
    #accepts parameters concurrent_activations to allow mulitple activations
    #actually let's make concurrent activations the default
    #and int_headers for privacy preserving headers
    
    #this differs from other string parsing functions in that
    #only complete entries are checked for presence as subsets in other entries
    '''
    
    suffixoverlap_results = {}
      
    if 'concurrent_activations' in params:
      concurrent_activations = params['concurrent_activations']
    else:
      concurrent_activations = True
      
    if 'int_headers' in params:
      int_headers = params['int_headers']
    else:
      int_headers = False

    if 'minsplit' in params:
      minsplit = params['minsplit']
    else:
      minsplit = 1
      
    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = '_sbst'
      
    if 'test_same_as_train' in params:
      test_same_as_train = params['test_same_as_train']
    else:
      test_same_as_train = False
    
    #first we find overlaps from mdf_train
    
    unique_list = list(mdf_train[column].unique())

    unique_list = list(map(str, unique_list))
    
    unique_list = sorted(unique_list, key=len, reverse=True)
    
#     maxlength = max(len(x) for x in unique_list)
    
#     minlength = min(len(x) for x in unique_list)
    
#     overlap_lengths = list(range(maxlength - 1, minlength, -1))

    overlap_dict = {}

    #we'll populate overlap_dict as
    #{extract_with_overlap : [list of associate categories with that overlap]}

    #we'll cycle through the overlap lengths and only record an overlap 
    #if it is not a subset of those already recorded
    
    #unique is what we are searching for
    for unique in unique_list:
      len_unique = len(unique)

      if len_unique >= minsplit:
      
        #unique2 is where we are searching
        for unique2 in unique_list:
          len_unique2 = len(unique2)
          
          if len_unique2 > len_unique:
            
            nbr_iterations = len_unique2 - len_unique + 1
            
            for i in range(nbr_iterations):
              
              extract = unique2[i:(len_unique+i)]
              
              extract_already_in_overlap_dict = False
                    
              if extract_already_in_overlap_dict is False:
                
                if extract == unique:
                  
                  if extract in overlap_dict:
                    
                    if unique2 not in overlap_dict[extract]:
                      
                      overlap_dict[extract].append(unique2)
                      
                      if concurrent_activations is False:

                        break
                        
                    # if unique not in overlap_dict[extract]:
                      
                    #   overlap_dict[extract].append(unique)

                    #   if concurrent_activations is False:

                    #     break
                        
                  #else if we don't have a key for extract
                  else:

                    overlap_dict.update({extract : [unique, unique2]})

                    if concurrent_activations is False:

                      break
                    
    #now for mdf_test we'll only consider those overlaps already identified from train set
    
    if test_same_as_train is True:
      test_overlap_dict = overlap_dict
    
    elif test_same_as_train is False:

      unique_list_test = list(mdf_test[column].unique())

      unique_list_test = list(map(str, unique_list_test))

      unique_list_test = sorted(unique_list_test, key=len, reverse=True)

      test_overlap_dict = {}

      train_keys = list(overlap_dict)

      train_keys.sort(key = len, reverse=True)

      for key in train_keys:

        test_overlap_dict.update({key:[]})

      for dict_key in train_keys:

        for unique_test in unique_list_test:

          len_key = len(dict_key)

          if len(unique_test) >= len_key:

            nbr_iterations4 = len(unique_test) - len_key + 1

            for l in range(nbr_iterations4):

              extract4 = unique_test[l:(len_key+l)]

              if extract4 == dict_key:

                test_overlap_dict[dict_key].append(unique_test)

                if concurrent_activations is False:

                  break
                
    newcolumns = []

    for dict_key in overlap_dict:

      newcolumn = column + suffix + '_' + dict_key
      
      mdf_train, suffixoverlap_results = \
      self.df_copy_train(mdf_train, column, newcolumn, suffixoverlap_results)
      
#       mdf_train[newcolumn] = mdf_train[column].copy()
  
      mdf_test[newcolumn] = mdf_test[column].copy()

      mdf_train[newcolumn] = mdf_train[newcolumn].astype(str)
      mdf_test[newcolumn] = mdf_test[newcolumn].astype(str)

      mdf_train[newcolumn] = mdf_train[newcolumn].isin(overlap_dict[dict_key])
      mdf_train[newcolumn] = mdf_train[newcolumn].astype(np.int8)
      
      mdf_test[newcolumn] = mdf_test[newcolumn].isin(test_overlap_dict[dict_key])
      mdf_test[newcolumn] = mdf_test[newcolumn].astype(np.int8)

      newcolumns.append(newcolumn)
      
    preint_newcolumns = newcolumns.copy()
      
    if int_headers is True:
      
      int_labels_dict = {}
      i = 0
      for entry in newcolumns:
        int_labels_dict.update({entry : column + suffix + '_' + str(i)})
        i += 1
        
      newcolumns = [int_labels_dict[entry] for entry in newcolumns]
        
      #now convert column headers from string to int convention
      suffixoverlap_results = \
      self.df_check_suffixoverlap(mdf_train, newcolumns, suffixoverlap_results)
      
      mdf_train = mdf_train.rename(columns=int_labels_dict)
      mdf_test  = mdf_test.rename(columns=int_labels_dict)

      inverse_int_labels_dict = {value:key for key,value in int_labels_dict.items()}
      for key in inverse_int_labels_dict:
        inverse_int_labels_dict[key] = inverse_int_labels_dict[key][len(column) + 1:]
        
    else:
      int_labels_dict = False
      inverse_int_labels_dict = False
    
    column_dict_list = []

    for tc in newcolumns:

      textnormalization_dict = {tc : {'suffix' : suffix, \
                                      'test_same_as_train' : test_same_as_train, \
                                      'overlap_dict' : overlap_dict, \
                                      'splt_newcolumns_sbst'   : newcolumns, \
                                      'minsplit' : minsplit, \
                                      'concurrent_activations' : concurrent_activations, \
                                      'preint_newcolumns' : preint_newcolumns, \
                                      'int_headers' : int_headers, \
                                      'int_labels_dict' : int_labels_dict, \
                                      'inverse_int_labels_dict' : inverse_int_labels_dict}}
      
      column_dict = {tc : {'category' : 'sbst', \
                           'origcategory' : category, \
                           'normalization_dict' : textnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : newcolumns, \
                           'categorylist' : newcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())
      
    if len(newcolumns) == 0:
      
      column_dict_list = []
    
    return mdf_train, mdf_test, column_dict_list

  def process_sbs3_class(self, mdf_train, mdf_test, column, category, \
                         postprocess_dict, params = {}):
    '''
    #process_sbst_class(mdf_train, mdf_test, column, category)
    #preprocess column with categorical entries as strings
    #identifies cases where a full unique value is present 
    #as a subset of a longer length unique value
    #and returns one-hot activations to aggregate those cases
    
    #accepts parameters concurrent_activations to allow mulitple activations
    #actually let's make concurrent activations the default
    #and int_headers for privacy preserving headers
    
    #this differs from other string parsing functions in that
    #only complete entries are checked for presence as subsets in other entries
    
    #sbs3 is comparable to sbst but with a returned binary encoding aggregation
    '''
    
    suffixoverlap_results = {}
      
    if 'concurrent_activations' in params:
      concurrent_activations = params['concurrent_activations']
    else:
      concurrent_activations = True
      
    if 'int_headers' in params:
      int_headers = params['int_headers']
    else:
      int_headers = False

    if 'minsplit' in params:
      minsplit = params['minsplit']
    else:
      minsplit = 1
      
    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = '_sbs3'
      
    if 'test_same_as_train' in params:
      test_same_as_train = params['test_same_as_train']
    else:
      test_same_as_train = False
    
    #first we find overlaps from mdf_train
    
    unique_list = list(mdf_train[column].unique())

    unique_list = list(map(str, unique_list))
    
    unique_list = sorted(unique_list, key=len, reverse=True)
    
#     maxlength = max(len(x) for x in unique_list)
    
#     minlength = min(len(x) for x in unique_list)
    
#     overlap_lengths = list(range(maxlength - 1, minlength, -1))

    overlap_dict = {}

    #we'll populate overlap_dict as
    #{extract_with_overlap : [list of associate categories with that overlap]}

    #we'll cycle through the overlap lengths and only record an overlap 
    #if it is not a subset of those already recorded
    
    #unique is what we are searching for
    for unique in unique_list:
      len_unique = len(unique)

      if len_unique >= minsplit:
      
        #unique2 is where we are searching
        for unique2 in unique_list:
          len_unique2 = len(unique2)
          
          if len_unique2 > len_unique:
            
            nbr_iterations = len_unique2 - len_unique + 1
            
            for i in range(nbr_iterations):
              
              extract = unique2[i:(len_unique+i)]
              
              extract_already_in_overlap_dict = False
                    
              if extract_already_in_overlap_dict is False:
                
                if extract == unique:
                  
                  if extract in overlap_dict:
                    
                    if unique2 not in overlap_dict[extract]:
                      
                      overlap_dict[extract].append(unique2)
                      
                      if concurrent_activations is False:

                        break
                        
                    # if unique not in overlap_dict[extract]:
                      
                    #   overlap_dict[extract].append(unique)

                    #   if concurrent_activations is False:

                    #     break
                        
                  #else if we don't have a key for extract
                  else:

                    overlap_dict.update({extract : [unique, unique2]})

                    if concurrent_activations is False:

                      break
                    
    #now for mdf_test we'll only consider those overlaps already identified from train set
    
    if test_same_as_train is True:
      test_overlap_dict = overlap_dict
    
    elif test_same_as_train is False:

      unique_list_test = list(mdf_test[column].unique())

      unique_list_test = list(map(str, unique_list_test))

      unique_list_test = sorted(unique_list_test, key=len, reverse=True)

      test_overlap_dict = {}

      train_keys = list(overlap_dict)

      train_keys.sort(key = len, reverse=True)

      for key in train_keys:

        test_overlap_dict.update({key:[]})

      for dict_key in train_keys:

        for unique_test in unique_list_test:

          len_key = len(dict_key)

          if len(unique_test) >= len_key:

            nbr_iterations4 = len(unique_test) - len_key + 1

            for l in range(nbr_iterations4):

              extract4 = unique_test[l:(len_key+l)]

              if extract4 == dict_key:

                test_overlap_dict[dict_key].append(unique_test)

                if concurrent_activations is False:

                  break
                
    newcolumns = []

    for dict_key in overlap_dict:

      newcolumn = column + '_sbst_' + dict_key
      
      mdf_train, suffixoverlap_results = \
      self.df_copy_train(mdf_train, column, newcolumn, suffixoverlap_results)
      
#       mdf_train[newcolumn] = mdf_train[column].copy()
  
      mdf_test[newcolumn] = mdf_test[column].copy()

      mdf_train[newcolumn] = mdf_train[newcolumn].astype(str)
      mdf_test[newcolumn] = mdf_test[newcolumn].astype(str)

      mdf_train[newcolumn] = mdf_train[newcolumn].isin(overlap_dict[dict_key])
      mdf_train[newcolumn] = mdf_train[newcolumn].astype(np.int8)
      
      mdf_test[newcolumn] = mdf_test[newcolumn].isin(test_overlap_dict[dict_key])
      mdf_test[newcolumn] = mdf_test[newcolumn].astype(np.int8)

      newcolumns.append(newcolumn)
      
    preint_newcolumns = newcolumns.copy()
      
    if int_headers is True:
      
      int_labels_dict = {}
      i = 0
      for entry in newcolumns:
        int_labels_dict.update({entry : column + '_sbst_' + str(i)})
        i += 1
        
      newcolumns = [int_labels_dict[entry] for entry in newcolumns]
        
      #now convert column headers from string to int convention
      suffixoverlap_results = \
      self.df_check_suffixoverlap(mdf_train, newcolumns, suffixoverlap_results)
      
      mdf_train = mdf_train.rename(columns=int_labels_dict)
      mdf_test  = mdf_test.rename(columns=int_labels_dict)

      inverse_int_labels_dict = {value:key for key,value in int_labels_dict.items()}
      for key in inverse_int_labels_dict:
        inverse_int_labels_dict[key] = inverse_int_labels_dict[key][len(column) + 1:]
        
    else:
      int_labels_dict = False
      inverse_int_labels_dict = False
    
    column_dict_list = []
    
    #begin binary encoding of set, leaving the int_headers here out of convenience, not really needed
    
    if len(newcolumns) > 0:
      
      sbs3_column = column + suffix

      #aggregate collection of activations as string set
      #the suffix 'activations_' is to avoid potential of overlap with binary encoding and aggregated activations
      mdf_train[sbs3_column] = 'activations_'
      mdf_test[sbs3_column] = 'activations_'

      for entry in newcolumns:
        mdf_train[sbs3_column] = mdf_train[sbs3_column] + mdf_train[entry].astype(str)
        mdf_test[sbs3_column] = mdf_test[sbs3_column] + mdf_test[entry].astype(str)

      #extract categories for column labels
      #note that .unique() extracts the labels as a numpy array
      labels_train = list(mdf_train[sbs3_column].unique())
      labels_train.sort()
      labels_test = list(mdf_test[sbs3_column].unique())
      labels_test.sort()

      #if infill not present in train set, insert
      if 'zzzinfill' not in labels_train:
        labels_train = labels_train + ['zzzinfill']
        labels_train.sort()
      if 'zzzinfill' not in labels_test:
        labels_test = labels_test + ['zzzinfill']
        labels_test.sort()

      #get length of the list
      listlength = len(labels_train)

      #calculate number of columns we'll need
      binary_column_count = int(np.ceil(np.log2(listlength)))

      #initialize dictionaryt to store encodings
      binary_encoding_dict = {}
      encoding_list = []

      for i in range(listlength):

        #this converts the integer i to binary encoding
        #where f is an f string for inserting the column coount into the string to designate length of encoding
        #0 is to pad out the encoding with 0's for the length
        #and b is telling it to convert to binary 
        #note this returns a string
        encoding = format(i, f"0{binary_column_count}b")

        if i < len(labels_train):

          #store the encoding in a dictionary
          binary_encoding_dict.update({labels_train[i] : encoding})

          #store the encoding in a list for checking in next step
          encoding_list.append(encoding)


      #clear up memory
      del encoding_list
  #     del overlap_list

      #new driftreport metric _1010_activations_dict
      _1010_activations_dict = {}
      for key in binary_encoding_dict:
        sumcalc = (mdf_train[sbs3_column] == key).sum() 
        ratio = sumcalc / mdf_train[sbs3_column].shape[0]
        _1010_activations_dict.update({key:ratio})


      #replace the cateogries in train set via ordinal trasnformation
      mdf_train[sbs3_column] = mdf_train[sbs3_column].replace(binary_encoding_dict) 

      #in test set, we'll need to strike any categories that weren't present in train
      #first let'/s identify what applies
      testspecificcategories = list(set(labels_test)-set(labels_train))

      #so we'll just replace those items with our plug value
      testplug_dict = dict(zip(testspecificcategories, ['zzzinfill'] * len(testspecificcategories)))
      mdf_test[sbs3_column] = mdf_test[sbs3_column].replace(testplug_dict)  

      #now we'll apply the 1010 transformation to the test set
      mdf_test[sbs3_column] = mdf_test[sbs3_column].replace(binary_encoding_dict)    

      #ok let's create a list of columns to store each entry of the binary encoding
      _1010_columnlist = []

      for i in range(binary_column_count):

        _1010_columnlist.append(column + suffix + '_' + str(i))

      suffixoverlap_results = \
      self.df_check_suffixoverlap(mdf_train, _1010_columnlist, suffixoverlap_results)

      #now let's store the encoding
      i=0
      for _1010_column in _1010_columnlist:

        mdf_train[_1010_column] = mdf_train[sbs3_column].str.slice(i,i+1).astype(np.int8)

        mdf_test[_1010_column] = mdf_test[sbs3_column].str.slice(i,i+1).astype(np.int8)

        i+=1

      #now delete the support column
      del mdf_train[sbs3_column]
      del mdf_test[sbs3_column]

      for entry in newcolumns:
        del mdf_train[entry]
        del mdf_test[entry]

      #now store the column_dict entries
      categorylist = _1010_columnlist

      column_dict_list = []

      for tc in categorylist:

  #                                   '_1010_overlap_replace' : overlap_replace, \
        normalization_dict = {tc : {'suffix' : suffix, \
                                    'test_same_as_train' : test_same_as_train, \
                                    '_1010_binary_encoding_dict' : binary_encoding_dict, \
                                    '_1010_binary_column_count' : binary_column_count, \
                                    '_1010_activations_dict' : _1010_activations_dict, \
                                    'categorylist' : categorylist, \
                                    'overlap_dict' : overlap_dict, \
                                    'splt_newcolumns_sbs3'   : newcolumns, \
                                    'concurrent_activations' : concurrent_activations, \
                                    'minsplit' : minsplit, \
                                    'preint_newcolumns' : preint_newcolumns, \
                                    'int_headers' : int_headers, \
                                    'int_labels_dict' : int_labels_dict, \
                                    'inverse_int_labels_dict' : inverse_int_labels_dict}}

        column_dict = {tc : {'category' : 'sbs3', \
                             'origcategory' : category, \
                             'normalization_dict' : normalization_dict, \
                             'origcolumn' : column, \
                             'inputcolumn' : column, \
                             'columnslist' : categorylist, \
                             'categorylist' : categorylist, \
                             'infillmodel' : False, \
                             'infillcomplete' : False, \
                             'suffixoverlap_results' : suffixoverlap_results, \
                             'deletecolumn' : False}}

        column_dict_list.append(column_dict.copy())
      
    else:
      
      column_dict_list = []
    
    return mdf_train, mdf_test, column_dict_list

  def process_srch_class(self, mdf_train, mdf_test, column, category, \
                         postprocess_dict, params = {}):
    """
    #process_srch_class(mdf_train, mdf_test, column, category)
    #preprocess column with categorical entries as strings
    #relies on user passed list of strings in search parameter
    #string parses unique entries to identify overlaps with search strings
    #when overlap found returns a column with boolean activation identifiers
    
    #note this differs from original srch in that makes use of pandas str.contains
    #which is expected to be more efficient for unbounded sets
    
    #for example, if a categoical set consisted of unique values 
    #['west', 'north', 'northwest']
    #and a user passed the search parameter as ['west']
    #then a new column would be returned 
    #with activations corresponding to entries of 'west' and 'northwest'
    
    #note that search parameter can include lists of search terms embedded in the list
    #which embedded lists will be aggregated to a single activation
    #for example if we want single activation for female names could pass search = [['Ms.', 'Miss', 'Mrs']] etc
    
    #note this returns all zeros in a column if search value not found
    
    #note returned coluymns are named by search term, e
    #e.g. column + '_srch_' + str(search)
    
    #note that search terms are converted to strings and compared to columns cast as strings

    #missing values are ignored by default
    """
    
    suffixoverlap_results = {}
        
    if 'search' in params:
      search = params['search']
    else:
      search = []
      
    if 'case' in params:
      case = params['case']
    else:
      case = True
      
    #we'll create mirror to account for any embdded lists of search terms for aggregation
    search_preflattening = search.copy()
    #this is kind of hacky just to reuse code below resetting this list to repopulate
    search = []
    aggregated_dict = {}
    
    for entry in search_preflattening:
      if type(entry) != type([]):
        search.append(str(entry))
      else:
        aggregated_dict.update({str(entry[-1]):[]})
        for entry2 in entry[0:-1]:
          search.append(entry2)
          aggregated_dict[str(entry[-1])].append(str(entry2))
        for entry2 in entry[-1:]:
          search.append(entry2)
    
    newcolumns = []
    search_dict = {}
    for searchitem in search:
      search_dict.update({column + '_srch_' + str(searchitem) : str(searchitem)})
      
    for newcolumn in search_dict:
      suffixoverlap_results = \
      self.df_check_suffixoverlap(mdf_train, newcolumn, suffixoverlap_results)
      
      mdf_train[newcolumn] = \
      np.where(mdf_train[column].astype(str).str.contains(search_dict[newcolumn], case=case, regex=False), 1, 0)
      
      mdf_test[newcolumn] = \
      np.where(mdf_test[column].astype(str).str.contains(search_dict[newcolumn], case=case, regex=False), 1, 0)
    
    newcolumns = list(search_dict)
    
    #now we'll address any aggregations fo search terms
    #from search parameter passed with embedded list of search terms
          
    #then after populating activations, we'll put this below
    #inverse_search_dict has key of search term and value of column for activations
    inverse_search_dict = {value:key for key,value in search_dict.items()}
    newcolumns_before_aggregation = newcolumns.copy()
    
    #now we consolidate activations
    #note that this only runs when aggregated_dict was populated with an embedded list of search terms
    for aggregated_dict_key in aggregated_dict:
      aggregated_dict_key_column = inverse_search_dict[aggregated_dict_key]
      
      for target_for_aggregation in aggregated_dict[aggregated_dict_key]:
        target_for_aggregation_column = inverse_search_dict[target_for_aggregation]
        
        mdf_train[aggregated_dict_key_column] = \
        np.where(mdf_train[target_for_aggregation_column] == 1, 1, mdf_train[aggregated_dict_key_column])
        mdf_test[aggregated_dict_key_column] = \
        np.where(mdf_test[target_for_aggregation_column] == 1, 1, mdf_test[aggregated_dict_key_column])
        
        del mdf_train[target_for_aggregation_column]
        del mdf_test[target_for_aggregation_column]
        
        newcolumns.remove(target_for_aggregation_column)
    
    for newcolumn in newcolumns:

      mdf_train[newcolumn] = mdf_train[newcolumn].astype(np.int8)
      mdf_test[newcolumn] = mdf_test[newcolumn].astype(np.int8)
    
    column_dict_list = []

    for tc in newcolumns:

      textnormalization_dict = {tc : {'search_dict' : search_dict, \
                                      'inverse_search_dict' : inverse_search_dict, \
                                      'srch_newcolumns_srch'   : newcolumns, \
                                      'newcolumns_before_aggregation' : newcolumns_before_aggregation, \
                                      'search' : search, \
                                      'search_preflattening' : search_preflattening, \
                                      'aggregated_dict' : aggregated_dict, \
                                      'case' : case}}
      
      column_dict = {tc : {'category' : 'srch', \
                           'origcategory' : category, \
                           'normalization_dict' : textnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : newcolumns, \
                           'categorylist' : newcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())
      
    if len(newcolumns) == 0:
      
      column_dict_list = []
    
    return mdf_train, mdf_test, column_dict_list

  def process_src2_class(self, mdf_train, mdf_test, column, category, \
                        postprocess_dict, params = {}):
    """
    #process_src2_class(mdf_train, mdf_test, column, category)
    #preprocess column with categorical entries as strings
    #relies on user passed list of strings in search parameter
    #string parses unique entries to identify overlaps with search strings
    #when overlap found returns a column with boolean activation identifiers
    
    #for example, if a categoical set consisted of unique values 
    #['west', 'north', 'northwest']
    #and a user passed the search parameter as ['west']
    #then a new column would be returned 
    #with activations corresponding to entries of 'west' and 'northwest'

    #missing values are ignored by default
    
    #assumes that unique values of test set are same or subset of train set
    #for more efficient application in postmunge
    """
    
    suffixoverlap_results = {}
        
    if 'search' in params:
      search = params['search']
    else:
      search = []
    
    #first we find overlaps from mdf_train
    
    unique_list = list(mdf_train[column].unique())

    unique_list = list(map(str, unique_list))
    
#     maxlength = max(len(x) for x in unique_list)
    
#     overlap_lengths = list(range(maxlength - 1, minsplit, -1))

    #we'll create mirror to account for any embdded lists of search terms for aggregation
    search_preflattening = search.copy()
    #this is kind of hacky just to reuse code below resetting this list to repopulate
    search = []
    aggregated_dict = {}
    
    for entry in search_preflattening:
      if type(entry) != type([]):
        search.append(str(entry))
      else:
        aggregated_dict.update({str(entry[-1]):[]})
        for entry2 in entry[0:-1]:
          search.append(entry2)
          aggregated_dict[str(entry[-1])].append(str(entry2))
        for entry2 in entry[-1:]:
          search.append(entry2)

    #we'll populate overlap_dict as
    #{search_string : [list of associate categories with that overlap found]}
    
    overlap_dict = {}
    
    for search_string in search:
      
      overlap_dict.update({search_string : []})
    
    for search_string in search:
      
      len_search_string = len(search_string)
    
      for unique in unique_list:
        
        len_unique = len(unique)
        
        if len_unique >= len_search_string:
          
          nbr_iterations = len_unique - len_search_string
          
          for i in range(nbr_iterations + 1):
            
            extract = unique[i:(len_search_string+i)]
            
            if extract in search:
              
              overlap_dict[extract].append(unique)
    
#     #now for mdf_test
    
#     unique_list_test = list(mdf_test[column].unique())

#     unique_list_test = list(map(str, unique_list_test))

#     test_overlap_dict = {}
    
#     for search_string in search:
      
#       test_overlap_dict.update({search_string : []})
    

#     train_keys = list(overlap_dict)

#     train_keys.sort(key = len, reverse=True)

#     for dict_key in train_keys:

#       for unique_test in unique_list_test:

#         len_key = len(dict_key)

#         if len(unique_test) >= len_key:

#           nbr_iterations4 = len(unique_test) - len_key

#           for l in range(nbr_iterations4 + 1):

#             extract4 = unique_test[l:(len_key+l)]

#             if extract4 == dict_key:

#               test_overlap_dict[dict_key].append(unique_test)
    
    newcolumns = []

    for dict_key in overlap_dict:
      
      if len(overlap_dict[dict_key]) > 0:

        newcolumn = column + '_src2_' + dict_key

        mdf_train, suffixoverlap_results = \
        self.df_copy_train(mdf_train, column, newcolumn, suffixoverlap_results)
        
        mdf_test[newcolumn] = mdf_test[column].copy()

        mdf_train[newcolumn] = mdf_train[newcolumn].astype(str)
        mdf_test[newcolumn] = mdf_test[newcolumn].astype(str)

        mdf_train[newcolumn] = mdf_train[newcolumn].isin(overlap_dict[dict_key])
#         mdf_train[newcolumn] = mdf_train[newcolumn].astype(np.int8)

        mdf_test[newcolumn] = mdf_test[newcolumn].isin(overlap_dict[dict_key])
#         mdf_test[newcolumn] = mdf_test[newcolumn].astype(np.int8)

        newcolumns.append(newcolumn)
    
    #now in case there are any aggregated activations, inspired by approach in srch
    inverse_search_dict = dict(zip(search, newcolumns))
    newcolumns_before_aggregation = newcolumns.copy()
    
    #now we consolidate activations
    #note that this only runs when aggregated_dict was populated with an embedded list of search terms
    for aggregated_dict_key in aggregated_dict:
      aggregated_dict_key_column = inverse_search_dict[aggregated_dict_key]
      
      for target_for_aggregation in aggregated_dict[aggregated_dict_key]:
        target_for_aggregation_column = inverse_search_dict[target_for_aggregation]
        
        mdf_train[aggregated_dict_key_column] = \
        np.where(mdf_train[target_for_aggregation_column] == 1, 1, mdf_train[aggregated_dict_key_column])
        mdf_test[aggregated_dict_key_column] = \
        np.where(mdf_test[target_for_aggregation_column] == 1, 1, mdf_test[aggregated_dict_key_column])
        
        del mdf_train[target_for_aggregation_column]
        del mdf_test[target_for_aggregation_column]
        
        newcolumns.remove(target_for_aggregation_column)
        
    for newcolumn in newcolumns:
      mdf_train[newcolumn] = mdf_train[newcolumn].astype(np.int8)
      mdf_test[newcolumn] = mdf_test[newcolumn].astype(np.int8)
    
    column_dict_list = []

    for tc in newcolumns:

      textnormalization_dict = {tc : {'overlap_dict' : overlap_dict, \
                                      'src2_newcolumns_src2'   : newcolumns, \
                                      'newcolumns_before_aggregation' : newcolumns_before_aggregation, \
                                      'search' : search, \
                                      'inverse_search_dict' : inverse_search_dict, \
                                      'aggregated_dict' : aggregated_dict, \
                                      'search_preflattening' : search_preflattening}}
      
      column_dict = {tc : {'category' : 'src2', \
                           'origcategory' : category, \
                           'normalization_dict' : textnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : newcolumns, \
                           'categorylist' : newcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())
      
    if len(newcolumns) == 0:
      
      column_dict_list = []
    
    return mdf_train, mdf_test, column_dict_list
  
  def process_src3_class(self, mdf_train, mdf_test, column, category, \
                         postprocess_dict, params = {}):
    """
    #process_src3_class(mdf_train, mdf_test, column, category)
    #preprocess column with categorical entries as strings
    #relies on user passed list of strings in search parameter
    #string parses unique entries to identify overlaps with search strings
    #when overlap found returns a column with boolean activation identifiers
    
    #for example, if a categoical set consisted of unique values 
    #['west', 'north', 'northwest']
    #and a user passed the search parameter as ['west']
    #then a new column would be returned 
    #with activations corresponding to entries of 'west' and 'northwest'

    #missing values are ignored by default
    
    #where srch is preferred for unbounded range of unique values
    
    #and src2 preferred when have bounded range of unique values for both train & test
    
    #and speculation is that src3 may be preferred when have a bounded
    #range of unique values but still want capacity to handle values in 
    #test set not found in train set
    """
    
    suffixoverlap_results = {}
        
    if 'search' in params:
      search = params['search']
    else:
      search = []
    
    #first we find overlaps from mdf_train
    
    unique_list = list(mdf_train[column].unique())

    unique_list = list(map(str, unique_list))
    
#     maxlength = max(len(x) for x in unique_list)
    
#     overlap_lengths = list(range(maxlength - 1, minsplit, -1))

    #we'll populate overlap_dict as
    #{search_string : [list of associate categories with that overlap found]}
    
    overlap_dict = {}
    
    for search_string in search:
      
      overlap_dict.update({search_string : []})
    
    for search_string in search:
      
      len_search_string = len(search_string)
    
      for unique in unique_list:
        
        len_unique = len(unique)
        
        if len_unique >= len_search_string:
          
          nbr_iterations = len_unique - len_search_string
          
          for i in range(nbr_iterations + 1):
            
            extract = unique[i:(len_search_string+i)]
            
            if extract in search:
              
              overlap_dict[extract].append(unique)
           
    #now for mdf_test
    
    unique_list_test = list(mdf_test[column].unique())

    unique_list_test = list(map(str, unique_list_test))

    test_overlap_dict = {}
    
    for search_string in search:
      
      test_overlap_dict.update({search_string : []})
    
    train_keys = list(overlap_dict)

    train_keys.sort(key = len, reverse=True)

    for dict_key in train_keys:

      for unique_test in unique_list_test:

        len_key = len(dict_key)

        if len(unique_test) >= len_key:

          nbr_iterations4 = len(unique_test) - len_key

          for l in range(nbr_iterations4 + 1):

            extract4 = unique_test[l:(len_key+l)]

            if extract4 == dict_key:

              test_overlap_dict[dict_key].append(unique_test)
    
    newcolumns = []

    for dict_key in overlap_dict:
      
      if len(overlap_dict[dict_key]) > 0:

        newcolumn = column + '_src3_' + dict_key

        mdf_train, suffixoverlap_results = \
        self.df_copy_train(mdf_train, column, newcolumn, suffixoverlap_results)
        
        mdf_test[newcolumn] = mdf_test[column].copy()

        mdf_train[newcolumn] = mdf_train[newcolumn].astype(str)
        mdf_test[newcolumn] = mdf_test[newcolumn].astype(str)

        mdf_train[newcolumn] = mdf_train[newcolumn].isin(overlap_dict[dict_key])
        mdf_train[newcolumn] = mdf_train[newcolumn].astype(np.int8)

        mdf_test[newcolumn] = mdf_test[newcolumn].isin(test_overlap_dict[dict_key])
        mdf_test[newcolumn] = mdf_test[newcolumn].astype(np.int8)

        newcolumns.append(newcolumn)
    
    column_dict_list = []

    for tc in newcolumns:

      textnormalization_dict = {tc : {'overlap_dict' : overlap_dict, \
                                      'srch_newcolumns_src3'   : newcolumns, \
                                      'search' : search}}
      
      column_dict = {tc : {'category' : 'src3', \
                           'origcategory' : category, \
                           'normalization_dict' : textnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : newcolumns, \
                           'categorylist' : newcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())
      
    if len(newcolumns) == 0:
      
      column_dict_list = []
    
    return mdf_train, mdf_test, column_dict_list

  def process_src4_class(self, mdf_train, mdf_test, column, category, \
                         postprocess_dict, params = {}):
    """
    #process_src4_class(mdf_train, mdf_test, column, category)
    #preprocess column with categorical entries as strings
    #relies on user passed list of strings in search parameter
    #string parses unique entries to identify overlaps with search strings
    #when overlap found returns a column with boolean activation identifiers
    
    #note this differs from original srch in that makes use of pandas str.contains
    #which is expected to be more efficient for unbounded sets
    
    #for example, if a categoical set consisted of unique values 
    #['west', 'north', 'northwest']
    #and a user passed the search parameter as ['west']
    #then a new column would be returned 
    #with activations corresponding to entries of 'west' and 'northwest'
    
    #note this returns all zeros in a column if search value not found
    
    #note returned coluymns are named by search term, e
    #e.g. column + '_srch_' + str(search)
    
    #note that search terms are converted to strings and compared to columns cast as strings

    #missing values are ignored by default
    
    #src4 builds on the srch by converting to an ordinal activation
    #with 0 reserved for no activations
    #note that if an entry was activated for multiple search terms
    #the order of entries in search parameter will dictate the final encoding
    #(e.g. entries at end of list are prioritized over beginning)
    """
    
    suffixoverlap_results = {}
        
    if 'search' in params:
      search = params['search']
    else:
      search = []
      
    if 'case' in params:
      case = params['case']
    else:
      case = True
      
    #we'll create mirror to account for any embdded lists of search terms for aggregation
    search_preflattening = search.copy()
    #this is kind of hacky just to reuse code below resetting this list to repopulate
    search = []
    aggregated_dict = {}
    
    for entry in search_preflattening:
      if type(entry) != type([]):
        search.append(str(entry))
      else:
        aggregated_dict.update({str(entry[-1]):[]})
        for entry2 in entry[0:-1]:
          search.append(entry2)
          aggregated_dict[str(entry[-1])].append(str(entry2))
        for entry2 in entry[-1:]:
          search.append(entry2)
    
    newcolumns = []
    search_dict = {}
    for searchitem in search:
      search_dict.update({column + '_src4_' + str(searchitem) : str(searchitem)})
      
    for newcolumn in search_dict:
      suffixoverlap_results = \
      self.df_check_suffixoverlap(mdf_train, newcolumn, suffixoverlap_results)
      
      mdf_train[newcolumn] = \
      np.where(mdf_train[column].astype(str).str.contains(search_dict[newcolumn], case=case, regex=False), 1, 0)
      
      mdf_test[newcolumn] = \
      np.where(mdf_test[column].astype(str).str.contains(search_dict[newcolumn], case=case, regex=False), 1, 0)
    
    newcolumns = list(search_dict)

#     for newcolumn in newcolumns:

#       mdf_train[newcolumn] = mdf_train[newcolumn].astype(np.int8)
#       mdf_test[newcolumn] = mdf_test[newcolumn].astype(np.int8)
      
    #ok now let's convert to ordinal for src4
    ordl_dict1 = {}
    ordl_dict2 = {}
    
    #reserve zero for no activations
    i = 1
    for newcolumn in newcolumns:
      ordl_dict1.update({i : newcolumn})
      ordl_dict2.update({newcolumn : i})
      i += 1
      
    suffixoverlap_results = \
    self.df_check_suffixoverlap(mdf_train, column + '_src4', suffixoverlap_results)
      
    mdf_train[column + '_src4'] = 0
    mdf_test[column + '_src4'] = 0
    
    for newcolumn in newcolumns:
      
      mdf_train[column + '_src4'] = \
      np.where(mdf_train[newcolumn] == 1, ordl_dict2[newcolumn], mdf_train[column + '_src4'])
      mdf_test[column + '_src4'] = \
      np.where(mdf_test[newcolumn] == 1, ordl_dict2[newcolumn], mdf_test[column + '_src4'])
      del mdf_train[newcolumn]
      del mdf_test[newcolumn]
      
    #now we'll address any aggregations fo search terms
    #from search parameter passed with embedded list of search terms
          
    #then after populating activations, we'll put this below
    #inverse_search_dict has key of search term and value of column for activations
    inverse_search_dict = {value:key for key,value in search_dict.items()}
#     newcolumns_before_aggregation = newcolumns.copy()
      
    #now we consolidate activations
    #note that this only runs when aggregated_dict was populated with an embedded list of search terms
    for aggregated_dict_key in aggregated_dict:
      aggregated_dict_key_column = inverse_search_dict[aggregated_dict_key]
      aggregated_dict_key_encoding = ordl_dict2[aggregated_dict_key_column]
      
      for target_for_aggregation in aggregated_dict[aggregated_dict_key]:
        target_for_aggregation_column = inverse_search_dict[target_for_aggregation]
        target_for_aggregation_encoding = ordl_dict2[target_for_aggregation_column]
        
        mdf_train[column + '_src4'] = \
        np.where(mdf_train[column + '_src4'] == target_for_aggregation_encoding, aggregated_dict_key_encoding, mdf_train[column + '_src4'])
        mdf_test[column + '_src4'] = \
        np.where(mdf_test[column + '_src4'] == target_for_aggregation_encoding, aggregated_dict_key_encoding, mdf_test[column + '_src4'])

    #we'll base the integer type on number of ordinal entries
    if len(ordl_dict1) < 254:
      mdf_train[column + '_src4'] = mdf_train[column + '_src4'].astype(np.uint8)
      mdf_test[column + '_src4'] = mdf_test[column + '_src4'].astype(np.uint8)
    elif len(ordl_dict1) < 65530:
      mdf_train[column + '_src4'] = mdf_train[column + '_src4'].astype(np.uint16)
      mdf_test[column + '_src4'] = mdf_test[column + '_src4'].astype(np.uint16)
    else:
      mdf_train[column + '_src4'] = mdf_train[column + '_src4'].astype(np.uint32)
      mdf_test[column + '_src4'] = mdf_test[column + '_src4'].astype(np.uint32)
    
    column_dict_list = []
    
    #newcolumns are based on the original srch transform
    #src4_newcolumns are after consolidating to ordinal encoding (single entry)
    src4_newcolumns = [column + '_src4']

    for tc in src4_newcolumns:

      textnormalization_dict = {tc : {'search_dict' : search_dict, \
                                      'inverse_search_dict' : inverse_search_dict, \
                                      'srch_newcolumns_src4' : newcolumns, \
                                      'src4_newcolumns' : src4_newcolumns, \
                                      'search' : search, \
                                      'search_preflattening' : search_preflattening, \
                                      'aggregated_dict' : aggregated_dict, \
                                      'case' : case, \
                                      'ordl_dict1' : ordl_dict1, \
                                      'activations_list' : list(ordl_dict1), \
                                      'ordl_dict2' : ordl_dict2}}
      
      column_dict = {tc : {'category' : 'src4', \
                           'origcategory' : category, \
                           'normalization_dict' : textnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : src4_newcolumns, \
                           'categorylist' : src4_newcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())

    return mdf_train, mdf_test, column_dict_list
  
  def process_aggt_class(self, df, column, category, postprocess_dict, params = {}):
    """
    #process_aggt_class(mdf_train, mdf_test, column, category)
    #preprocess column with categorical entries as strings
    #and aggregates differently spelled duplicates into single representation
    #based on user passed parameter 'aggregate'
    #which is a list of lists, where sublists are the aggregation groups
    #and the final representation will be the final item in list
    #note also supports passing aggregate as a single list of terms without embedded lists
    """
    
    suffixoverlap_results = {}
    
    if 'aggregate' in params:
      aggregate = params['aggregate']
    else:
      aggregate = [[]]
      
    df, suffixoverlap_results = \
    self.df_copy_train(df, column, column + '_aggt', suffixoverlap_results)

    for sublist in aggregate:
      
      if not isinstance(sublist, list):
        
        sublist = aggregate
      
        length_sublist = len(sublist)

        for i in range(length_sublist-1):

          df[column + '_aggt'] = np.where(df[column + '_aggt'] == sublist[i], sublist[-1], df[column + '_aggt'])
          
        break
      
      else:
        
        length_sublist = len(sublist)

        for i in range(length_sublist-1):

          df[column + '_aggt'] = np.where(df[column + '_aggt'] == sublist[i], sublist[-1], df[column + '_aggt'])

    normalization_dict = {column + '_aggt' : {'aggregate' : aggregate}}
    
    nmbrcolumns = [column + '_aggt']
    
    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in nmbrcolumns:
      
      column_dict = { nc : {'category' : 'aggt', \
                           'origcategory' : category, \
                           'normalization_dict' : normalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())
        
    return df, column_dict_list

  def process_strn_class(self, df, column, category, postprocess_dict, params = {}):
    """
    #process_strn_class(df, column, category, postprocess_dict)
    #parses string entries and if any strings present returns longest string
    #i.e. character subsets excluding numerical entries
    #entries without strings present subject to infill
    """
    
    suffixoverlap_results = {}
    
    unique_list = list(df[column].unique())

    unique_list = list(map(str, unique_list))
    
    maxlength = max(len(x) for x in unique_list)
    
    overlap_lengths = list(range(maxlength, 0, -1))

    overlap_dict = {}
    
    for overlap_length in overlap_lengths:

      for unique in unique_list:
        
        if unique not in overlap_dict:

          len_unique = len(unique)

          if len_unique >= overlap_length:
            
            if overlap_length > 1:

              nbr_iterations = len_unique - overlap_length

              for i in range(nbr_iterations + 1):
                
                if unique not in overlap_dict:

                  extract = unique[i:(overlap_length+i)]
                  
                  has_number = False
                  
                  for j in range(len(extract)):
                    
                    if self.is_number(extract[j]):
                      
                      has_number = True

  #                 extract_already_in_overlap_dict = False

                  if has_number is False:

                    overlap_dict.update({unique : extract})
                
            #else if overlap_length == 1    
            else:
              
              nbr_iterations = len_unique - overlap_length
              
              in_dict = False

              for i in range(nbr_iterations + 1):
                
                if unique not in overlap_dict:

                  extract = unique[i:(overlap_length+i)]

  #                 extract_already_in_overlap_dict = False
  
                  has_number = False
                  
                  for j in range(len(extract)):
                    
                    if self.is_number(extract[j]):
                      
                      has_number = True

  #                 extract_already_in_overlap_dict = False

                  if has_number is False:
      
                    in_dict = True

                    overlap_dict.update({unique : extract})
                  
              if in_dict is False:

                overlap_dict.update({unique : np.nan})
    
    suffixoverlap_results = \
    self.df_check_suffixoverlap(df, column + '_strn', suffixoverlap_results)
    
    df[column + '_strn'] = df[column].astype(str)
    df[column + '_strn'] = df[column + '_strn'].replace(overlap_dict)

    #replace missing data with training set mean as default infill
    df[column + '_strn'] = df[column + '_strn'].fillna('zzzinfill')
    
#     #a few more metrics collected for driftreport
#     #get maximum value of training column
#     maximum = df[column + '_nmrc'].max()
#     #get minimum value of training column
#     minimum = df[column + '_nmrc'].min()
    
    #create list of columns
    nmbrcolumns = [column + '_strn']

    nmbrnormalization_dict = {column + '_strn' : {'overlap_dict' : overlap_dict}}
#                                                   'mean' : mean, \
#                                                   'maximum' : maximum, \
#                                                   'minimum' : minimum }}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []
    
    for nc in nmbrcolumns:

      column_dict = { nc : {'category' : 'strn', \
                           'origcategory' : category, \
                           'normalization_dict' : nmbrnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())
        
    return df, column_dict_list

  def process_strg_class(self, df, column, category, postprocess_dict, params = {}):
    '''
    #str function
    #accepts input of integer categoric sets, such as from an ordinal transform
    #and converts to strings for purposes of categoric recognition in some downstream libaries
    #(eg some libraries will treat integer label sets as targets for regression instead of classificaiton)
    #does not perform infill, just converts entries to string
    '''
    
    suffixoverlap_results = {}
      
    strg_column = column + '_strg'
    
    df, suffixoverlap_results = \
    self.df_copy_train(df, column, strg_column, suffixoverlap_results)
    
    df[strg_column] = df[strg_column].astype(str)

    column_dict_list = []

    column_dict = {strg_column : {'category' : 'strg', \
                                 'origcategory' : category, \
                                 'normalization_dict' : {strg_column:{}}, \
                                 'origcolumn' : column, \
                                 'inputcolumn' : column, \
                                 'columnslist' : [strg_column], \
                                 'categorylist' : [strg_column], \
                                 'infillmodel' : False, \
                                 'infillcomplete' : False, \
                                 'suffixoverlap_results' : suffixoverlap_results, \
                                 'deletecolumn' : False}}
    
    #now append column_dict onto postprocess_dict
    column_dict_list.append(column_dict.copy())

    return df, column_dict_list

  def process_nmrc_class(self, df, column, category, postprocess_dict, params = {}):
    """
    #process_nmrc_class(df, column, category, postprocess_dict)
    #parses string entries and if any numbers present returns numbers
    #entries without numbers present subject to infill
    #accepts parameters 
    #convention as numbers/commas/spaces
    #suffix for column suffix identifier
    """
    
    suffixoverlap_results = {}
    
    if 'convention' in params:
      #accepts numbers/commas/spaces
      convention = params['convention']
    else:
      convention = 'numbers'
      
    if 'suffix' in params:
      #accepts string for suffix appender
      suffix = params['suffix']
    else:
      suffix = '_nmrc'
      
    nmrc_column = column + suffix
    
    df, suffixoverlap_results = \
    self.df_copy_train(df, column, nmrc_column, suffixoverlap_results)
    
    unique_list = list(df[nmrc_column].unique())

    unique_list = list(map(str, unique_list))
    
    maxlength = max(len(x) for x in unique_list)
    
    overlap_lengths = list(range(maxlength, 0, -1))

    overlap_dict = {}
    
    for overlap_length in overlap_lengths:

      for unique in unique_list:
        
        if unique not in overlap_dict:

          len_unique = len(unique)

          if len_unique >= overlap_length:
            
            if overlap_length > 1:

              nbr_iterations = len_unique - overlap_length

              for i in range(nbr_iterations + 1):
                
                if unique not in overlap_dict:

                  extract = unique[i:(overlap_length+i)]

  #                 extract_already_in_overlap_dict = False
                  
                  if convention == 'numbers':
                  
                    if self.is_number(extract):

                      overlap_dict.update({unique : float(extract)})
              
                  elif convention == 'commas':
                  
                    if self.is_number_comma(extract):

                      overlap_dict.update({unique : float(extract.replace(',',''))})
                      
                  elif convention == 'spaces':
                  
                    if self.is_number_EU(extract):

                      overlap_dict.update({unique : float(extract[0] + extract[1:-1].replace(' ','').replace('.','').replace(',','.') + extract[-1])})
                      
            #else if overlap_length == 1    
            else:
              
              nbr_iterations = len_unique - overlap_length
              
              in_dict = False

              for i in range(nbr_iterations + 1):
                
                if unique not in overlap_dict:

                  extract = unique[i:(overlap_length+i)]

  #                 extract_already_in_overlap_dict = False

                  if self.is_number(extract):

                    in_dict = True

                    overlap_dict.update({unique : float(extract)})

              if in_dict is False:

                overlap_dict.update({unique : np.nan})
    
    df[nmrc_column] = df[nmrc_column].astype(str)
    df[nmrc_column] = df[nmrc_column].replace(overlap_dict)

    df[nmrc_column] = pd.to_numeric(df[nmrc_column], errors='coerce')
    
    #get mean of training data
    mean = df[nmrc_column].mean()
    if mean != mean:
      mean = 0
      
    #replace missing data with training set mean as default infill
    df[nmrc_column] = df[nmrc_column].fillna(mean)
    
    #a few more metrics collected for driftreport
    #get maximum value of training column
    maximum = df[nmrc_column].max()
    #get minimum value of training column
    minimum = df[nmrc_column].min()
    
    #create list of columns
    nmbrcolumns = [nmrc_column]

    #populate data structures
    nmbrnormalization_dict = {nmrc_column : {'overlap_dict' : overlap_dict, \
                                            'mean' : mean, \
                                            'maximum' : maximum, \
                                            'minimum' : minimum, \
                                            'convention' : convention, \
                                            'suffix' : suffix }}
    
    column_dict_list = []
    
    for nc in nmbrcolumns:

      column_dict = { nc : {'category' : 'nmrc', \
                           'origcategory' : category, \
                           'normalization_dict' : nmbrnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())
    
    return df, column_dict_list

  def process_nmr4_class(self, mdf_train, mdf_test, column, category, \
                         postprocess_dict, params = {}):
    """
    #extract numeric partitions from categoric entries, test treated differently than train
    #accepts parameters
    #convention as numbers/commas/spaces
    #suffix for column suffix identifier
    #test_same_as_train as True/False
    #where True copiues overlap_dict from train for test, False parses test entries not found in train
    """
    
    suffixoverlap_results = {}
    
    if 'convention' in params:
      #accepts numbers/commas/spaces
      convention = params['convention']
    else:
      convention = 'numbers'
      
    if 'suffix' in params:
      #accepts string for suffix appender
      suffix = params['suffix']
    else:
      suffix = '_nmr4'
      
    if 'test_same_as_train' in params:
      #accepts boolean
      test_same_as_train = params['test_same_as_train']
    else:
      test_same_as_train = True
      
    nmrc_column = column + suffix
    
    mdf_train, suffixoverlap_results = \
    self.df_copy_train(mdf_train, column, nmrc_column, suffixoverlap_results)
    
    mdf_test[nmrc_column] = mdf_test[column].copy()
    
    #begin parsing train set
    
    unique_list = list(mdf_train[nmrc_column].unique())

    unique_list = list(map(str, unique_list))
    
    maxlength = max(len(x) for x in unique_list)
    
    overlap_lengths = list(range(maxlength, 0, -1))

    overlap_dict = {}
    
    for overlap_length in overlap_lengths:

      for unique in unique_list:
        
        if unique not in overlap_dict:

          len_unique = len(unique)

          if len_unique >= overlap_length:
            
            if overlap_length > 1:

              nbr_iterations = len_unique - overlap_length

              for i in range(nbr_iterations + 1):
                
                if unique not in overlap_dict:

                  extract = unique[i:(overlap_length+i)]

  #                 extract_already_in_overlap_dict = False
                  
                  if convention == 'numbers':
                  
                    if self.is_number(extract):

                      overlap_dict.update({unique : float(extract)})
              
                  elif convention == 'commas':
                  
                    if self.is_number_comma(extract):

                      overlap_dict.update({unique : float(extract.replace(',',''))})
                      
                  elif convention == 'spaces':
                  
                    if self.is_number_EU(extract):

                      overlap_dict.update({unique : float(extract[0] + extract[1:-1].replace(' ','').replace('.','').replace(',','.') + extract[-1])})
                      
            #else if overlap_length == 1    
            else:
              
              nbr_iterations = len_unique - overlap_length
              
              in_dict = False

              for i in range(nbr_iterations + 1):
                
                if unique not in overlap_dict:

                  extract = unique[i:(overlap_length+i)]

  #                 extract_already_in_overlap_dict = False
  
                  if self.is_number(extract):

                    in_dict = True

                    overlap_dict.update({unique : float(extract)})

              if in_dict is False:

                overlap_dict.update({unique : np.nan})
                
    mdf_train[nmrc_column] = mdf_train[nmrc_column].astype(str)
    mdf_train[nmrc_column] = mdf_train[nmrc_column].replace(overlap_dict)
    
    #now test set
    test_unique_list = list(mdf_test[nmrc_column].unique())
    test_unique_list = list(map(str, test_unique_list))
    extra_test_unique = list(set(test_unique_list) - set(unique_list))

    test_overlap_dict = deepcopy(overlap_dict)
    
    if test_same_as_train is True:
      
      for test_unique in extra_test_unique:
        test_overlap_dict.update({str(test_unique) : np.nan})
      
    elif test_same_as_train is False:
      
      testmaxlength = max(len(x) for x in unique_list)

      overlap_lengths = list(range(testmaxlength, 0, -1))

  #     overlap_dict = {}

      for overlap_length in overlap_lengths:

        for unique in extra_test_unique:

          if unique not in test_overlap_dict:

            len_unique = len(unique)

            if len_unique >= overlap_length:

              if overlap_length > 1:

                nbr_iterations = len_unique - overlap_length

                for i in range(nbr_iterations + 1):

                  if unique not in test_overlap_dict:

                    extract = unique[i:(overlap_length+i)]

    #                 extract_already_in_overlap_dict = False
                    
                    if convention == 'numbers':
                    
                      if self.is_number(extract):

                        test_overlap_dict.update({unique : float(extract)})
                  
                    elif convention == 'commas':
                    
                      if self.is_number_comma(extract):

                        test_overlap_dict.update({unique : float(extract.replace(',',''))})
                        
                    elif convention == 'spaces':
                    
                      if self.is_number_EU(extract):

                        test_overlap_dict.update({unique : float(extract[0] + extract[1:-1].replace(' ','').replace('.','').replace(',','.') + extract[-1])})

              #else if overlap_length == 1    
              else:

                nbr_iterations = len_unique - overlap_length

                in_dict = False

                for i in range(nbr_iterations + 1):

                  if unique not in test_overlap_dict:

                    extract = unique[i:(overlap_length+i)]

    #                 extract_already_in_overlap_dict = False
                    
                    if self.is_number(extract):

                      in_dict = True

                      test_overlap_dict.update({unique : float(extract)})
                    
                if in_dict is False:

                  test_overlap_dict.update({unique : np.nan})
    
    #great now that test_overlap_dict is populated
    mdf_test[nmrc_column] = mdf_test[nmrc_column].astype(str)
    mdf_test[nmrc_column] = mdf_test[nmrc_column].replace(test_overlap_dict)

    mdf_train[nmrc_column] = pd.to_numeric(mdf_train[nmrc_column], errors='coerce')
    mdf_test[nmrc_column] = pd.to_numeric(mdf_test[nmrc_column], errors='coerce')

    #get mean of training data
    mean = mdf_train[nmrc_column].mean()
    if mean != mean:
      mean = 0

    #replace missing data with training set mean as default infill
    mdf_train[nmrc_column] = mdf_train[nmrc_column].fillna(mean)
    mdf_test[nmrc_column] = mdf_test[nmrc_column].fillna(mean)
    
    #a few more metrics collected for driftreport
    maximum = mdf_train[nmrc_column].max()
    minimum = mdf_train[nmrc_column].min()
    
    #create list of columns
    nmbrcolumns = [nmrc_column]
    
    #populate data structures
    nmbrnormalization_dict = {nmrc_column : {'overlap_dict' : overlap_dict, \
                                            'mean' : mean, \
                                            'maximum' : maximum, \
                                            'minimum' : minimum, \
                                            'unique_list' : unique_list, \
                                            'maxlength' : maxlength, \
                                            'convention' : convention, \
                                            'suffix' : suffix, \
                                            'test_same_as_train' : test_same_as_train}}
    
    column_dict_list = []
    
    for nc in nmbrcolumns:

      column_dict = { nc : {'category' : 'nmr4', \
                           'origcategory' : category, \
                           'normalization_dict' : nmbrnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())
      
    return mdf_train, mdf_test, column_dict_list
  
  def process_ordl_class(self, mdf_train, mdf_test, column, category, \
                         postprocess_dict, params = {}):
    '''
    #process_ordl_class(mdf_train, mdf_test, column, category)
    #preprocess column with categories into ordinal (sequentuial integer) sets
    #corresponding to (sorted) categories
    #adresses infill with new point which we arbitrarily set as 'zzzinfill'
    #intended to show up as last point in set alphabetically
    #for categories presetn in test set not present in train set use this 'zzz' category
    #as implemented this function seperately encodes numbers and string equivalent (eg 2 != '2')
    '''
    
    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    #adjinfill accepts True/False to change default infill from mean inputation to adjacent cell
    if 'adjinfill' in params:
      adjinfill = params['adjinfill']
    else:
      adjinfill = False
      
    #ordered_overide is boolean to indicate if order of integer encoding basis will 
    #defer to cases when a column is a pandas categorical ordered set
    if 'ordered_overide' in params:
      ordered_overide = params['ordered_overide']
    else:
      ordered_overide = True
      
    #str_convert provides consistent encodings between numbers and string equivalent, eg 2 == '2'
    if 'str_convert' in params:
      str_convert = params['str_convert']
    else:
      str_convert = False
    
    if inplace is not True:
      
      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.df_copy_train(mdf_train, column, column + '_ordl', suffixoverlap_results)

      mdf_test[column + '_ordl'] = mdf_test[column].copy()
    
    else:
      
      suffixoverlap_results = \
      self.df_check_suffixoverlap(mdf_train, column + '_ordl', suffixoverlap_results)
      
      mdf_train.rename(columns = {column : column + '_ordl'}, inplace = True)
      mdf_test.rename(columns = {column : column + '_ordl'}, inplace = True)
      
    ordered = False
    if ordered_overide:
      if mdf_train[column + '_ordl'].dtype.name == 'category':
        if mdf_train[column + '_ordl'].cat.ordered:
          ordered = True
          labels_train = list(mdf_train[column + '_ordl'].cat.categories)
          if mdf_test[column + '_ordl'].dtype.name == 'category':
            if mdf_test[column + '_ordl'].cat.ordered:
              labels_test = list(mdf_test[column + '_ordl'].cat.categories)
            else:
              ordered = False
          else:
            ordered = False
    
    #convert column to category if it isn't already
    mdf_train[column + '_ordl'] = mdf_train[column + '_ordl'].astype('category')
    mdf_test[column + '_ordl'] = mdf_test[column + '_ordl'].astype('category')

    #if set is categorical we'll need the plug value for missing values included
    if 'zzzinfill' not in mdf_train[column + '_ordl'].cat.categories:
      mdf_train[column + '_ordl'] = mdf_train[column + '_ordl'].cat.add_categories(['zzzinfill'])
    if 'zzzinfill' not in mdf_test[column + '_ordl'].cat.categories:
      mdf_test[column + '_ordl'] = mdf_test[column + '_ordl'].cat.add_categories(['zzzinfill'])
      
    if adjinfill is True:
      mdf_train[column + '_ordl'] = mdf_train[column + '_ordl'].fillna(method='ffill')
      mdf_test[column + '_ordl'] = mdf_test[column + '_ordl'].fillna(method='ffill')
      mdf_train[column + '_ordl'] = mdf_train[column + '_ordl'].fillna(method='bfill')
      mdf_test[column + '_ordl'] = mdf_test[column + '_ordl'].fillna(method='bfill')

    #replace NA with a dummy variable
    mdf_train[column + '_ordl'] = mdf_train[column + '_ordl'].fillna('zzzinfill')
    mdf_test[column + '_ordl'] = mdf_test[column + '_ordl'].fillna('zzzinfill')

    #replace numerical with string equivalent
    if str_convert is True:
      mdf_train[column + '_ordl'] = mdf_train[column + '_ordl'].astype(str)
      mdf_test[column + '_ordl'] = mdf_test[column + '_ordl'].astype(str)
      if ordered is True:
        labels_train = [str(x) for x in labels_train]
        labels_test = [str(x) for x in labels_test]
    else:
      mdf_train[column + '_ordl'] = mdf_train[column + '_ordl'].astype('object')
      mdf_test[column + '_ordl'] = mdf_test[column + '_ordl'].astype('object')
            
    if ordered is False:
      
      #extract categories for column labels
      #note that .unique() extracts the labels as a numpy array
      labels_train = list(mdf_train[column + '_ordl'].unique())
      labels_train = sorted(labels_train, key=str)
      labels_test = list(mdf_test[column + '_ordl'].unique())
      labels_test = sorted(labels_test, key=str)

    #if infill not present in train set, insert
    if 'zzzinfill' not in labels_train:
      labels_train = labels_train + ['zzzinfill']
#       labels_train.sort()
    if 'zzzinfill' not in labels_test:
      labels_test = labels_test + ['zzzinfill']
#       labels_test.sort()
    
    listlength = len(labels_train)
    
    #____
    #quick check if there are any overlaps between binary encodings and prior unique values in the column
    #as would interfere with the replacement operation
    
    overlap_list = []
    overlap_replace = {}
    for value in labels_train:
      if value in range(listlength):
        overlap_list.append(value)
        
        #here's what we'll replace with, the string suffix is arbitrary and intended as not likely to be in set
        overlap_replace.update({value : str(value) + 'encoding_overlap'})
    
    #here we replace the overlaps with version with jibberish suffix
    if len(overlap_list) > 0:
      
      #then we'll redo the encodings
      
      if ordered is True:
        #this replaces entries with overlap while retaining order
        for foundoverlap in overlap_replace:
          labels_train = [overlap_replace[foundoverlap] if x == foundoverlap else x for x in labels_train]
          labels_test = [overlap_replace[foundoverlap] if x == foundoverlap else x for x in labels_test]
          
        #then replace encoding overlap entries in the returned column

        mdf_train[column + '_ordl'] = mdf_train[column + '_ordl'].replace(overlap_replace)
        mdf_test[column + '_ordl'] = mdf_test[column + '_ordl'].replace(overlap_replace)

      if ordered is False:

        mdf_train[column + '_ordl'] = mdf_train[column + '_ordl'].replace(overlap_replace)
        mdf_test[column + '_ordl'] = mdf_test[column + '_ordl'].replace(overlap_replace)

        #extract categories for column labels
        #note that .unique() extracts the labels as a numpy array
        labels_train = list(mdf_train[column + '_ordl'].unique())
        labels_train = sorted(labels_train, key=str)
        labels_test = list(mdf_test[column + '_ordl'].unique())
        labels_test = sorted(labels_test, key=str)

      #if infill not present in train set, insert
      if 'zzzinfill' not in labels_train:
        labels_train = labels_train + ['zzzinfill']
      if 'zzzinfill' not in labels_test:
        labels_test = labels_test + ['zzzinfill']
      
    #clear up memory
    del overlap_list
    
    #____
    
    #get length of the list, then zip a dictionary from list and range(length)
    #the range values will be our ordinal points to replace the categories
    listlength = len(labels_train)
    ordinal_dict = dict(zip(labels_train, range(listlength)))
    
    #dtype operation is to address edge case if object type drifted to numeric which impacts replace
    if mdf_train[column + '_ordl'].dtype.name != 'object':
      mdf_train[column + '_ordl'] = mdf_train[column + '_ordl'].astype('object')
    
    #replace the cateogries in train set via ordinal trasnformation
    mdf_train[column + '_ordl'] = mdf_train[column + '_ordl'].replace(ordinal_dict)
    
    #in test set, we'll need to strike any categories that weren't present in train
    #first let'/s identify what applies
    testspecificcategories = list(set(labels_test)-set(labels_train))
    
    #so we'll just replace those items with our plug value
    testplug_dict = dict(zip(testspecificcategories, ['zzzinfill'] * len(testspecificcategories)))
    if mdf_test[column + '_ordl'].dtype.name != 'object':
      mdf_test[column + '_ordl'] = mdf_test[column + '_ordl'].astype('object')
    mdf_test[column + '_ordl'] = mdf_test[column + '_ordl'].replace(testplug_dict)
    
    #now we'll apply the ordinal transformation to the test set
    if mdf_test[column + '_ordl'].dtype.name != 'object':
      mdf_test[column + '_ordl'] = mdf_test[column + '_ordl'].astype('object')
    mdf_test[column + '_ordl'] = mdf_test[column + '_ordl'].replace(ordinal_dict)
    
    #just want to make sure these arent' being saved as floats for memory considerations
    if len(ordinal_dict) < 254:
      mdf_train[column + '_ordl'] = mdf_train[column + '_ordl'].astype(np.uint8)
      mdf_test[column + '_ordl'] = mdf_test[column + '_ordl'].astype(np.uint8)
    elif len(ordinal_dict) < 65530:
      mdf_train[column + '_ordl'] = mdf_train[column + '_ordl'].astype(np.uint16)
      mdf_test[column + '_ordl'] = mdf_test[column + '_ordl'].astype(np.uint16)
    else:
      mdf_train[column + '_ordl'] = mdf_train[column + '_ordl'].astype(np.uint32)
      mdf_test[column + '_ordl'] = mdf_test[column + '_ordl'].astype(np.uint32)
    
#     #convert column to category
#     mdf_train[column + '_ordl'] = mdf_train[column + '_ordl'].astype('category')
#     mdf_test[column + '_ordl'] = mdf_test[column + '_ordl'].astype('category')

#     #change data type for memory savings
#     mdf_train[column + '_ordl'] = mdf_train[column + '_ordl'].astype(np.int32)
#     mdf_test[column + '_ordl'] = mdf_test[column + '_ordl'].astype(np.int32)

    #new driftreport metric ordl_activations_dict
    ordl_activations_dict = {}
    for key in ordinal_dict:
      sumcalc = (mdf_train[column+'_ordl'] == ordinal_dict[key]).sum() 
      ratio = sumcalc / mdf_train[column+'_ordl'].shape[0]
      ordl_activations_dict.update({key:ratio})

    inverse_ordinal_dict = {value:key for key,value in ordinal_dict.items()}
    activations_list = list(inverse_ordinal_dict)
    
    categorylist = [column + '_ordl']  
        
    column_dict_list = []
    
    for tc in categorylist:
        
      normalization_dict = {tc : {'ordinal_dict' : ordinal_dict, \
                                  'inverse_ordinal_dict' : inverse_ordinal_dict, \
                                  'activations_list' : activations_list, \
                                  'ordinal_overlap_replace' : overlap_replace, \
                                  'ordl_activations_dict' : ordl_activations_dict, \
                                  'adjinfill' : adjinfill, \
                                  'ordered_overide' : ordered_overide, \
                                  'ordered' : ordered, \
                                  'str_convert' : str_convert}}
    
      column_dict = {tc : {'category' : 'ordl', \
                           'origcategory' : category, \
                           'normalization_dict' : normalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : categorylist, \
                           'categorylist' : categorylist, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())
    
    return mdf_train, mdf_test, column_dict_list

  def process_ord3_class(self, mdf_train, mdf_test, column, category, \
                         postprocess_dict, params = {}):
    '''
    #process_ord3_class(mdf_train, mdf_test, column, category)
    #preprocess column with categories into ordinal (sequentuial integer) sets
    #corresponding to categories sorted by frequency of occurance
    #adresses infill with new point which we arbitrarily set as 'zzzinfill'
    #intended to show up as last point in set alphabetically
    #for categories presetn in test set not present in train set use this 'zzz' category
    #as implemented this function seperately encodes numbers and string equivalent (eg 2 != '2')
    '''
    
    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    #adjinfill accepts True/False to change default infill from mean inputation to adjacent cell
    if 'adjinfill' in params:
      adjinfill = params['adjinfill']
    else:
      adjinfill = False
      
    #ordered_overide is boolean to indicate if order of integer encoding basis will 
    #defer to cases when a column is a pandas categorical ordered set
    if 'ordered_overide' in params:
      ordered_overide = params['ordered_overide']
    else:
      ordered_overide = True
      
    #str_convert provides consistent encodings between numbers and string equivalent, eg 2 == '2'
    if 'str_convert' in params:
      str_convert = params['str_convert']
    else:
      str_convert = False
    
    if inplace is not True:
      
      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.df_copy_train(mdf_train, column, column + '_ord3', suffixoverlap_results)

      mdf_test[column + '_ord3'] = mdf_test[column].copy()
    
    else:
      
      suffixoverlap_results = \
      self.df_check_suffixoverlap(mdf_train, column + '_ord3', suffixoverlap_results)
      
      mdf_train.rename(columns = {column : column + '_ord3'}, inplace = True)
      mdf_test.rename(columns = {column : column + '_ord3'}, inplace = True)
    
    ordered = False
    if ordered_overide:
      if mdf_train[column + '_ord3'].dtype.name == 'category':
        if mdf_train[column + '_ord3'].cat.ordered:
          ordered = True
          labels_train = list(mdf_train[column + '_ord3'].cat.categories)
          if mdf_test[column + '_ord3'].dtype.name == 'category':
            if mdf_test[column + '_ord3'].cat.ordered:
              labels_test = list(mdf_test[column + '_ord3'].cat.categories)
            else:
              ordered = False
          else:
            ordered = False
    
    #convert column to category if it isn't already
    mdf_train[column + '_ord3'] = mdf_train[column + '_ord3'].astype('category')
    mdf_test[column + '_ord3'] = mdf_test[column + '_ord3'].astype('category')

    #if set is categorical we'll need the plug value for missing values included
    if 'zzzinfill' not in mdf_train[column + '_ord3'].cat.categories:
      mdf_train[column + '_ord3'] = mdf_train[column + '_ord3'].cat.add_categories(['zzzinfill'])
    if 'zzzinfill' not in mdf_test[column + '_ord3'].cat.categories:
      mdf_test[column + '_ord3'] = mdf_test[column + '_ord3'].cat.add_categories(['zzzinfill'])
      
    if adjinfill is True:
      mdf_train[column + '_ord3'] = mdf_train[column + '_ord3'].fillna(method='ffill')
      mdf_test[column + '_ord3'] = mdf_test[column + '_ord3'].fillna(method='ffill')
      mdf_train[column + '_ord3'] = mdf_train[column + '_ord3'].fillna(method='bfill')
      mdf_test[column + '_ord3'] = mdf_test[column + '_ord3'].fillna(method='bfill')

    #replace NA with a dummy variable
    mdf_train[column + '_ord3'] = mdf_train[column + '_ord3'].fillna('zzzinfill')
    mdf_test[column + '_ord3'] = mdf_test[column + '_ord3'].fillna('zzzinfill')
    
    if str_convert is True:
      #replace numerical with string equivalent (this operation changes dtype from category to object)
      mdf_train[column + '_ord3'] = mdf_train[column + '_ord3'].astype(str)
      mdf_test[column + '_ord3'] = mdf_test[column + '_ord3'].astype(str)
      if ordered is True:
        labels_train = [str(x) for x in labels_train]
        labels_test = [str(x) for x in labels_test]
    else:
      mdf_train[column + '_ord3'] = mdf_train[column + '_ord3'].astype('object')
      mdf_test[column + '_ord3'] = mdf_test[column + '_ord3'].astype('object')
            
    if ordered is False:
      
      #extract categories for column labels
      #with values sorted by frequency of occurance from most to least
      labels_train = pd.DataFrame(mdf_train[column + '_ord3'].value_counts())
      labels_train = labels_train.rename_axis('zzzinfill').sort_values(by = [column + '_ord3', 'zzzinfill'], ascending = [False, True])
      labels_train = list(labels_train.index)
      
      labels_test = list(mdf_test[column + '_ord3'].unique())

    #if infill not present in train set, insert
    if 'zzzinfill' not in labels_train:
      labels_train = labels_train + ['zzzinfill']
    if 'zzzinfill' not in labels_test:
      labels_test = labels_test + ['zzzinfill']
    
    listlength = len(labels_train)
    
    #____
    #quick check if there are any overlaps between binary encodings and prior unique values in the column
    #as would interfere with the replacement operation
    
    overlap_list = []
    overlap_replace = {}
    for value in labels_train:
      if value in range(listlength):
        overlap_list.append(value)
        
        #here's what we'll replace with, the string suffix is arbitrary and intended as not likely to be in set
        overlap_replace.update({value : str(value) + 'encoding_overlap'})
    
    #here we replace the overlaps with version with jibberish suffix
    if len(overlap_list) > 0:
      
      if ordered is True:
        #this replaces entries with overlap while retaining order
        for foundoverlap in overlap_replace:
          labels_train = [overlap_replace[foundoverlap] if x == foundoverlap else x for x in labels_train]
          labels_test = [overlap_replace[foundoverlap] if x == foundoverlap else x for x in labels_test]
          
        #then replace encoding overlap entries in the returned column
        mdf_train[column + '_ord3'] = mdf_train[column + '_ord3'].replace(overlap_replace)
        mdf_test[column + '_ord3'] = mdf_test[column + '_ord3'].replace(overlap_replace)

      if ordered is False:
            
        mdf_train[column + '_ord3'] = mdf_train[column + '_ord3'].replace(overlap_replace)
        mdf_test[column + '_ord3'] = mdf_test[column + '_ord3'].replace(overlap_replace)

        #then we'll redo the encodings

        #extract categories for column labels
        #note that .unique() extracts the labels as a numpy array
        labels_train = pd.DataFrame(mdf_train[column + '_ord3'].value_counts())
        labels_train = labels_train.rename_axis('zzzinfill').sort_values(by = [column + '_ord3', 'zzzinfill'], ascending = [False, True])
        labels_train = list(labels_train.index)

        labels_test = list(mdf_test[column + '_ord3'].unique())
        
      #if infill not present in train set, insert
      if 'zzzinfill' not in labels_train:
        labels_train = labels_train + ['zzzinfill']
      if 'zzzinfill' not in labels_test:
        labels_test = labels_test + ['zzzinfill']
      
    #clear up memory
    del overlap_list
    
    #____
    
    #get length of the list, then zip a dictionary from list and range(length)
    #the range values will be our ordinal points to replace the categories
    listlength = len(labels_train)
    ordinal_dict = dict(zip(labels_train, range(listlength)))
    
    #there is an edge case for replace operation is dtyp drifted from object such as to numeric
    if mdf_train[column + '_ord3'].dtype.name != 'object':
      mdf_train[column + '_ord3'] = mdf_train[column + '_ord3'].astype('object')
    
    #replace the cateogries in train set via ordinal trasnformation
    mdf_train[column + '_ord3'] = mdf_train[column + '_ord3'].replace(ordinal_dict)
    
    #in test set, we'll need to strike any categories that weren't present in train
    #first let'/s identify what applies
    testspecificcategories = list(set(labels_test)-set(labels_train))
    
    #so we'll just replace those items with our plug value
    testplug_dict = dict(zip(testspecificcategories, ['zzzinfill'] * len(testspecificcategories)))
    if mdf_test[column + '_ord3'].dtype.name != 'object':
      mdf_test[column + '_ord3'] = mdf_test[column + '_ord3'].astype('object')
    mdf_test[column + '_ord3'] = mdf_test[column + '_ord3'].replace(testplug_dict)
    
    #now we'll apply the ordinal transformation to the test set
    if mdf_test[column + '_ord3'].dtype.name != 'object':
      mdf_test[column + '_ord3'] = mdf_test[column + '_ord3'].astype('object')
    mdf_test[column + '_ord3'] = mdf_test[column + '_ord3'].replace(ordinal_dict)
    
    #just want to make sure these arent' being saved as floats for memory considerations
    if len(ordinal_dict) < 254:
      mdf_train[column + '_ord3'] = mdf_train[column + '_ord3'].astype(np.uint8)
      mdf_test[column + '_ord3'] = mdf_test[column + '_ord3'].astype(np.uint8)
    elif len(ordinal_dict) < 65530:
      mdf_train[column + '_ord3'] = mdf_train[column + '_ord3'].astype(np.uint16)
      mdf_test[column + '_ord3'] = mdf_test[column + '_ord3'].astype(np.uint16)
    else:
      mdf_train[column + '_ord3'] = mdf_train[column + '_ord3'].astype(np.uint32)
      mdf_test[column + '_ord3'] = mdf_test[column + '_ord3'].astype(np.uint32)
    
#     #convert column to category
#     mdf_train[column + '_ordl'] = mdf_train[column + '_ordl'].astype('category')
#     mdf_test[column + '_ordl'] = mdf_test[column + '_ordl'].astype('category')

#     #change data type for memory savings
#     mdf_train[column + '_ordl'] = mdf_train[column + '_ordl'].astype(np.int32)
#     mdf_test[column + '_ordl'] = mdf_test[column + '_ordl'].astype(np.int32)

    #new driftreport metric ordl_activations_dict
    ordl_activations_dict = {}
    for key in ordinal_dict:
      sumcalc = (mdf_train[column+'_ord3'] == ordinal_dict[key]).sum() 
      ratio = sumcalc / mdf_train[column+'_ord3'].shape[0]
      ordl_activations_dict.update({key:ratio})

    inverse_ordinal_dict = {value:key for key,value in ordinal_dict.items()}
    activations_list = list(inverse_ordinal_dict)
    
    categorylist = [column + '_ord3']  
        
    column_dict_list = []
    
    for tc in categorylist:
        
      normalization_dict = {tc : {'ordinal_dict' : ordinal_dict, \
                                  'inverse_ordinal_dict' : inverse_ordinal_dict, \
                                  'activations_list' : activations_list, \
                                  'ordinal_overlap_replace' : overlap_replace, \
                                  'ordl_activations_dict' : ordl_activations_dict, \
                                  'adjinfill' : adjinfill, \
                                  'ordered_overide' : ordered_overide, \
                                  'ordered' : ordered, \
                                  'str_convert' : str_convert}}
    
      column_dict = {tc : {'category' : 'ord3', \
                           'origcategory' : category, \
                           'normalization_dict' : normalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : categorylist, \
                           'categorylist' : categorylist, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())
    
    return mdf_train, mdf_test, column_dict_list
  
  def process_ucct_class(self, mdf_train, mdf_test, column, category, \
                         postprocess_dict, params = {}):
    '''
    #process_ucct_class(mdf_train, mdf_test, column, category)
    #preprocess column with categories into unique class count sets
    #normalized by total row count
    #e.g. for each class in train set, 
    #counts instances and divides by total train set row count
    #(so values will fall in range 0-1)
    #test sets recive comparable encoding
    '''
    
    suffixoverlap_results = {}
    
    #create new column for trasnformation
    mdf_train, suffixoverlap_results = \
    self.df_copy_train(mdf_train, column, column + '_ucct', suffixoverlap_results)
    
    mdf_test[column + '_ucct'] = mdf_test[column].copy()
    
    #convert column to category
    mdf_train[column + '_ucct'] = mdf_train[column + '_ucct'].astype('category')
    mdf_test[column + '_ucct'] = mdf_test[column + '_ucct'].astype('category')

    #if set is categorical we'll need the plug value for missing values included
    if 'zzzinfill' not in mdf_train[column + '_ucct'].cat.categories:
      mdf_train[column + '_ucct'] = mdf_train[column + '_ucct'].cat.add_categories(['zzzinfill'])
    if 'zzzinfill' not in mdf_test[column + '_ucct'].cat.categories:
      mdf_test[column + '_ucct'] = mdf_test[column + '_ucct'].cat.add_categories(['zzzinfill'])

    #replace NA with a dummy variable
    mdf_train[column + '_ucct'] = mdf_train[column + '_ucct'].fillna('zzzinfill')
    mdf_test[column + '_ucct'] = mdf_test[column + '_ucct'].fillna('zzzinfill')

    #replace numerical with string equivalent
    mdf_train[column + '_ucct'] = mdf_train[column + '_ucct'].astype(str)
    mdf_test[column + '_ucct'] = mdf_test[column + '_ucct'].astype(str)
    
    #extract categories for column labels
    #with values sorted by frequency of occurance from most to least
    labels_train = pd.DataFrame(mdf_train[column + '_ucct'].value_counts())
    labels_train = labels_train.rename_axis('zzzinfill').sort_values(by = [column + '_ucct', 'zzzinfill'], ascending = [False, True])
    labels_train = list(labels_train.index)
    
#     labels_train = list(mdf_train[column + '_ordl'].unique())
#     labels_train.sort()
    labels_test = list(mdf_test[column + '_ucct'].unique())
    labels_test.sort()

    #if infill not present in train set, insert
    if 'zzzinfill' not in labels_train:
      labels_train = labels_train + ['zzzinfill']
#       labels_train.sort()
    if 'zzzinfill' not in labels_test:
      labels_test = labels_test + ['zzzinfill']
      labels_test.sort()
    
    listlength = len(labels_train)
    
    #____
    #quick check if there are any overlaps between binary encodings and prior unique values in the column
    #as would interfere with the replacement operation
    #(I know this is an outlier scenario, just trying to be thorough)
    
    overlap_list = []
    overlap_replace = {}
    for value in labels_train:
      if value in range(listlength):
        overlap_list.append(value)
        
        #here's what we'll replace with, the string suffix is arbitrary and intended as not likely to be in set
        overlap_replace.update({value : value + 'encoding_overlap'})
    
    #here we replace the overlaps with version with jibberish suffix
    if len(overlap_list) > 0:
      mdf_train[column + '_ucct'] = mdf_train[column + '_ucct'].replace(overlap_replace)
      mdf_test[column + '_ucct'] = mdf_test[column + '_ucct'].replace(overlap_replace)
      
      #then we'll redo the encodings
      
      #extract categories for column labels
      #note that .unique() extracts the labels as a numpy array
      labels_train = pd.DataFrame(mdf_train[column + '_ucct'].value_counts())
      labels_train = labels_train.rename_axis('zzzinfill').sort_values(by = [column + '_ucct', 'zzzinfill'], ascending = [False, True])
      labels_train = list(labels_train.index)
      
#       labels_train = list(mdf_train[column + '_ord2'].unique())
#       labels_train.sort()
      labels_test = list(mdf_test[column + '_ucct'].unique())
      labels_test.sort()
      
    #clear up memory
    del overlap_list
    
    #____
    
    #assemble the ordinal_dict
    #with key of class and value of normalized unique class count
    ordinal_dict = {}
    rowcount = mdf_train.shape[0]
    
    for item in labels_train:
      item_count = mdf_train[mdf_train[column + '_ucct'] == item].shape[0]
      ordinal_dict.update({item: item_count / rowcount})
    
    #replace the cateogries in train set via ordinal trasnformation
    mdf_train[column + '_ucct'] = mdf_train[column + '_ucct'].replace(ordinal_dict)
    
    #in test set, we'll need to strike any categories that weren't present in train
    #first let'/s identify what applies
    testspecificcategories = list(set(labels_test)-set(labels_train))
    
    #so we'll just replace those items with our plug value
    testplug_dict = dict(zip(testspecificcategories, ['zzzinfill'] * len(testspecificcategories)))
    mdf_test[column + '_ucct'] = mdf_test[column + '_ucct'].replace(testplug_dict)
    
    #now we'll apply the ordinal transformation to the test set
    mdf_test[column + '_ucct'] = mdf_test[column + '_ucct'].replace(ordinal_dict)

    #new driftreport metric ordl_activations_dict
    ordl_activations_dict = {}
    for key in ordinal_dict:
      sumcalc = (mdf_train[column+'_ucct'] == ordinal_dict[key]).sum() 
      ratio = sumcalc / mdf_train[column+'_ucct'].shape[0]
      ordl_activations_dict.update({key:ratio})
    
    categorylist = [column + '_ucct']  
        
    column_dict_list = []
    
    for tc in categorylist:
        
      normalization_dict = {tc : {'ordinal_dict' : ordinal_dict, \
                                  'ordinal_overlap_replace' : overlap_replace, \
                                  'ordl_activations_dict' : ordl_activations_dict}}
    
      column_dict = {tc : {'category' : 'ucct', \
                           'origcategory' : category, \
                           'normalization_dict' : normalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : categorylist, \
                           'categorylist' : categorylist, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())
    
    return mdf_train, mdf_test, column_dict_list
  
  def process_1010_class(self, mdf_train, mdf_test, column, category, \
                         postprocess_dict, params = {}):
    '''
    #process_1010_class(mdf_train, mdf_test, column, category)
    #preprocess column with categories into binary encoded sets
    #corresponding to (sorted) categories of >2 values
    #adresses infill with new point which we arbitrarily set as 'zzzinfill'
    #intended to show up as last point in set alphabetically
    #for categories present in test set not present in train set uses this 'zzzinfill' category
    '''
    
    suffixoverlap_results = {}
    
    #adjinfill accepts True/False to change default infill from mean inputation to adjacent cell
    if 'adjinfill' in params:
      adjinfill = params['adjinfill']
    else:
      adjinfill = False
      
    #str_convert provides consistent encodings between numbers and string equivalent, eg 2 == '2'
    if 'str_convert' in params:
      str_convert = params['str_convert']
    else:
      str_convert = False
    
    #create new column for trasnformation
    mdf_train, suffixoverlap_results = \
    self.df_copy_train(mdf_train, column, column + '_1010', suffixoverlap_results)
    
    mdf_test[column + '_1010'] = mdf_test[column].copy()
    
    #convert column to category
    mdf_train[column + '_1010'] = mdf_train[column + '_1010'].astype('category')
    mdf_test[column + '_1010'] = mdf_test[column + '_1010'].astype('category')

    #if set is categorical we'll need the plug value for missing values included
    if 'zzzinfill' not in mdf_train[column + '_1010'].cat.categories:
      mdf_train[column + '_1010'] = mdf_train[column + '_1010'].cat.add_categories(['zzzinfill'])
    if 'zzzinfill' not in mdf_test[column + '_1010'].cat.categories:
      mdf_test[column + '_1010'] = mdf_test[column + '_1010'].cat.add_categories(['zzzinfill'])
      
    if adjinfill is True:
      mdf_train[column + '_1010'] = mdf_train[column + '_1010'].fillna(method='ffill')
      mdf_test[column + '_1010'] = mdf_test[column + '_1010'].fillna(method='ffill')
      mdf_train[column + '_1010'] = mdf_train[column + '_1010'].fillna(method='bfill')
      mdf_test[column + '_1010'] = mdf_test[column + '_1010'].fillna(method='bfill')

    #replace NA with a dummy variable
    mdf_train[column + '_1010'] = mdf_train[column + '_1010'].fillna('zzzinfill')
    mdf_test[column + '_1010'] = mdf_test[column + '_1010'].fillna('zzzinfill')

    if str_convert is True:
      #replace numerical with string equivalent
      mdf_train[column + '_1010'] = mdf_train[column + '_1010'].astype(str)
      mdf_test[column + '_1010'] = mdf_test[column + '_1010'].astype(str)
    else:
      mdf_train[column + '_1010'] = mdf_train[column + '_1010'].astype('object')
      mdf_test[column + '_1010'] = mdf_test[column + '_1010'].astype('object')
    
    #extract categories for column labels
    #note that .unique() extracts the labels as a numpy array
    labels_train = list(mdf_train[column + '_1010'].unique())
#     labels_train.sort()
    labels_train = sorted(labels_train, key=str)
    labels_test = list(mdf_test[column + '_1010'].unique())
#     labels_test.sort()
    labels_test = sorted(labels_test, key=str)

    #if infill not present in train set, insert
    if 'zzzinfill' not in labels_train:
      labels_train = labels_train + ['zzzinfill']
      labels_train = sorted(labels_train, key=str)
#       labels_train.sort()
    if 'zzzinfill' not in labels_test:
      labels_test = labels_test + ['zzzinfill']
      labels_test = sorted(labels_test, key=str)
#       labels_test.sort()
    
    #get length of the list
    listlength = len(labels_train)
    
    #calculate number of columns we'll need
    #currently using numk;py since already imported, this could also be done with math library
    binary_column_count = int(np.ceil(np.log2(listlength)))
    
    #initialize dictionaryt to store encodings
    binary_encoding_dict = {}
    encoding_list = []
    
    for i in range(listlength):
      
      #this converts the integer i to binary encoding
      #where f is an f string for inserting the column coount into the string to designate length of encoding
      #0 is to pad out the encoding with 0's for the length
      #and b is telling it to convert to binary 
      #note this returns a string
      encoding = format(i, f"0{binary_column_count}b")
      
      if i < len(labels_train):

        #store the encoding in a dictionary
        binary_encoding_dict.update({labels_train[i] : encoding})

        #store the encoding in a list for checking in next step
        encoding_list.append(encoding)

    #____
    #quick check if there are any overlaps between binary encodings and prior unique values in the column
    #as would interfere with the replacement operation
    #(I know this is an outlier scenario, just trying to be thorough)
    
    overlap_list = []
    overlap_replace = {}
    for value in labels_train:
      if value in encoding_list:
        overlap_list.append(value)
        
        #since overlapreplace will add suffix to the category overlapped with a binary encoding
        #let's quickly check if that category plus suffix is already present in the set
        #if so we'll keep adding digits until a unique entry
        encoding_overlap_suffix = 'encoding_overlap'
        for i in range(111):
          j = random.randint(0,9)
          if value + encoding_overlap_suffix in labels_train:
            encoding_overlap_suffix += str(j)
          else:
            break
        
        #here's what we'll replace with, the string suffix is arbitrary and intended as not likely to be in set
        overlap_replace.update({value : value + encoding_overlap_suffix})

    #here we replace the overlaps with version with jibberish suffix
    if len(overlap_list) > 0:
      
      mdf_train[column + '_1010'] = mdf_train[column + '_1010'].replace(overlap_replace)
      mdf_test[column + '_1010'] = mdf_test[column + '_1010'].replace(overlap_replace)
      
      #then we'll redo the encodings
      
      #extract categories for column labels
      #note that .unique() extracts the labels as a numpy array
      labels_train = list(mdf_train[column + '_1010'].unique())
      labels_train = sorted(labels_train, key=str)
#       labels_train.sort()
      
      labels_test = list(mdf_test[column + '_1010'].unique())
      labels_test = sorted(labels_test, key=str)
#       labels_test.sort()

      #if infill not present in train set, insert
      if 'zzzinfill' not in labels_train:
        labels_train = labels_train + ['zzzinfill']
        labels_train = sorted(labels_train, key=str)
  #       labels_train.sort()
      if 'zzzinfill' not in labels_test:
        labels_test = labels_test + ['zzzinfill']
        labels_test = sorted(labels_test, key=str)
  #       labels_test.sort()
      
      #initialize dictionaryt to store encodings
      binary_encoding_dict = {}
      encoding_list = []

      for i in range(listlength):

        #this converts the integer i to binary encoding
        #where f is an f string for inserting the column coount into the string to designate length of encoding
        #0 is to pad out the encoding with 0's for the length
        #and b is telling it to convert to binary 
        #note this returns a string
        encoding = format(i, f"0{binary_column_count}b")
        
        if i < len(labels_train):

          #store the encoding in a dictionary
          binary_encoding_dict.update({labels_train[i] : encoding})

          #store the encoding in a list for checking in next step
          encoding_list.append(encoding)

    #clear up memory
    del encoding_list
    del overlap_list
    
    #new driftreport metric _1010_activations_dict
    _1010_activations_dict = {}
    for key in binary_encoding_dict:
      sumcalc = (mdf_train[column+'_1010'] == key).sum() 
      ratio = sumcalc / mdf_train[column+'_1010'].shape[0]
      _1010_activations_dict.update({key:ratio})
    
    #____
    
    #replace the cateogries in train set via ordinal trasnformation
    
    if mdf_train[column + '_1010'].dtype.name != 'object':
      mdf_train[column + '_1010'] = mdf_train[column + '_1010'].astype('object')
    
    mdf_train[column + '_1010'] = mdf_train[column + '_1010'].replace(binary_encoding_dict)      
    
    #in test set, we'll need to strike any categories that weren't present in train
    #first let'/s identify what applies
    testspecificcategories = list(set(labels_test)-set(labels_train))
    
    #so we'll just replace those items with our plug value
    if mdf_test[column + '_1010'].dtype.name != 'object':
      mdf_test[column + '_1010'] = mdf_test[column + '_1010'].astype('object')
    testplug_dict = dict(zip(testspecificcategories, ['zzzinfill'] * len(testspecificcategories)))
    mdf_test[column + '_1010'] = mdf_test[column + '_1010'].replace(testplug_dict)    
    
    #now we'll apply the 1010 transformation to the test set
    if mdf_test[column + '_1010'].dtype.name != 'object':
      mdf_test[column + '_1010'] = mdf_test[column + '_1010'].astype('object')
    mdf_test[column + '_1010'] = mdf_test[column + '_1010'].replace(binary_encoding_dict)    

    #ok let's create a list of columns to store each entry of the binary encoding
    _1010_columnlist = []
    
    for i in range(binary_column_count):
      
      _1010_columnlist.append(column + '_1010_' + str(i))
      
    suffixoverlap_results = \
    self.df_check_suffixoverlap(mdf_train, _1010_columnlist, suffixoverlap_results)
      
    #now let's store the encoding
    i=0
    for _1010_column in _1010_columnlist:
      
      mdf_train[_1010_column] = mdf_train[column + '_1010'].str.slice(i,i+1).astype(np.int8)
      
      mdf_test[_1010_column] = mdf_test[column + '_1010'].str.slice(i,i+1).astype(np.int8)
      
      i+=1
  
    #now delete the support column
    del mdf_train[column + '_1010']
    del mdf_test[column + '_1010']
    
    #now store the column_dict entries
    
    categorylist = _1010_columnlist
        
    column_dict_list = []
    
    for tc in categorylist:
        
      normalization_dict = {tc : {'_1010_binary_encoding_dict' : binary_encoding_dict, \
                                  '_1010_overlap_replace' : overlap_replace, \
                                  '_1010_binary_column_count' : binary_column_count, \
                                  '_1010_activations_dict' : _1010_activations_dict, \
                                  'adjinfill' : adjinfill, \
                                  'str_convert' : str_convert}}
    
      column_dict = {tc : {'category' : '1010', \
                           'origcategory' : category, \
                           'normalization_dict' : normalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : categorylist, \
                           'categorylist' : categorylist, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())
    
    return mdf_train, mdf_test, column_dict_list

  def process_bshr_class(self, df, column, category, postprocess_dict, params = {}):
    '''
    #processing funciton depending on input format of datetime data 
    #that creates a boolean column indicating 1 for rows
    #corresponding to traditional business hours in source column
    #note this is a "singleprocess" function since is applied to single dataframe
    '''
    
    suffixoverlap_results = {}
    
    #initialize parameters
    if 'start' in params:
      start = params['start']
    else:
      start = 9
      
    if 'end' in params:
      end = params['end']
    else:
      end = 17
      
    suffixoverlap_results = \
    self.df_check_suffixoverlap(df, column+'_bshr', suffixoverlap_results)
    
    #convert improperly formatted values to datetime in new column
    df[column+'_bshr'] = pd.to_datetime(df[column], errors = 'coerce')
    
    #This is kind of hack for whole hour increments, if we were needing
    #to evlauate hour ranges between seperate days a different metod
    #would be required
    #For now we'll defer to Dollly Parton
    df[column+'_bshr'] = df[column+'_bshr'].dt.hour
    df[column+'_bshr'] = df[column+'_bshr'].between(start, end)
    
    #reduce memory footprint
    df[column+'_bshr'] = df[column+'_bshr'].astype(np.int8)
    
    #create list of columns
    datecolumns = [column + '_bshr']

    #grab some driftreport metrics
    activationratio = df[column + '_bshr'].sum() / df[column + '_bshr'].shape[0]

    #create normalization dictionary
    normalization_dict = {column + '_bshr' : {'activationratio' : activationratio}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for dc in datecolumns:

      column_dict = { dc : {'category' : 'bshr', \
                           'origcategory' : category, \
                           'normalization_dict' : normalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : datecolumns, \
                           'categorylist' : datecolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())

    return df, column_dict_list

  def process_wkdy_class(self, df, column, category, postprocess_dict, params = {}):
    '''
    #processing funciton depending on input format of datetime data 
    #that creates a boolean column indicating 1 for rows
    #corresponding to weekdays in source column
    #note this is a "singleprocess" function since is applied to single dataframe
    '''
    
    suffixoverlap_results = {}
    
    suffixoverlap_results = \
    self.df_check_suffixoverlap(df, column+'_wkdy', suffixoverlap_results)
    
    #convert improperly formatted values to datetime in new column
    df[column+'_wkdy'] = pd.to_datetime(df[column], errors = 'coerce')
    
    #This is kind of hack for whole hour increments, if we were needing
    #to evlauate hour ranges between seperate days a different metod
    #would be required
    #For now we'll defer to Dollly Parton
    df[column+'_wkdy'] = pd.DatetimeIndex(df[column+'_wkdy']).dayofweek
    
    df[column+'_wkdy'] = df[column+'_wkdy'].between(0,4)
    
    #reduce memory footprint
    df[column+'_wkdy'] = df[column+'_wkdy'].astype(np.int8)
    
    #create list of columns
    datecolumns = [column+'_wkdy']

    #grab some driftreport metrics
    activationratio = df[column + '_wkdy'].sum() / df[column + '_wkdy'].shape[0]

    #create normalization dictionary
    normalization_dict = {column + '_wkdy' : {'activationratio' : activationratio}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for dc in datecolumns:

      column_dict = { dc : {'category' : 'wkdy', \
                           'origcategory' : category, \
                           'normalization_dict' : normalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : datecolumns, \
                           'categorylist' : datecolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'deletecolumn' : False, \
                           'suffixoverlap_results' : suffixoverlap_results}}

      column_dict_list.append(column_dict.copy())

    return df, column_dict_list

  def process_hldy_class(self, df, column, category, postprocess_dict, params = {}):
    '''
    #processing funciton depending on input format of datetime data 
    #that creates a boolean column indicating 1 for rows
    #corresponding to US Federal Holidays in source column
    #note this is a "singleprocess" function since is applied to single dataframe
    '''
    
    suffixoverlap_results = {}
    
    #initialize parameters
    if 'holiday_list' in params:
      holiday_list = params['holiday_list']
    else:
      holiday_list = []
    
    if len(holiday_list) > 0:
    
      #reformat holiday_list
      holiday_list = pd.to_datetime(pd.DataFrame(holiday_list)[0], errors = 'coerce')

      #reform holiday_list again
      timestamp_list = []

      for row in range(holiday_list.shape[0]):
        timestamp = pd.Timestamp(holiday_list[row])
        timestamp_list += [timestamp]
      timestamp_list
      
    else:
      timestamp_list = []
      
    suffixoverlap_results = \
    self.df_check_suffixoverlap(df, column+'_hldy', suffixoverlap_results)
    
    #convert improperly formatted values to datetime in new column
    df[column+'_hldy'] = pd.to_datetime(df[column], errors = 'coerce')
    
    df[column+'_hldy'] = df[column+'_hldy'].dt.date
    
    df[column+'_hldy'] = pd.to_datetime(df[column+'_hldy'], errors = 'coerce')
    
    #grab list of holidays from import
    holidays = USFederalHolidayCalendar().holidays().tolist()

    holidays += timestamp_list
    
    #activate boolean identifier for holidays
    df[column+'_hldy'] = df[column+'_hldy'].isin(holidays)

    #reduce memory footprint
    df[column+'_hldy'] = df[column+'_hldy'].astype(np.int8)
    
    #create list of columns
    datecolumns = [column + '_hldy']

    #grab some driftreport metrics
    activationratio = df[column + '_hldy'].sum() / df[column + '_hldy'].shape[0]

    #create normalization dictionary
    normalization_dict = {column + '_hldy' : {'activationratio' : activationratio}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for dc in datecolumns:

      column_dict = { dc : {'category' : 'hldy', \
                           'origcategory' : category, \
                           'normalization_dict' : normalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : datecolumns, \
                           'categorylist' : datecolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())

    return df, column_dict_list
  
  def process_wkds_class(self, df, column, category, postprocess_dict, params = {}):
    '''
    #processing funciton depending on input format of datetime data 
    #that creates a categorical column 
    #corresponding to weekdays in source column
    #note this is a "singleprocess" function since is applied to single dataframe
    #defdault infill is eight days a week
    '''
    
    suffixoverlap_results = {}
    
    suffixoverlap_results = \
    self.df_check_suffixoverlap(df, column+'_wkds', suffixoverlap_results)
    
    #convert improperly formatted values to datetime in new column
    df[column+'_wkds'] = pd.to_datetime(df[column], errors = 'coerce')
    
    #This is kind of hack for whole hour increments, if we were needing
    #to evlauate hour ranges between seperate days a different metod
    #would be required
    #For now we'll defer to Dollly Parton
    df[column+'_wkds'] = pd.DatetimeIndex(df[column+'_wkds']).dayofweek
    
#     df[column+'_wkdy'] = df[column+'_wkdy'].between(0,4)

    #we'll use convention for default infill of eight days a week
    df[column + '_wkds'] = df[column + '_wkds'].fillna(7)
    
    #reduce memory footprint
    df[column+'_wkds'] = df[column+'_wkds'].astype(int)
    
    #create list of columns
    datecolumns = [column+'_wkds']

    #grab some driftreport metrics
    numberofrows = df[column + '_wkds'].shape[0]
    mon_ratio = df[df[column + '_wkds'] == 0].shape[0] / numberofrows
    tue_ratio = df[df[column + '_wkds'] == 1].shape[0] / numberofrows
    wed_ratio = df[df[column + '_wkds'] == 2].shape[0] / numberofrows
    thr_ratio = df[df[column + '_wkds'] == 3].shape[0] / numberofrows
    fri_ratio = df[df[column + '_wkds'] == 4].shape[0] / numberofrows
    sat_ratio = df[df[column + '_wkds'] == 5].shape[0] / numberofrows
    sun_ratio = df[df[column + '_wkds'] == 6].shape[0] / numberofrows
    infill_ratio = df[df[column + '_wkds'] == 7].shape[0] / numberofrows
  
  
    #create normalization dictionary
    normalization_dict = {column+'_wkds' : {'mon_ratio' : mon_ratio, \
                                            'tue_ratio' : tue_ratio, \
                                            'wed_ratio' : wed_ratio, \
                                            'thr_ratio' : thr_ratio, \
                                            'fri_ratio' : fri_ratio, \
                                            'sat_ratio' : sat_ratio, \
                                            'sun_ratio' : sun_ratio, \
                                            'infill_ratio' : infill_ratio}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for dc in datecolumns:

      column_dict = { dc : {'category' : 'wkds', \
                           'origcategory' : category, \
                           'normalization_dict' : normalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : datecolumns, \
                           'categorylist' : datecolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())

    return df, column_dict_list
  
  def process_mnts_class(self, df, column, category, postprocess_dict, params = {}):
    '''
    #processing funciton depending on input format of datetime data 
    #that creates a categorical column 
    #corresponding to months in source column
    #note this is a "singleprocess" function since is applied to single dataframe
    #default infill is 0
    '''
    
    suffixoverlap_results = {}
    
    suffixoverlap_results = \
    self.df_check_suffixoverlap(df, column+'_mnts', suffixoverlap_results)
    
    #convert improperly formatted values to datetime in new column
    df[column+'_mnts'] = pd.to_datetime(df[column], errors = 'coerce')
    
    #This is kind of hack for whole hour increments, if we were needing
    #to evlauate hour ranges between seperate days a different metod
    #would be required
    #For now we'll defer to Dollly Parton
    df[column+'_mnts'] = pd.DatetimeIndex(df[column+'_mnts']).month
    
#     df[column+'_wkdy'] = df[column+'_wkdy'].between(0,4)

    #we'll use convention for default infill of eight days a week
    #jan-dec is 1-12, 0 is default infill
    df[column + '_mnts'] = df[column + '_mnts'].fillna(0)
    
    #reduce memory footprint
    df[column+'_mnts'] = df[column+'_mnts'].astype(np.int8)
    
    #create list of columns
    datecolumns = [column+'_mnts']

    #grab some driftreport metrics
    numberofrows = df[column + '_mnts'].shape[0]
    infill_ratio = df[df[column + '_mnts'] == 0].shape[0] / numberofrows
    jan_ratio = df[df[column + '_mnts'] == 1].shape[0] / numberofrows
    feb_ratio = df[df[column + '_mnts'] == 2].shape[0] / numberofrows
    mar_ratio = df[df[column + '_mnts'] == 3].shape[0] / numberofrows
    apr_ratio = df[df[column + '_mnts'] == 4].shape[0] / numberofrows
    may_ratio = df[df[column + '_mnts'] == 5].shape[0] / numberofrows
    jun_ratio = df[df[column + '_mnts'] == 6].shape[0] / numberofrows
    jul_ratio = df[df[column + '_mnts'] == 7].shape[0] / numberofrows
    aug_ratio = df[df[column + '_mnts'] == 8].shape[0] / numberofrows
    sep_ratio = df[df[column + '_mnts'] == 9].shape[0] / numberofrows
    oct_ratio = df[df[column + '_mnts'] == 10].shape[0] / numberofrows
    nov_ratio = df[df[column + '_mnts'] == 11].shape[0] / numberofrows
    dec_ratio = df[df[column + '_mnts'] == 12].shape[0] / numberofrows
  
    #create normalization dictionary
    normalization_dict = {column+'_mnts' : {'infill_ratio' : infill_ratio, \
                                            'jan_ratio' : jan_ratio, \
                                            'feb_ratio' : feb_ratio, \
                                            'mar_ratio' : mar_ratio, \
                                            'apr_ratio' : apr_ratio, \
                                            'may_ratio' : may_ratio, \
                                            'jun_ratio' : jun_ratio, \
                                            'jul_ratio' : jul_ratio, \
                                            'aug_ratio' : aug_ratio, \
                                            'sep_ratio' : sep_ratio, \
                                            'oct_ratio' : oct_ratio, \
                                            'nov_ratio' : nov_ratio, \
                                            'dec_ratio' : dec_ratio}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for dc in datecolumns:

      column_dict = { dc : {'category' : 'mnts', \
                           'origcategory' : category, \
                           'normalization_dict' : normalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : datecolumns, \
                           'categorylist' : datecolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())

    return df, column_dict_list

  def process_tmsc_class(self, mdf_train, mdf_test, column, category, \
                         postprocess_dict, params = {}):
    """
    #time data segregated by time scale
    #with sin or cos applied to address periodicity
    #such as may be useful to return both by seperate transformation categories
    #accepts parameter 'scale' to distinguish between year/month/day/hour/minute/second
    #note that some scales can be returned combined by passing 
    #monthday/dayhourminute/hourminutesecond/minutesecond
    #accepts parameter 'suffix' for returned column header suffix
    #accets parameter 'function' to distinguish between sin/cos
    """
    
    suffixoverlap_results = {}
    
    if 'scale' in params:
      #accepts year/month/day/hour/minute/second
      scale = params['scale']
    else:
      scale = 'monthday'
      
    if 'suffix' in params:
      #accepts column header suffix appender
      suffix = params['suffix']
    else:
      suffix = '_mdsn'
      
    if 'function' in params:
      #accepts sin/cos
      function = params['function']
    else:
      function = 'sin'
    
    time_column = column + suffix
    
    mdf_train, suffixoverlap_results = \
    self.df_copy_train(mdf_train, column, time_column, suffixoverlap_results)
    
    mdf_test[time_column] = mdf_test[column].copy()
    
    #apply pd.to_datetime to column, note that the errors = 'coerce' needed for messy data
    mdf_train[time_column] = pd.to_datetime(mdf_train[time_column], errors = 'coerce')
    mdf_test[time_column] = pd.to_datetime(mdf_test[time_column], errors = 'coerce')
    
    #access time scale from one of year/month/day/hour/minute/second
    #monthday/dayhourminute/hourminutesecond/minutesecond
    if scale == 'year':
      mdf_train[time_column] = mdf_train[time_column].dt.year
      mdf_test[time_column] = mdf_test[time_column].dt.year
      
      #we'll scale periodicity by decade
      mdf_train[time_column] = (mdf_train[time_column]) * 2 * np.pi / 10
      mdf_test[time_column] = (mdf_test[time_column]) * 2 * np.pi / 10
      
    elif scale == 'month':
      mdf_train[time_column] = mdf_train[time_column].dt.month
      mdf_test[time_column] = mdf_test[time_column].dt.month
      
      #we'll scale periodicity by year
      mdf_train[time_column] = (mdf_train[time_column]) * 2 * np.pi / 12
      mdf_test[time_column] = (mdf_test[time_column]) * 2 * np.pi / 12
      
    elif scale == 'day':
      mdf_train[time_column] = mdf_train[time_column].dt.day
      mdf_test[time_column] = mdf_test[time_column].dt.day
      
      #we'll scale periodicity by week
      mdf_train[time_column] = (mdf_train[time_column]) * 2 * np.pi / 7
      mdf_test[time_column] = (mdf_test[time_column]) * 2 * np.pi / 7
      
    elif scale == 'hour':
      mdf_train[time_column] = mdf_train[time_column].dt.hour
      mdf_test[time_column] = mdf_test[time_column].dt.hour
      
      #we'll scale periodicity by day
      mdf_train[time_column] = (mdf_train[time_column]) * 2 * np.pi / 24
      mdf_test[time_column] = (mdf_test[time_column]) * 2 * np.pi / 24
      
    elif scale == 'minute':
      mdf_train[time_column] = mdf_train[time_column].dt.minute
      mdf_test[time_column] = mdf_test[time_column].dt.minute
      
      #we'll scale periodicity by hour
      mdf_train[time_column] = (mdf_train[time_column]) * 2 * np.pi / 60
      mdf_test[time_column] = (mdf_test[time_column]) * 2 * np.pi / 60
      
    elif scale == 'second':
      mdf_train[time_column] = mdf_train[time_column].dt.second
      mdf_test[time_column] = mdf_test[time_column].dt.second
      
      #we'll scale periodicity by minute
      mdf_train[time_column] = (mdf_train[time_column]) * 2 * np.pi / 60
      mdf_test[time_column] = (mdf_test[time_column]) * 2 * np.pi / 60
    
    elif scale == 'monthday':
      tempcolumn1 = time_column + '_tmp1'
      tempcolumn2 = time_column + '_tmp2'
      
      suffixoverlap_results = \
      self.df_check_suffixoverlap(mdf_train, [tempcolumn1, tempcolumn2], suffixoverlap_results)
      
      #temp1 is for number of days in month, temp2 is to handle leap year support
      mdf_train[tempcolumn1] = mdf_train[time_column].copy()
      mdf_train[tempcolumn2] = mdf_train[time_column].copy()
      
      mdf_train[tempcolumn1] = mdf_train[tempcolumn1].dt.month
      mdf_train[tempcolumn2] = mdf_train[tempcolumn2].dt.is_leap_year
      
      mdf_train[tempcolumn2] = \
      np.where(mdf_train[tempcolumn2], 29, 28)
      
      mdf_train[tempcolumn1] = \
      np.where(mdf_train[tempcolumn1].isin([1,3,5,7,8,10,12]), 31, mdf_train[tempcolumn1].values)
      
      mdf_train[tempcolumn1] = \
      np.where(mdf_train[tempcolumn1].isin([4,6,9,11]), 30, mdf_train[tempcolumn1].values)
      
      mdf_train[tempcolumn1] = \
      np.where(mdf_train[tempcolumn1].isin([2]), mdf_train[tempcolumn2], \
      mdf_train[tempcolumn1].values)
      
      #do same for test set
      mdf_test[tempcolumn1] = mdf_test[time_column].copy()
      mdf_test[tempcolumn2] = mdf_test[time_column].copy()
      
      mdf_test[tempcolumn1] = mdf_test[tempcolumn1].dt.month
      mdf_test[tempcolumn2] = mdf_test[tempcolumn2].dt.is_leap_year
      
      mdf_test[tempcolumn2] = \
      np.where(mdf_test[tempcolumn2], 29, 28)
      
      mdf_test[tempcolumn1] = \
      np.where(mdf_test[tempcolumn1].isin([1,3,5,7,8,10,12]), 31, mdf_test[tempcolumn1].values)
      
      mdf_test[tempcolumn1] = \
      np.where(mdf_test[tempcolumn1].isin([4,6,9,11]), 30, mdf_test[tempcolumn1].values)
      
      mdf_test[tempcolumn1] = \
      np.where(mdf_test[tempcolumn1].isin([2]), mdf_test[tempcolumn2], \
      mdf_test[tempcolumn1].values)
      
      #combine month and day, scale for trigonomic transform, periodicity by year
      mdf_train[time_column] = (mdf_train[time_column].dt.month + mdf_train[time_column].dt.day / \
      mdf_train[tempcolumn1]) * 2 * np.pi / 12
      
      mdf_test[time_column] = (mdf_test[time_column].dt.month + mdf_test[time_column].dt.day / \
      mdf_test[tempcolumn1]) * 2 * np.pi / 12
      
      #delete the support columns 
      del mdf_train[tempcolumn1]
      del mdf_test[tempcolumn1]

      del mdf_train[tempcolumn2]
      del mdf_test[tempcolumn2]
      
    elif scale == 'dayhourminute':
      #we'll scale periodicity by week
      mdf_train[time_column] = (mdf_train[time_column].dt.day + mdf_train[time_column].dt.hour / 24 + mdf_train[time_column].dt.minute / 24 / 60) * 2 * np.pi / 7
      mdf_test[time_column] = (mdf_test[time_column].dt.day + mdf_test[time_column].dt.hour / 24 + mdf_test[time_column].dt.minute / 24 / 60) * 2 * np.pi / 7

    elif scale == 'hourminutesecond':
      #we'll scale periodicity by day
      mdf_train[time_column] = (mdf_train[time_column].dt.hour + mdf_train[time_column].dt.minute / 60 + mdf_train[time_column].dt.second / 60 / 60) * 2 * np.pi / 24
      mdf_test[time_column] = (mdf_test[time_column].dt.hour + mdf_test[time_column].dt.minute / 60 + mdf_test[time_column].dt.second / 60 / 60) * 2 * np.pi / 24

    elif scale == 'minutesecond':
      #we'll scale periodicity by hour
      mdf_train[time_column] = (mdf_train[time_column].dt.minute + mdf_train[time_column].dt.second / 60) * 2 * np.pi / 60
      mdf_test[time_column] = (mdf_test[time_column].dt.minute + mdf_test[time_column].dt.second / 60) * 2 * np.pi / 60
      
    #grab a few drift metrics, we'll evaluate prior to trigometric transform
    timemean = mdf_train[time_column].mean()
    timemax = mdf_train[time_column].max()
    timemin = mdf_train[time_column].min()
    timestd = mdf_train[time_column].std()
    
    #default infill is adjacent cell
    mdf_train[time_column] = mdf_train[time_column].fillna(method='ffill')
    mdf_train[time_column] = mdf_train[time_column].fillna(method='bfill')
    
    mdf_test[time_column] = mdf_test[time_column].fillna(method='ffill')
    mdf_test[time_column] = mdf_test[time_column].fillna(method='bfill')
    
    #backup default infill for cases without valid entries
    mdf_train[time_column] = mdf_train[time_column].fillna(0)
    mdf_test[time_column] = mdf_test[time_column].fillna(0)
    
    #apply trigometric transform
    
    if function == 'sin':
      
      mdf_train[time_column] = np.sin(mdf_train[time_column])
      mdf_test[time_column] = np.sin(mdf_test[time_column])
      
    if function == 'cos':

      mdf_train[time_column] = np.cos(mdf_train[time_column])
      mdf_test[time_column] = np.cos(mdf_test[time_column])
      
    #populate data structures
    column_dict_list = []
    categorylist = [time_column]
    
    for tc in categorylist:
      norm_dict = {tc : {'scale'         : scale, \
                         'suffix'        : suffix, \
                         'function'      : function, \
                         'timemean'      : timemean, \
                         'timemax'       : timemax, \
                         'timemin'       : timemin, \
                         'timestd'       : timestd}}
      
      column_dict = {tc : {'category' : 'tmsc', \
                           'origcategory' : category, \
                           'normalization_dict' : norm_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : categorylist, \
                           'categorylist' : categorylist, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())
    
    return mdf_train, mdf_test, column_dict_list

  def process_time_class(self, mdf_train, mdf_test, column, category, \
                         postprocess_dict, params = {}):
    """
    #z-score normalized time data segregated by a particular time scale
    #accepts parameter 'scale' to distinguish between year/month/day/hour/minute/second
    #accepts parameter 'suffix' for returned column header suffix
    #accepts parameter 'normalization' to distinguish between zscore/minmax/unscaled
    """
    
    suffixoverlap_results = {}
    
    if 'scale' in params:
      #accepts year/month/day/hour/minute/second
      scale = params['scale']
    else:
      scale = 'year'
      
    if 'suffix' in params:
      #accepts column header suffix appender
      suffix = params['suffix']
    else:
      suffix = '_year'
      
    if 'normalization' in params:
      #accepts zscore/minmax/unscaled
      normalization = params['normalization']
    else:
      normalization = 'zscore'
      
    time_column = column + suffix
    
    mdf_train, suffixoverlap_results = \
    self.df_copy_train(mdf_train, column, time_column, suffixoverlap_results)
    
    mdf_test[time_column] = mdf_test[column].copy()
    
    #apply pd.to_datetime to column, note that the errors = 'coerce' needed for messy data
    mdf_train[time_column] = pd.to_datetime(mdf_train[time_column], errors = 'coerce')
    mdf_test[time_column] = pd.to_datetime(mdf_test[time_column], errors = 'coerce')
    
    #access time scale from one of year/month/day/hour/minute/second
    if scale == 'year':
      mdf_train[time_column] = mdf_train[time_column].dt.year
      mdf_test[time_column] = mdf_test[time_column].dt.year
    elif scale == 'month':
      mdf_train[time_column] = mdf_train[time_column].dt.month
      mdf_test[time_column] = mdf_test[time_column].dt.month
    elif scale == 'day':
      mdf_train[time_column] = mdf_train[time_column].dt.day
      mdf_test[time_column] = mdf_test[time_column].dt.day
    elif scale == 'hour':
      mdf_train[time_column] = mdf_train[time_column].dt.hour
      mdf_test[time_column] = mdf_test[time_column].dt.hour
    elif scale == 'minute':
      mdf_train[time_column] = mdf_train[time_column].dt.minute
      mdf_test[time_column] = mdf_test[time_column].dt.minute
    elif scale == 'second':
      mdf_train[time_column] = mdf_train[time_column].dt.second
      mdf_test[time_column] = mdf_test[time_column].dt.second
      
    #default infill is adjacent cell
    mdf_train[time_column] = mdf_train[time_column].fillna(method='ffill')
    mdf_train[time_column] = mdf_train[time_column].fillna(method='bfill')
    
    mdf_test[time_column] = mdf_test[time_column].fillna(method='ffill')
    mdf_test[time_column] = mdf_test[time_column].fillna(method='bfill')
      
    #grab a few drift metrics
    timemean = mdf_train[time_column].mean()
    timemax = mdf_train[time_column].max()
    timemin = mdf_train[time_column].min()
    timestd = mdf_train[time_column].std()

    maxminusmin = timemax - timemin
    
    #backup default infill for cases without valid entries
    mdf_train[time_column] = mdf_train[time_column].fillna(0)
    mdf_test[time_column] = mdf_test[time_column].fillna(0)
      
    #formula for scaling is (x - scaler) / divisor
    #normalizaiton is either zscore/minmax/unscaled
    if normalization == 'zscore':
      scaler = timemean
      divisor = timestd
    if normalization == 'minmax':
      scaler = timemin
      divisor = maxminusmin
    if normalization == 'unscaled':
      scaler = 0
      divisor = 1
    
    if divisor == 0 or divisor != divisor:
      divisor = 1
      
    if scaler != scaler:
      scaler = 0
      
    #apply normalization
    if normalization != 'unscaled':
      mdf_train[time_column] = (mdf_train[time_column] - scaler) / divisor
      mdf_test[time_column] = (mdf_test[time_column] - scaler) / divisor
      
    #populate data structures
    column_dict_list = []
    categorylist = [time_column]
    
    for tc in categorylist:
      norm_dict = {tc : {'scale'         : scale, \
                         'suffix'        : suffix, \
                         'normalization' : normalization, \
                         'scaler'        : scaler, \
                         'divisor'       : divisor, \
                         'timemean'      : timemean, \
                         'timemax'       : timemax, \
                         'timemin'       : timemin, \
                         'timestd'       : timestd, \
                         'maxminusmin'   : maxminusmin}}
      
      column_dict = {tc : {'category' : 'time', \
                           'origcategory' : category, \
                           'normalization_dict' : norm_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : categorylist, \
                           'categorylist' : categorylist, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())
    
    return mdf_train, mdf_test, column_dict_list

  def process_bxcx_class(self, mdf_train, mdf_test, column, category, \
                         postprocess_dict, params = {}):
    '''
    Applies Box-Cox transform to an all-positive numerical set.
    '''
    
    suffixoverlap_results = {}
    
    #df_train, nmbrcolumns, nmbrnormalization_dict, categorylist = \
    mdf_train, column_dict_list = \
    self.process_bxcx_support(mdf_train, column, category, 1, bxcx_lmbda = None, \
                              trnsfrm_mean = None)

    #grab the normalization_dict associated with the bxcx category
    columnkeybxcx = column + '_bxcx'
    for column_dict in column_dict_list:
      if columnkeybxcx in column_dict:
        bxcxnormalization_dict = column_dict[columnkeybxcx]['normalization_dict'][columnkeybxcx]

    #df_test, nmbrcolumns, _1, _2 = \
    mdf_test, _1 = \
    self.process_bxcx_support(mdf_test, column, category, 1, bxcx_lmbda = \
                             bxcxnormalization_dict['bxcx_lmbda'], \
                             trnsfrm_mean = bxcxnormalization_dict['trnsfrm_mean'])

    return mdf_train, mdf_test, column_dict_list

  def process_bxcx_support(self, df, column, category, bxcxerrorcorrect, bxcx_lmbda = None, trnsfrm_mean = None):
    '''                      
    #process_bxcx_class(df, column, bxcx_lmbda = None, trnsfrm_mean = None, trnsfrm_std = None)
    #function that takes as input a dataframe with numnerical column for purposes
    #of applying a box-cox transformation. If lmbda = None it will infer a suitable
    #lambda value by minimizing log likelihood using SciPy's stats boxcox call. If
    #we pass a mean or std value it will apply the mean for the initial infill and 
    #use the values to apply postprocess_numerical_class function. 
    #Returns transformed dataframe, a list nmbrcolumns of the associated columns,
    #and a normalization dictionary nmbrnormalization_dict which we'll use for our
    #postprocess_dict, and the parameter lmbda that was used
    #expect this approach works better than our prior numerical address when the 
    #distribution is less thin tailed
    '''
    
    suffixoverlap_results = {}
    
    bxcxcolumn = column + '_bxcx'

    df, suffixoverlap_results = \
    self.df_copy_train(df, column, bxcxcolumn, suffixoverlap_results)

    #convert all values to either numeric or NaN
    df[bxcxcolumn] = pd.to_numeric(df[bxcxcolumn], errors='coerce')
    #convert non-positive values to nan
    df.loc[df[bxcxcolumn] <= 0, (bxcxcolumn)] = np.nan

    #get the mean value to apply to infill
    if trnsfrm_mean == None:
      #get mean of training data
      mean = df[bxcxcolumn].mean()  

    else:
      mean = trnsfrm_mean

    #edge case
    if mean != mean or mean <= 0:
      mean = 0
      bxcx_lmbda = False

    #replace missing data with training set mean
    df[bxcxcolumn] = df[bxcxcolumn].fillna(mean)
    
    #edge case to avoid stats.boxcox error
    if df[bxcxcolumn].nunique() == 1:
      df[bxcxcolumn] = 0
      
      #we'll use convention that if training data is set to 0 then so will all subsequent data
      bxcx_lmbda = False
      
    else:

      #apply box-cox transformation to generate a new column
      #note the returns are different based on whether we passed a lmbda value

      if bxcx_lmbda == None:

        df[bxcxcolumn], bxcx_lmbda = stats.boxcox(df[bxcxcolumn])
        df[bxcxcolumn] *= bxcxerrorcorrect
        
      elif bxcx_lmbda is False:
        
        df[bxcxcolumn] = 0

      else:

        df[bxcxcolumn] = stats.boxcox(df[bxcxcolumn], lmbda = bxcx_lmbda)
        df[bxcxcolumn] *= bxcxerrorcorrect

    #this is to address an error when bxcx transofrm produces overflow
    #I'm not sure of cause, showed up in the housing set)
    bxcxerrorcorrect = 1
    if max(df[bxcxcolumn]) > (2 ** 31 - 1):
      bxcxerrorcorrect = 0
      df[bxcxcolumn] = 0
      bxcxcolumn = bxcxcolumn
      print("overflow condition found in boxcox transofrm, column set to 0: ", bxcxcolumn)

#     #replace original column
#     del df[column]

#     df[column] = df[column + '_temp'].copy()

#     del df[column + '_temp']

#     #change data type for memory savings
#     df[column + '_bxcx'] = df[column + '_bxcx'].astype(np.float32)

    #output of a list of the created column names
    #nmbrcolumns = [column + '_nmbr', column + '_bxcx', column + '_NArw']
    nmbrcolumns = [bxcxcolumn]

    #create list of columns associated with categorical transform (blank for now)
    categorylist = []

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in nmbrcolumns:

      #save a dictionary of the associated column mean and std

      normalization_dict = {nc : {'trnsfrm_mean' : mean, \
                                  'bxcx_lmbda' : bxcx_lmbda, \
                                  'bxcxerrorcorrect' : bxcxerrorcorrect, \
                                  'mean' : mean}}

      column_dict = { nc : {'category' : 'bxcx', \
                           'origcategory' : category, \
                           'normalization_dict' : normalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())

    #return df, nmbrcolumns, nmbrnormalization_dict, categorylist
    return df, column_dict_list

  def process_log0_class(self, mdf_train, mdf_test, column, category, \
                         postprocess_dict, params = {}):
    '''
    #process_log0_class(mdf_train, mdf_test, column, category)
    #function to apply logatrithmic transform
    #takes as arguement pandas dataframe of training and test data (mdf_train), (mdf_test)\
    #and the name of the column string ('column') and parent category (category)
    #applies a logarithmic transform (base 10)
    #replaces zeros, negative, and missing or improperly formatted data with post-log mean as default infill
    #returns same dataframes with new column of name column + '_log0'
    '''
    
    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    if inplace is not True:
      
      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.df_copy_train(mdf_train, column, column + '_log0', suffixoverlap_results)

      mdf_test[column + '_log0'] = mdf_test[column].copy()
    
    else:
      
      suffixoverlap_results = \
      self.df_check_suffixoverlap(mdf_train, column + '_log0', suffixoverlap_results)
      
      mdf_train.rename(columns = {column : column + '_log0'}, inplace = True)
      mdf_test.rename(columns = {column : column + '_log0'}, inplace = True)

    #convert all values to either numeric or NaN
    mdf_train[column + '_log0'] = pd.to_numeric(mdf_train[column + '_log0'], errors='coerce')
    mdf_test[column + '_log0'] = pd.to_numeric(mdf_test[column + '_log0'], errors='coerce')
    
#     #replace all zeros with nan for the log operation
#     zeroreplace = {0 : np.nan}
#     mdf_train[column + '_log0'] = mdf_train[column + '_log0'].replace(zeroreplace)
#     mdf_test[column + '_log0'] = mdf_test[column + '_log0'].replace(zeroreplace)
    
    #replace all non-positive with nan for the log operation
    mdf_train.loc[mdf_train[column + '_log0'] <= 0, (column + '_log0')] = np.nan
    mdf_test.loc[mdf_test[column + '_log0'] <= 0, (column + '_log0')] = np.nan
    
    #log transform column
    #note that this replaces negative values with nan which we will infill with mean
    mdf_train[column + '_log0'] = np.log10(mdf_train[column + '_log0'])
    mdf_test[column + '_log0'] = np.log10(mdf_test[column + '_log0'])
    
    #get mean of train set
    meanlog = mdf_train[column + '_log0'].mean()
    
    if meanlog != meanlog:
      meanlog = 0

    #replace missing data with training set mean
    mdf_train[column + '_log0'] = mdf_train[column + '_log0'].fillna(meanlog)
    mdf_test[column + '_log0'] = mdf_test[column + '_log0'].fillna(meanlog)

#     #replace missing data with 0
#     mdf_train[column + '_log0'] = mdf_train[column + '_log0'].fillna(0)
#     mdf_test[column + '_log0'] = mdf_test[column + '_log0'].fillna(0)

#     #change data type for memory savings
#     mdf_train[column + '_log0'] = mdf_train[column + '_log0'].astype(np.float32)
#     mdf_test[column + '_log0'] = mdf_test[column + '_log0'].astype(np.float32)

    #create list of columns
    nmbrcolumns = [column + '_log0']

    nmbrnormalization_dict = {column + '_log0' : {'meanlog' : meanlog}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in nmbrcolumns:

      column_dict = { nc : {'category' : 'log0', \
                           'origcategory' : category, \
                           'normalization_dict' : nmbrnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())
        
    return mdf_train, mdf_test, column_dict_list
  
  def process_logn_class(self, mdf_train, mdf_test, column, category, \
                         postprocess_dict, params = {}):
    '''
    #process_logn_class(mdf_train, mdf_test, column, category)
    #function to apply natural logatrithmic transform
    #takes as arguement pandas dataframe of training and test data (mdf_train), (mdf_test)\
    #and the name of the column string ('column') and parent category (category)
    #applies a logarithmic transform (base e)
    #replaces zeros, negative, and missing or improperly formatted data with post-log mean as default infill
    #returns same dataframes with new column of name column + '_logn'
    '''
    
    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    if inplace is not True:
      
      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.df_copy_train(mdf_train, column, column + '_logn', suffixoverlap_results)

      mdf_test[column + '_logn'] = mdf_test[column].copy()
    
    else:
      
      suffixoverlap_results = \
      self.df_check_suffixoverlap(mdf_train, column + '_logn', suffixoverlap_results)
      
      mdf_train.rename(columns = {column : column + '_logn'}, inplace = True)
      mdf_test.rename(columns = {column : column + '_logn'}, inplace = True)

    #convert all values to either numeric or NaN
    mdf_train[column + '_logn'] = pd.to_numeric(mdf_train[column + '_logn'], errors='coerce')
    mdf_test[column + '_logn'] = pd.to_numeric(mdf_test[column + '_logn'], errors='coerce')
    
#     #replace all zeros with nan for the log operation
#     zeroreplace = {0 : np.nan}
#     mdf_train[column + '_log0'] = mdf_train[column + '_log0'].replace(zeroreplace)
#     mdf_test[column + '_log0'] = mdf_test[column + '_log0'].replace(zeroreplace)
    
    #replace all non-positive with nan for the log operation
    mdf_train.loc[mdf_train[column + '_logn'] <= 0, (column + '_logn')] = np.nan
    mdf_test.loc[mdf_test[column + '_logn'] <= 0, (column + '_logn')] = np.nan
    
    #log transform column
    #note that this replaces negative values with nan which we will infill with mean
    mdf_train[column + '_logn'] = np.log(mdf_train[column + '_logn'])
    mdf_test[column + '_logn'] = np.log(mdf_test[column + '_logn'])
    
    #get mean of train set
    meanlog = mdf_train[column + '_logn'].mean()
    
    if meanlog != meanlog:
      meanlog = 0

    #replace missing data with training set mean
    mdf_train[column + '_logn'] = mdf_train[column + '_logn'].fillna(meanlog)
    mdf_test[column + '_logn'] = mdf_test[column + '_logn'].fillna(meanlog)

#     #replace missing data with 0
#     mdf_train[column + '_log0'] = mdf_train[column + '_log0'].fillna(0)
#     mdf_test[column + '_log0'] = mdf_test[column + '_log0'].fillna(0)

#     #change data type for memory savings
#     mdf_train[column + '_log0'] = mdf_train[column + '_log0'].astype(np.float32)
#     mdf_test[column + '_log0'] = mdf_test[column + '_log0'].astype(np.float32)

    #create list of columns
    nmbrcolumns = [column + '_logn']

    nmbrnormalization_dict = {column + '_logn' : {'meanlog' : meanlog}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in nmbrcolumns:

      column_dict = { nc : {'category' : 'logn', \
                           'origcategory' : category, \
                           'normalization_dict' : nmbrnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())
    
    return mdf_train, mdf_test, column_dict_list
  
  def process_sqrt_class(self, mdf_train, mdf_test, column, category, \
                         postprocess_dict, params = {}):
    '''
    #process_sqrt_class(mdf_train, mdf_test, column, category)
    #function to apply square root transform
    #takes as arguement pandas dataframe of training and test data (mdf_train), (mdf_test)\
    #and the name of the column string ('column') and parent category (category)
    #applies a square root transform
    #replaces zeros, negative, and missing or improperly formatted data with post-log mean as default infill
    #returns same dataframes with new column of name column + '_log0'
    '''
    
    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    if inplace is not True:
      
      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.df_copy_train(mdf_train, column, column + '_sqrt', suffixoverlap_results)

      mdf_test[column + '_sqrt'] = mdf_test[column].copy()
    
    else:
      
      suffixoverlap_results = \
      self.df_check_suffixoverlap(mdf_train, column + '_sqrt', suffixoverlap_results)
      
      mdf_train.rename(columns = {column : column + '_sqrt'}, inplace = True)
      mdf_test.rename(columns = {column : column + '_sqrt'}, inplace = True)

    #convert all values to either numeric or NaN
    mdf_train[column + '_sqrt'] = pd.to_numeric(mdf_train[column + '_sqrt'], errors='coerce')
    mdf_test[column + '_sqrt'] = pd.to_numeric(mdf_test[column + '_sqrt'], errors='coerce')
    
#     #replace all zeros with nan for the log operation
#     zeroreplace = {0 : np.nan}
#     mdf_train[column + '_log0'] = mdf_train[column + '_log0'].replace(zeroreplace)
#     mdf_test[column + '_log0'] = mdf_test[column + '_log0'].replace(zeroreplace)
    
    #replace all non-positive with nan for the log operation
    mdf_train.loc[mdf_train[column + '_sqrt'] < 0, (column + '_sqrt')] = np.nan
    mdf_test.loc[mdf_test[column + '_sqrt'] < 0, (column + '_sqrt')] = np.nan
    
    #log transform column
    #note that this replaces negative values with nan which we will infill with mean
    mdf_train[column + '_sqrt'] = np.sqrt(mdf_train[column + '_sqrt'])
    mdf_test[column + '_sqrt'] = np.sqrt(mdf_test[column + '_sqrt'])
    
    #get mean of train set
    meansqrt = mdf_train[column + '_sqrt'].mean()
    
    if meansqrt != meansqrt:
      meansqrt = 0

    #replace missing data with training set mean
    mdf_train[column + '_sqrt'] = mdf_train[column + '_sqrt'].fillna(meansqrt)
    mdf_test[column + '_sqrt'] = mdf_test[column + '_sqrt'].fillna(meansqrt)

#     #replace missing data with 0
#     mdf_train[column + '_log0'] = mdf_train[column + '_log0'].fillna(0)
#     mdf_test[column + '_log0'] = mdf_test[column + '_log0'].fillna(0)

#     #change data type for memory savings
#     mdf_train[column + '_log0'] = mdf_train[column + '_log0'].astype(np.float32)
#     mdf_test[column + '_log0'] = mdf_test[column + '_log0'].astype(np.float32)

    #create list of columns
    nmbrcolumns = [column + '_sqrt']

    nmbrnormalization_dict = {column + '_sqrt' : {'meansqrt' : meansqrt}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in nmbrcolumns:

      column_dict = { nc : {'category' : 'sqrt', \
                           'origcategory' : category, \
                           'normalization_dict' : nmbrnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())
    
    return mdf_train, mdf_test, column_dict_list
  
  def process_addd_class(self, mdf_train, mdf_test, column, category, \
                         postprocess_dict, params = {}):
    '''
    #process_addd_class(.)
    #function to apply addition transform
    #takes as arguement pandas dataframe of training and test data (mdf_train), (mdf_test)\
    #and the name of the column string ('column') and parent category (category)
    #accepts parameter 'add' for amount of addition, otherwise defaults to adding 1
    #applies an addition transform
    #replaces non-numeric entries with set mean after addition
    #returns same dataframes with new column of name column + '_addd'
    '''
    
    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    if 'add' in params:
        
      add = params['add']
    
    else:
      
      add = 1
    
    if inplace is not True:
      
      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.df_copy_train(mdf_train, column, column + '_addd', suffixoverlap_results)

      mdf_test[column + '_addd'] = mdf_test[column].copy()
    
    else:
      
      suffixoverlap_results = \
      self.df_check_suffixoverlap(mdf_train, column + '_addd', suffixoverlap_results)
      
      mdf_train.rename(columns = {column : column + '_addd'}, inplace = True)
      mdf_test.rename(columns = {column : column + '_addd'}, inplace = True)

    #convert all values to either numeric or NaN
    mdf_train[column + '_addd'] = pd.to_numeric(mdf_train[column + '_addd'], errors='coerce')
    mdf_test[column + '_addd'] = pd.to_numeric(mdf_test[column + '_addd'], errors='coerce')
    
    
    #apply addition
    mdf_train[column + '_addd'] = mdf_train[column + '_addd'] + add
    mdf_test[column + '_addd'] = mdf_test[column + '_addd'] + add
    
    #get mean of train set
    mean = mdf_train[column + '_addd'].mean()
    
    if mean != mean:
      mean = 0

    #replace missing data with training set mean
    mdf_train[column + '_addd'] = mdf_train[column + '_addd'].fillna(mean)
    mdf_test[column + '_addd'] = mdf_test[column + '_addd'].fillna(mean)

    #create list of columns
    nmbrcolumns = [column + '_addd']

    nmbrnormalization_dict = {column + '_addd' : {'mean' : mean, \
                                                  'add' : add}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in nmbrcolumns:

      column_dict = { nc : {'category' : 'addd', \
                           'origcategory' : category, \
                           'normalization_dict' : nmbrnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())
        
    return mdf_train, mdf_test, column_dict_list
  
  def process_sbtr_class(self, mdf_train, mdf_test, column, category, \
                         postprocess_dict, params = {}):
    '''
    #process_sbtr_class(.)
    #function to apply subtraction transform
    #takes as arguement pandas dataframe of training and test data (mdf_train), (mdf_test)\
    #and the name of the column string ('column') and parent category (category)
    #accepts parameter 'subtract' for amount of subtraction, otherwise defaults to subtracting 1
    #applies a subtraction transform
    #replaces non-numeric entries with set mean after subtraction
    #returns same dataframes with new column of name column + '_sbtr'
    '''
    
    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    if 'subtract' in params:
        
      subtract = params['subtract']
    
    else:
      
      subtract = 1
    
    if inplace is not True:
      
      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.df_copy_train(mdf_train, column, column + '_sbtr', suffixoverlap_results)

      mdf_test[column + '_sbtr'] = mdf_test[column].copy()
    
    else:
      
      suffixoverlap_results = \
      self.df_check_suffixoverlap(mdf_train, column + '_sbtr', suffixoverlap_results)
      
      mdf_train.rename(columns = {column : column + '_sbtr'}, inplace = True)
      mdf_test.rename(columns = {column : column + '_sbtr'}, inplace = True)

    #convert all values to either numeric or NaN
    mdf_train[column + '_sbtr'] = pd.to_numeric(mdf_train[column + '_sbtr'], errors='coerce')
    mdf_test[column + '_sbtr'] = pd.to_numeric(mdf_test[column + '_sbtr'], errors='coerce')
    
    #apply subtraction
    mdf_train[column + '_sbtr'] = mdf_train[column + '_sbtr'] - subtract
    mdf_test[column + '_sbtr'] = mdf_test[column + '_sbtr'] - subtract
    
    #get mean of train set
    mean = mdf_train[column + '_sbtr'].mean()
    
    if mean != mean:
      mean = 0

    #replace missing data with training set mean
    mdf_train[column + '_sbtr'] = mdf_train[column + '_sbtr'].fillna(mean)
    mdf_test[column + '_sbtr'] = mdf_test[column + '_sbtr'].fillna(mean)

    #create list of columns
    nmbrcolumns = [column + '_sbtr']

    nmbrnormalization_dict = {column + '_sbtr' : {'mean' : mean, \
                                                  'subtract' : subtract}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in nmbrcolumns:

      column_dict = { nc : {'category' : 'sbtr', \
                           'origcategory' : category, \
                           'normalization_dict' : nmbrnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())
        
    return mdf_train, mdf_test, column_dict_list
  
  def process_mltp_class(self, mdf_train, mdf_test, column, category, \
                         postprocess_dict, params = {}):
    '''
    #process_mltp_class(.)
    #function to apply multiplication transform
    #takes as arguement pandas dataframe of training and test data (mdf_train), (mdf_test)\
    #and the name of the column string ('column') and parent category (category)
    #accepts parameter 'multiply' for amount of addition, otherwise defaults to multiplying 2
    #applies an multiplication transform
    #replaces non-numeric entries with set mean after addition
    #returns same dataframes with new column of name column + '_mltp'
    '''
    
    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    if 'multiply' in params:
        
      multiply = params['multiply']
    
    else:
      
      multiply = 2
    
    if inplace is not True:
      
      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.df_copy_train(mdf_train, column, column + '_mltp', suffixoverlap_results)

      mdf_test[column + '_mltp'] = mdf_test[column].copy()
    
    else:
      
      suffixoverlap_results = \
      self.df_check_suffixoverlap(mdf_train, column + '_mltp', suffixoverlap_results)
      
      mdf_train.rename(columns = {column : column + '_mltp'}, inplace = True)
      mdf_test.rename(columns = {column : column + '_mltp'}, inplace = True)

    #convert all values to either numeric or NaN
    mdf_train[column + '_mltp'] = pd.to_numeric(mdf_train[column + '_mltp'], errors='coerce')
    mdf_test[column + '_mltp'] = pd.to_numeric(mdf_test[column + '_mltp'], errors='coerce')
    
    #apply multiplication
    mdf_train[column + '_mltp'] = mdf_train[column + '_mltp'] * multiply
    mdf_test[column + '_mltp'] = mdf_test[column + '_mltp'] * multiply
    
    #get mean of train set
    mean = mdf_train[column + '_mltp'].mean()
    
    if mean != mean:
      mean = 0

    #replace missing data with training set mean
    mdf_train[column + '_mltp'] = mdf_train[column + '_mltp'].fillna(mean)
    mdf_test[column + '_mltp'] = mdf_test[column + '_mltp'].fillna(mean)

    #create list of columns
    nmbrcolumns = [column + '_mltp']


    nmbrnormalization_dict = {column + '_mltp' : {'mean' : mean, \
                                                  'multiply' : multiply}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in nmbrcolumns:

      column_dict = { nc : {'category' : 'mltp', \
                           'origcategory' : category, \
                           'normalization_dict' : nmbrnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())
        
    return mdf_train, mdf_test, column_dict_list
  
  def process_divd_class(self, mdf_train, mdf_test, column, category, \
                         postprocess_dict, params = {}):
    '''
    #process_divd_class(.)
    #function to apply division transform
    #accepts parameter 'divide' for amount of division, otherwise defaults to dividing by 2
    #applies an division transform
    #replaces non-numeric entries with set mean after division
    #returns same dataframes with new column of name column + '_divd'
    '''
    
    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    if 'divide' in params:
        
      divide = params['divide']
    
    else:
      
      divide = 2
      
    #special case override to avoid div by 0
    if divide == 0:
      divide = 1
    
    if inplace is not True:
      
      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.df_copy_train(mdf_train, column, column + '_divd', suffixoverlap_results)

      mdf_test[column + '_divd'] = mdf_test[column].copy()
    
    else:
      
      suffixoverlap_results = \
      self.df_check_suffixoverlap(mdf_train, column + '_divd', suffixoverlap_results)
      
      mdf_train.rename(columns = {column : column + '_divd'}, inplace = True)
      mdf_test.rename(columns = {column : column + '_divd'}, inplace = True)

    #convert all values to either numeric or NaN
    mdf_train[column + '_divd'] = pd.to_numeric(mdf_train[column + '_divd'], errors='coerce')
    mdf_test[column + '_divd'] = pd.to_numeric(mdf_test[column + '_divd'], errors='coerce')
    
    
    #apply multiplication
    mdf_train[column + '_divd'] = mdf_train[column + '_divd'] / divide
    mdf_test[column + '_divd'] = mdf_test[column + '_divd'] / divide
    
    #get mean of train set
    mean = mdf_train[column + '_divd'].mean()
    
    if mean != mean:
      mean = 0

    #replace missing data with training set mean
    mdf_train[column + '_divd'] = mdf_train[column + '_divd'].fillna(mean)
    mdf_test[column + '_divd'] = mdf_test[column + '_divd'].fillna(mean)


    #create list of columns
    nmbrcolumns = [column + '_divd']


    nmbrnormalization_dict = {column + '_divd' : {'mean' : mean, \
                                                  'divide' : divide}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in nmbrcolumns:

      column_dict = { nc : {'category' : 'divd', \
                           'origcategory' : category, \
                           'normalization_dict' : nmbrnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())
        
    return mdf_train, mdf_test, column_dict_list
  
  def process_rais_class(self, mdf_train, mdf_test, column, category, \
                         postprocess_dict, params = {}):
    '''
    #process_rais_class(.)
    #function to apply raise to a power transform
    #accepts parameter 'raiser' for amount of power, otherwise defaults to square (raise by 2)
    #applies an raise transform
    #replaces non-numeric entries with set mean after raise
    #returns same dataframes with new column of name column + '_rais'
    '''
    
    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    if 'raiser' in params:
        
      raiser = params['raiser']
    
    else:
      
      raiser = 2
    
    if inplace is not True:
      
      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.df_copy_train(mdf_train, column, column + '_rais', suffixoverlap_results)

      mdf_test[column + '_rais'] = mdf_test[column].copy()
    
    else:
      
      suffixoverlap_results = \
      self.df_check_suffixoverlap(mdf_train, column + '_rais', suffixoverlap_results)
      
      mdf_train.rename(columns = {column : column + '_rais'}, inplace = True)
      mdf_test.rename(columns = {column : column + '_rais'}, inplace = True)

    #convert all values to either numeric or NaN
    mdf_train[column + '_rais'] = pd.to_numeric(mdf_train[column + '_rais'], errors='coerce')
    mdf_test[column + '_rais'] = pd.to_numeric(mdf_test[column + '_rais'], errors='coerce')
    
    #apply addition
    mdf_train[column + '_rais'] = mdf_train[column + '_rais'] ** raiser
    mdf_test[column + '_rais'] = mdf_test[column + '_rais'] ** raiser
    
    #get mean of train set
    mean = mdf_train[column + '_rais'].mean()
    
    if mean != mean:
      mean = 0

    #replace missing data with training set mean
    mdf_train[column + '_rais'] = mdf_train[column + '_rais'].fillna(mean)
    mdf_test[column + '_rais'] = mdf_test[column + '_rais'].fillna(mean)

    #create list of columns
    nmbrcolumns = [column + '_rais']

    nmbrnormalization_dict = {column + '_rais' : {'mean' : mean, \
                                                  'raiser' : raiser}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in nmbrcolumns:

      column_dict = { nc : {'category' : 'rais', \
                           'origcategory' : category, \
                           'normalization_dict' : nmbrnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())
        
    return mdf_train, mdf_test, column_dict_list
  
  def process_absl_class(self, mdf_train, mdf_test, column, category, \
                         postprocess_dict, params = {}):
    '''
    #process_absl_class(.)
    #function to apply absolute transform
    #does not accept paraemters
    #applies an absolute transform
    #replaces non-numeric entries with set mean after transform
    #returns same dataframes with new column of name column + '_absl'
    '''
    
    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    if inplace is not True:
      
      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.df_copy_train(mdf_train, column, column + '_absl', suffixoverlap_results)

      mdf_test[column + '_absl'] = mdf_test[column].copy()
    
    else:
      
      suffixoverlap_results = \
      self.df_check_suffixoverlap(mdf_train, column + '_absl', suffixoverlap_results)
      
      mdf_train.rename(columns = {column : column + '_absl'}, inplace = True)
      mdf_test.rename(columns = {column : column + '_absl'}, inplace = True)

    #convert all values to either numeric or NaN
    mdf_train[column + '_absl'] = pd.to_numeric(mdf_train[column + '_absl'], errors='coerce')
    mdf_test[column + '_absl'] = pd.to_numeric(mdf_test[column + '_absl'], errors='coerce')
    
    #apply addition
    mdf_train[column + '_absl'] = mdf_train[column + '_absl'].abs()
    mdf_test[column + '_absl'] = mdf_test[column + '_absl'].abs()
    
    #get mean of train set
    mean = mdf_train[column + '_absl'].mean()
    
    if mean != mean:
      mean = 0

    #replace missing data with training set mean
    mdf_train[column + '_absl'] = mdf_train[column + '_absl'].fillna(mean)
    mdf_test[column + '_absl'] = mdf_test[column + '_absl'].fillna(mean)

    #create list of columns
    nmbrcolumns = [column + '_absl']

    nmbrnormalization_dict = {column + '_absl' : {'mean' : mean}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in nmbrcolumns:

      column_dict = { nc : {'category' : 'absl', \
                           'origcategory' : category, \
                           'normalization_dict' : nmbrnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())
        
    return mdf_train, mdf_test, column_dict_list

  def process_pwrs_class(self, mdf_train, mdf_test, column, category, postprocess_dict, params = {}):
    '''
    #processes a numerical set by creating bins corresponding to powers
    #of ten in one hot encoded columns
    
    #pwrs will be intended for a raw set that is not yet normalized
    
    #postiive values encoded under column 'column' + '_10^#' where # is power of 10
    #0 and negative values considered infill with no activations
    
    #if all values are infill no columns returned
    
    #accepts boolean 'negvalues' parameter, defaults False, True activates encoding for values <0
    '''
    
    suffixoverlap_results = {}
    
    if 'negvalues' in params:
      negvalues = params['negvalues']
    else:
      negvalues = False
    
    tempcolumn = column + '_-10^'

    #store original column for later reversion
    mdf_train, suffixoverlap_results = \
    self.df_copy_train(mdf_train, column, tempcolumn, suffixoverlap_results)
    
    mdf_test[tempcolumn] = mdf_test[column].copy()

    #convert all values to either numeric or NaN
    mdf_train[tempcolumn] = pd.to_numeric(mdf_train[tempcolumn], errors='coerce')
    mdf_test[tempcolumn] = pd.to_numeric(mdf_test[tempcolumn], errors='coerce')
    
    #create copy with negative values
    negtempcolumn = column + '_negtemp'
    mdf_train, suffixoverlap_results = \
    self.df_copy_train(mdf_train, tempcolumn, negtempcolumn, suffixoverlap_results)
    
    mdf_test[negtempcolumn] = mdf_test[tempcolumn].copy()
    
    #convert all values in negtempcolumn >= 0 to Nan
    mdf_train[negtempcolumn] = \
    np.where(mdf_train[negtempcolumn] >= 0, np.nan, mdf_train[negtempcolumn].values)
    mdf_test[negtempcolumn] = \
    np.where(mdf_test[negtempcolumn] >= 0, np.nan, mdf_test[negtempcolumn].values)
    
    #convert all values <= 0 to Nan
    mdf_train[tempcolumn] = \
    np.where(mdf_train[tempcolumn] <= 0, np.nan, mdf_train[tempcolumn].values)
    mdf_test[tempcolumn] = \
    np.where(mdf_test[tempcolumn] <= 0, np.nan, mdf_test[tempcolumn].values)
    
    #log transform column
    
    #take abs value of negtempcolumn
    mdf_train[negtempcolumn] = mdf_train[negtempcolumn].abs()
    mdf_test[negtempcolumn] = mdf_test[negtempcolumn].abs()
    
    mdf_train[negtempcolumn] = \
    np.where(mdf_train[negtempcolumn] != np.nan, np.floor(np.log10(mdf_train[negtempcolumn])), mdf_train[negtempcolumn].values)
    mdf_test[negtempcolumn] = \
    np.where(mdf_test[negtempcolumn] != np.nan, np.floor(np.log10(mdf_test[negtempcolumn])), mdf_test[negtempcolumn].values)
    
    train_neg_dict = {}
    newunique_list = []
    negunique = mdf_train[negtempcolumn].unique()
    for unique in negunique:
      if unique != unique:
        newunique = np.nan
      else:
        #this is update for difference between pwr2 and pwrs
        if negvalues:
          newunique = column + '_-10^' + str(int(unique))
        else:
          newunique = np.nan
      train_neg_dict.update({unique : newunique})
      newunique_list.append(newunique)
      
    test_neg_dict = {}
    negunique = mdf_test[negtempcolumn].unique()
    for unique in negunique:
      if unique != unique:
        newunique = np.nan
      else:
        #this is update for difference between pwr2 and pwrs
        if negvalues:
          newunique = column + '_-10^' + str(int(unique))
        else:
          newunique = np.nan
      if newunique in newunique_list and newunique == newunique:
        test_neg_dict.update({unique : newunique})
      else:
        test_neg_dict.update({unique : np.nan})
        
    mdf_train[negtempcolumn] = mdf_train[negtempcolumn].replace(train_neg_dict)
    mdf_test[negtempcolumn] = mdf_test[negtempcolumn].replace(test_neg_dict)
    
    #now log trasnform positive values in column column 

    mdf_train[tempcolumn] = \
    np.where(mdf_train[tempcolumn] != np.nan, np.floor(np.log10(mdf_train[tempcolumn])), mdf_train[tempcolumn].values)
    mdf_test[tempcolumn] = \
    np.where(mdf_test[tempcolumn] != np.nan, np.floor(np.log10(mdf_test[tempcolumn])), mdf_test[tempcolumn].values)

    train_pos_dict = {}
    newposunique_list = []
    posunique = mdf_train[tempcolumn].unique()
    for unique in posunique:
      if unique != unique:
        newunique = np.nan
      else:
        newunique = column + '_10^' + str(int(unique))
      train_pos_dict.update({unique : newunique})
      newposunique_list.append(newunique)
      
    test_pos_dict = {}
    posunique = mdf_test[tempcolumn].unique()
    for unique in posunique:
      if unique != unique:
        newunique = np.nan
      else:
        newunique = column + '_10^' + str(int(unique))
      if newunique in newposunique_list and newunique == newunique:
        test_pos_dict.update({unique : newunique})
      else:
        test_pos_dict.update({unique : np.nan})
    
    mdf_train[tempcolumn] = mdf_train[tempcolumn].replace(train_pos_dict)
    mdf_test[tempcolumn] = mdf_test[tempcolumn].replace(test_pos_dict)
    
    #combine the two columns
    mdf_train[tempcolumn] = mdf_train[negtempcolumn].where(mdf_train[negtempcolumn] == mdf_train[negtempcolumn], mdf_train[tempcolumn])
    mdf_test[tempcolumn] = mdf_test[negtempcolumn].where(mdf_test[negtempcolumn] == mdf_test[negtempcolumn], mdf_test[tempcolumn])
    
    #pandas one hot encoder
    df_train_cat = pd.get_dummies(mdf_train[tempcolumn])
    df_test_cat = pd.get_dummies(mdf_test[tempcolumn])
    
    labels_train = list(df_train_cat)
    labels_test = list(df_test_cat)

    #Get missing columns in test set that are present in training set
    missing_cols = set( df_train_cat.columns ) - set( df_test_cat.columns )
    
    #Add a missing column in test set with default value equal to 0
    for c in missing_cols:
        df_test_cat[c] = 0
    #Ensure the order of column in the test set is in the same order than in train set
    #Note this also removes categories in test set that aren't present in training set
    df_test_cat = df_test_cat[df_train_cat.columns]
    
    suffixoverlap_results = \
    self.df_check_suffixoverlap(mdf_train, list(df_train_cat), suffixoverlap_results)
    
    #concatinate the sparse set with the rest of our training data
    mdf_train = pd.concat([mdf_train, df_train_cat], axis=1)
    mdf_test = pd.concat([mdf_test, df_test_cat], axis=1)
    
    #replace original column from training data
    
    del mdf_train[negtempcolumn]    
    del mdf_test[negtempcolumn]
    
    del mdf_train[tempcolumn]    
    del mdf_test[tempcolumn]
    
    #create output of a list of the created column names
#     NAcolumn = columnNAr2
    labels_train = list(df_train_cat)
#     if NAcolumn in labels_train:
#       labels_train.remove(NAcolumn)
    powercolumns = labels_train
  
    #change data type for memory savings
    for powercolumn in powercolumns:
      mdf_train[powercolumn] = mdf_train[powercolumn].astype(np.int8)
      mdf_test[powercolumn] = mdf_test[powercolumn].astype(np.int8)
    
    normalizationdictvalues = labels_train
    normalizationdictkeys = powercolumns
    
    normalizationdictkeys.sort()
    normalizationdictvalues.sort()
    
    powerlabelsdict = dict(zip(normalizationdictkeys, normalizationdictvalues))
    
    #store some values in the text_dict{} for use later in ML infill methods
    column_dict_list = []
    
    for pc in powercolumns:
      
      #new parameter collected for driftreport
      tc_ratio = pc + '_ratio'
      tcratio = mdf_train[pc].sum() / mdf_train[pc].shape[0]

      powernormalization_dict = {pc : {'powerlabelsdict_pwrs' : powerlabelsdict, \
                                       'labels_train' : labels_train, \
                                       'missing_cols' : missing_cols, \
                                       'negvalues' : negvalues, \
                                       tc_ratio : tcratio}}
    
      column_dict = {pc : {'category' : 'pwrs', \
                           'origcategory' : category, \
                           'normalization_dict' : powernormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : powercolumns, \
                           'categorylist' : powercolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}
        
      column_dict_list.append(column_dict.copy())
    
    return mdf_train, mdf_test, column_dict_list
  
  def process_pwor_class(self, mdf_train, mdf_test, column, category, postprocess_dict, params = {}):
    '''
    #processes a numerical set by creating bins coresponding to powers
    #of ten in ordinal encoded columns
    
    #pwrs will be intended for a raw set that is not yet normalized
    
    #infill has 0, other designations are based on the data
    
    #negative values based on negvalues parameter, makes comparable to por2
    '''
    
    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    if 'negvalues' in params:
      negvalues = params['negvalues']
    else:
      negvalues = False
    
    pworcolumn = column + '_pwor'
    
    if inplace is not True:
      
      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.df_copy_train(mdf_train, column, pworcolumn, suffixoverlap_results)
      
      mdf_test[pworcolumn] = mdf_test[column].copy()
    
    else:
      
      suffixoverlap_results = \
      self.df_check_suffixoverlap(mdf_train, pworcolumn, suffixoverlap_results)
      
      mdf_train.rename(columns = {column : pworcolumn}, inplace = True)
      mdf_test.rename(columns = {column : pworcolumn}, inplace = True)
    
    #convert all values to either numeric or NaN
    mdf_train[pworcolumn] = pd.to_numeric(mdf_train[pworcolumn], errors='coerce')
    mdf_test[pworcolumn] = pd.to_numeric(mdf_test[pworcolumn], errors='coerce')
    
    #copy set for negative values
    negtempcolumn = column + '_negtempcolumn'
    
    mdf_train, suffixoverlap_results = \
    self.df_copy_train(mdf_train, pworcolumn, negtempcolumn, suffixoverlap_results)
    
    mdf_test[negtempcolumn] = mdf_test[pworcolumn].copy()
    
    #convert all values >= 0 to Nan
    mdf_train[negtempcolumn] = \
    np.where(mdf_train[negtempcolumn] >= 0, np.nan, mdf_train[negtempcolumn].values)
    mdf_test[negtempcolumn] = \
    np.where(mdf_test[negtempcolumn] >= 0, np.nan, mdf_test[negtempcolumn].values)
    
    #take abs value of negtempcolumn
    mdf_train[negtempcolumn] = mdf_train[negtempcolumn].abs()
    mdf_test[negtempcolumn] = mdf_test[negtempcolumn].abs()
    
    #convert all values <= 0 in column to Nan
    mdf_train[pworcolumn] = \
    np.where(mdf_train[pworcolumn] <= 0, np.nan, mdf_train[pworcolumn].values)
    mdf_test[pworcolumn] = \
    np.where(mdf_test[pworcolumn] <= 0, np.nan, mdf_test[pworcolumn].values)

    mdf_train[pworcolumn] = \
    np.where(mdf_train[pworcolumn] != np.nan, np.floor(np.log10(mdf_train[pworcolumn])), mdf_train[pworcolumn].values)
    mdf_test[pworcolumn] = \
    np.where(mdf_test[pworcolumn] != np.nan, np.floor(np.log10(mdf_test[pworcolumn])), mdf_test[pworcolumn].values)
    
    #do same for negtempcolumn
    mdf_train[negtempcolumn] = \
    np.where(mdf_train[negtempcolumn] != np.nan, np.floor(np.log10(mdf_train[negtempcolumn])), mdf_train[negtempcolumn].values)
    mdf_test[negtempcolumn] = \
    np.where(mdf_test[negtempcolumn] != np.nan, np.floor(np.log10(mdf_test[negtempcolumn])), mdf_test[negtempcolumn].values)

    train_neg_dict = {}
    newunique_list = []
    negunique = mdf_train[negtempcolumn].unique()
    for unique in negunique:
      if unique != unique:
        newunique = np.nan
      else:
        #this is update for difference between pwr2 and pwrs
        if negvalues:
          newunique = column + '_-10^' + str(int(unique))
        else:
          newunique = np.nan
      train_neg_dict.update({unique : newunique})
      newunique_list.append(newunique)
      
    test_neg_dict = {}
    negunique = mdf_test[negtempcolumn].unique()
    for unique in negunique:
      if unique != unique:
        newunique = np.nan
      else:
        #this is update for difference between pwr2 and pwrs
        if negvalues:
          newunique = column + '_-10^' + str(int(unique))
        else:
          newunique = np.nan
      if newunique in newunique_list and newunique == newunique:
        test_neg_dict.update({unique : newunique})
      else:
        test_neg_dict.update({unique : np.nan})
        
    mdf_train[negtempcolumn] = mdf_train[negtempcolumn].replace(train_neg_dict)
    mdf_test[negtempcolumn] = mdf_test[negtempcolumn].replace(test_neg_dict)
    
    #now do same for column
    train_pos_dict = {}
    newposunique_list = []
    posunique = mdf_train[pworcolumn].unique()
    for unique in posunique:
      if unique != unique:
        newunique = np.nan
      else:
        newunique = column + '_10^' + str(int(unique))
      train_pos_dict.update({unique : newunique})
      newposunique_list.append(newunique)
      
    test_pos_dict = {}
    posunique = mdf_test[pworcolumn].unique()
    for unique in posunique:
      if unique != unique:
        newunique = np.nan
      else:
        newunique = column + '_10^' + str(int(unique))
      if newunique in newposunique_list and newunique == newunique:
        test_pos_dict.update({unique : newunique})
      else:
        test_pos_dict.update({unique : np.nan})
    
    mdf_train[pworcolumn] = mdf_train[pworcolumn].replace(train_pos_dict)
    mdf_test[pworcolumn] = mdf_test[pworcolumn].replace(test_pos_dict)
    
    #combine the two columns
    mdf_train[pworcolumn] = mdf_train[negtempcolumn].where(mdf_train[negtempcolumn] == mdf_train[negtempcolumn], mdf_train[pworcolumn])
    mdf_test[pworcolumn] = mdf_test[negtempcolumn].where(mdf_test[negtempcolumn] == mdf_test[negtempcolumn], mdf_test[pworcolumn])
    
    train_unique = mdf_train[pworcolumn].unique()
    test_unique = mdf_test[pworcolumn].unique()
  
    #Get missing entries in test set that are present in training set
    missing_cols = set( list(train_unique) ) - set( list(test_unique) )
    
    extra_cols = set( list(test_unique) ) - set( list(train_unique) )
    
    train_replace_dict = {}
    train_len = len(train_unique)
    for i in range(train_len):
      if train_unique[i] != train_unique[i]:
        train_replace_dict.update({train_unique[i] : 0})
      else:
        train_replace_dict.update({train_unique[i] : i+1})
    if np.nan not in train_replace_dict:
      train_replace_dict.update({np.nan : 0})
      
    test_replace_dict = {}
    for testunique in test_unique:
      if testunique in train_unique:
        test_replace_dict.update({testunique : train_replace_dict[testunique]})
      else:
        test_replace_dict.update({testunique : 0})
    
#     pworcolumn = column + '_por2'
#     mdf_train[pworcolumn] = mdf_train[column].copy()
#     mdf_test[pworcolumn] = mdf_test[column].copy()
    
    mdf_train[pworcolumn] = mdf_train[pworcolumn].replace(train_replace_dict)
    mdf_test[pworcolumn] = mdf_test[pworcolumn].replace(test_replace_dict)
    
    #replace original column from training data
    del mdf_train[negtempcolumn]    
    del mdf_test[negtempcolumn]    
    
#     del mdf_train[column]    
#     del mdf_test[column]
    
#     mdf_train[column] = mdf_train[column + '_temp'].copy()
#     mdf_test[column] = mdf_test[column + '_temp'].copy()

#     del mdf_train[column + '_temp']    
#     del mdf_test[column + '_temp']
        
    #store some values in the text_dict{} for use later in ML infill methods
    column_dict_list = []
    
    powercolumns = [pworcolumn]
    
    #new driftreport metric ordl_activations_dict
    ordl_activations_dict = {}
    for unique in mdf_train[pworcolumn].unique():
      sumcalc = (mdf_train[pworcolumn] == unique).sum() 
      ratio = sumcalc / mdf_train[pworcolumn].shape[0]
      ordl_activations_dict.update({unique:ratio})

    inverse_train_replace_dict = {value:key for key,value in train_replace_dict.items()}
    activations_list = list(inverse_train_replace_dict)
    
    for pc in powercolumns:

      powernormalization_dict = {pc : {'train_replace_dict' : train_replace_dict, \
                                       'inverse_train_replace_dict' : inverse_train_replace_dict, \
                                       'activations_list' : activations_list, \
                                       'test_replace_dict' : test_replace_dict, \
                                       'ordl_activations_dict' : ordl_activations_dict, \
                                       'negvalues' : negvalues}}
    
      column_dict = {pc : {'category' : 'pwor', \
                           'origcategory' : category, \
                           'normalization_dict' : powernormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : powercolumns, \
                           'categorylist' : powercolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}
        
      column_dict_list.append(column_dict.copy())
    
    return mdf_train, mdf_test, column_dict_list

  def process_bins_class(self, mdf_train, mdf_test, column, category, \
                         postprocess_dict, params = {}):
    '''
    #bins processes a numerical set by creating bins coresponding to post z score
    #normalization of <-2, -2-1, -10, 01, 12, >2 in one hot encoded columns
    
    #bins will be intended for a raw set that is not normalized
    #bint will be intended for a previously normalized set
    
    #bins accepts a parameter bincount 
    #as integer for number of bins
    #where if bincount is an odd number the center bin straddles the mean
    #and if bincount is even the center two bins straddle the mean
    #defaults to 6 bins
    #suffix appender is '_bins_#'' where # is integer for bin id
    '''
    
    suffixoverlap_results = {}
    
    if 'bincount' in params:
      bincount = params['bincount']
    else:
      bincount = 6
    
    #if data is known to be z-score normalized we'll reduce the computational overhead
    if 'normalizedinput' in params:
      normalizedinput = params['normalizedinput']
    else:
      normalizedinput = False
    
    binscolumn = column + '_bins'
    
    if bincount > 0:

      mdf_train, suffixoverlap_results = \
      self.df_copy_train(mdf_train, column, binscolumn, suffixoverlap_results)

      mdf_test[binscolumn] = mdf_test[column].copy()

      #convert all values to either numeric or NaN
      mdf_train[binscolumn] = pd.to_numeric(mdf_train[binscolumn], errors='coerce')
      mdf_test[binscolumn] = pd.to_numeric(mdf_test[binscolumn], errors='coerce')
      
      if normalizedinput is False:

        #get mean of training data
        mean = mdf_train[binscolumn].mean()

        if mean != mean:
          mean = 0

        #get standard deviation of training data
        std = mdf_train[binscolumn].std()

        #special case, if standard deviation is 0 we'll set it to 1 to avoid division by 0
        if std == 0:
          std = 1

        if std != std:
          std = 1
      
      else:
        mean = 0
        std = 1
      
      #replace missing data with training set mean
      mdf_train[binscolumn] = mdf_train[binscolumn].fillna(mean)
      mdf_test[binscolumn] = mdf_test[binscolumn].fillna(mean)
      
      if normalizedinput is False:

        #z-score normalize
        mdf_train[binscolumn] = (mdf_train[binscolumn] - mean) / std
        mdf_test[binscolumn] = (mdf_test[binscolumn] - mean) / std

      #derive cuts based on bincount

      bincuts = []

      mincut = - (bincount - 2) / 2
      bincuts.append(-float('inf'))

      for i in range(bincount - 1):
        bincuts.append(mincut)
        mincut += 1

      bincuts.append(float('inf'))

      binlabels = list(range(bincount))
      binlabels = list(map(str, binlabels))

      #create bins based on standard deviation increments
  #     binscolumn = column + '_bins'
      mdf_train[binscolumn] = \
      pd.cut( mdf_train[binscolumn], bins = bincuts,  \
             labels = binlabels, precision=4)
      mdf_test[binscolumn] = \
      pd.cut( mdf_test[binscolumn], bins = bincuts,  \
             labels = binlabels, precision=4)

      #returned column headers
      textcolumns = []
      for binlabel in binlabels:
        textcolumns.append(binscolumn + '_' + binlabel)

      suffixoverlap_results = \
      self.df_check_suffixoverlap(mdf_train, textcolumns, suffixoverlap_results)

      #process bins as a categorical set
      mdf_train = \
      self.postprocess_textsupport_class(mdf_train, binscolumn, {}, 'tempkey', {'textcolumns':textcolumns})
      mdf_test = \
      self.postprocess_textsupport_class(mdf_test, binscolumn, {}, 'tempkey', {'textcolumns':textcolumns})

      #change data type for memory savings
      for textcolumn in textcolumns:
        mdf_train[textcolumn] = mdf_train[textcolumn].astype(np.int8)
        mdf_test[textcolumn] = mdf_test[textcolumn].astype(np.int8)

      #delete the support column
      del mdf_train[binscolumn]
      del mdf_test[binscolumn]
    
      #store some values in the nmbr_dict{} for use later in ML infill methods
      column_dict_list = []
      
      for nc in textcolumns:

        #new parameter collected for driftreport
        tc_ratio = nc + '_ratio'
        tcratio = mdf_train[nc].sum() / mdf_train[nc].shape[0]

        nmbrnormalization_dict = {nc : {'bincuts' : bincuts, \
                                        'binlabels' : binlabels, \
                                        'binscolumns' : textcolumns, \
                                        'bincount' : bincount, \
                                        'binsmean' : mean, \
                                        'binsstd' : std, \
                                        'normalizedinput' : normalizedinput, \
                                        tc_ratio : tcratio}}

        column_dict = { nc : {'category' : 'bins', \
                              'origcategory' : category, \
                              'normalization_dict' : nmbrnormalization_dict, \
                              'origcolumn' : column, \
                              'inputcolumn' : column, \
                              'columnslist' : textcolumns, \
                              'categorylist' : textcolumns, \
                              'infillmodel' : False, \
                              'infillcomplete' : False, \
                              'suffixoverlap_results' : suffixoverlap_results, \
                              'deletecolumn' : False}}

        column_dict_list.append(column_dict.copy())
          
    else:
      
      column_dict_list = []

    return mdf_train, mdf_test, column_dict_list

  def process_bsor_class(self, mdf_train, mdf_test, column, category, postprocess_dict, params = {}):
    '''
    #bins processes a numerical set by creating bins coresponding to post z score
    #normalization of <-2, -2-1, -10, 01, 12, >2 in one hot encoded columns
    
    #bins accepts a parameter bincount 
    #as integer for number of bins
    #where if bincount is an odd number the center bin straddles the mean
    #and if bincount is even the center two bins straddle the mean
    #defaults to 5 bins
    #suffix appender is '_bins_#'' where # is integer for bin id
    
    #bsor is comparable to bins but returns ordinal encoded column
    '''
    
    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    if 'bincount' in params:
      bincount = params['bincount']
    else:
      bincount = 6
      
    #if data is known to be z-score normalized we'll reduce the computational overhead
    if 'normalizedinput' in params:
      normalizedinput = params['normalizedinput']
    else:
      normalizedinput = False
    
    binscolumn = column + '_bsor'
    
    if bincount > 0:

      if inplace is not True:

        #copy source column into new column
        mdf_train, suffixoverlap_results = \
        self.df_copy_train(mdf_train, column, binscolumn, suffixoverlap_results)

        mdf_test[binscolumn] = mdf_test[column].copy()

      else:

        suffixoverlap_results = \
        self.df_check_suffixoverlap(mdf_train, binscolumn, suffixoverlap_results)

        mdf_train.rename(columns = {column : binscolumn}, inplace = True)
        mdf_test.rename(columns = {column : binscolumn}, inplace = True)
      
      #convert all values to either numeric or NaN
      mdf_train[binscolumn] = pd.to_numeric(mdf_train[binscolumn], errors='coerce')
      mdf_test[binscolumn] = pd.to_numeric(mdf_test[binscolumn], errors='coerce')
      
      if normalizedinput is False:

        #get mean of training data
        mean = mdf_train[binscolumn].mean()
        if mean != mean:
          mean = 0

        #get standard deviation of training data
        std = mdf_train[binscolumn].std()
        if std == 0:
          std = 1
        if std != std:
          std = 1
          
      else:
        
        mean = 0
        std = 1

      #replace missing data with training set mean
      mdf_train[binscolumn] = mdf_train[binscolumn].fillna(mean)
      mdf_test[binscolumn] = mdf_test[binscolumn].fillna(mean)
      
      if normalizedinput is False:

        #z-score normalize
        mdf_train[binscolumn] = (mdf_train[binscolumn] - mean) / std
        mdf_test[binscolumn] = (mdf_test[binscolumn] - mean) / std
      
      #derive cuts based on bincount

      bincuts = []

      mincut = - (bincount - 2) / 2
      bincuts.append(-float('inf'))

      for i in range(bincount - 1):
        bincuts.append(mincut)
        mincut += 1

      bincuts.append(float('inf'))

      binlabels = list(range(bincount))

  #     binscolumn = column + '_bsor'
      mdf_train[binscolumn] = \
      pd.cut( mdf_train[binscolumn], bins = bincuts,  \
             labels = binlabels, precision=4)
      mdf_test[binscolumn] = \
      pd.cut( mdf_test[binscolumn], bins = bincuts,  \
             labels = binlabels, precision=4)

      ordinal_dict = {}
      for binlabel in binlabels:
        ordinal_dict.update({str(binlabel) : binlabel})

      #new driftreport metric ordl_activations_dict
      ordl_activations_dict = {}
      for key in ordinal_dict:
        sumcalc = (mdf_train[binscolumn] == ordinal_dict[key]).sum() 
        ratio = sumcalc / mdf_train[binscolumn].shape[0]
        ordl_activations_dict.update({key:ratio})

      inverse_ordinal_dict = {value:key for key,value in ordinal_dict.items()}
      activations_list = list(inverse_ordinal_dict)

      mdf_train[binscolumn] = mdf_train[binscolumn].astype(np.int8)
      mdf_test[binscolumn] = mdf_test[binscolumn].astype(np.int8)

      #create list of columns
      nmbrcolumns = [binscolumn]

      #nmbrnormalization_dict = {'mean' : mean, 'std' : std}

      #store some values in the nmbr_dict{} for use later in ML infill methods
      column_dict_list = []

      for nc in nmbrcolumns:

        nmbrnormalization_dict = {nc : {'ordinal_dict' : ordinal_dict, \
                                        'inverse_ordinal_dict' : inverse_ordinal_dict, \
                                        'activations_list' : activations_list, \
                                        'ordl_activations_dict' : ordl_activations_dict, \
                                        'binsmean' : mean, \
                                        'binsstd' : std, \
                                        'normalizedinput' : normalizedinput, \
                                        'bincount' : bincount, \
                                        'bincuts' : bincuts, \
                                        'binlabels' : binlabels}}

        column_dict = { nc : {'category' : 'bsor', \
                              'origcategory' : category, \
                              'normalization_dict' : nmbrnormalization_dict, \
                              'origcolumn' : column, \
                              'inputcolumn' : column, \
                              'columnslist' : nmbrcolumns, \
                              'categorylist' : nmbrcolumns, \
                              'infillmodel' : False, \
                              'infillcomplete' : False, \
                              'suffixoverlap_results' : suffixoverlap_results, \
                              'deletecolumn' : False}}

        column_dict_list.append(column_dict.copy())

    else:
      
      column_dict_list = []
    
    return mdf_train, mdf_test, column_dict_list
  
  def process_bnwd_class(self, mdf_train, mdf_test, column, category, postprocess_dict, params = {}):
    '''
    #processes a numerical set by creating equal width bins coresponding to 
    #parameter 'width' which defaults to 1
    #and returning in one-hot encoded set
    
    #deletes columns without activations
    
    #can be applied top either a raw set not yet normalized or after normalization
    #such as after z-score normalization)
    '''
    
    suffixoverlap_results = {}
    
    if 'width' in params:
      bn_width = params['width']
    else:
      bn_width = 1
      
    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = '_bnwd'
      
    binscolumn = column + suffix

    #store original column for later reversion
    mdf_train, suffixoverlap_results = \
    self.df_copy_train(mdf_train, column, binscolumn, suffixoverlap_results)
    
    mdf_test[binscolumn] = mdf_test[column].copy()

    #convert all values to either numeric or NaN
    mdf_train[binscolumn] = pd.to_numeric(mdf_train[binscolumn], errors='coerce')
    mdf_test[binscolumn] = pd.to_numeric(mdf_test[binscolumn], errors='coerce')

    #get mean of training data
    mean = mdf_train[binscolumn].mean()
    
    if mean != mean:
      mean = 0

    #replace missing data with training set mean
    mdf_train[binscolumn] = mdf_train[binscolumn].fillna(mean)
    mdf_test[binscolumn] = mdf_test[binscolumn].fillna(mean)

    #evaluate train set for transformation parameters
    bn_min = mdf_train[binscolumn].min()
    bn_max = mdf_train[binscolumn].max()
    bn_delta = bn_max - bn_min
    if bn_delta == 0:
      bn_delta = 1
    bn_count = int(np.ceil(bn_delta / bn_width))
    
    bins_id = []
    for i in range(bn_count):
      bins_id.append(str(bn_width) + '_' + str(i))
      
    bins_cuts = [-float('inf'), float('inf')]
    for i in range(bn_count-1):
      bins_cuts.insert(-1,(bn_min + (i+1) * bn_width))
      
    #create bins based on standard deviation increments
#     binscolumn = column + '_bnwd'
    mdf_train[binscolumn] = \
    pd.cut(mdf_train[binscolumn], bins = bins_cuts,  \
           labels = bins_id, precision=len(str(bn_count)))
    mdf_test[binscolumn] = \
    pd.cut(mdf_test[binscolumn], bins = bins_cuts,  \
           labels = bins_id, precision=len(str(bn_count)))

    foundinset = mdf_train[binscolumn].unique()
    
    textcolumns = []
    for i in foundinset:
      textcolumns.append(binscolumn + '_' + str(i))
      
    #postprocess_textsupport_class will return columns in alphabetical order
    textcolumns.sort()
    
    suffixoverlap_results = \
    self.df_check_suffixoverlap(mdf_train, textcolumns, suffixoverlap_results)
    
    #process bins as a categorical set
    mdf_train = \
    self.postprocess_textsupport_class(mdf_train, binscolumn, {}, 'tempkey', {'textcolumns':textcolumns})
    mdf_test = \
    self.postprocess_textsupport_class(mdf_test, binscolumn, {}, 'tempkey', {'textcolumns':textcolumns})
    
    #change data type for memory savings
    for textcolumn in textcolumns:
      mdf_train[textcolumn] = mdf_train[textcolumn].astype(np.int8)
      mdf_test[textcolumn] = mdf_test[textcolumn].astype(np.int8)
    
    #delete the support column
    del mdf_train[binscolumn]
    del mdf_test[binscolumn]

    #nmbrnormalization_dict = {'mean' : mean, 'std' : std}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in textcolumns:
      
      #new parameter collected for driftreport
      tc_ratio = nc + '_ratio'
      tcratio = mdf_train[nc].sum() / mdf_train[nc].shape[0]

      nmbrnormalization_dict = {nc : {'suffix' : suffix, \
                                      'binsmean' : mean, \
                                      'bn_min' : bn_min, \
                                      'bn_max' : bn_max, \
                                      'bn_delta' : bn_delta, \
                                      'bn_count' : bn_count, \
                                      'bins_id' : bins_id, \
                                      'bins_cuts' : bins_cuts, \
                                      'bn_width_bnwd' : bn_width, \
                                      'textcolumns' : textcolumns, \
                                      tc_ratio : tcratio}}

      column_dict = { nc : {'category' : 'bnwd', \
                            'origcategory' : category, \
                            'normalization_dict' : nmbrnormalization_dict, \
                            'origcolumn' : column, \
                            'inputcolumn' : column, \
                            'columnslist' : textcolumns, \
                            'categorylist' : textcolumns, \
                            'infillmodel' : False, \
                            'infillcomplete' : False, \
                            'suffixoverlap_results' : suffixoverlap_results, \
                            'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())
       
    return mdf_train, mdf_test, column_dict_list

  def process_bnwo_class(self, mdf_train, mdf_test, column, category, postprocess_dict, params = {}):
    '''
    #processes a numerical set by creating equal width bins coresponding to 
    #parameter 'width' which defaults to 1
    #and returning in ordinal encoded set
    
    #segments without activations are included
    
    #can be applied top either a raw set not yet normalized or after normalization
    #such as after z-score normalization)
    '''
    
    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    if 'width' in params:
      bn_width = params['width']
    else:
      bn_width = 1
      
    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = '_bnwo'
      
    binscolumn = column + suffix

    if inplace is not True:

      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.df_copy_train(mdf_train, column, binscolumn, suffixoverlap_results)

      mdf_test[binscolumn] = mdf_test[column].copy()

    else:

      suffixoverlap_results = \
      self.df_check_suffixoverlap(mdf_train, binscolumn, suffixoverlap_results)

      mdf_train.rename(columns = {column : binscolumn}, inplace = True)
      mdf_test.rename(columns = {column : binscolumn}, inplace = True)
    
    #convert all values to either numeric or NaN
    mdf_train[binscolumn] = pd.to_numeric(mdf_train[binscolumn], errors='coerce')
    mdf_test[binscolumn] = pd.to_numeric(mdf_test[binscolumn], errors='coerce')

    #get mean of training data
    mean = mdf_train[binscolumn].mean()
    
    if mean != mean:
      mean = 0

    #replace missing data with training set mean
    mdf_train[binscolumn] = mdf_train[binscolumn].fillna(mean)
    mdf_test[binscolumn] = mdf_test[binscolumn].fillna(mean)

    #evaluate train set for transformation parameters
    bn_min = mdf_train[binscolumn].min()
    bn_max = mdf_train[binscolumn].max()
    bn_delta = bn_max - bn_min
    if bn_delta == 0:
      bn_delta = 1
    bn_count = int(np.ceil(bn_delta / bn_width))
    
    bins_id = []
    for i in range(bn_count):
      bins_id.append(i)
      
    bins_cuts = [-float('inf'), float('inf')]
    for i in range(bn_count-1):
      bins_cuts.insert(-1,(bn_min + (i+1) * bn_width))
      
    #create bins based on standard deviation increments
#     binscolumn = column + '_bnwo'
    mdf_train[binscolumn] = \
    pd.cut(mdf_train[binscolumn], bins = bins_cuts,  \
           labels = bins_id, precision=len(str(bn_count)))
    mdf_test[binscolumn] = \
    pd.cut(mdf_test[binscolumn], bins = bins_cuts,  \
           labels = bins_id, precision=len(str(bn_count)))
    
    #change column dtype
    mdf_train[binscolumn] = mdf_train[binscolumn].astype(int)
    mdf_test[binscolumn] = mdf_test[binscolumn].astype(int)

    #create list of columns
    nmbrcolumns = [binscolumn]
    
    #new driftreport metric ordl_activations_dict
    ordl_activations_dict = {}
    for unique in mdf_train[binscolumn].unique():
      sumcalc = (mdf_train[binscolumn] == unique).sum() 
      ratio = sumcalc / mdf_train[binscolumn].shape[0]
      ordl_activations_dict.update({unique:ratio})

    activations_list = list(ordl_activations_dict)

    #nmbrnormalization_dict = {'mean' : mean, 'std' : std}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in nmbrcolumns:

      nmbrnormalization_dict = {nc : {'suffix' : suffix, \
                                      'binsmean' : mean, \
                                      'bn_min' : bn_min, \
                                      'bn_max' : bn_max, \
                                      'bn_delta' : bn_delta, \
                                      'bn_count' : bn_count, \
                                      'bins_id' : bins_id, \
                                      'bins_cuts' : bins_cuts, \
                                      'bn_width' : bn_width, \
                                      'activations_list' : activations_list, \
                                      'ordl_activations_dict' : ordl_activations_dict}}

      column_dict = { nc : {'category' : 'bnwo', \
                            'origcategory' : category, \
                            'normalization_dict' : nmbrnormalization_dict, \
                            'origcolumn' : column, \
                            'inputcolumn' : column, \
                            'columnslist' : nmbrcolumns, \
                            'categorylist' : nmbrcolumns, \
                            'infillmodel' : False, \
                            'infillcomplete' : False, \
                            'suffixoverlap_results' : suffixoverlap_results, \
                            'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())

    return mdf_train, mdf_test, column_dict_list

  def process_bnep_class(self, mdf_train, mdf_test, column, category, postprocess_dict, params = {}):
    '''
    #processes a numerical set by creating equal population bins coresponding to 
    #parameter 'bincount' which defaults to 5
    #and returning in one-hot encoded set
    
    #default infill is to have no activations in a row
    
    #can be applied top either a raw set not yet normalized or after normalization
    #such as after z-score normalization)
    '''
    
    suffixoverlap_results = {}
    
    if 'bincount' in params:
      bincount = params['bincount']
    else:
      bincount = 5
      
    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = '_bnep'
      
    binscolumn = column + suffix

    #copy original column
    mdf_train, suffixoverlap_results = \
    self.df_copy_train(mdf_train, column, binscolumn, suffixoverlap_results)
    
    mdf_test[binscolumn] = mdf_test[column].copy()

    #convert all values to either numeric or NaN
    mdf_train[binscolumn] = pd.to_numeric(mdf_train[binscolumn], errors='coerce')
    mdf_test[binscolumn] = pd.to_numeric(mdf_test[binscolumn], errors='coerce')

    #get mean of training data
    mean = mdf_train[binscolumn].mean()
    
    if mean != mean:
      mean = 0

#     #replace missing data with training set mean
#     mdf_train[binscolumn] = mdf_train[binscolumn].fillna(mean)
#     mdf_test[binscolumn] = mdf_test[binscolumn].fillna(mean)

    #evaluate train set for transformation parameters
    bn_min = mdf_train[binscolumn].min()
    bn_max = mdf_train[binscolumn].max()
    bn_delta = bn_max - bn_min
#     if bn_delta == 0:
#       bn_delta = 1
#     bn_count = int(np.ceil(bn_delta / bn_width))

#     if bn_delta > 0 and bn_min == bn_min:

    if bn_delta > 0 and bn_min == bn_min:

      #grab the intervals using qcut based on equal population in train set
      intervalset = pd.qcut(mdf_train[binscolumn].values, bincount, duplicates='drop').unique()

      #note we're sorting here, and scrubbing any nan
      intervalset = sorted([interval for interval in intervalset if interval == interval])

      #we'll make the bottom interval open-ended at negative end
      firstinterval = pd.Interval(-np.inf, intervalset[0].right, closed='right')

      #and the last interval open-ended at positive end
      lastinterval = pd.Interval(intervalset[-1].left, np.inf, closed='right')

      #now create a list to apply in cut operatoin
      newinterval_list = []

      #now we'll assemble a list of intervals to prepare for the cut operation, 
      #replacing first and last with the open-ended
      for i in range(len(intervalset)):
        if i == 0:
          newinterval_list.append(firstinterval)
        elif i == len(intervalset)-1:
          newinterval_list.append(lastinterval)
        else:
          newinterval_list.append(intervalset[i])

      #now translate intervals to the list of boundaries for cut operatoin
      cutintervals = []

      for interval in newinterval_list:

        cutintervals.append(interval.left)

      cutintervals.append(np.inf)

      bn_count = len(newinterval_list)

      bins_id = []
      for i in range(bn_count):
        bins_id.append(str(i))

      bins_cuts = cutintervals

      #create bins based on standard deviation increments
  #     binscolumn = column + '_bnwd'
      mdf_train[binscolumn] = \
      pd.cut(mdf_train[binscolumn], bins = bins_cuts,  \
             labels = bins_id, precision=len(str(bn_count)), duplicates='drop')
      mdf_test[binscolumn] = \
      pd.cut(mdf_test[binscolumn], bins = bins_cuts,  \
             labels = bins_id, precision=len(str(bn_count)), duplicates='drop')

      foundinset = mdf_train[binscolumn].unique()

      textcolumns = []
      for i in foundinset:
        if i == i:
          textcolumns.append(binscolumn + '_' + str(i))

      #postprocess_textsupport_class will return columns in alphabetical order
      textcolumns.sort()
      
      suffixoverlap_results = \
      self.df_check_suffixoverlap(mdf_train, textcolumns, suffixoverlap_results)

      #process bins as a categorical set
      mdf_train = \
      self.postprocess_textsupport_class(mdf_train, binscolumn, {}, 'tempkey', {'textcolumns':textcolumns})
      mdf_test = \
      self.postprocess_textsupport_class(mdf_test, binscolumn, {}, 'tempkey', {'textcolumns':textcolumns})

      #change data type for memory savings
      for textcolumn in textcolumns:
        mdf_train[textcolumn] = mdf_train[textcolumn].astype(np.int8)
        mdf_test[textcolumn] = mdf_test[textcolumn].astype(np.int8)

      #delete the support column
      del mdf_train[binscolumn]
      del mdf_test[binscolumn]
      
    else:
      mdf_train[binscolumn] = 0
      mdf_test[binscolumn] = 0
      
      textcolumns = [binscolumn]
      
      bn_count = bincount
      bins_id = False
      bins_cuts = False

    #nmbrnormalization_dict = {'mean' : mean, 'std' : std}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in textcolumns:
      
      #new parameter collected for driftreport
      tc_ratio = nc + '_ratio'
      tcratio = mdf_train[nc].sum() / mdf_train[nc].shape[0]

      nmbrnormalization_dict = {nc : {'suffix' : suffix, \
                                      'binsmean' : mean, \
                                      'bn_min' : bn_min, \
                                      'bn_max' : bn_max, \
                                      'bn_delta' : bn_delta, \
                                      'bn_count' : bn_count, \
                                      'bins_id' : bins_id, \
                                      'bins_cuts' : bins_cuts, \
                                      'bincount_bnep' : bincount, \
                                      'textcolumns' : textcolumns, \
                                      tc_ratio : tcratio}}

      column_dict = { nc : {'category' : 'bnep', \
                            'origcategory' : category, \
                            'normalization_dict' : nmbrnormalization_dict, \
                            'origcolumn' : column, \
                            'inputcolumn' : column, \
                            'columnslist' : textcolumns, \
                            'categorylist' : textcolumns, \
                            'infillmodel' : False, \
                            'infillcomplete' : False, \
                            'suffixoverlap_results' : suffixoverlap_results, \
                            'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())
       
    return mdf_train, mdf_test, column_dict_list

  def process_bneo_class(self, mdf_train, mdf_test, column, category, postprocess_dict, params = {}):
    '''
    #processes a numerical set by creating equal population bins coresponding to 
    #parameter 'bincount' which defaults to 5
    #and returning in ordinal encoded set
    
    #default infill is adjacent cell infill
    
    #can be applied top either a raw set not yet normalized or after normalization
    #such as after z-score normalization)
    '''
    
    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    if 'bincount' in params:
      bincount = params['bincount']
    else:
      bincount = 5
      
    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = '_bneo'
      
    binscolumn = column + suffix

    if inplace is not True:

      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.df_copy_train(mdf_train, column, binscolumn, suffixoverlap_results)

      mdf_test[binscolumn] = mdf_test[column].copy()

    else:

      suffixoverlap_results = \
      self.df_check_suffixoverlap(mdf_train, binscolumn, suffixoverlap_results)

      mdf_train.rename(columns = {column : binscolumn}, inplace = True)
      mdf_test.rename(columns = {column : binscolumn}, inplace = True)

    #convert all values to either numeric or NaN
    mdf_train[binscolumn] = pd.to_numeric(mdf_train[binscolumn], errors='coerce')
    mdf_test[binscolumn] = pd.to_numeric(mdf_test[binscolumn], errors='coerce')

    #get mean of training data
    mean = mdf_train[binscolumn].mean()
    
    if mean != mean:
      mean = 0

#     #replace missing data with training set mean
#     mdf_train[binscolumn] = mdf_train[binscolumn].fillna(mean)
#     mdf_test[binscolumn] = mdf_test[binscolumn].fillna(mean)

    #evaluate train set for transformation parameters
    bn_min = mdf_train[binscolumn].min()
    bn_max = mdf_train[binscolumn].max()
    bn_delta = bn_max - bn_min
#     if bn_delta == 0:
#       bn_delta = 1
#     bn_count = int(np.ceil(bn_delta / bn_width))

    if bn_delta > 0 and bn_min == bn_min:

      #grab the intervals using qcut based on equal population in train set
      intervalset = pd.qcut(mdf_train[binscolumn].values, bincount, duplicates='drop').unique()

      #note we're sorting here, and scrubbing any nan
      intervalset = sorted([interval for interval in intervalset if interval == interval])

      #we'll make the bottom interval open-ended at negative end
      firstinterval = pd.Interval(-np.inf, intervalset[0].right, closed='right')

      #and the last interval open-ended at positive end
      lastinterval = pd.Interval(intervalset[-1].left, np.inf, closed='right')

      #now create a list to apply in cut operation
      newinterval_list = []

      #now we'll assemble a list of intervals to prepare for the cut operation, 
      #replacing first and last with the open-ended
      for i in range(len(intervalset)):
        if i == 0:
          newinterval_list.append(firstinterval)
        elif i == len(intervalset)-1:
          newinterval_list.append(lastinterval)
        else:
          newinterval_list.append(intervalset[i])

      #now translate intervals to the list of boundaries for cut operatoin
      cutintervals = []

      for interval in newinterval_list:

        cutintervals.append(interval.left)

      cutintervals.append(np.inf)

      bn_count = len(newinterval_list)

      bins_id = []
      for i in range(bn_count):
        bins_id.append(i)

      bins_cuts = cutintervals

      #create bins based on prepared increments
      mdf_train[binscolumn] = \
      pd.cut(mdf_train[binscolumn], bins = bins_cuts,  \
             labels = bins_id, precision=len(str(bn_count)), duplicates='drop')
      mdf_test[binscolumn] = \
      pd.cut(mdf_test[binscolumn], bins = bins_cuts,  \
             labels = bins_id, precision=len(str(bn_count)), duplicates='drop')

      #apply ffill to replace NArows with value from adjacent cell in pre4ceding row
      mdf_train[binscolumn] = mdf_train[binscolumn].fillna(method='ffill')
      mdf_test[binscolumn] = mdf_test[binscolumn].fillna(method='ffill')

      #we'll follow with a bfill just in case first row had a nan
      mdf_train[binscolumn] = mdf_train[binscolumn].fillna(method='bfill')
      mdf_test[binscolumn] = mdf_test[binscolumn].fillna(method='bfill')

      #and if the entire set was nan we'll infill with a 0 plug
      mdf_train[binscolumn] = mdf_train[binscolumn].fillna(0)
      mdf_test[binscolumn] = mdf_test[binscolumn].fillna(0)

      #change column dtype
      mdf_train[binscolumn] = mdf_train[binscolumn].astype(int)
      mdf_test[binscolumn] = mdf_test[binscolumn].astype(int)
      
    else:
      
      mdf_train[binscolumn] = 0
      mdf_test[binscolumn] = 0
      
      bn_count = bincount
      bins_id = False
      bins_cuts = False

    #create list of columns
    nmbrcolumns = [binscolumn]

    #new driftreport metric ordl_activations_dict
    ordl_activations_dict = {}
    for unique in mdf_train[binscolumn].unique():
      sumcalc = (mdf_train[binscolumn] == unique).sum() 
      ratio = sumcalc / mdf_train[binscolumn].shape[0]
      ordl_activations_dict.update({unique:ratio})

    #nmbrnormalization_dict = {'mean' : mean, 'std' : std}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in nmbrcolumns:

      nmbrnormalization_dict = {nc : {'suffix' : suffix, \
                                      'binsmean' : mean, \
                                      'bn_min' : bn_min, \
                                      'bn_max' : bn_max, \
                                      'bn_delta' : bn_delta, \
                                      'bn_count' : bn_count, \
                                      'bins_id' : bins_id, \
                                      'activations_list' : bins_id, \
                                      'bins_cuts' : bins_cuts, \
                                      'bincount' : bincount, \
                                      'ordl_activations_dict' : ordl_activations_dict}}

      column_dict = { nc : {'category' : 'bneo', \
                            'origcategory' : category, \
                            'normalization_dict' : nmbrnormalization_dict, \
                            'origcolumn' : column, \
                            'inputcolumn' : column, \
                            'columnslist' : nmbrcolumns, \
                            'categorylist' : nmbrcolumns, \
                            'infillmodel' : False, \
                            'infillcomplete' : False, \
                            'suffixoverlap_results' : suffixoverlap_results, \
                            'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())

    return mdf_train, mdf_test, column_dict_list
  
  def process_tlbn_class(self, mdf_train, mdf_test, column, category, postprocess_dict, params = {}):
    '''
    #processes a numerical set by creating equal population bins coresponding to 
    #parameter 'bincount' which defaults to 9
    #and returning in one-hot encoded set
    
    #how this differs from bnep in that the activated bins are replaced with
    #min-max scaling for source column values found in that bin, and then other values as -1
    
    #note that for the bottom bin order reversed to accomodate subsequent values out of range 
    #and still use -1 register
    
    #default infill is to have no activations in a row
    
    #can be applied top either a raw set not yet normalized or after normalization
    #such as after z-score normalization)
    '''
    
    suffixoverlap_results = {}
    
    if 'bincount' in params:
        
      bincount = params['bincount']
    
    else:
      
      bincount = 9

    binscolumn = column + '_tlbn'

    #copy original column
    mdf_train, suffixoverlap_results = \
    self.df_copy_train(mdf_train, column, binscolumn, suffixoverlap_results)
    
    mdf_test[binscolumn] = mdf_test[column].copy()

    #convert all values to either numeric or NaN
    mdf_train[binscolumn] = pd.to_numeric(mdf_train[binscolumn], errors='coerce')
    mdf_test[binscolumn] = pd.to_numeric(mdf_test[binscolumn], errors='coerce')

    #get mean of training data
    mean = mdf_train[binscolumn].mean()
    
    if mean != mean:
      mean = 0

#     #replace missing data with training set mean
#     mdf_train[binscolumn] = mdf_train[binscolumn].fillna(mean)
#     mdf_test[binscolumn] = mdf_test[binscolumn].fillna(mean)

    #evaluate train set for transformation parameters
    bn_min = mdf_train[binscolumn].min()
    bn_max = mdf_train[binscolumn].max()
    bn_delta = bn_max - bn_min
#     if bn_delta == 0:
#       bn_delta = 1
#     bn_count = int(np.ceil(bn_delta / bn_width))

    if bn_delta > 0 and bn_min == bn_min:

      #grab the intervals using qcut based on equal population in train set
      intervalset = pd.qcut(mdf_train[binscolumn].values, bincount, duplicates='drop').unique()

      #note we're sorting here, and scrubbing any nan
      intervalset = sorted([interval for interval in intervalset if interval == interval])

      #we'll make the bottom interval open-ended at negative end
      firstinterval = pd.Interval(-np.inf, intervalset[0].right, closed='right')

      #and the last interval open-ended at positive end
      lastinterval = pd.Interval(intervalset[-1].left, np.inf, closed='right')

      #now create a list to apply in cut operatoin
      newinterval_list = []

      #now we'll assemble a list of intervals to prepare for the cut operation, 
      #replacing first and last with the open-ended
      for i in range(len(intervalset)):
        if i == 0:
          newinterval_list.append(firstinterval)
        elif i == len(intervalset)-1:
          newinterval_list.append(lastinterval)
        else:
          newinterval_list.append(intervalset[i])

      #now translate intervals to the list of boundaries for cut operatoin
      cutintervals = []

      for interval in newinterval_list:

        cutintervals.append(interval.left)

      cutintervals.append(np.inf)

      bn_count = len(newinterval_list)

      bins_id = []
      for i in range(bn_count):
        bins_id.append(str(i))

      bins_cuts = cutintervals

      #create bins based on standard deviation increments
  #     binscolumn = column + '_bnwd'
      mdf_train[binscolumn] = \
      pd.cut(mdf_train[binscolumn], bins = bins_cuts,  \
             labels = bins_id, precision=len(str(bn_count)), duplicates='drop')
      mdf_test[binscolumn] = \
      pd.cut(mdf_test[binscolumn], bins = bins_cuts,  \
             labels = bins_id, precision=len(str(bn_count)), duplicates='drop')

      foundinset = mdf_train[binscolumn].unique()

      textcolumns = []
      for i in foundinset:
        if i == i:
          textcolumns.append(binscolumn + '_' + str(i))

      #postprocess_textsupport_class will return columns in alphabetical order
      textcolumns.sort()
      
      suffixoverlap_results = \
      self.df_check_suffixoverlap(mdf_train, textcolumns, suffixoverlap_results)

      #process bins as a categorical set
      mdf_train = \
      self.postprocess_textsupport_class(mdf_train, binscolumn, {}, 'tempkey', {'textcolumns':textcolumns})
      mdf_test = \
      self.postprocess_textsupport_class(mdf_test, binscolumn, {}, 'tempkey', {'textcolumns':textcolumns})
      
      #initialize binscolumn once more
      mdf_train[binscolumn] = mdf_train[column].copy()
      mdf_test[binscolumn] = mdf_test[column].copy()
      
      mdf_train[binscolumn] = pd.to_numeric(mdf_train[binscolumn], errors='coerce')
      mdf_test[binscolumn] = pd.to_numeric(mdf_test[binscolumn], errors='coerce')
      
      if len(textcolumns) > 1:

        #for i in range(bincount):
        for i in range(len(textcolumns)):

          tlbn_column = binscolumn + '_' + str(i)

          if i == 0:

            mdf_train[tlbn_column] = \
            np.where(mdf_train[tlbn_column] == 1, \
                    (bins_cuts[i+1] - mdf_train[binscolumn]) / (bins_cuts[i+1] - bn_min), -1)

            mdf_test[tlbn_column] = \
            np.where(mdf_test[tlbn_column] == 1, \
                    (bins_cuts[i+1] - mdf_test[binscolumn]) / (bins_cuts[i+1] - bn_min), -1)

          elif i == bincount - 1:

            mdf_train[tlbn_column] = \
            np.where(mdf_train[tlbn_column] == 1, \
                    (mdf_train[binscolumn] - bins_cuts[i]) / (bn_max - bins_cuts[i]), -1)

            mdf_test[tlbn_column] = \
            np.where(mdf_test[tlbn_column] == 1, \
                    (mdf_test[binscolumn] - bins_cuts[i]) / (bn_max - bins_cuts[i]), -1)

          else:

            mdf_train[tlbn_column] = \
            np.where(mdf_train[tlbn_column] == 1, \
                    (mdf_train[binscolumn] - bins_cuts[i]) / (bins_cuts[i+1] - bins_cuts[i]), -1)

            mdf_test[tlbn_column] = \
            np.where(mdf_test[tlbn_column] == 1, \
                    (mdf_test[binscolumn] - bins_cuts[i]) / (bins_cuts[i+1] - bins_cuts[i]), -1)

#       #change data type for memory savings
#       for textcolumn in textcolumns:
#         mdf_train[textcolumn] = mdf_train[textcolumn].astype(np.int8)
#         mdf_test[textcolumn] = mdf_test[textcolumn].astype(np.int8)

      #delete the support column
      del mdf_train[binscolumn]
      del mdf_test[binscolumn]
      
      
    else:
      mdf_train[binscolumn] = 0
      mdf_test[binscolumn] = 0
      
      textcolumns = [binscolumn]
      
      bn_count = bincount
      bins_id = False
      bins_cuts = False

    #nmbrnormalization_dict = {'mean' : mean, 'std' : std}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in textcolumns:

      #new parameter collected for driftreport
      tc_ratio = nc + '_ratio'
      tcratio = mdf_train[nc].sum() / mdf_train[nc].shape[0]

      nmbrnormalization_dict = {nc : {'binsmean' : mean, \
                                      'bn_min' : bn_min, \
                                      'bn_max' : bn_max, \
                                      'bn_delta' : bn_delta, \
                                      'bn_count' : bn_count, \
                                      'bins_id' : bins_id, \
                                      'bins_cuts' : bins_cuts, \
                                      'bincount_tlbn' : bincount, \
                                      'textcolumns' : textcolumns, \
                                      tc_ratio : tcratio}}

      column_dict = { nc : {'category' : 'tlbn', \
                            'origcategory' : category, \
                            'normalization_dict' : nmbrnormalization_dict, \
                            'origcolumn' : column, \
                            'inputcolumn' : column, \
                            'columnslist' : textcolumns, \
                            'categorylist' : textcolumns, \
                            'infillmodel' : False, \
                            'infillcomplete' : False, \
                            'suffixoverlap_results' : suffixoverlap_results, \
                            'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())
       
    return mdf_train, mdf_test, column_dict_list
  
  def process_bkt1_class(self, mdf_train, mdf_test, column, category, postprocess_dict, params = {}):
    '''
    #processes a numerical set by creating custom bins coresponding to 
    #parameter 'buckets' which defaults to [0,1]
    #first and last buckets unconstrained
    #and returning in one-hot encoded set
    
    #removes buckets without activations in train set
    '''
    
    suffixoverlap_results = {}
    
    if 'buckets' in params:
        
      buckets = params['buckets']
    
    else:
      
      buckets = [0,1,2]
      
    binscolumn = column + '_bkt1'

    #store original column for later reversion
    mdf_train, suffixoverlap_results = \
    self.df_copy_train(mdf_train, column, binscolumn, suffixoverlap_results)
    
    mdf_test[binscolumn] = mdf_test[column].copy()

    #convert all values to either numeric or NaN
    mdf_train[binscolumn] = pd.to_numeric(mdf_train[binscolumn], errors='coerce')
    mdf_test[binscolumn] = pd.to_numeric(mdf_test[binscolumn], errors='coerce')

    #get mean of training data
    mean = mdf_train[binscolumn].mean()
    
    if mean != mean:
      mean = 0

    # #replace missing data with training set mean
    # mdf_train[binscolumn] = mdf_train[binscolumn].fillna(mean)
    # mdf_test[binscolumn] = mdf_test[binscolumn].fillna(mean)

    #assemble buckets  
    bins_cuts = buckets.copy()
    bins_cuts.insert(0, -np.inf)
    bins_cuts.insert(len(bins_cuts), np.inf)
    
    #create labels for bins
    bins_id = list(range(len(bins_cuts)-1))
    
    #create bins based on increments
#     binscolumn = column + '_bnwd'
    mdf_train[binscolumn] = \
    pd.cut(mdf_train[binscolumn], bins = bins_cuts,  \
           labels = bins_id, precision=len(str(len(bins_id))))
    mdf_test[binscolumn] = \
    pd.cut(mdf_test[binscolumn], bins = bins_cuts,  \
           labels = bins_id, precision=len(str(len(bins_id))))

    foundinset = mdf_train[binscolumn].unique()
    
    textcolumns = []
    for i in foundinset:
      textcolumns.append(binscolumn + '_' + str(i))
      
    #postprocess_textsupport_class will return columns in alphabetical order
    textcolumns.sort()
    
    #remove nan for cases where value did not fall within range
    textcolumns = [x for x in textcolumns if x[-3:] != 'nan']
    
    suffixoverlap_results = \
    self.df_check_suffixoverlap(mdf_train, textcolumns, suffixoverlap_results)
    
    #process bins as a categorical set
    mdf_train = \
    self.postprocess_textsupport_class(mdf_train, binscolumn, {}, 'tempkey', {'textcolumns':textcolumns})
    mdf_test = \
    self.postprocess_textsupport_class(mdf_test, binscolumn, {}, 'tempkey', {'textcolumns':textcolumns})
    
    #change data type for memory savings
    for textcolumn in textcolumns:
      mdf_train[textcolumn] = mdf_train[textcolumn].astype(np.int8)
      mdf_test[textcolumn] = mdf_test[textcolumn].astype(np.int8)
    
    #delete the support column
    del mdf_train[binscolumn]
    del mdf_test[binscolumn]

    #create list of columns
    nmbrcolumns = textcolumns

    #nmbrnormalization_dict = {'mean' : mean, 'std' : std}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in textcolumns:
      
      #new parameter collected for driftreport
      tc_ratio = nc + '_ratio'
      tcratio = mdf_train[nc].sum() / mdf_train[nc].shape[0]

      nmbrnormalization_dict = {nc : {'binsmean' : mean, \
                                      'buckets_bkt1' : buckets, \
                                      'bins_cuts' : bins_cuts, \
                                      'bins_id' : bins_id, \
                                      'textcolumns' : textcolumns, \
                                       tc_ratio : tcratio}}

      column_dict = { nc : {'category' : 'bkt1', \
                            'origcategory' : category, \
                            'normalization_dict' : nmbrnormalization_dict, \
                            'origcolumn' : column, \
                            'inputcolumn' : column, \
                            'columnslist' : textcolumns, \
                            'categorylist' : textcolumns, \
                            'infillmodel' : False, \
                            'infillcomplete' : False, \
                            'suffixoverlap_results' : suffixoverlap_results, \
                            'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())
       
    return mdf_train, mdf_test, column_dict_list
  
  def process_bkt2_class(self, mdf_train, mdf_test, column, category, postprocess_dict, params = {}):
    '''
    #processes a numerical set by creating custom bins coresponding to 
    #parameter 'buckets' which defaults to [0,1]
    #first and last buckets bounded
    #and returning in one-hot encoded set
    
    #removes buckets without activations in train set
    '''
    
    suffixoverlap_results = {}
    
    if 'buckets' in params:
        
      buckets = params['buckets']
    
    else:
      
      buckets = [0,1,2]
      
    binscolumn = column + '_bkt2'

    #store original column for later reversion
    mdf_train, suffixoverlap_results = \
    self.df_copy_train(mdf_train, column, binscolumn, suffixoverlap_results)
    
    mdf_test[binscolumn] = mdf_test[column].copy()

    #convert all values to either numeric or NaN
    mdf_train[binscolumn] = pd.to_numeric(mdf_train[binscolumn], errors='coerce')
    mdf_test[binscolumn] = pd.to_numeric(mdf_test[binscolumn], errors='coerce')

    #get mean of training data
    mean = mdf_train[binscolumn].mean()
    
    if mean != mean:
      mean = 0

    # #replace missing data with training set mean
    # mdf_train[binscolumn] = mdf_train[binscolumn].fillna(mean)
    # mdf_test[binscolumn] = mdf_test[binscolumn].fillna(mean)

    #assemble buckets  
    bins_cuts = buckets.copy()
#     bins_cuts.insert(0, -np.inf)
#     bins_cuts.insert(len(bins_cuts), np.inf)

    #create labels for bins
    bins_id = list(range(len(bins_cuts)-1))
    
    #create bins based on increments
#     binscolumn = column + '_bnwd'
    mdf_train[binscolumn] = \
    pd.cut(mdf_train[binscolumn], bins = bins_cuts,  \
           labels = bins_id, precision=len(str(len(bins_id))))
    mdf_test[binscolumn] = \
    pd.cut(mdf_test[binscolumn], bins = bins_cuts,  \
           labels = bins_id, precision=len(str(len(bins_id))))

    foundinset = mdf_train[binscolumn].unique()
    
    textcolumns = []
    for i in foundinset:
      textcolumns.append(binscolumn + '_' + str(i))
      
    #postprocess_textsupport_class will return columns in alphabetical order
    textcolumns.sort()
    
    #remove nan for cases where value did not fall within range
    textcolumns = [x for x in textcolumns if x[-3:] != 'nan']
    
    suffixoverlap_results = \
    self.df_check_suffixoverlap(mdf_train, textcolumns, suffixoverlap_results)
    
    #process bins as a categorical set
    mdf_train = \
    self.postprocess_textsupport_class(mdf_train, binscolumn, {}, 'tempkey', {'textcolumns':textcolumns})
    mdf_test = \
    self.postprocess_textsupport_class(mdf_test, binscolumn, {}, 'tempkey', {'textcolumns':textcolumns})
    
    #change data type for memory savings
    for textcolumn in textcolumns:
      mdf_train[textcolumn] = mdf_train[textcolumn].astype(np.int8)
      mdf_test[textcolumn] = mdf_test[textcolumn].astype(np.int8)
    
    #delete the support column
    del mdf_train[binscolumn]
    del mdf_test[binscolumn]

    #nmbrnormalization_dict = {'mean' : mean, 'std' : std}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in textcolumns:
      
      #new parameter collected for driftreport
      tc_ratio = nc + '_ratio'
      tcratio = mdf_train[nc].sum() / mdf_train[nc].shape[0]

      nmbrnormalization_dict = {nc : {'binsmean' : mean, \
                                      'buckets_bkt2' : buckets, \
                                      'bins_cuts' : bins_cuts, \
                                      'bins_id' : bins_id, \
                                      'textcolumns' : textcolumns, \
                                       tc_ratio : tcratio}}

      column_dict = { nc : {'category' : 'bkt2', \
                            'origcategory' : category, \
                            'normalization_dict' : nmbrnormalization_dict, \
                            'origcolumn' : column, \
                            'inputcolumn' : column, \
                            'columnslist' : textcolumns, \
                            'categorylist' : textcolumns, \
                            'infillmodel' : False, \
                            'infillcomplete' : False, \
                            'suffixoverlap_results' : suffixoverlap_results, \
                            'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())
       
    return mdf_train, mdf_test, column_dict_list
  
  def process_bkt3_class(self, mdf_train, mdf_test, column, category, postprocess_dict, params = {}):
    '''
    #processes a numerical set by creating custom bins coresponding to 
    #parameter 'buckets' which defaults to [0,1]
    #first and last buckets unconstrained
    #and returning in ordinal encoded set
    
    #segments without activations are included
    
    '''
    
    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    if 'buckets' in params:
        
      buckets = params['buckets']
    
    else:
      
      buckets = [0,1,2]
      
    binscolumn = column + '_bkt3'

    if inplace is not True:

      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.df_copy_train(mdf_train, column, binscolumn, suffixoverlap_results)

      mdf_test[binscolumn] = mdf_test[column].copy()

    else:

      suffixoverlap_results = \
      self.df_check_suffixoverlap(mdf_train, binscolumn, suffixoverlap_results)

      mdf_train.rename(columns = {column : binscolumn}, inplace = True)
      mdf_test.rename(columns = {column : binscolumn}, inplace = True)

    #convert all values to either numeric or NaN
    mdf_train[binscolumn] = pd.to_numeric(mdf_train[binscolumn], errors='coerce')
    mdf_test[binscolumn] = pd.to_numeric(mdf_test[binscolumn], errors='coerce')

    #get mean of training data
    mean = mdf_train[binscolumn].mean()
    
    if mean != mean:
      mean = 0

    # #replace missing data with training set mean
    # mdf_train[binscolumn] = mdf_train[binscolumn].fillna(mean)
    # mdf_test[binscolumn] = mdf_test[binscolumn].fillna(mean)

    #assemble buckets  
    bins_cuts = buckets.copy()
    bins_cuts.insert(0, -np.inf)
    bins_cuts.insert(len(bins_cuts), np.inf)
    
    #create labels for bins
    bins_id = list(range(len(bins_cuts)-1))

    infill_activation = len(bins_cuts)-1
      
    #create bins based on standard deviation increments
#     binscolumn = column + '_bnwo'
    mdf_train[binscolumn] = \
    pd.cut(mdf_train[binscolumn], bins = bins_cuts,  \
           labels = bins_id, precision=len(str(len(bins_id))))
    mdf_test[binscolumn] = \
    pd.cut(mdf_test[binscolumn], bins = bins_cuts,  \
           labels = bins_id, precision=len(str(len(bins_id))))

    mdf_train[binscolumn] = mdf_train[binscolumn].astype(float)
    mdf_test[binscolumn] = mdf_test[binscolumn].astype(float)
    
    #replace missing data with infill_activation
    mdf_train[binscolumn] = mdf_train[binscolumn].fillna(infill_activation)
    mdf_test[binscolumn] = mdf_test[binscolumn].fillna(infill_activation)
    
    #change column dtype
    mdf_train[binscolumn] = mdf_train[binscolumn].astype(int)
    mdf_test[binscolumn] = mdf_test[binscolumn].astype(int)

    #create list of columns
    nmbrcolumns = [binscolumn]
    
    #new driftreport metric ordl_activations_dict
    ordl_activations_dict = {}
    for unique in mdf_train[binscolumn].unique():
      sumcalc = (mdf_train[binscolumn] == unique).sum() 
      ratio = sumcalc / mdf_train[binscolumn].shape[0]
      ordl_activations_dict.update({unique:ratio})

    #nmbrnormalization_dict = {'mean' : mean, 'std' : std}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in nmbrcolumns:

      nmbrnormalization_dict = {nc : {'binsmean' : mean, \
                                      'buckets' : buckets, \
                                      'bins_cuts' : bins_cuts, \
                                      'bins_id' : bins_id, \
                                      'activations_list' : bins_id, \
                                      'infill_activation' : infill_activation, \
                                      'ordl_activations_dict' : ordl_activations_dict}}

      column_dict = { nc : {'category' : 'bkt3', \
                            'origcategory' : category, \
                            'normalization_dict' : nmbrnormalization_dict, \
                            'origcolumn' : column, \
                            'inputcolumn' : column, \
                            'columnslist' : nmbrcolumns, \
                            'categorylist' : nmbrcolumns, \
                            'infillmodel' : False, \
                            'infillcomplete' : False, \
                            'suffixoverlap_results' : suffixoverlap_results, \
                            'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())

    return mdf_train, mdf_test, column_dict_list
  
  def process_bkt4_class(self, mdf_train, mdf_test, column, category, postprocess_dict, params = {}):
    '''
    #processes a numerical set by creating custom bins coresponding to 
    #parameter 'buckets' which defaults to [0,1]
    #first and last buckets bounded
    #and returning in ordinal encoded set
    
    #segments without activations are included
    '''
    
    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    if 'buckets' in params:
        
      buckets = params['buckets']
    
    else:
      
      buckets = [0,1,2]
      
    binscolumn = column + '_bkt4'

    if inplace is not True:

      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.df_copy_train(mdf_train, column, binscolumn, suffixoverlap_results)

      mdf_test[binscolumn] = mdf_test[column].copy()

    else:

      suffixoverlap_results = \
      self.df_check_suffixoverlap(mdf_train, binscolumn, suffixoverlap_results)

      mdf_train.rename(columns = {column : binscolumn}, inplace = True)
      mdf_test.rename(columns = {column : binscolumn}, inplace = True)

    #convert all values to either numeric or NaN
    mdf_train[binscolumn] = pd.to_numeric(mdf_train[binscolumn], errors='coerce')
    mdf_test[binscolumn] = pd.to_numeric(mdf_test[binscolumn], errors='coerce')
    
    #set all values that fall outside of bounded buckets to nan for replacement with mean
    mdf_train.loc[mdf_train[binscolumn] <= buckets[0], (binscolumn)] = np.nan
    mdf_test.loc[mdf_test[binscolumn] <= buckets[0], (binscolumn)] = np.nan
    
    mdf_train.loc[mdf_train[binscolumn] > buckets[-1], (binscolumn)] = np.nan
    mdf_test.loc[mdf_test[binscolumn] > buckets[-1], (binscolumn)] = np.nan

    #get mean of training data
    mean = mdf_train[binscolumn].mean()
    
    if mean != mean:
      mean = 0
      
    #edge case, if mean does nto fall within buckets range, we'll apply infill to top bucket
    #this edge case specific to bkt4
    #this assumes buckets was passed with sorted values
    if mean < buckets[0] or mean > buckets[-1]:
      mean = buckets[-1]

    # #replace missing data with training set mean
    # mdf_train[binscolumn] = mdf_train[binscolumn].fillna(mean)
    # mdf_test[binscolumn] = mdf_test[binscolumn].fillna(mean)

    #assemble buckets  
    bins_cuts = buckets.copy()
#     bins_cuts.insert(0, -np.inf)
#     bins_cuts.insert(len(bins_cuts), np.inf)

    #create labels for bins
    bins_id = list(range(len(bins_cuts)-1))

    infill_activation = len(bins_cuts)-1
      
    #create bins based on standard deviation increments
#     binscolumn = column + '_bnwo'
    mdf_train[binscolumn] = \
    pd.cut(mdf_train[binscolumn], bins = bins_cuts,  \
           labels = bins_id, precision=len(str(len(bins_id))))
    mdf_test[binscolumn] = \
    pd.cut(mdf_test[binscolumn], bins = bins_cuts,  \
           labels = bins_id, precision=len(str(len(bins_id))))
    
    mdf_train[binscolumn] = mdf_train[binscolumn].astype(float)
    mdf_test[binscolumn] = mdf_test[binscolumn].astype(float)
    
    #replace missing data with infill_activation
    mdf_train[binscolumn] = mdf_train[binscolumn].fillna(infill_activation)
    mdf_test[binscolumn] = mdf_test[binscolumn].fillna(infill_activation)
    
    #change column dtype
    mdf_train[binscolumn] = mdf_train[binscolumn].astype(int)
    mdf_test[binscolumn] = mdf_test[binscolumn].astype(int)

    #create list of columns
    nmbrcolumns = [binscolumn]
    
    #new driftreport metric ordl_activations_dict
    ordl_activations_dict = {}
    for unique in mdf_train[binscolumn].unique():
      sumcalc = (mdf_train[binscolumn] == unique).sum() 
      ratio = sumcalc / mdf_train[binscolumn].shape[0]
      ordl_activations_dict.update({unique:ratio})

    #nmbrnormalization_dict = {'mean' : mean, 'std' : std}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in nmbrcolumns:

      nmbrnormalization_dict = {nc : {'binsmean' : mean, \
                                      'buckets' : buckets, \
                                      'bins_cuts' : bins_cuts, \
                                      'bins_id' : bins_id, \
                                      'activations_list' : bins_id, \
                                      'infill_activation' : infill_activation, \
                                      'ordl_activations_dict' : ordl_activations_dict}}

      column_dict = { nc : {'category' : 'bkt4', \
                            'origcategory' : category, \
                            'normalization_dict' : nmbrnormalization_dict, \
                            'origcolumn' : column, \
                            'inputcolumn' : column, \
                            'columnslist' : nmbrcolumns, \
                            'categorylist' : nmbrcolumns, \
                            'infillmodel' : False, \
                            'infillcomplete' : False, \
                            'suffixoverlap_results' : suffixoverlap_results, \
                            'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())

    return mdf_train, mdf_test, column_dict_list

  def process_DPnb_class(self, mdf_train, mdf_test, column, category, postprocess_dict, params = {}):
    '''
    #process_DPnb_class(mdf_train, mdf_test, column, category, postprocess_dict, params = {})
    #function to inject noise to training data, such as for differential privacy purposes
    #assumes input is numeric data with z-score normalization to mean 0 and sigma 1
    #adds data sampled from normal distribution with mean 0 and sigma 0.06 by default
    #where noise only injected to a subset of data based on flip_prob defaulting to 0.03
    #the noise properties may be customized with parameters 'mu', 'sigma', 'flip_prob'
    #note that the noise is only injected into the designated training data of df_train
    #for test data this is a pass-through operation
    #note this assumes clean data as input since this will be intended for downstream applicaiton
    #in family trees, so no infill is performed
    #note that for postprocess function in postmunge, determination of whether to treat
    #df_test as train or test data is based on the traindata entry in postprocess_dict
    #in automunge df_test is treated as test data by default
    '''
    
    suffixoverlap_results = {}
    
    #initialize parameters
    if 'mu' in params:
      mu = params['mu']
    else:
      mu = 0.0
      
    if 'sigma' in params:
      sigma = params['sigma']
    else:
      sigma = 0.06
      
    if 'flip_prob' in params:
      flip_prob = params['flip_prob']
    else:
      flip_prob = 1.0
      
    if 'noisedistribution' in params:
      noisedistribution = params['noisedistribution']
    else:
      #can pass as 'normal' or 'laplace'
      noisedistribution = 'normal'
      
    DPnm_column = column + '_DPnb'
    
    suffixoverlap_results = \
    self.df_check_suffixoverlap(mdf_train, DPnm_column, suffixoverlap_results)
      
    #first we'll derive our sampled noise for injection
    if noisedistribution == 'normal':
      normal_samples = np.random.normal(loc=mu, scale=sigma, size=(mdf_train.shape[0]))
    elif noisedistribution == 'laplace':
      normal_samples = np.random.laplace(loc=mu, scale=sigma, size=(mdf_train.shape[0]))
      
    binomial_samples = np.random.binomial(n=1, p=flip_prob, size=(mdf_train.shape[0]))
    
    mdf_train[DPnm_column] = pd.DataFrame(normal_samples) * pd.DataFrame(binomial_samples)
    
    #now inject noise
    mdf_train[DPnm_column] = mdf_train[DPnm_column] + mdf_train[column]
    
    #for test data is just pass-through
    mdf_test[DPnm_column] = mdf_test[column]
    
    #create list of columns
    nmbrcolumns = [DPnm_column]

    nmbrnormalization_dict = {DPnm_column : {'mu' : mu, 'sigma' : sigma, 'flip_prob' : flip_prob, \
                                             'noisedistribution' : noisedistribution}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in nmbrcolumns:

      column_dict = { nc : {'category' : 'DPnb', \
                           'origcategory' : category, \
                           'normalization_dict' : nmbrnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())
        
    return mdf_train, mdf_test, column_dict_list

  def process_DPmm_class(self, mdf_train, mdf_test, column, category, postprocess_dict, params = {}):
    '''
    #process_DPmm_class(mdf_train, mdf_test, column, category, postprocess_dict, params = {})
    #function to inject noise to training data, such as for differential privacy purposes
    #assumes input is numeric data min-max scaled within range 0-1
    #adds data sampled from normal distribution with mean 0 and sigma 0.03 by default
    #the noise properties may be customized with parameters 'mu', 'sigma'
    #also accepts parameter 'flip_prob' for ratio of data that will be adjusted (defaults to 1.)
    #noise is scaled based on the recieved points to keep within range 0-1
    #(e.g. for recieved data point 0.1, noise is scaled so as not to fall below -0.1)
    #gaussian noise source is also capped to maintain the range -0.5 to 0.5 (rare outlier points)
    #note that the noise is only injected into the designated training data of df_train
    #for test data this is a pass-through operation
    #note this assumes clean data as input since this will be intended for downstream applicaiton
    #in family trees, so no infill is performed
    #note that for postprocess function in postmunge, determination of whether to treat
    #df_test as train or test data is based on the traindata entry in postprocess_dict
    #in automunge df_test is treated as test data by default
    '''
    
    suffixoverlap_results = {}
    
    #initialize parameters
    if 'mu' in params:
      mu = params['mu']
    else:
      mu = 0.0
      
    if 'sigma' in params:
      sigma = params['sigma']
    else:
      sigma = 0.03
      
    if 'flip_prob' in params:
      flip_prob = params['flip_prob']
    else:
      flip_prob = 1.
      
    if 'noisedistribution' in params:
      noisedistribution = params['noisedistribution']
    else:
      #can pass as 'normal' or 'laplace'
      noisedistribution = 'normal'
      
    DPmm_column = column + '_DPmm'
    DPmm_column_temp1 = column + '_DPmm' + '_tmp1'
    
    suffixoverlap_results = \
    self.df_check_suffixoverlap(mdf_train, [DPmm_column, DPmm_column_temp1], suffixoverlap_results)
      
    #first we'll derive our sampled noise for injection
    if noisedistribution == 'normal':
      normal_samples = np.random.normal(loc=mu, scale=sigma, size=(mdf_train.shape[0]))
    elif noisedistribution == 'laplace':
      normal_samples = np.random.laplace(loc=mu, scale=sigma, size=(mdf_train.shape[0]))
    binomial_samples = np.random.binomial(n=1, p=flip_prob, size=(mdf_train.shape[0]))
    
    mdf_train[DPmm_column] = pd.DataFrame(normal_samples) * pd.DataFrame(binomial_samples)
    
    #cap outliers
    mdf_train[DPmm_column] = np.where(mdf_train[DPmm_column] < -0.5, np.nan, mdf_train[DPmm_column])
    mdf_train[DPmm_column] = np.where(mdf_train[DPmm_column] > 0.5, np.nan, mdf_train[DPmm_column])
    
    #adjacent cell infill
    mdf_train[DPmm_column] = mdf_train[DPmm_column].fillna(method='ffill')
    mdf_train[DPmm_column] = mdf_train[DPmm_column].fillna(method='bfill')
    
    #support column to signal sign of noise, 0 is neg, 1 is pos
    mdf_train[DPmm_column_temp1] = 0
    mdf_train[DPmm_column_temp1] = np.where(mdf_train[DPmm_column] >= 0., 1, mdf_train[DPmm_column_temp1])
    
    #now inject noise, with scaled noise to maintain range 0-1
    #(so if mnmx value <0.5, and neg noise, we scale noise to maintain ratio as if minmax was 0.5, similarly for >0.5 mnmx)
    mdf_train[DPmm_column] = np.where(mdf_train[column] < 0.5, \
                                      mdf_train[column] + \
                                      (1 - mdf_train[DPmm_column_temp1]) * (mdf_train[DPmm_column] * mdf_train[column] / 0.5) + \
                                      (mdf_train[DPmm_column_temp1]) * (mdf_train[DPmm_column]), \
                                      mdf_train[DPmm_column])
    
    mdf_train[DPmm_column] = np.where(mdf_train[column] >= 0.5, \
                                      mdf_train[column] + \
                                      (1 - mdf_train[DPmm_column_temp1]) * (mdf_train[DPmm_column]) + \
                                      (mdf_train[DPmm_column_temp1]) * (mdf_train[DPmm_column] * (1 - mdf_train[column]) / 0.5), \
                                      mdf_train[DPmm_column])
    
    #remove support column
    del mdf_train[DPmm_column_temp1]
    
    #for test data is just pass-through
    mdf_test[DPmm_column] = mdf_test[column]
    
    #create list of columns
    nmbrcolumns = [DPmm_column]

    nmbrnormalization_dict = {DPmm_column : {'mu' : mu, 'sigma' : sigma, 'flip_prob' : flip_prob, \
                                             'noisedistribution' : noisedistribution}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in nmbrcolumns:

      column_dict = { nc : {'category' : 'DPmm', \
                           'origcategory' : category, \
                           'normalization_dict' : nmbrnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())
        
    return mdf_train, mdf_test, column_dict_list

  def process_DPrt_class(self, mdf_train, mdf_test, column, category, postprocess_dict, params = {}):
    """
    #process_DPrt_class
    #function to scale data as follows:
    
    # if max >= 0 and min <= 0:
    #   #scaling based on 
    #   x = x / (max - min)

    # elif max >= 0 and min >= 0:
    #   #traditional min/max
    #   x = (x - min) / (max - min)
    
    # elif max <= 0 and min <= 0:
    #   #max/min (retains negative values)
    #   x = (x - max) / (max - min)
    
    #followed by a noise injection similar to DPmm based based on this set's retn range
    
    #replaces missing or improperly formatted data with mean of remaining values
    #(prior to noise injection)
    
    #returns same dataframes with new column of name column + '_DPrt'
    #note this is a "dualprocess" function since is applied to both dataframes
    
    #note with parameters divisor can also be set as standard deviation
    #also aprameters accepted for cap/floor/mulitplier/offset
    #where cap/floor based on pretransform values
    #multiplier/offset based on posttransform values, muoltiplier applied betfore offset
    """
    
    suffixoverlap_results = {}
    
    #accepts divisor parameters of 'minmax' or 'std', eg divisor for normalization equation
    #note that standard deviation doesn't have same properties for sign retention when all values > or < 0
    if 'divisor' in params:
      divisor = params['divisor']
    else:
      divisor = 'minmax'
    
    #offset is just an added constant applied after multiplier
    if 'offset' in params:
      offset = params['offset']
    else:
      offset = 0
      
    #multiplier scales the set by multiplication prior to offset
    if 'multiplier' in params:
      multiplier = params['multiplier']
    else:
      multiplier = 1
    
    #cap can be passed as True for max of training data or as a specific value prior to normalization, False for no cap
    if 'cap' in params:
      cap = params['cap']
    else:
      cap = False
      
    #floor can be passed as True for min of training data or as a specific value prior to normalization, False for no floor
    if 'floor' in params:
      floor = params['floor']
    else:
      floor = False
      
    #adjinfill accepts True/False to change default infill from mean inputation to adjacent cell
    if 'adjinfill' in params:
      adjinfill = params['adjinfill']
    else:
      adjinfill = False
      
    #here are differential privacy parameters
    if 'mu' in params:
      mu = params['mu']
    else:
      mu = 0.0
      
    if 'sigma' in params:
      sigma = params['sigma']
    else:
      sigma = 0.03
      
    if 'flip_prob' in params:
      flip_prob = params['flip_prob']
    else:
      flip_prob = 1.
      
    if 'noisedistribution' in params:
      noisedistribution = params['noisedistribution']
    else:
      #can pass as 'normal' or 'laplace'
      noisedistribution = 'normal'
      
    DPrt_column = column + '_DPrt'
    DPrt_column_temp1 = column + '_DPrt' + '_tmp1'
    DPrt_column_temp2 = column + '_DPrt' + '_tmp2'
    
    newcolumns = [DPrt_column, DPrt_column_temp1, DPrt_column_temp2]
    
    suffixoverlap_results = \
    self.df_check_suffixoverlap(mdf_train, newcolumns, suffixoverlap_results)
    
    #copy source column into new column
    mdf_train[DPrt_column] = mdf_train[column].copy()
    mdf_test[DPrt_column] = mdf_test[column].copy()

    #convert all values to either numeric or NaN
    mdf_train[DPrt_column] = pd.to_numeric(mdf_train[DPrt_column], errors='coerce')
    mdf_test[DPrt_column] = pd.to_numeric(mdf_test[DPrt_column], errors='coerce')
    
    #a few more metrics collected for driftreport
    #get standard deviation of training data
    std = mdf_train[DPrt_column].std()
    
    mad = mdf_train[DPrt_column].mad()
    
    #get maximum value of training column
    maximum = mdf_train[DPrt_column].max()
    
    #get minimum value of training column
    minimum = mdf_train[DPrt_column].min()
    
    #avoid outlier div by zero when max = min
    maxminusmin = maximum - minimum
    if maxminusmin == 0 or maxminusmin != maxminusmin:
      maxminusmin = 1
      
    if std != std or std == 0:
      std = 1
      
    if mad != mad or mad == 0:
      mad = 1
      
    #if cap < maximum, maximum = cap
    if cap is not False and cap is not True:
      if cap < maximum:
        maximum = cap
    if floor is not False and floor is not True:
      if floor > minimum:
        minimum = floor
        
    #cap and floor application
    if cap is True:
      cap = maximum
    if floor is True:
      floor = minimum
      
    if cap is not False:
      #replace values in test > cap with cap
      mdf_train.loc[mdf_train[DPrt_column] > cap, (DPrt_column)] \
      = cap
      
      mdf_test.loc[mdf_test[DPrt_column] > cap, (DPrt_column)] \
      = cap
    
    if floor is not False:
      #replace values in test < floor with floor
      mdf_train.loc[mdf_train[DPrt_column] < floor, (DPrt_column)] \
      = floor
      
      mdf_test.loc[mdf_test[DPrt_column] < floor, (DPrt_column)] \
      = floor
      
    #get mean of training data
    mean = mdf_train[DPrt_column].mean()
    if mean != mean:
      mean = 0
    
    if adjinfill is True:
      mdf_train[DPrt_column] = mdf_train[DPrt_column].fillna(method='ffill')
      mdf_test[DPrt_column] = mdf_test[DPrt_column].fillna(method='ffill')
      mdf_train[DPrt_column] = mdf_train[DPrt_column].fillna(method='bfill')
      mdf_test[DPrt_column] = mdf_test[DPrt_column].fillna(method='bfill')
    
    #replace missing data with training set mean
    mdf_train[DPrt_column] = mdf_train[DPrt_column].fillna(mean)
    mdf_test[DPrt_column] = mdf_test[DPrt_column].fillna(mean)
    
    #edge case (only neccesary so scalingapproach is assigned)
    if maximum != maximum:
      maximum = 0
    if minimum != minimum:
      minimum = 0
      
    #divisor
    if divisor not in ['minmax', 'std', 'mad']:
      print("Error: retn transform parameter 'divisor' only accepts entries of 'minmax' 'mad' or 'std'")
    if divisor == 'minmax':
      divisor = maxminusmin
    elif divisor == 'mad':
      divisor = mad
    else:
      divisor = std
      
    if divisor == 0 or divisor != divisor:
      divisor = 1
    
    #driftreport metric scalingapproach returned as 'retn' or 'mnmx' or 'mxmn'
    #where mnmx is for cases where all values in train set are positive
    #mxmn is for cases where all values in train set are negative
    
    if maximum >= 0 and minimum <= 0:
      
      mdf_train[DPrt_column] = (mdf_train[DPrt_column]) / \
                                    (divisor) * multiplier + offset
      
      mdf_test[DPrt_column] = (mdf_test[DPrt_column]) / \
                                    (divisor) * multiplier + offset
      
      scalingapproach = 'retn'
      
    elif maximum >= 0 and minimum >= 0:
    
      #perform min-max scaling to train and test sets using values from train
      mdf_train[DPrt_column] = (mdf_train[DPrt_column] - minimum) / \
                                    (divisor) * multiplier + offset

      mdf_test[DPrt_column] = (mdf_test[DPrt_column] - minimum) / \
                                   (divisor) * multiplier + offset
      
      scalingapproach = 'mnmx'
      
    elif maximum <= 0 and minimum <= 0:
    
      #perform min-max scaling to train and test sets using values from train
      mdf_train[DPrt_column] = (mdf_train[DPrt_column] - maximum) / \
                                    (divisor) * multiplier + offset

      mdf_test[DPrt_column] = (mdf_test[DPrt_column] - maximum) / \
                                   (divisor) * multiplier + offset
      
      scalingapproach = 'mxmn'
      
      
    #now apply noise injection
    
    #first we'll derive our sampled noise for injection
    if noisedistribution == 'normal':
      normal_samples = np.random.normal(loc=mu, scale=sigma, size=(mdf_train.shape[0]))
    elif noisedistribution == 'laplace':
      normal_samples = np.random.laplace(loc=mu, scale=sigma, size=(mdf_train.shape[0]))
      
    binomial_samples = np.random.binomial(n=1, p=flip_prob, size=(mdf_train.shape[0]))
    
    mdf_train[DPrt_column_temp2] = pd.DataFrame(normal_samples) * pd.DataFrame(binomial_samples)
    
    #cap outliers
    mdf_train[DPrt_column_temp2] = np.where(mdf_train[DPrt_column_temp2] < -0.5, np.nan, mdf_train[DPrt_column_temp2])
    mdf_train[DPrt_column_temp2] = np.where(mdf_train[DPrt_column_temp2] > 0.5, np.nan, mdf_train[DPrt_column_temp2])
    
    #adjacent cell infill
    mdf_train[DPrt_column_temp2] = mdf_train[DPrt_column_temp2].fillna(method='ffill')
    mdf_train[DPrt_column_temp2] = mdf_train[DPrt_column_temp2].fillna(method='bfill')
    
    #support column to signal sign of noise, 0 is neg, 1 is pos
    mdf_train[DPrt_column_temp1] = 0
    mdf_train[DPrt_column_temp1] = np.where(mdf_train[DPrt_column_temp2] >= 0., 1, mdf_train[DPrt_column_temp1])
    
    #for noise injection we'll first move data into range 0-1 and then revert after injection
    if scalingapproach == 'retn':
      mdf_train[DPrt_column] = (mdf_train[DPrt_column] - (minimum / divisor) ) / multiplier - offset
    elif scalingapproach == 'mnmx':
      mdf_train[DPrt_column] = (mdf_train[DPrt_column]) / multiplier - offset
    elif scalingapproach == 'mxmn':
      mdf_train[DPrt_column] = (mdf_train[DPrt_column] + (maximum - minimum) / divisor) / multiplier - offset
      
    
    #now inject noise, with scaled noise to maintain range 0-1
    #(so if mnmx value <0.5, and neg noise, we scale noise to maintain ratio as if minmax was 0.5, similarly for >0.5 mnmx)
    mdf_train[DPrt_column] = np.where(mdf_train[DPrt_column] < 0.5, \
                                      mdf_train[DPrt_column] + \
                                      (1 - mdf_train[DPrt_column_temp1]) * (mdf_train[DPrt_column_temp2] * mdf_train[DPrt_column] / 0.5) + \
                                      (mdf_train[DPrt_column_temp1]) * (mdf_train[DPrt_column_temp2]), \
                                      mdf_train[DPrt_column])
    
    mdf_train[DPrt_column] = np.where(mdf_train[DPrt_column] >= 0.5, \
                                      mdf_train[DPrt_column] + \
                                      (1 - mdf_train[DPrt_column_temp1]) * (mdf_train[DPrt_column_temp2]) + \
                                      (mdf_train[DPrt_column_temp1]) * (mdf_train[DPrt_column_temp2] * (1 - mdf_train[DPrt_column]) / 0.5), \
                                      mdf_train[DPrt_column])
    
    #remove support columns
    del mdf_train[DPrt_column_temp1]
    del mdf_train[DPrt_column_temp2]
    
    #for noise injection we'll first move data into range 0-1 and then revert after injection
    if scalingapproach == 'retn':
      mdf_train[DPrt_column] = (mdf_train[DPrt_column] + (minimum / divisor) ) * multiplier + offset
    elif scalingapproach == 'mnmx':
      mdf_train[DPrt_column] = (mdf_train[DPrt_column]) * multiplier + offset
    elif scalingapproach == 'mxmn':
      mdf_train[DPrt_column] = (mdf_train[DPrt_column] - (maximum - minimum) / divisor) * multiplier + offset
    
    #for test data is just pass-through
    #mdf_test[DPrt_column] = mdf_test[DPrt_column]
    
    #create list of columns
    nmbrcolumns = [DPrt_column]
    
    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []
    
    nmbrnormalization_dict = {DPrt_column : {'mu' : mu, \
                                             'sigma' : sigma, \
                                             'flip_prob' : flip_prob, \
                                             'noisedistribution' : noisedistribution, \
                                             'minimum' : minimum, \
                                             'maximum' : maximum, \
                                             'mean' : mean, \
                                             'std' : std, \
                                             'mad' : mad, \
                                             'scalingapproach' : scalingapproach, \
                                             'offset' : offset, \
                                             'multiplier': multiplier, \
                                             'cap' : cap, \
                                             'floor' : floor, \
                                             'divisor' : divisor, \
                                             'adjinfill' : adjinfill, \
                                            }}
    
    for nc in nmbrcolumns:

      column_dict = { nc : {'category' : 'DPrt', \
                           'origcategory' : category, \
                           'normalization_dict' : nmbrnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())
        
    return mdf_train, mdf_test, column_dict_list

  def process_DPbn_class(self, mdf_train, mdf_test, column, category, postprocess_dict, params = {}):
    '''
    #process_DPbn_class(mdf_train, mdf_test, column, category, postprocess_dict, params = {})
    #function to inject noise to training data, such as for differential privacy purposes
    #assumes input is bnry encoded data (i.e. boolean integers in single column)
    #adds data sampled from Bernoulli distribution with flip_prob 0.03 by default
    #the noise properties may be customized with parameter 'flip_prob'
    #note that the noise is only injected into the designated training data of df_train
    #for test data this is a pass-through operation
    #note this assumes clean data as input since this will be intended for downstream applicaiton
    #in family trees, so no infill is performed
    #note that for postprocess function in postmunge, determination of whether to treat
    #df_test as train or test data is based on the traindata entry in postprocess_dict
    #in automunge df_test is treated as test data by default
    '''
    
    suffixoverlap_results = {}
    
    #initialize parameters
    if 'flip_prob' in params:
      flip_prob = params['flip_prob']
    else:
      flip_prob = 0.03
      
    DPbn_column = column + '_DPbn'
    
    suffixoverlap_results = \
    self.df_check_suffixoverlap(mdf_train, DPbn_column, suffixoverlap_results)
      
    #first we'll derive our sampled noise for injection
    mdf_train[DPbn_column] = pd.DataFrame(np.random.binomial(n=1, p=flip_prob, size=(mdf_train.shape[0])))
    
    #now inject noise
    mdf_train[DPbn_column] = abs(mdf_train[column] - mdf_train[DPbn_column])
    
    #for test data is just pass-through
    mdf_test[DPbn_column] = mdf_test[column]
    
    #create list of columns
    nmbrcolumns = [DPbn_column]

    nmbrnormalization_dict = {DPbn_column : {'flip_prob' : flip_prob}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in nmbrcolumns:

      column_dict = { nc : {'category' : 'DPbn', \
                           'origcategory' : category, \
                           'normalization_dict' : nmbrnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())
        
    return mdf_train, mdf_test, column_dict_list

  def process_DPod_class(self, mdf_train, mdf_test, column, category, postprocess_dict, params = {}):
    '''
    #process_DPod_class(mdf_train, mdf_test, column, category, postprocess_dict, params = {})
    #function to inject noise to training data, such as for differential privacy purposes
    #assumes input is ordinal encoded data (i.e. categoric by integer in single column)
    #adds data sampled from Bernoulli distribution with flip_prob 0.03 by default
    #the noise properties may be customized with parameter 'flip_prob'
    #when flip activated selects from the set of encodings per level random draw
    #(including potenitally the current encoding for no flip)
    #note that the noise is only injected into the designated training data of df_train
    #for test data this is a pass-through operation
    #note this assumes clean data as input since this will be intended for downstream applicaiton
    #in family trees, so no infill is performed
    #note that for postprocess function in postmunge, determination of whether to treat
    #df_test as train or test data is based on the traindata entry in postprocess_dict
    #in automunge df_test is treated as test data by default
    '''
    
    suffixoverlap_results = {}
    
    #initialize parameters
    if 'flip_prob' in params:
      flip_prob = params['flip_prob']
    else:
      flip_prob = 0.03
      
    DPod_column = column + '_DPod'
    DPod_tempcolumn1 = column + '_DPod_tmp1'
    DPod_tempcolumn2 = column + '_DPod_tmp2'
    
    newcolumns = [DPod_column, DPod_tempcolumn1, DPod_tempcolumn2]
    
    suffixoverlap_results = \
    self.df_check_suffixoverlap(mdf_train, newcolumns, suffixoverlap_results)
    
    #we'll want to know the set of activations present in column, for automunge this is unique values
    ord_encodings = mdf_train[column].unique()
      
    #first we'll derive our sampled noise for injection
    mdf_train[DPod_tempcolumn1] = pd.DataFrame(np.random.binomial(n=1, p=flip_prob, size=(mdf_train.shape[0])))
    mdf_train[DPod_tempcolumn2] = pd.DataFrame(np.random.choice(ord_encodings, size=(mdf_train.shape[0])))
    
    #now inject noise
    
    #initialize return column
    mdf_train[DPod_column] = 0
    
    for entry in list(ord_encodings):
      
      #this returns column value when DPod_tempcolumn1 is 0 or DPod_tempcolumn2 when DPod_tempcolumn1 is 1
      mdf_train[DPod_column] = \
      np.where(mdf_train[column] == entry, \
               mdf_train[column] * (1 - mdf_train[DPod_tempcolumn1]) + mdf_train[DPod_tempcolumn1] * mdf_train[DPod_tempcolumn2], \
               mdf_train[DPod_column])
      
    del mdf_train[DPod_tempcolumn1]
    del mdf_train[DPod_tempcolumn2]
    
    #for test data is just pass-through
    mdf_test[DPod_column] = mdf_test[column]
    
    #create list of columns
    nmbrcolumns = [DPod_column]

    nmbrnormalization_dict = {DPod_column : {'flip_prob' : flip_prob}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in nmbrcolumns:

      column_dict = { nc : {'category' : 'DPod', \
                           'origcategory' : category, \
                           'normalization_dict' : nmbrnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())
        
    return mdf_train, mdf_test, column_dict_list
  
  def process_null_class(self, df, column, category, postprocess_dict, params = {}):
    '''
    #here we'll delete any columns that returned a 'null' category
    #note this is a. singleprocess transform
    '''
    
    suffixoverlap_results = {}
    
    #df = df.drop([column], axis=1)
    #deletion takes place elsewhere

    column_dict_list = []

    column_dict = {column + '_null' : {'category' : 'null', \
                                      'origcategory' : category, \
                                      'normalization_dict' : {column + '_null':{}}, \
                                      'origcolumn' : column, \
                                      'inputcolumn' : column, \
                                      'columnslist' : [], \
                                      'categorylist' : [], \
                                      'infillmodel' : False, \
                                      'infillcomplete' : False, \
                                      'suffixoverlap_results' : suffixoverlap_results, \
                                      'deletecolumn' : False}}
    
    #now append column_dict onto postprocess_dict
    column_dict_list.append(column_dict.copy())

    return df, column_dict_list
  
  def process_copy_class(self, df, column, category, postprocess_dict, params = {}):
    '''
    #copy function
    #accepts parameter 'suffix' for suffix appender
    #useful if want to apply same function more than once with different parameters
    '''
    
    suffixoverlap_results = {}
    
    if 'suffix' in params:
        
      copy_column = column + params['suffix']
    
    else:
      
      copy_column = column + '_copy'
    
    df, suffixoverlap_results = \
    self.df_copy_train(df, column, copy_column, suffixoverlap_results)

    column_dict_list = []

    column_dict = {copy_column : {'category' : 'copy', \
                                 'origcategory' : category, \
                                 'normalization_dict' : {copy_column:{}}, \
                                 'origcolumn' : column, \
                                 'inputcolumn' : column, \
                                 'columnslist' : [copy_column], \
                                 'categorylist' : [copy_column], \
                                 'infillmodel' : False, \
                                 'infillcomplete' : False, \
                                 'suffixoverlap_results' : suffixoverlap_results, \
                                 'deletecolumn' : False}}
    
    #now append column_dict onto postprocess_dict
    column_dict_list.append(column_dict.copy())

    return df, column_dict_list

  def process_excl_class(self, df, column, category, postprocess_dict, params = {}):
    """
    #here we'll address any columns that returned a 'excl' category
    #note this is a. singleprocess transform
    #we'll simply maintain the same column but with a suffix to the header
    #the excl trasnform is a very special exception, and this suffix is later
    #removed when automunge(*.)parameter excl_suffix passed as False
    
    #note that excl transform is also special in that it may only be applied
    #as a supplement primitive in a family tree (eg cousins)
    #as it replaces the source column internally (by a simple rename)
    
    #Note that the function check_transformdict(.) works 'under the hood'
    #to translate user passed excl transforms in family trees
    #from replacement primitives to corresponding supplement primitives
    """
    
    suffixoverlap_results = {}

    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    exclcolumn = column + '_excl'

    if inplace is not True:

      df, suffixoverlap_results = \
      self.df_copy_train(df, column, exclcolumn, suffixoverlap_results)

    else:
    
      suffixoverlap_results = \
      self.df_check_suffixoverlap(df, exclcolumn, suffixoverlap_results)
      
      df.rename(columns = {column : exclcolumn}, inplace = True)

    #decided against this to maximize efficiency
    #there are some workflow scenarios where a lot of excl columns in postmunge
    # #this moves exclcolumn to end of dataframe to maintain column order correspondance
    # df_columns = list(df)
    # df_columns.remove(exclcolumn)
    # df_columns.append(exclcolumn)
    # df = df.reindex(columns = df_columns)
    
    column_dict_list = []

    column_dict = {exclcolumn : {'category' : 'excl', \
                                 'origcategory' : category, \
                                 'normalization_dict' : {exclcolumn:{}}, \
                                 'origcolumn' : column, \
                                 'inputcolumn' : column, \
                                 'columnslist' : [exclcolumn], \
                                 'categorylist' : [exclcolumn], \
                                 'infillmodel' : False, \
                                 'infillcomplete' : False, \
                                 'suffixoverlap_results' : suffixoverlap_results, \
                                 'deletecolumn' : False}}
    
    #now append column_dict onto postprocess_dict
    column_dict_list.append(column_dict.copy())

    return df, column_dict_list

  def process_exc2_class(self, mdf_train, mdf_test, column, category, \
                         postprocess_dict, params = {}):
    '''
    #here we'll address any columns that returned a 'excl' category
    #note this is a. singleprocess transform
    #we'll simply maintain the same column but with a suffix to the header
    '''
    
    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    exclcolumn = column + '_exc2'
    
    if inplace is not True:

      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.df_copy_train(mdf_train, column, exclcolumn, suffixoverlap_results)

      mdf_test[exclcolumn] = mdf_test[column].copy()

    else:

      suffixoverlap_results = \
      self.df_check_suffixoverlap(mdf_train, exclcolumn, suffixoverlap_results)

      mdf_train.rename(columns = {column : exclcolumn}, inplace = True)
      mdf_test.rename(columns = {column : exclcolumn}, inplace = True)
    
    #del df[column]
    
    mdf_train[exclcolumn] = pd.to_numeric(mdf_train[exclcolumn], errors='coerce')
    mdf_test[exclcolumn] = pd.to_numeric(mdf_test[exclcolumn], errors='coerce')
    
    if len(mdf_train[exclcolumn].mode())<1:
      fillvalue = mdf_train[exclcolumn].mean()
    else:
      fillvalue = mdf_train[exclcolumn].mode()[0]
      
    #special case if column didn't have any numeric entries
    if fillvalue != fillvalue:
      fillvalue = 0
    
    #replace missing data with fill value
    mdf_train[exclcolumn] = mdf_train[exclcolumn].fillna(fillvalue)
    mdf_test[exclcolumn] = mdf_test[exclcolumn].fillna(fillvalue)
    
    exc2_normalization_dict = {exclcolumn : {'fillvalue' : fillvalue}}
    
    column_dict_list = []

    column_dict = {exclcolumn : {'category' : 'exc2', \
                                 'origcategory' : category, \
                                 'normalization_dict' : exc2_normalization_dict, \
                                 'origcolumn' : column, \
                                 'inputcolumn' : column, \
                                 'columnslist' : [exclcolumn], \
                                 'categorylist' : [exclcolumn], \
                                 'infillmodel' : False, \
                                 'infillcomplete' : False, \
                                 'suffixoverlap_results' : suffixoverlap_results, \
                                 'deletecolumn' : False}}
    
    #now append column_dict onto postprocess_dict
    column_dict_list.append(column_dict.copy())

    return mdf_train, mdf_test, column_dict_list
  
  def process_exc5_class(self, mdf_train, mdf_test, column, category, \
                         postprocess_dict, params = {}):
    '''
    #here we'll address any columns that returned a 'excl' category
    #note this is a. singleprocess transform
    #we'll simply maintain the same column but with a suffix to the header
    '''
    
    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    exclcolumn = column + '_exc5'
    
    if inplace is not True:

      #copy source column into new column
      mdf_train, suffixoverlap_results = \
      self.df_copy_train(mdf_train, column, exclcolumn, suffixoverlap_results)

      mdf_test[exclcolumn] = mdf_test[column].copy()

    else:

      suffixoverlap_results = \
      self.df_check_suffixoverlap(mdf_train, exclcolumn, suffixoverlap_results)

      mdf_train.rename(columns = {column : exclcolumn}, inplace = True)
      mdf_test.rename(columns = {column : exclcolumn}, inplace = True)
    
    #del df[column]
    
    mdf_train[exclcolumn] = pd.to_numeric(mdf_train[exclcolumn], errors='coerce')
    mdf_test[exclcolumn] = pd.to_numeric(mdf_test[exclcolumn], errors='coerce')
    
    #non integers are subject to infill
    mdf_train[exclcolumn] = np.where(mdf_train[exclcolumn] == mdf_train[exclcolumn].round(), mdf_train[exclcolumn], np.nan)
    mdf_test[exclcolumn] = np.where(mdf_test[exclcolumn] == mdf_test[exclcolumn].round(), mdf_test[exclcolumn], np.nan)

    if len(mdf_train[exclcolumn].mode())<1:
      fillvalue = mdf_train[exclcolumn].mean()
    else:
      fillvalue = mdf_train[exclcolumn].mode()[0]
      
    #special case if column didn't have any numeric entries
    if fillvalue != fillvalue:
      fillvalue = 0
    
    #replace missing data with fill value
    mdf_train[exclcolumn] = mdf_train[exclcolumn].fillna(fillvalue)
    mdf_test[exclcolumn] = mdf_test[exclcolumn].fillna(fillvalue)
    
    exc2_normalization_dict = {exclcolumn : {'fillvalue' : fillvalue}}
    
    column_dict_list = []

    column_dict = {exclcolumn : {'category' : 'exc5', \
                                 'origcategory' : category, \
                                 'normalization_dict' : exc2_normalization_dict, \
                                 'origcolumn' : column, \
                                 'inputcolumn' : column, \
                                 'columnslist' : [exclcolumn], \
                                 'categorylist' : [exclcolumn], \
                                 'infillmodel' : False, \
                                 'infillcomplete' : False, \
                                 'suffixoverlap_results' : suffixoverlap_results, \
                                 'deletecolumn' : False}}
    
    #now append column_dict onto postprocess_dict
    column_dict_list.append(column_dict.copy())

    return mdf_train, mdf_test, column_dict_list
  
  def process_shfl_class(self, df, column, category, postprocess_dict, params = {}):
    '''
    #function to shuffle data in a column
    #non-numeric entries allowed
    #for missing values, uses adjacent cell infill as default
    '''
    
    suffixoverlap_results = {}
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    if inplace is not True:
      
      #copy source column into new column
      df, suffixoverlap_results = \
      self.df_copy_train(df, column, column + '_shfl', suffixoverlap_results)
    
    else:
      
      suffixoverlap_results = \
      self.df_check_suffixoverlap(df, column + '_shfl', suffixoverlap_results)
      
      df.rename(columns = {column : column + '_shfl'}, inplace = True)
    
    #we've introduced that randomseed is now accessible throughout in the postprocess_dict
    random = postprocess_dict['randomseed']
    
    #uses support function
    df = self.df_shuffle_series(df, column + '_shfl', random)
    
    #we'll do the adjacent cell infill after the shuffle operation
    
    #apply ffill to replace NArows with value from adjacent cell in preceding row
    df[column + '_shfl'] = df[column + '_shfl'].fillna(method='ffill')
    
    #we'll follow with a bfill just in case first row had a nan
    df[column + '_shfl'] = df[column + '_shfl'].fillna(method='bfill')
    
    
    #create list of columns
    nmbrcolumns = [column + '_shfl']


    nmbrnormalization_dict = {column + '_shfl' : {}}

    #store some values in the nmbr_dict{} for use later in ML infill methods
    column_dict_list = []

    for nc in nmbrcolumns:

      column_dict = { nc : {'category' : 'shfl', \
                           'origcategory' : category, \
                           'normalization_dict' : nmbrnormalization_dict, \
                           'origcolumn' : column, \
                           'inputcolumn' : column, \
                           'columnslist' : nmbrcolumns, \
                           'categorylist' : nmbrcolumns, \
                           'infillmodel' : False, \
                           'infillcomplete' : False, \
                           'suffixoverlap_results' : suffixoverlap_results, \
                           'deletecolumn' : False}}

      column_dict_list.append(column_dict.copy())
        
    return df, column_dict_list

  def evalcategory(self, df_source, column, randomseed, eval_ratio, \
                   numbercategoryheuristic, powertransform, labels = False):
    '''
    #evalcategory(df, column)
    #Function that dakes as input a dataframe and associated column id \
    #evaluates the contents of cells and classifies the column into one of four categories
    #category 1, 'bnry', is for columns with only two categorys of text or integer
    #category 2, 'nmbr', is for columns with ndumerical integer or float values
    #category 3: 'bxcx', is for nmbr category with all positive values
    #category 4, 'text', is for columns with multiple categories appropriate for one-hot
    #category 5, 'date', is for columns with Timestamp data
    #category 6, 'null', is for columns with >85% null values (arbitrary figure)
    #returns category id as a string
    '''
    
    #we'll introduce convention of special values for powertransform to change default
    #we'll allow powertransform == 'excl' to signal that nonassigned columns should
    #be left untouched (a simpler version of existing functionality of assigning excl in assigncat)
    if powertransform == 'excl':
      category = 'excl'
      
    #or powertransform == 'exc2' for unprocessed but subject to force to numeric and modeinfill
    elif powertransform == 'exc2':
      category = 'exc2'
    
    else:
      
      #_____
      #a few default categories
      
      #default categorical
      #defaultcategorical = 'text'
      defaultcategorical = '1010'
      
      defaultordinal = 'ord3'

      defaultordinal_allunique = 'ord5'
      
      defaultnumerical = 'nmbr'
      #defaultnumerical = 'mean'
      
      defaultdatetime = 'dat6'
      
      #_____

      rowcount = df_source.shape[0]

      #we'll have convention that eval_ratio only applied for sets with >2,000 rows
      if rowcount < 2000:
        eval_ratio = 1.
      else:
        if eval_ratio > 0 and eval_ratio <= 1:
          eval_ratio = eval_ratio
        else:
          eval_ratio = eval_ratio / rowcount
      
      #take a random sample of rows for evaluation based on eval_ratio heuristic
      df = pd.DataFrame(df_source[column]).sample(frac=eval_ratio, random_state=randomseed)

      #I couldn't find a good pandas tool for evaluating data class, \
      #So will produce an array containing data types of each cell and \
      #evaluate for most common variable using the collections library

      type1_df = df[column].apply(lambda x: type(x)).values
      
      #c = collections.Counter(type1_df)
      c = Counter(type1_df)
      mc2 = c.most_common(2)
      mc = [mc2[0]]
      
      #count number of unique values
      nunique = df[column].nunique()

      #check if nan present for cases where nunique == 3
      nanpresent = False
      if nunique == 3:
        for unique in df[column].unique():
          if unique != unique:
            nanpresent = True

      #free memory (dtypes are memory hogs)
      del type1_df

      #additional array needed to check for time series

      #df['typecolumn2'] = df[column].apply(lambda x: type(pd.to_datetime(x, errors = 'coerce')))
      type2_df = df[column].apply(lambda x: type(pd.to_datetime(x, errors = 'coerce'))).values

      #datec = collections.Counter(type2_df)
      datec = Counter(type2_df)
      datemc = datec.most_common(1)
      datemc2 = datec.most_common(2)

      #this is to address scenario where only one value so we can still call mc2[1][0]
      if len(datemc2) == len(datemc):
        datemc2 = datemc + datemc

      #free memory (dtypes are memory hogs)
      del type2_df

      #an extension of this approach could be for those columns that produce a text\
      #category to implement an additional text to determine the number of \
      #common groupings / or the amount of uniquity. For example if every row has\
      #a unique value then one-hot-encoding would not be appropriate. It would \
      #probably be apopropraite to either return an error message if this is found \
      #or alternatively find a furhter way to automate this processing such as \
      #look for contextual clues to groupings that can be inferred.

      #This is kind of hack to evaluate class by comparing these with output of mc
      checkint = 1
      checkfloat = 1.1
      checkstring = 'string'
      checkNAN = np.nan

      #there's probably easier way to do this, here will create a check for date
      df_checkdate = pd.DataFrame([{'checkdate' : '7/4/2018'}])
      df_checkdate['checkdate'] = pd.to_datetime(df_checkdate['checkdate'], errors = 'coerce')

      #create dummy variable to store determined class (default is text class)
      category = defaultcategorical

      #if most common in column is string and > two values, set category to text
      if isinstance(checkstring, mc[0][0]) and nunique > 2:
        category = defaultcategorical

      #if most common is date, set category to date
      if isinstance(df_checkdate['checkdate'][0], datemc[0][0]):
        category = defaultdatetime

      if df[column].dtype.name == 'category':
        if nunique <= 2:
          category = 'bnry'
        else:
          category = defaultcategorical

      #if most common in column is integer and > two values, set category to number of bxcx
      if isinstance(checkint, mc[0][0]) and nunique > 2:

        if df[column].dtype.name == 'category':
          if nunique <= 2:
            category = 'bnry'
          else:
            category = defaultcategorical

        #take account for numbercategoryheuristic
        #if df[column].nunique() / df[column].shape[0] < numbercategoryheuristic:
        #if nunique < numbercategoryheuristic:
        if nunique <= 3:
          if nunique == 3:
            category = 'text'
          else:
            category = 'bnry'
  #       if True is False:
  #         pass

        else:
          category = defaultnumerical

      #if most common in column is float, set category to number or bxcx
      if isinstance(checkfloat, mc[0][0]):

        #take account for numbercategoryheuristic
        #if df[column].nunique() / df[column].shape[0] < numbercategoryheuristic \
  #       if nunique < numbercategoryheuristic \
  #       or df[column].dtype.name == 'category':
  #       if df[column].dtype.name == 'category':
        if df[column].dtype.name == 'category':
          if nunique <= 2:
            category = 'bnry'
          else:
            category = defaultcategorical

        elif nunique <= 3:
          if nunique == 3:
            category = 'text'
          elif nunique <= 2:
            category = 'bnry'

        else:
          category = defaultnumerical

      #if most common in column is integer and <= two values, set category to binary
      if isinstance(checkint, mc[0][0]) and nunique <= 2:
        category = 'bnry'

      #if most common in column is string and <= two values, set category to binary
      if isinstance(checkstring, mc[0][0]) and nunique <= 2:
        category = 'bnry'

      #else if most common in column is NaN, re-evaluate using the second most common type
      #(I suspect the below might be impacted if there are three dtypes instead of two,
      #in which case the 50% ratio rule may not be valid, that is kind of remote edge case)
      #elif df[column].isna().sum() >= df.shape[0] / 2:
      if len(mc2) > 1:
      
        if df[column].isna().sum() >= df.shape[0] / 2:


          #if 2nd most common in column is string and two values, set category to binary
          if isinstance(checkstring, mc2[1][0]) and nunique == 2:
            category = 'bnry'

          #if 2nd most common in column is string and > two values, set category to text
          if isinstance(checkstring, mc2[1][0]) and nunique > 2:
            category = defaultcategorical

          #if 2nd most common is date, set category to date   
          if isinstance(df_checkdate['checkdate'][0], datemc2[1][0]):
            category = defaultdatetime

          #if 2nd most common in column is integer and > two values, set category to number
          if isinstance(checkint, mc2[1][0]) and nunique > 2:

            if df[column].dtype.name == 'category':
              if nunique <= 2:
                category = 'bnry'
              else:
                category = defaultcategorical

    #         #take account for numbercategoryheuristic
    #         #if df[column].nunique() / df[column].shape[0] < numbercategoryheuristic:
            if nunique <= 3:

              if nunique == 3:
                category = defaultcategorical
              else:
                category = 'bnry'

    #         if True is False:
    #           pass

            else:

              category = defaultnumerical

          #if 2nd most common in column is float, set category to number
          if isinstance(checkfloat, mc2[1][0]):

    #         #take account for numbercategoryheuristic
    #         #if df[column].nunique() / df[column].shape[0] < numbercategoryheuristic:
    #         if df[column].nunique() < numbercategoryheuristic:

    #           category = 'text'

    #         else:

            if df[column].dtype.name == 'category':
              if nunique <= 2:
                category = 'bnry'
              else:
                category = defaultcategorical

            if df[column].nunique() <= 3:

              if nunique == 3:
                category = 'text'
              else:
                category = 'bnry'

            else:

              category = defaultnumerical

          #if 2nd most common in column is integer and <= two values, set category to binary
          if isinstance(checkint, mc2[1][0]) and nunique <= 2:
            category = 'bnry'

          #if 2nd most common in column is string and <= two values, set category to binary
          if isinstance(checkstring, mc2[1][0]) and nunique <= 2:
            category = 'bnry'

      if df[column].isna().sum() == df.shape[0]:
        category = 'null'

      #if category == 'text':
      if category == defaultcategorical:
        if nunique > numbercategoryheuristic:
          category = defaultordinal
        if nunique == df.shape[0]:
          category = defaultordinal_allunique

      #new statistical tests for numerical sets from v2.25
      #I don't consider mytself an expert here, these are kind of a placeholder while I conduct more research

  #     #default to 'nmbr' category instead of 'bxcx'
  #     if category == 'bxcx' and powertransform is False:
  #       category = 'nmbr'

      if category in ['nmbr', 'bxcx', defaultnumerical] \
      and powertransform is True:
        
        if df[pd.to_numeric(df[column], errors='coerce').notnull()][column].astype(float).nunique() >= 3:

          #shapiro tests for normality, we'll use a common threshold p<0.05 to reject the normality hypothesis
          stat, p = shapiro(df[pd.to_numeric(df[column], errors='coerce').notnull()][column].astype(float))
          #a typical threshold to test for normality is >0.05, let's try a lower bar for this application
          if p > 0.025:
            category = 'nmbr'
          if p <= 0.025:
            #skewness helps recognize exponential distributions, reference wikipedia
            #reference from wikipedia
    #       A normal distribution and any other symmetric distribution with finite third moment has a skewness of 0
    #       A half-normal distribution has a skewness just below 1
    #       An exponential distribution has a skewness of 2
    #       A lognormal distribution can have a skewness of any positive value, depending on its parameters
            #skewness = skew(df[column])
            skewness = skew(df[pd.to_numeric(df[column], errors='coerce').notnull()][column].astype(float))
            if skewness < 1.5:
              category = 'mnmx'
            else:
              #if powertransform is True:
              if category in ['nmbr', 'bxcx']:

                #note we'll only allow bxcx category if all values greater than a clip value
                #>0 (currently set at 0.1) since there is an asymptote for box-cox at 0
                if (df[pd.to_numeric(df[column], errors='coerce').notnull()][column].astype(float) >= 0.1).all():
                  category = 'bxcx'

                else:
                  category = 'nmbr'

              else:
                category = 'MAD3'

      del df
      
      #special cases for evlauation of labels column
      if labels is True:

        #(defaultnumerical = 'nmbr')
        if category == defaultnumerical:
          category = 'lbnm'
          
        #(defaultcategorical = '1010')
        if category == defaultcategorical:
          category = 'lb10'
          
        #(defaultordinal = 'ord3')
        if category == defaultordinal:
          category = 'lbor'

        #(defaultordinal_allunique = 'ord5')
        if category == defaultordinal_allunique:
          category = 'lbo5'
          
        if category == 'text':
          category = 'lbte'
          
        if category == 'bnry':
          category = 'lbbn'
          
        #(defaultdatetime = 'dat6')
        if category == defaultdatetime:
          category = 'lbda'
    
    return category

  def getNArows(self, df2, column, category, postprocess_dict, \
                drift_dict = {}, driftassess = False):
    '''
    #NArows(df, column), function that when fed a dataframe, \
    #column id, and category label outputs a single column dataframe composed of \
    #True and False with the same number of rows as the input and the True's \
    #coresponding to those rows of the input that had missing or NaN data. This \
    #output can later be used to identify which rows for a column to infill with ML\
    # derived plug data
    
    #also accepts a dictionary to store results of a drfit assessment available
    #by passing driftassess = True
    #if drift assessment performed returns an updated dictionary withj results
    
    #by default all NArowtypes recognize np.inf as NaN
    #(option activated external to this function)
    '''
    
    NArowtype = postprocess_dict['process_dict'][category]['NArowtype']
    
    #originally these evaluations were performed on a copy of the received column
    #struck that approach to reduce memory overhead from copy operation
    #small tradeoff in that edits here (such as cast to numeric) 
    #are preserved outside of this function
    # df2 = pd.DataFrame(df[column].copy())
    
    #if category == 'text':
    if NArowtype in ['justNaN']:
      
      if driftassess is True:
        
        nunique = df2[column].nunique()
        
        #this is to ensure postprocess_dict file size doesn't get out of control so 
        #only collect unique entries in source column drift stats
        #if number of unique is below a threshold (arbrily set to 500)
        if nunique < 500:

          drift_dict.update({column : {'unique' : df2[column].unique(), \
                                       'nunique' : nunique, \
                                       'nanratio' : pd.isna(df2[column]).sum() / df2[column].shape[0]}})
          
        else:
          
          drift_dict.update({column : {'nunique' : nunique, \
                                       'nanratio' : pd.isna(df2[column]).sum() / df2[column].shape[0]}})
          
      #returns dataframe of True and False, where True coresponds to the NaN's
      #renames column name to column + '_NArows'
      NArows = pd.isna(df2[column])
      NArows = pd.DataFrame(NArows)
      NArows = NArows.rename(columns = {column:column+'_NArows'})

#     if category == 'bnry':

#       #returns dataframe of True and False, where True coresponds to the NaN's
#       #renames column name to column + '_NArows'
#       NArows = pd.isna(df2[column])
#       NArows = pd.DataFrame(NArows)
#       NArows = NArows.rename(columns = {column:column+'_NArows'})

    #if category == 'nmbr' or category == 'bxcx':
    #if category in ['nmbr', 'bxcx', 'nbr2']:
    if NArowtype in ['numeric']:

      #convert all values to either numeric or NaN
      df2[column] = pd.to_numeric(df2[column], errors='coerce')
      
      if driftassess is True:
        
        if df2[column].notnull().nunique() > 2:
          W, p = shapiro(df2[df2[column].notnull()][column].astype(float))
          skew_stat = skew(df2[df2[column].notnull()][column].astype(float))
        else:
          W = np.nan
          p = np.nan
          skew_stat = np.nan
          
#         W, p = shapiro(df2[df2[column].notnull()][column].astype(float))
#         skew_stat = skew(df2[df2[column].notnull()][column].astype(float))
        
        drift_dict.update({column : {'max' : df2[column].max(), \
                                     'quantile_99' : df2[column].quantile(0.99), \
                                     'quantile_90' : df2[column].quantile(0.90), \
                                     'quantile_66' : df2[column].quantile(0.66), \
                                     'median' : df2[column].median(), \
                                     'quantile_33' : df2[column].quantile(0.33), \
                                     'quantile_10' : df2[column].quantile(0.10), \
                                     'quantile_01' : df2[column].quantile(0.01), \
                                     'min' : df2[column].min(), \
                                     'mean' : df2[column].mean(), \
                                     'std' : df2[column].std(), \
                                     'MAD' : df2[column].mad(), \
                                     'skew' : skew_stat, \
                                     'shapiro_W' : W, \
                                     'shapiro_p' : p, \
                                     'nan_ratio' : pd.isna(df2[column]).sum() / df2[column].shape[0]}})

      #returns dataframe of True and False, where True coresponds to the NaN's
      #renames column name to column + '_NArows'
      NArows = pd.isna(df2[column])
      NArows = pd.DataFrame(NArows)
      NArows = NArows.rename(columns = {column:column+'_NArows'})
      
    if NArowtype in ['integer']:

      #convert all values to either numeric or NaN
      df2[column] = pd.to_numeric(df2[column], errors='coerce')
      
      #non integers are subject to infill
      df2[column] = np.where(df2[column] == df2[column].round(), df2[column], np.nan)
      
      if driftassess is True:
        
        if df2[column].notnull().nunique() > 2:
          W, p = shapiro(df2[df2[column].notnull()][column].astype(float))
          skew_stat = skew(df2[df2[column].notnull()][column].astype(float))
        else:
          W = np.nan
          p = np.nan
          skew_stat = np.nan
          
#         W, p = shapiro(df2[df2[column].notnull()][column].astype(float))
#         skew_stat = skew(df2[df2[column].notnull()][column].astype(float))
        
        drift_dict.update({column : {'max' : df2[column].max(), \
                                     'quantile_99' : df2[column].quantile(0.99), \
                                     'quantile_90' : df2[column].quantile(0.90), \
                                     'quantile_66' : df2[column].quantile(0.66), \
                                     'median' : df2[column].median(), \
                                     'quantile_33' : df2[column].quantile(0.33), \
                                     'quantile_10' : df2[column].quantile(0.10), \
                                     'quantile_01' : df2[column].quantile(0.01), \
                                     'min' : df2[column].min(), \
                                     'mean' : df2[column].mean(), \
                                     'std' : df2[column].std(), \
                                     'MAD' : df2[column].mad(), \
                                     'skew' : skew_stat, \
                                     'shapiro_W' : W, \
                                     'shapiro_p' : p, \
                                     'nan_ratio' : pd.isna(df2[column]).sum() / df2[column].shape[0]}})

      #returns dataframe of True and False, where True coresponds to the NaN's
      #renames column name to column + '_NArows'
      NArows = pd.isna(df2[column])
      NArows = pd.DataFrame(NArows)
      NArows = NArows.rename(columns = {column:column+'_NArows'})
      
    if NArowtype in ['positivenumeric']:
      
      #convert all values to either numeric or NaN
      df2[column] = pd.to_numeric(df2[column], errors='coerce')
      nonpositive_ratio = df2[df2[column] <= 0].sum()[0] / df2[column].shape[0]
      
      df2.loc[df2[column] <= 0, (column)] = np.nan
      
      if driftassess is True:
        
        if df2[column].notnull().nunique() > 2:
          W, p = shapiro(df2[df2[column].notnull()][column].astype(float))
          skew_stat = skew(df2[df2[column].notnull()][column].astype(float))
        else:
          W = np.nan
          p = np.nan
          skew_stat = np.nan
        
        drift_dict.update({column : {'max' : df2[column].max(), \
                                     'quantile_99' : df2[column].quantile(0.99), \
                                     'quantile_90' : df2[column].quantile(0.90), \
                                     'quantile_66' : df2[column].quantile(0.66), \
                                     'median' : df2[column].median(), \
                                     'quantile_33' : df2[column].quantile(0.33), \
                                     'quantile_10' : df2[column].quantile(0.10), \
                                     'quantile_01' : df2[column].quantile(0.01), \
                                     'min' : df2[column].min(), \
                                     'mean' : df2[column].mean(), \
                                     'std' : df2[column].std(), \
                                     'MAD' : df2[column].mad(), \
                                     'skew' : skew_stat, \
                                     'shapiro_W' : W, \
                                     'shapiro_p' : p, \
                                     'nonpositive_ratio' : nonpositive_ratio, \
                                     'nan_ratio' : pd.isna(df2[column]).sum() / df2[column].shape[0]}})
    
      #returns dataframe of True and False, where True coresponds to the NaN's
      #renames column name to column + '_NArows'
      NArows = pd.isna(df2[column])
      NArows = pd.DataFrame(NArows)
      NArows = NArows.rename(columns = {column:column+'_NArows'})
      
    if NArowtype in ['nonnegativenumeric']:
      
      #convert all values to either numeric or NaN
      df2[column] = pd.to_numeric(df2[column], errors='coerce')
      negative_ratio = df2[df2[column] < 0].sum()[0] / df2[column].shape[0]
      
      df2.loc[df2[column] < 0, (column)] = np.nan
      
      if driftassess is True:
        
        if df2[column].notnull().nunique() > 2:
          W, p = shapiro(df2[df2[column].notnull()][column].astype(float))
          skew_stat = skew(df2[df2[column].notnull()][column].astype(float))
        else:
          W = np.nan
          p = np.nan
          skew_stat = np.nan
        
        drift_dict.update({column : {'max' : df2[column].max(), \
                                     'quantile_99' : df2[column].quantile(0.99), \
                                     'quantile_90' : df2[column].quantile(0.90), \
                                     'quantile_66' : df2[column].quantile(0.66), \
                                     'median' : df2[column].median(), \
                                     'quantile_33' : df2[column].quantile(0.33), \
                                     'quantile_10' : df2[column].quantile(0.10), \
                                     'quantile_01' : df2[column].quantile(0.01), \
                                     'min' : df2[column].min(), \
                                     'mean' : df2[column].mean(), \
                                     'std' : df2[column].std(), \
                                     'MAD' : df2[column].mad(), \
                                     'skew' : skew_stat, \
                                     'shapiro_W' : W, \
                                     'shapiro_p' : p, \
                                     'negative_ratio' : negative_ratio, \
                                     'nan_ratio' : pd.isna(df2[column]).sum() / df2[column].shape[0]}})

      #returns dataframe of True and False, where True coresponds to the NaN's
      #renames column name to column + '_NArows'
      NArows = pd.isna(df2[column])
      NArows = pd.DataFrame(NArows)
      NArows = NArows.rename(columns = {column:column+'_NArows'})
      
    if NArowtype in ['nonzeronumeric']:

      #convert all values to either numeric or NaN
      df2[column] = pd.to_numeric(df2[column], errors='coerce')
      zero_ratio = df2[df2[column] == 0].sum()[0] / df2[column].shape[0]
      
      df2.loc[df2[column] == 0, (column)] = np.nan
      
      if driftassess is True:
        
        if df2[column].notnull().nunique() > 2:
          W, p = shapiro(df2[df2[column].notnull()][column].astype(float))
          skew_stat = skew(df2[df2[column].notnull()][column].astype(float))
        else:
          W = np.nan
          p = np.nan
          skew_stat = np.nan
        
        drift_dict.update({column : {'max' : df2[column].max(), \
                                     'quantile_99' : df2[column].quantile(0.99), \
                                     'quantile_90' : df2[column].quantile(0.90), \
                                     'quantile_66' : df2[column].quantile(0.66), \
                                     'median' : df2[column].median(), \
                                     'quantile_33' : df2[column].quantile(0.33), \
                                     'quantile_10' : df2[column].quantile(0.10), \
                                     'quantile_01' : df2[column].quantile(0.01), \
                                     'min' : df2[column].min(), \
                                     'mean' : df2[column].mean(), \
                                     'std' : df2[column].std(), \
                                     'MAD' : df2[column].mad(), \
                                     'skew' : skew_stat, \
                                     'shapiro_W' : W, \
                                     'shapiro_p' : p, \
                                     'zero_ratio' : zero_ratio, \
                                     'nan_ratio' : pd.isna(df2[column]).sum() / df2[column].shape[0]}})

      #returns dataframe of True and False, where True coresponds to the NaN's
      #renames column name to column + '_NArows'
      NArows = pd.isna(df2[column])
      NArows = pd.DataFrame(NArows)
      NArows = NArows.rename(columns = {column:column+'_NArows'})
      
    if NArowtype in ['parsenumeric']:
      
      NArows = self.parsenumeric(df2, column)

      drift_dict.update({column : {'nunique' : df2[column].nunique(), \
                                   'nanratio' : pd.isna(df2[column]).sum() / df2[column].shape[0]}})      
      
    if NArowtype in ['parsenumeric_commas']:
      
      NArows = self.parsenumeric_commas(df2, column)

      drift_dict.update({column : {'nunique' : df2[column].nunique(), \
                                   'nanratio' : pd.isna(df2[column]).sum() / df2[column].shape[0]}})
      
    if NArowtype in ['parsenumeric_EU']:
      
      NArows = self.parsenumeric_EU(df2, column)

      drift_dict.update({column : {'nunique' : df2[column].nunique(), \
                                   'nanratio' : pd.isna(df2[column]).sum() / df2[column].shape[0]}})
      
    if NArowtype in ['datetime']:
      
      df2[column] = pd.to_datetime(df2[column], errors = 'coerce')

      if driftassess is True:
        drift_dict.update({column : {'nanratio' : pd.isna(df2[column]).sum() / df2[column].shape[0]}})
      
      NArows = pd.isna(df2[column])
      NArows = pd.DataFrame(NArows)
      NArows = NArows.rename(columns = {column:column+'_NArows'})
      
#       NArows = self.parsedate(df2, column)
      
    #if category in ['excl']:
    if NArowtype in ['exclude', 'boolexclude', 'totalexclude']:
      
      if driftassess is True:
        drift_dict.update({column : {}})
      
#       NArows = pd.DataFrame(np.zeros((df2.shape[0], 1)), columns=[column+'_NArows'])
      #NArows = NArows.rename(columns = {column:column+'_NArows'})
      
      NArows = pd.DataFrame(df2[column].copy())
      NArows[column] = False
      NArows = NArows.rename(columns = {column:column+'_NArows'})
#       NArows[column+'_NArows'] = False
      
    del df2
    
    if driftassess is False:
      
      return NArows
    
    else:
      
      return NArows, drift_dict
  
  def parsedate(self, df, column):
    """
    #support function for NArows
    #parses datetime entries and returns a column with boolean identification
    #for entries that aren't registering as datetime objects
    #wherein activations are 0 if a datetime is present and 1 if not
    """
    
    df[column] = pd.to_datetime(df[column], errors = 'coerce')

    NArows = pd.isna(df[column])
    NArows = pd.DataFrame(NArows)
    NArows = NArows.rename(columns = {column:column+'_NArows'})
    
    return NArows
  
  def is_number(self, s):
    """
    #support function for numeric parsing
    #tests if a string s is numeric
    #partly inspired by stack overflow discussion 
    #https://stackoverflow.com/questions/354038/how-do-i-check-if-a-string-is-a-number-float
    """
    try:
      s = float(s)
    except ValueError:
      return False
    if s == s and not np.isinf(s):
      return True
    else:
      #(nan will be subject to infill)
      return False

  def is_number_comma(self, s):
    """
    #support function for numeric parsing
    #tests if a string s is numeric after stripping out commas
    #partly inspired by stack overflow discussion 
    #https://stackoverflow.com/questions/354038/how-do-i-check-if-a-string-is-a-number-float
    """
    try:
      #strips out commas
      s = float(s.replace(',',''))
    except ValueError:
      return False
    if s == s and not np.isinf(s):
      return True
    else:
      #(nan will be subject to infill)
      return False
    
  def is_number_EU(self, s):
    """
    #support function for numeric parsing
    #tests if a string s is numeric after stripping out periods
    #and replacing commas with periods
    #such as to convert from international conventions to US
    #this is relavent for cases where working with international data on US OS
    #I expect if working on international OS a different convention may be appropriate
    #based on what is recognized as a float
    """
    try:
      #strips out spaces, periods other than first and last character, replaces commas with periods, cast as float
      s = float(s[0] + s[1:-1].replace(' ','').replace('.','').replace(',','.') + s[-1])
    except ValueError:
      return False
    if s == s and not np.isinf(s):
      return True
    else:
      #(nan will be subject to infill)
      return False
    
  def parsenumeric(self, df, column):
    """
    #support function for process_nmrc and NArows
    #parses string entries and returns a column with boolean identification
    #for entries that include numeric string portions
    #wherein activations are 0 if a number is present and 1 if not
    #treats numeric entries as number as well
    """
    
    #first we find overlaps from mdf_train
    
    unique_list = list(df[column].unique())

    unique_list = list(map(str, unique_list))
    
    maxlength = max(len(x) for x in unique_list)
    
    overlap_lengths = list(range(maxlength, 0, -1))

    overlap_dict = {}
    
    for overlap_length in overlap_lengths:

      for unique in unique_list:
        
        if unique not in overlap_dict:

          len_unique = len(unique)

          if len_unique >= overlap_length:
            
            if overlap_length > 1:

              nbr_iterations = len_unique - overlap_length

              for i in range(nbr_iterations + 1):
                
                if unique not in overlap_dict:

                  extract = unique[i:(overlap_length+i)]

  #                 extract_already_in_overlap_dict = False

                  if self.is_number(extract):
        
                    overlap_dict.update({unique : False})
                
            #else if overlap_length == 1    
            else:
              
              nbr_iterations = len_unique - overlap_length
              
              in_dict = False

              for i in range(nbr_iterations + 1):
                
                if unique not in overlap_dict:

                  extract = unique[i:(overlap_length+i)]

  #                 extract_already_in_overlap_dict = False

                  if self.is_number(extract):

                    in_dict = True

                    overlap_dict.update({unique : False})
                  
              if in_dict is False:

                overlap_dict.update({unique : True})
    
    NArows = pd.DataFrame(df[column].copy())

    NArows[column] = NArows[column].astype(str)
    NArows[column] = NArows[column].replace(overlap_dict)
#     df[column] = df[column].astype(np.int8)
    
    NArows.columns = [column+'_NArows']
    
    return NArows
  
  def parsenumeric_commas(self, df, column):
    """
    #support function for process_nmrc and NArows
    #parses string entries and returns a column with boolean identification
    #for entries that include numeric string portions (after stripping commas)
    #wherein activations are 0 if a number is present and 1 if not
    #treats numeric entries as number as well
    """
    
    #first we find overlaps from mdf_train
    
    unique_list = list(df[column].unique())

    unique_list = list(map(str, unique_list))
    
    maxlength = max(len(x) for x in unique_list)
    
    overlap_lengths = list(range(maxlength, 0, -1))

    overlap_dict = {}
    
    for overlap_length in overlap_lengths:

      for unique in unique_list:
        
        if unique not in overlap_dict:

          len_unique = len(unique)

          if len_unique >= overlap_length:
            
            if overlap_length > 1:

              nbr_iterations = len_unique - overlap_length

              for i in range(nbr_iterations + 1):
                
                if unique not in overlap_dict:

                  extract = unique[i:(overlap_length+i)]

  #                 extract_already_in_overlap_dict = False

                  if self.is_number_comma(extract):

                    overlap_dict.update({unique : False})
                
            #else if overlap_length == 1    
            else:
              
              nbr_iterations = len_unique - overlap_length
              
              in_dict = False

              for i in range(nbr_iterations + 1):
                
                if unique not in overlap_dict:

                  extract = unique[i:(overlap_length+i)]

  #                 extract_already_in_overlap_dict = False

                  if self.is_number_comma(extract):

                    in_dict = True

                    overlap_dict.update({unique : False})
                  
              if in_dict is False:

                overlap_dict.update({unique : True})
    
    NArows = pd.DataFrame(df[column].copy())

    NArows[column] = NArows[column].astype(str)
    NArows[column] = NArows[column].replace(overlap_dict)
#     df[column] = df[column].astype(np.int8)
    
    NArows.columns = [column+'_NArows']
    
    return NArows
  
  def parsenumeric_EU(self, df, column):
    """
    #support function for process_nmEU and NArows
    #parses string entries and returns a column with boolean identification
    #for entries that include numeric string portions (after converting from international format)
    #wherein activations are 0 if a number is present and 1 if not
    #treats numeric entries as number as well
    """
    
    #first we find overlaps from mdf_train
    
    unique_list = list(df[column].unique())

    unique_list = list(map(str, unique_list))
    
    maxlength = max(len(x) for x in unique_list)
    
    overlap_lengths = list(range(maxlength, 0, -1))

    overlap_dict = {}
    
    for overlap_length in overlap_lengths:

      for unique in unique_list:
        
        if unique not in overlap_dict:

          len_unique = len(unique)

          if len_unique >= overlap_length:
            
            if overlap_length > 1:

              nbr_iterations = len_unique - overlap_length

              for i in range(nbr_iterations + 1):
                
                if unique not in overlap_dict:

                  extract = unique[i:(overlap_length+i)]

  #                 extract_already_in_overlap_dict = False

                  if self.is_number_EU(extract):

                    overlap_dict.update({unique : False})
                
            #else if overlap_length == 1    
            else:
              
              nbr_iterations = len_unique - overlap_length
              
              in_dict = False

              for i in range(nbr_iterations + 1):
                
                if unique not in overlap_dict:

                  extract = unique[i:(overlap_length+i)]

  #                 extract_already_in_overlap_dict = False

                  if self.is_number_EU(extract):

                    in_dict = True

                    overlap_dict.update({unique : False})
                  
              if in_dict is False:

                overlap_dict.update({unique : True})
    
    NArows = pd.DataFrame(df[column].copy())

    NArows[column] = NArows[column].astype(str)
    NArows[column] = NArows[column].replace(overlap_dict)
#     df[column] = df[column].astype(np.int8)
    
    NArows.columns = [column+'_NArows']
    
    return NArows

  def populateMLinfilldefaults(self, randomseed):
    '''
    populates a dictionary with default values for ML infill,
    currently based on Random Forest Regressor and Random Forest Classifier 
    (Each based on ScikitLearn default values)
  
    note that n_estimators set at 100 (default for version 0.22)
    '''
  
    MLinfilldefaults = {'RandomForestClassifier':{}, 'RandomForestRegressor':{}}
    
    MLinfilldefaults['RandomForestClassifier'].update({'n_estimators':100, \
                                                       'criterion':'gini', \
                                                       'max_depth':None, \
                                                       'min_samples_split':2, \
                                                       'min_samples_leaf':1, \
                                                       'min_weight_fraction_leaf':0.0, \
                                                       'max_features':'auto', \
                                                       'max_leaf_nodes':None, \
                                                       'min_impurity_decrease':0.0, \
                                                       'min_impurity_split':None, \
                                                       'bootstrap':True, \
                                                       'oob_score':False, \
                                                       'n_jobs':None, \
                                                       'random_state':randomseed, \
                                                       'verbose':0, \
                                                       'warm_start':False, \
                                                       'class_weight':None})
  
    MLinfilldefaults['RandomForestRegressor'].update({'n_estimators':100, \
                                                      'criterion':'mse', \
                                                      'max_depth':None, \
                                                      'min_samples_split':2, \
                                                      'min_samples_leaf':1, \
                                                      'min_weight_fraction_leaf':0.0, \
                                                      'max_features':'auto', \
                                                      'max_leaf_nodes':None, \
                                                      'min_impurity_decrease':0.0, \
                                                      'min_impurity_split':None, \
                                                      'bootstrap':True, \
                                                      'oob_score':False, \
                                                      'n_jobs':None, \
                                                      'random_state':randomseed, \
                                                      'verbose':0, \
                                                      'warm_start':False})

    return MLinfilldefaults

  def initRandomForestClassifier(self, ML_cmnd, MLinfilldefaults):
    '''
    function that assigns appropriate parameters based on defaults and user inputs
    and then initializes a RandomForestClassifier model
    '''
    
    #populate ML_cmnd if stuff not already present
    if 'MLinfill_cmnd' not in ML_cmnd:
      ML_cmnd.update({'MLinfill_cmnd':{}})
    if 'RandomForestClassifier' not in ML_cmnd['MLinfill_cmnd']:
      ML_cmnd['MLinfill_cmnd'].update({'RandomForestClassifier':{}})
      

    #MLinfilldefaults['RandomForestClassifier']
    if 'n_estimators' in ML_cmnd['MLinfill_cmnd']['RandomForestClassifier']:
      n_estimators = ML_cmnd['MLinfill_cmnd']['RandomForestClassifier']['n_estimators']
    else:
      n_estimators = MLinfilldefaults['RandomForestClassifier']['n_estimators']

    if 'criterion' in ML_cmnd['MLinfill_cmnd']['RandomForestClassifier']:
      criterion = ML_cmnd['MLinfill_cmnd']['RandomForestClassifier']['criterion']
    else:
      criterion = MLinfilldefaults['RandomForestClassifier']['criterion']

    if 'max_depth' in ML_cmnd['MLinfill_cmnd']['RandomForestClassifier']:
      max_depth = ML_cmnd['MLinfill_cmnd']['RandomForestClassifier']['max_depth']
    else:
      max_depth = MLinfilldefaults['RandomForestClassifier']['max_depth']

    if 'min_samples_split' in ML_cmnd['MLinfill_cmnd']['RandomForestClassifier']:
      min_samples_split = ML_cmnd['MLinfill_cmnd']['RandomForestClassifier']['min_samples_split']
    else:
      min_samples_split = MLinfilldefaults['RandomForestClassifier']['min_samples_split']

    if 'min_samples_leaf' in ML_cmnd['MLinfill_cmnd']['RandomForestClassifier']:
      min_samples_leaf = ML_cmnd['MLinfill_cmnd']['RandomForestClassifier']['min_samples_leaf']
    else:
      min_samples_leaf = MLinfilldefaults['RandomForestClassifier']['min_samples_leaf']

    if 'min_weight_fraction_leaf' in ML_cmnd['MLinfill_cmnd']['RandomForestClassifier']:
      min_weight_fraction_leaf = ML_cmnd['MLinfill_cmnd']['RandomForestClassifier']['min_weight_fraction_leaf']
    else:
      min_weight_fraction_leaf = MLinfilldefaults['RandomForestClassifier']['min_weight_fraction_leaf']

    if 'max_features' in ML_cmnd['MLinfill_cmnd']['RandomForestClassifier']:
      max_features = ML_cmnd['MLinfill_cmnd']['RandomForestClassifier']['max_features']
    else:
      max_features = MLinfilldefaults['RandomForestClassifier']['max_features']

    if 'max_leaf_nodes' in ML_cmnd['MLinfill_cmnd']['RandomForestClassifier']:
      max_leaf_nodes = ML_cmnd['MLinfill_cmnd']['RandomForestClassifier']['max_leaf_nodes']
    else:
      max_leaf_nodes = MLinfilldefaults['RandomForestClassifier']['max_leaf_nodes']

    if 'min_impurity_decrease' in ML_cmnd['MLinfill_cmnd']['RandomForestClassifier']:
      min_impurity_decrease = ML_cmnd['MLinfill_cmnd']['RandomForestClassifier']['min_impurity_decrease']
    else:
      min_impurity_decrease = MLinfilldefaults['RandomForestClassifier']['min_impurity_decrease']

    if 'min_impurity_split' in ML_cmnd['MLinfill_cmnd']['RandomForestClassifier']:
      min_impurity_split = ML_cmnd['MLinfill_cmnd']['RandomForestClassifier']['min_impurity_split']
    else:
      min_impurity_split = MLinfilldefaults['RandomForestClassifier']['min_impurity_split']

    if 'bootstrap' in ML_cmnd['MLinfill_cmnd']['RandomForestClassifier']:
      bootstrap = ML_cmnd['MLinfill_cmnd']['RandomForestClassifier']['bootstrap']
    else:
      bootstrap = MLinfilldefaults['RandomForestClassifier']['bootstrap']

    if 'oob_score' in ML_cmnd['MLinfill_cmnd']['RandomForestClassifier']:
      oob_score = ML_cmnd['MLinfill_cmnd']['RandomForestClassifier']['oob_score']
    else:
      oob_score = MLinfilldefaults['RandomForestClassifier']['oob_score']

    if 'n_jobs' in ML_cmnd['MLinfill_cmnd']['RandomForestClassifier']:
      n_jobs = ML_cmnd['MLinfill_cmnd']['RandomForestClassifier']['n_jobs']
    else:
      n_jobs = MLinfilldefaults['RandomForestClassifier']['n_jobs']

    if 'random_state' in ML_cmnd['MLinfill_cmnd']['RandomForestClassifier']:
      random_state = ML_cmnd['MLinfill_cmnd']['RandomForestClassifier']['random_state']
    else:
      random_state = MLinfilldefaults['RandomForestClassifier']['random_state']

    if 'verbose' in ML_cmnd['MLinfill_cmnd']['RandomForestClassifier']:
      verbose = ML_cmnd['MLinfill_cmnd']['RandomForestClassifier']['verbose']
    else:
      verbose = MLinfilldefaults['RandomForestClassifier']['verbose']

    if 'warm_start' in ML_cmnd['MLinfill_cmnd']['RandomForestClassifier']:
      warm_start = ML_cmnd['MLinfill_cmnd']['RandomForestClassifier']['warm_start']
    else:
      warm_start = MLinfilldefaults['RandomForestClassifier']['warm_start']

    if 'class_weight' in ML_cmnd['MLinfill_cmnd']['RandomForestClassifier']:
      class_weight = ML_cmnd['MLinfill_cmnd']['RandomForestClassifier']['class_weight']
    else:
      class_weight = MLinfilldefaults['RandomForestClassifier']['class_weight']

    #do other stuff?

    #then initialize RandomForestClassifier model
    model = RandomForestClassifier(n_estimators = n_estimators, \
                                   criterion = criterion, \
                                   max_depth = max_depth, \
                                   min_samples_split = min_samples_split, \
                                   min_samples_leaf = min_samples_leaf, \
                                   min_weight_fraction_leaf = min_weight_fraction_leaf, \
                                   max_features = max_features, \
                                   max_leaf_nodes = max_leaf_nodes, \
                                   min_impurity_decrease = min_impurity_decrease, \
                                   min_impurity_split = min_impurity_split, \
                                   bootstrap = bootstrap, \
                                   oob_score = oob_score, \
                                   n_jobs = n_jobs, \
                                   random_state = random_state, \
                                   verbose = verbose, \
                                   warm_start = warm_start, \
                                   class_weight = class_weight)

    return model

  def initRandomForestRegressor(self, ML_cmnd, MLinfilldefaults):
    '''
    function that assigns appropriate parameters based on defaults and user inputs
    and then initializes a RandomForestRegressor model
    '''
    
    #populate ML_cmnd if stuff not already present
    if 'MLinfill_cmnd' not in ML_cmnd:
      ML_cmnd.update({'MLinfill_cmnd':{}})
    if 'RandomForestRegressor' not in ML_cmnd['MLinfill_cmnd']:
      ML_cmnd['MLinfill_cmnd'].update({'RandomForestRegressor':{}})
      
    #MLinfilldefaults['RandomForestRegressor']
    if 'n_estimators' in ML_cmnd['MLinfill_cmnd']['RandomForestRegressor']:
      n_estimators = ML_cmnd['MLinfill_cmnd']['RandomForestRegressor']['n_estimators']
    else:
      n_estimators = MLinfilldefaults['RandomForestRegressor']['n_estimators']

    if 'criterion' in ML_cmnd['MLinfill_cmnd']['RandomForestRegressor']:
      criterion = ML_cmnd['MLinfill_cmnd']['RandomForestRegressor']['criterion']
    else:
      criterion = MLinfilldefaults['RandomForestRegressor']['criterion']

    if 'max_depth' in ML_cmnd['MLinfill_cmnd']['RandomForestRegressor']:
      max_depth = ML_cmnd['MLinfill_cmnd']['RandomForestRegressor']['max_depth']
    else:
      max_depth = MLinfilldefaults['RandomForestRegressor']['max_depth']

    if 'min_samples_split' in ML_cmnd['MLinfill_cmnd']['RandomForestRegressor']:
      min_samples_split = ML_cmnd['MLinfill_cmnd']['RandomForestRegressor']['min_samples_split']
    else:
      min_samples_split = MLinfilldefaults['RandomForestRegressor']['min_samples_split']

    if 'min_samples_leaf' in ML_cmnd['MLinfill_cmnd']['RandomForestRegressor']:
      min_samples_leaf = ML_cmnd['MLinfill_cmnd']['RandomForestRegressor']['min_samples_leaf']
    else:
      min_samples_leaf = MLinfilldefaults['RandomForestRegressor']['min_samples_leaf']

    if 'min_weight_fraction_leaf' in ML_cmnd['MLinfill_cmnd']['RandomForestRegressor']:
      min_weight_fraction_leaf = ML_cmnd['MLinfill_cmnd']['RandomForestRegressor']['min_weight_fraction_leaf']
    else:
      min_weight_fraction_leaf = MLinfilldefaults['RandomForestRegressor']['min_weight_fraction_leaf']

    if 'max_features' in ML_cmnd['MLinfill_cmnd']['RandomForestRegressor']:
      max_features = ML_cmnd['MLinfill_cmnd']['RandomForestRegressor']['max_features']
    else:
      max_features = MLinfilldefaults['RandomForestRegressor']['max_features']

    if 'max_leaf_nodes' in ML_cmnd['MLinfill_cmnd']['RandomForestRegressor']:
      max_leaf_nodes = ML_cmnd['MLinfill_cmnd']['RandomForestRegressor']['max_leaf_nodes']
    else:
      max_leaf_nodes = MLinfilldefaults['RandomForestRegressor']['max_leaf_nodes']

    if 'min_impurity_decrease' in ML_cmnd['MLinfill_cmnd']['RandomForestRegressor']:
      min_impurity_decrease = ML_cmnd['MLinfill_cmnd']['RandomForestRegressor']['min_impurity_decrease']
    else:
      min_impurity_decrease = MLinfilldefaults['RandomForestRegressor']['min_impurity_decrease']

    if 'min_impurity_split' in ML_cmnd['MLinfill_cmnd']['RandomForestRegressor']:
      min_impurity_split = ML_cmnd['MLinfill_cmnd']['RandomForestRegressor']['min_impurity_split']
    else:
      min_impurity_split = MLinfilldefaults['RandomForestRegressor']['min_impurity_split']

    if 'bootstrap' in ML_cmnd['MLinfill_cmnd']['RandomForestRegressor']:
      bootstrap = ML_cmnd['MLinfill_cmnd']['RandomForestRegressor']['bootstrap']
    else:
      bootstrap = MLinfilldefaults['RandomForestRegressor']['bootstrap']

    if 'oob_score' in ML_cmnd['MLinfill_cmnd']['RandomForestRegressor']:
      oob_score = ML_cmnd['MLinfill_cmnd']['RandomForestRegressor']['oob_score']
    else:
      oob_score = MLinfilldefaults['RandomForestRegressor']['oob_score']

    if 'n_jobs' in ML_cmnd['MLinfill_cmnd']['RandomForestRegressor']:
      n_jobs = ML_cmnd['MLinfill_cmnd']['RandomForestRegressor']['n_jobs']
    else:
      n_jobs = MLinfilldefaults['RandomForestClassifier']['n_jobs']

    if 'random_state' in ML_cmnd['MLinfill_cmnd']['RandomForestRegressor']:
      random_state = ML_cmnd['MLinfill_cmnd']['RandomForestRegressor']['random_state']
    else:
      random_state = MLinfilldefaults['RandomForestRegressor']['random_state']

    if 'verbose' in ML_cmnd['MLinfill_cmnd']['RandomForestRegressor']:
      verbose = ML_cmnd['MLinfill_cmnd']['RandomForestRegressor']['verbose']
    else:
      verbose = MLinfilldefaults['RandomForestRegressor']['verbose']

    if 'warm_start' in ML_cmnd['MLinfill_cmnd']['RandomForestRegressor']:
      warm_start = ML_cmnd['MLinfill_cmnd']['RandomForestRegressor']['warm_start']
    else:
      warm_start = MLinfilldefaults['RandomForestRegressor']['warm_start']

    #do other stuff?

    #then initialize RandomForestRegressor model 
    model = RandomForestRegressor(n_estimators = n_estimators, \
                                  criterion = criterion, \
                                  max_depth = max_depth, \
                                  min_samples_split = min_samples_split, \
                                  min_samples_leaf = min_samples_leaf, \
                                  min_weight_fraction_leaf = min_weight_fraction_leaf, \
                                  max_features = max_features, \
                                  max_leaf_nodes = max_leaf_nodes, \
                                  min_impurity_decrease = min_impurity_decrease, \
                                  min_impurity_split = min_impurity_split, \
                                  bootstrap = bootstrap, \
                                  oob_score = oob_score, \
                                  n_jobs = n_jobs, \
                                  random_state = random_state, \
                                  verbose = verbose, \
                                  warm_start = warm_start)

    return model
  
  def inspect_ML_cmnd(self, ML_cmnd, autoML_type, MLinfill_alg):
    """
    #Inspects ML_cmnd to determine if any of the parameters passed
    #for regressor or classifier are passed as lists instead of distinct
    #values, in which case they will be evaluated via grid search
    #or in a future extension random search or other hyperparameter tuning methods
    
    #takes as input user-passed ML_cmnd, returns tune_marker
    #where tune_marker = True indicates sets were passed, else False
    
    #MLinfill_type refers to type of predcitive algorithm applied,
    #currently only support for scikit Random Forest via 'default'
    #intent is to build in additional options
    
    #MLinfill_alg refers to the target algorithm for passed parameters
    #currently supports 'RandomForestRegressor' & 'RandomForestClassifier'
    """
    
    #initialize tune_marker to default
    tune_marker = False
    
    if autoML_type == 'randomforest':
    
      if 'MLinfill_cmnd' in ML_cmnd:

        if MLinfill_alg in ML_cmnd['MLinfill_cmnd']:

          for key in ML_cmnd['MLinfill_cmnd'][MLinfill_alg]:

            #if passed parameter is a set
            if type(ML_cmnd['MLinfill_cmnd'][MLinfill_alg][key]) \
            in [type([1]), type(range(1)), type(stats.expon(1))]:

              tune_marker = True
    
    return tune_marker
  
  def assemble_param_sets(self, ML_cmnd, autoML_type, MLinfill_alg):
    """
    #assembles ML_cmnd passed parameters into two sets
    #for hyoeroparameter tuning operation
    
    #those parameters that were passed as sets 
    #will be saved in tune_params dictionary
    
    #those parameters that were otherwise passed
    #will be saved in static_params dictionary
    
    #returns those two dictionaries tune_params & static_params
    
    #MLinfill_type refers to type of predcitive algorithm applied,
    #currently only support for scikit Random Forest via 'default'
    #intent is to build in additional options
    
    #MLinfill_alg refers to the target algorithm for passed parameters
    #currently supports 'RandomForestRegressor' & 'RandomForestClassifier'
    """
    
    #initialize returned dictionaries
    static_params = {}
    tune_params = {}
    
    if autoML_type == 'randomforest':
      
      if 'MLinfill_cmnd' in ML_cmnd:
        
        if MLinfill_alg in ML_cmnd['MLinfill_cmnd']:
          
          for key in ML_cmnd['MLinfill_cmnd'][MLinfill_alg]:
            
            #if passed parameter is a set
            if type(ML_cmnd['MLinfill_cmnd'][MLinfill_alg][key]) \
            in [type([1]), type(range(1)), type(stats.expon(1))]:
              
              #add set to tune_params which will be targeted for grid search
              tune_params.update({key : ML_cmnd['MLinfill_cmnd'][MLinfill_alg][key]})
              
            else:
              
              #else add to static_params which will overwrite defaults
              static_params.update({key : ML_cmnd['MLinfill_cmnd'][MLinfill_alg][key]})
        
    return static_params, tune_params

  def predictinfill(self, category, df_train_filltrain, df_train_filllabel, \
                    df_train_fillfeatures, df_test_fillfeatures, randomseed, \
                    postprocess_dict, ML_cmnd, autoMLer, printstatus, categorylist = []):
    '''
    #predictinfill(category, df_train_filltrain, df_train_filllabel, \
    #df_train_fillfeatures, df_test_fillfeatures, randomseed, categorylist), \
    #function that takes as input \
    #a category string, the output of createMLinfillsets(.), a seed for randomness \
    #and a list of columns produced by a text class preprocessor when applicable and 
    #returns predicted infills for the train and test feature sets as df_traininfill, \
    #df_testinfill based on derivations using scikit-learn, with the lenth of \
    #infill consistent with the number of True values from NArows, and the trained \
    #model
    #accepts autoMLer populated with architecture options which is applied based on entries to ML_cmnd
    '''

    #if autoML_type not specified than we'll apply default (randomforest)
    #note this is only a temporary update to ML_cmnd and is not returned from function call
    if 'autoML_type' not in ML_cmnd:
      ML_cmnd.update({'autoML_type' : 'randomforest'})
    
    #grab autoML_type from ML_cmnd, this will be one of our keys for autoMLer dictionary
    autoML_type = ML_cmnd['autoML_type']
  
    #MLinfilltype distinguishes between classifier/regressor, single/multi column, ordinal/onehot/binary, etc
    #see potential values documented in assembleprocessdict function
    MLinfilltype = postprocess_dict['process_dict'][category]['MLinfilltype']
    
    #if a numeric target set
    if MLinfilltype in ['numeric', 'concurrent_nmbr']:
      
      #edge case if training data has zero rows (such as if column was all NaN) 
      if df_train_filltrain.shape[0] == 0:
        df_traininfill = np.zeros(shape=(1,len(categorylist)))
        df_testinfill = np.zeros(shape=(1,len(categorylist)))

        model = False
      
      else:
        
        #now call our training function
        #which handles tuning if applicable, model initialization, and training
        
        #ML_application is another key to access the function, distinguishes between classification and regression
        ML_application = 'regression'
        
        model = \
        autoMLer[autoML_type][ML_application]['train'](ML_cmnd, df_train_filltrain, df_train_filllabel, randomseed, printstatus)
        
        #only run following if we have any train rows needing infill
        if df_train_fillfeatures.shape[0] > 0:
          df_traininfill = autoMLer[autoML_type][ML_application]['predict'](ML_cmnd, model, df_train_fillfeatures, printstatus, categorylist)
        else:
          df_traininfill = np.array([0])

        #only run following if we have any test rows needing infill
        if df_test_fillfeatures.shape[0] > 0:
          df_testinfill = autoMLer[autoML_type][ML_application]['predict'](ML_cmnd, model, df_test_fillfeatures, printstatus, categorylist)
        else:
          df_testinfill = np.array([0])

      #convert infill values to dataframe
      df_traininfill = pd.DataFrame(df_traininfill, columns = ['infill'])
      df_testinfill = pd.DataFrame(df_testinfill, columns = ['infill'])
      
    #if target is categoric, such as ordinal or boolean integers
    if MLinfilltype in ['singlct', 'binary', 'concurrent_act']:
      
      #edge case if training data has zero rows (such as if column was all NaN) 
      if df_train_filltrain.shape[0] == 0:
        df_traininfill = np.zeros(shape=(1,len(categorylist)))
        df_testinfill = np.zeros(shape=(1,len(categorylist)))

        model = False

      else:
        
        #now call our training function
        #which handles tuning if applicable, model initialization, and training
        
        #ML_application is another key to access the function, distinguishes between classification and regression
        if MLinfilltype == 'singlct':
          ML_application = 'ordinalclassification'
        else:
          ML_application = 'booleanclassification'
        
        model = \
        autoMLer[autoML_type][ML_application]['train'](ML_cmnd, df_train_filltrain, df_train_filllabel, randomseed, printstatus)
        
        #only run following if we have any train rows needing infill
        if df_train_fillfeatures.shape[0] > 0:
          df_traininfill = autoMLer[autoML_type][ML_application]['predict'](ML_cmnd, model, df_train_fillfeatures, printstatus, categorylist)
        else:
          df_traininfill = np.array([0])

        #only run following if we have any test rows needing infill
        if df_test_fillfeatures.shape[0] > 0:
          df_testinfill = autoMLer[autoML_type][ML_application]['predict'](ML_cmnd, model, df_test_fillfeatures, printstatus, categorylist)
        else:
          df_testinfill = np.array([0])

      #convert infill values to dataframe
      df_traininfill = pd.DataFrame(df_traininfill, columns = ['infill'])
      df_testinfill = pd.DataFrame(df_testinfill, columns = ['infill'])
      
    #if target is multi column categoric (onehot encoded) / (binary encoded handled seperately)
    if MLinfilltype in ['multirt']:

      if df_train_filltrain.shape[0] == 0:
        df_traininfill = np.zeros(shape=(1,len(categorylist)))
        df_testinfill = np.zeros(shape=(1,len(categorylist)))

        model = False

      else:
        
        #future extension - Label Smoothing for ML infill
        #(might incorporate this into the training function to be activated by ML_cmnd)
          
        #now call our training function
        #which handles tuning if applicable, model initialization, and training
        
        #ML_application is another key to access the function, distinguishes between classification and regression
        ML_application = 'onehotclassification'
        
        model = \
        autoMLer[autoML_type][ML_application]['train'](ML_cmnd, df_train_filltrain, df_train_filllabel, randomseed, printstatus)
        
        #only run following if we have any train rows needing infill
        if df_train_fillfeatures.shape[0] > 0:
          df_traininfill = autoMLer[autoML_type][ML_application]['predict'](ML_cmnd, model, df_train_fillfeatures, printstatus, categorylist)
        else:
          #this needs to have same number of columns as text category
          df_traininfill = np.zeros(shape=(1,len(categorylist)))
        
        #only run following if we have any test rows needing infill
        if df_test_fillfeatures.shape[0] > 0:
          df_testinfill = autoMLer[autoML_type][ML_application]['predict'](ML_cmnd, model, df_test_fillfeatures, printstatus, categorylist)
        else:
          #this needs to have same number of columns as text category
          df_testinfill = np.zeros(shape=(1,len(categorylist)))
          
      #convert infill values to dataframe (this column labeleling also works for single column case)
      df_traininfill = pd.DataFrame(df_traininfill, columns = categorylist)
      df_testinfill = pd.DataFrame(df_testinfill, columns = categorylist)
    
    #if target is a binary encoded categoric set
    if MLinfilltype in ['1010']:
      
      if df_train_filltrain.shape[0] == 0:

        df_traininfill = np.zeros(shape=(1,len(categorylist)))
        df_testinfill = np.zeros(shape=(1,len(categorylist)))

        model = False

      else:

        #convert from binary to one-hot encoding
        df_train_filllabel = \
        self.convert_1010_to_onehot(df_train_filllabel)
          
        #now call our training function
        #which handles tuning if applicable, model initialization, and training
        
        #ML_application is another key to access the function, distinguishes between classification and regression
        ML_application = 'onehotclassification'
        
        model = \
        autoMLer[autoML_type][ML_application]['train'](ML_cmnd, df_train_filltrain, df_train_filllabel, randomseed, printstatus)

        #this is to support 1010 infill predictions in postmunge
        for entry in categorylist:
          postprocess_dict['column_dict'][entry].update({'_1010_categorylist_proxy_for_postmunge_MLinfill' : list(range(df_train_filllabel.shape[1]))})
        
        #only run following if we have any train rows needing infill
        if df_train_fillfeatures.shape[0] > 0:
          df_traininfill = autoMLer[autoML_type][ML_application]['predict'](ML_cmnd, model, df_train_fillfeatures, printstatus, list(range(df_train_filllabel.shape[1])))

          df_traininfill = \
          self.convert_onehot_to_1010(df_traininfill)

        else:
          #this needs to have same number of columns as text category
          df_traininfill = np.zeros(shape=(1,len(categorylist)))
        
        #only run following if we have any test rows needing infill
        if df_test_fillfeatures.shape[0] > 0:
          df_testinfill = autoMLer[autoML_type][ML_application]['predict'](ML_cmnd, model, df_test_fillfeatures, printstatus, list(range(df_train_filllabel.shape[1])))

          df_testinfill = \
          self.convert_onehot_to_1010(df_testinfill)

        else:
          #this needs to have same number of columns as text category
          df_testinfill = np.zeros(shape=(1,len(categorylist)))

      #convert infill values to dataframe
      df_traininfill = pd.DataFrame(df_traininfill, columns = categorylist)
      df_testinfill = pd.DataFrame(df_testinfill, columns = categorylist)
      
    #if target category excluded from ML infill:
    if MLinfilltype in ['exclude', 'boolexclude', 'totalexclude']:

      #create empty sets for now
      #an extension of this method would be to implement a comparable infill \
      #method for the time category, based on the columns output from the \
      #preprocessing
      df_traininfill = pd.DataFrame({'infill' : [0]}) 
      df_testinfill = pd.DataFrame({'infill' : [0]}) 

      model = False
    
    return df_traininfill, df_testinfill, model, postprocess_dict

  def createMLinfillsets(self, df_train, df_test, column, trainNArows, testNArows, \
                         category, randomseed, postprocess_dict, columnslist = [], \
                         categorylist = []):
    '''
    #update createMLinfillsets as follows:
    #instead of diferientiation by category, do a test for whether categorylist = []
    #if so do a single column transform excluding those other columns from columnslist
    #in the sets comparable to , otherwise do a transform comparable to text category
    #createMLinfillsets(df_train, df_test, column, trainNArows, testNArows, \
    #category, columnslist = []) function that when fed dataframes of train and\
    #test sets, column id, df of True/False corresponding to rows from original \
    #sets with missing values, a string category of 'text', 'date', 'nmbr', or \
    #'bnry', and a list of column id's for the text category if applicable. The \
    #function returns a seris of dataframes which can be applied to training a \
    #machine learning model to predict apppropriate infill values for those points \
    #that had missing values from the original sets, indlucing returns of \
    #df_train_filltrain, df_train_filllabel, df_train_fillfeatures, \
    #and df_test_fillfeatures
    '''
    
    MLinfilltype = postprocess_dict['process_dict'][category]['MLinfilltype']
    
    #create 3 new dataframes for each train column - the train and labels \
    #for rows not needing infill, and the features for rows needing infill \
    #also create a test features column 
    
    #categories are nmbr, bnry, text, date, bxcx, bins, bint, NArw, null
    #if category in ['nmbr', 'bxcx', 'bnry', 'text', 'bins', 'bint']:
    
    #if category in ['nmbr', 'nbr2', 'bxcx', 'bnry', 'text', 'bins', 'bint']:
    if MLinfilltype in ['numeric', 'singlct', 'binary', \
                        'multirt', '1010', \
                        'concurrent_act', 'concurrent_nmbr']:

      #if this is a single column set or concurrent_act
      if len(categorylist) == 1 or \
      postprocess_dict['process_dict'][category]['MLinfilltype'] in ['concurrent_act', 'concurrent_nmbr']:

        #first concatinate the NArows True/False designations to df_train & df_test
        df_train = pd.concat([df_train, trainNArows], axis=1)
        df_test = pd.concat([df_test, testNArows], axis=1)

        #create copy of df_train to serve as training set for fill
        df_train_filltrain = df_train.copy()
        #now delete rows coresponding to True
        df_train_filltrain = df_train_filltrain[df_train_filltrain[trainNArows.columns[0]] == False]

        #now delete columns = columnslist and the NA labels (orig column+'_NArows') from this df
        df_train_filltrain = df_train_filltrain.drop(columnslist, axis=1)
        df_train_filltrain = df_train_filltrain.drop([trainNArows.columns[0]], axis=1)

        #create a copy of df_train[column] for fill train labels
        df_train_filllabel = pd.DataFrame(df_train[column].copy())
        #concatinate with the NArows
        df_train_filllabel = pd.concat([df_train_filllabel, trainNArows], axis=1)
        #drop rows corresponding to True
        df_train_filllabel = df_train_filllabel[df_train_filllabel[trainNArows.columns[0]] == False]

        #delete the NArows column
        df_train_filllabel = df_train_filllabel.drop([trainNArows.columns[0]], axis=1)

        #create features df_train for rows needing infill
        #create copy of df_train (note it already has NArows included)
        df_train_fillfeatures = df_train.copy()
        #delete rows coresponding to False
        df_train_fillfeatures = df_train_fillfeatures[(df_train_fillfeatures[trainNArows.columns[0]])]
        #delete columnslist and column+'_NArows'
        df_train_fillfeatures = df_train_fillfeatures.drop(columnslist, axis=1)
        df_train_fillfeatures = df_train_fillfeatures.drop([trainNArows.columns[0]], axis=1)

        #create features df_test for rows needing infill
        #create copy of df_test (note it already has NArows included)
        df_test_fillfeatures = df_test.copy()
        #delete rows coresponding to False
        df_test_fillfeatures = df_test_fillfeatures[(df_test_fillfeatures[testNArows.columns[0]])]
        #delete column and column+'_NArows'
        df_test_fillfeatures = df_test_fillfeatures.drop(columnslist, axis=1)
        df_test_fillfeatures = df_test_fillfeatures.drop([testNArows.columns[0]], axis=1)

        #delete NArows from df_train, df_test
        df_train = df_train.drop([trainNArows.columns[0]], axis=1)
        df_test = df_test.drop([testNArows.columns[0]], axis=1)

      #else if categorylist wasn't single entry
      else:

        #create a list of columns representing columnslist exlucding elements from
        #categorylist
        noncategorylist = columnslist[:]
        #this removes categorylist elements from noncategorylist
        noncategorylist = list(set(noncategorylist).difference(set(categorylist)))

        #first concatinate the NArows True/False designations to df_train & df_test
        df_train = pd.concat([df_train, trainNArows], axis=1)
        df_test = pd.concat([df_test, testNArows], axis=1)

        #create copy of df_train to serve as training set for fill
        df_train_filltrain = df_train.copy()
        #now delete rows coresponding to True
        df_train_filltrain = df_train_filltrain[df_train_filltrain[trainNArows.columns[0]] == False]

        #now delete columns = columnslist and the NA labels (orig column+'_NArows') from this df
        df_train_filltrain = df_train_filltrain.drop(columnslist, axis=1)
        df_train_filltrain = df_train_filltrain.drop([trainNArows.columns[0]], axis=1)

        #create a copy of df_train[categorylist] for fill train labels
        df_train_filllabel = df_train[categorylist].copy()
        #concatinate with the NArows
        df_train_filllabel = pd.concat([df_train_filllabel, trainNArows], axis=1)
        #drop rows corresponding to True
        df_train_filllabel = df_train_filllabel[df_train_filllabel[trainNArows.columns[0]] == False]

        #delete the NArows column
        df_train_filllabel = df_train_filllabel.drop([trainNArows.columns[0]], axis=1)

        #create features df_train for rows needing infill
        #create copy of df_train (note it already has NArows included)
        df_train_fillfeatures = df_train.copy()
        #delete rows coresponding to False
        df_train_fillfeatures = df_train_fillfeatures[(df_train_fillfeatures[trainNArows.columns[0]])]
        #delete columnslist and column+'_NArows'
        df_train_fillfeatures = df_train_fillfeatures.drop(columnslist, axis=1)
        df_train_fillfeatures = df_train_fillfeatures.drop([trainNArows.columns[0]], axis=1)

        #create features df_test for rows needing infill
        #create copy of df_test (note it already has NArows included)
        df_test_fillfeatures = df_test.copy()
        #delete rows coresponding to False
        df_test_fillfeatures = df_test_fillfeatures[(df_test_fillfeatures[testNArows.columns[0]])]
        #delete column and column+'_NArows'
        df_test_fillfeatures = df_test_fillfeatures.drop(columnslist, axis=1)
        df_test_fillfeatures = df_test_fillfeatures.drop([testNArows.columns[0]], axis=1)

        #delete NArows from df_train, df_test
        df_train = df_train.drop([trainNArows.columns[0]], axis=1)
        df_test = df_test.drop([testNArows.columns[0]], axis=1)

    #if MLinfilltype in ['exclude']:
    else:

      #create empty sets for now
      #an extension of this method would be to implement a comparable method \
      #for the time category, based on the columns output from the preprocessing
      df_train_filltrain = pd.DataFrame({'foo' : []}) 
      df_train_filllabel = pd.DataFrame({'foo' : []})
      df_train_fillfeatures = pd.DataFrame({'foo' : []})
      df_test_fillfeatures = pd.DataFrame({'foo' : []})
    
    return df_train_filltrain, df_train_filllabel, df_train_fillfeatures, df_test_fillfeatures

  def insertinfill(self, df, column, infill, category, NArows, postprocess_dict, \
                   columnslist = [], categorylist = [], singlecolumncase = False):
    '''
    #uses the boolean indicators for presence of infill in NArows to apply infill
    #passed in infill dataframe to df[column]
    #note that infill dataframe is multicolumn when categorylist length > 1
    #and singlecolumn case is False
    #singlecolumn case is for special case (used in adjinfill) when we want to 
    #override the categorylist >1 methods
    '''
    
    MLinfilltype = postprocess_dict['process_dict'][category]['MLinfilltype']
    
    #NArows column name uses original column name + _NArows as key
    NArowcolumn = NArows.columns[0]

    #if category in ['nmbr', 'nbr2', 'bxcx', 'bnry', 'text']:
    if MLinfilltype in ['numeric', 'singlct', 'binary', \
                        'multirt', '1010', \
                        'concurrent_act', 'concurrent_nmbr']:

      #if this is a single column set (not categorical)
      if len(categorylist) == 1 or singlecolumncase is True \
      or MLinfilltype in ['concurrent_act', 'concurrent_nmbr']:
        
        #create new dataframe for infills wherein the infill values are placed in \
        #rows coresponding to NArows True values and rows coresponding to NArows \
        #False values are filled with a 0    

        #assign index values to a column
        NArows['tempindex1'] = df.index

        #create list of index numbers coresponding to the NArows True values
        infillindex = NArows.loc[NArows[NArowcolumn]]['tempindex1']

        #create a dictionary for use to insert infill using df's index as the key
        infill_dict = dict(zip(infillindex, infill.values))

        #replace 'tempindex1' column with infill in rows where NArows is True
        NArows['tempindex1'] = np.where(NArows[NArowcolumn], NArows['tempindex1'].replace(infill_dict), 0)

        #now carry that infill over to the target column for rows where NArows is True
        df[column] = np.where(NArows[NArowcolumn], NArows['tempindex1'], df[column])

      #else if categorylist wasn't single value
      else:

        for textcolumnname in categorylist:
          
          #assign index values to a column
          NArows['tempindex1'] = df.index

          #create list of index numbers coresponding to the NArows True values
          infillindex = NArows.loc[NArows[NArowcolumn]]['tempindex1']

          #create a dictionary for use to insert infill using df's index as the key
          infill_dict = dict(zip(infillindex, infill[textcolumnname].values))

          #replace 'tempindex1' column with infill in rows where NArows is True
          NArows['tempindex1'] = np.where(NArows[NArowcolumn], NArows['tempindex1'].replace(infill_dict), 0)

          #now carry that infill over to the target column for rows where NArows is True
          df[column] = np.where(NArows[NArowcolumn], NArows['tempindex1'], df[column])

    #if category == 'date':
    if MLinfilltype in ['exclude', 'boolexclude', 'totalexclude']:
      pass

    return df

  def MLinfillfunction (self, df_train, df_test, column, postprocess_dict, \
                        masterNArows_train, masterNArows_test, randomseed, \
                        ML_cmnd, printstatus):
    '''
    #new function ML infill, generalizes the MLinfill application between categories
    #def MLinfill (df_train, df_test, column, postprocess_dict, \
    #masterNArows_train, masterNArows_test, randomseed)
    #function that applies series of functions of createMLinfillsets, 
    #predictinfill, and insertinfill to a categorical encoded set.
    #for the record I'm sure that the conversion of the single column
    #series to a dataframe is counter to the intent of pandas
    #it's probably less memory efficient but it's the current basis of
    #the functions so we're going to maintain that approach for now
    #the revision of these functions to accept pandas series is a
    #possible future extension
    '''
    
    if postprocess_dict['column_dict'][column]['infillcomplete'] is False:

      columnslist = postprocess_dict['column_dict'][column]['columnslist']
      categorylist = postprocess_dict['column_dict'][column]['categorylist']
      origcolumn = postprocess_dict['column_dict'][column]['origcolumn']
      category = postprocess_dict['column_dict'][column]['category']
      autoMLer = postprocess_dict['autoMLer']
      
      if len(categorylist) == 1 or \
      postprocess_dict['process_dict'][postprocess_dict['column_dict'][column]['category']]['MLinfilltype'] \
      in ['concurrent_act', 'concurrent_nmbr']:
        #copy the datatype to ensure returned set is consistent
        df_temp_dtype = pd.DataFrame(df_train[column][:0]).copy()

      elif len(categorylist) > 1:
        #copy the datatype to ensure returned set is consistent
        df_temp_dtype = pd.DataFrame(df_train[categorylist][:0]).copy()

      #createMLinfillsets
      df_train_filltrain, df_train_filllabel, df_train_fillfeatures, df_test_fillfeatures = \
      self.createMLinfillsets(df_train, \
                         df_test, column, \
                         pd.DataFrame(masterNArows_train[origcolumn+'_NArows']), \
                         pd.DataFrame(masterNArows_test[origcolumn+'_NArows']), \
                         category, randomseed, postprocess_dict, \
                         columnslist = columnslist, \
                         categorylist = categorylist)

      #predict infill values using defined function predictinfill(.)
      df_traininfill, df_testinfill, model, postprocess_dict = \
      self.predictinfill(category, df_train_filltrain, df_train_filllabel, \
                        df_train_fillfeatures, df_test_fillfeatures, randomseed, \
                        postprocess_dict, ML_cmnd, autoMLer, printstatus, categorylist = categorylist)

      #now we'll add our trained model to the postprocess_dict
      postprocess_dict['column_dict'][column]['infillmodel'] \
      = model

      #note: we're only saving trained model in the postprocess_dict for one 
      #of columns from multicolumn set to reduce file size
      
      #only insert infill if we have a valid model
      if model is not False:

        #apply the function insertinfill(.) to insert missing value predicitons \
        #to df's associated column
        df_train = self.insertinfill(df_train, column, df_traininfill, category, \
                              pd.DataFrame(masterNArows_train[origcolumn+'_NArows']), \
                              postprocess_dict, columnslist = columnslist, \
                              categorylist = categorylist)

        #if we don't train the train set model on any features, that we won't be able 
        #to apply the model to predict the test set infill. 

        if any(x == True for x in masterNArows_train[origcolumn+'_NArows']):

          df_test = self.insertinfill(df_test, column, df_testinfill, category, \
                             pd.DataFrame(masterNArows_test[origcolumn+'_NArows']), \
                             postprocess_dict, columnslist = columnslist, \
                             categorylist = categorylist)

      #now change the infillcomplete marker in the text_dict for each \
      #associated text column unless in concurrent_activations MLinfilltype
      if postprocess_dict['process_dict'][postprocess_dict['column_dict'][column]['category']]['MLinfilltype'] \
      in ['concurrent_act', 'concurrent_nmbr']:
        
        postprocess_dict['column_dict'][column]['infillcomplete'] = True

        #now we'll add our trained text model to the postprocess_dict
        postprocess_dict['column_dict'][column]['infillmodel'] \
        = model
        
      else:
        
        for columnname in categorylist:
          postprocess_dict['column_dict'][columnname]['infillcomplete'] = True

          #now we'll add our trained text model to the postprocess_dict
          postprocess_dict['column_dict'][columnname]['infillmodel'] \
          = model

#         #now change the infillcomplete marker in the dict for each associated column
#         for columnname in categorylist:
#           postprocess_dict['column_dict'][columnname]['infillcomplete'] = True

      #reset data type to ensure returned data is consistent with what was passed
      if len(categorylist) == 1 or \
      postprocess_dict['process_dict'][postprocess_dict['column_dict'][column]['category']]['MLinfilltype'] \
      in ['concurrent_act', 'concurrent_nmbr']:
        df_train[column] = \
        df_train[column].astype({column:df_temp_dtype[column].dtypes})
        
        df_test[column] = \
        df_test[column].astype({column:df_temp_dtype[column].dtypes})

      elif len(categorylist) > 1:
        for dtype_column in categorylist:
          df_train[dtype_column] = \
          df_train[dtype_column].astype({dtype_column:df_temp_dtype[dtype_column].dtypes})
          
          df_test[dtype_column] = \
          df_test[dtype_column].astype({dtype_column:df_temp_dtype[dtype_column].dtypes})

    return df_train, df_test, postprocess_dict

  def assemble_autoMLer(self):
    """
    #populates the "autoMLer" data structure that supports application of autoML for ML infill
    #first tier is platform e.g. 'randomforest', 'autoML_1', 'auto_ML2'
    #(currently just randomforest supported, intent is to build in support for a few autoML platforms)
    #second tier is application i.e. 'classification', 'regression'
    #third tier is action i.e. 'train', 'predict'
    #third tier is populated with associated functions, which follow convention
    
    #train:
    #model = train_function(ML_cmnd, df_train_filltrain, df_train_filllabel, randomseed, printstatus)
    
    #predict:
    #infill = predict_function(ML_cmnd, model, df_train_fillfeatures, printstatus)

    #the intent is to incorproate some additional autoML options here in future extension
    
    #note that binary encoded sets use onehotclassification by way of 1010->text conversion in predictinfill function
    """
    
    autoMLer = {}
    
    autoMLer.update({'randomforest' : {'booleanclassification'  : {'train'   : self.train_randomforest_classifier, \
                                                                   'predict' : self.predict_randomforest_classifier}, \
                                       'ordinalclassification'  : {'train'   : self.train_randomforest_classifier, \
                                                                   'predict' : self.predict_randomforest_classifier}, \
                                       'onehotclassification'   : {'train'   : self.train_randomforest_classifier, \
                                                                   'predict' : self.predict_randomforest_classifier}, \
                                       'regression'             : {'train'   : self.train_randomforest_regressor, \
                                                                   'predict' : self.predict_randomforest_regressor}}, 
                     'autogluon'    : {'booleanclassification'  : {'train'   : self.train_autogluon, \
                                                                   'predict' : self.predict_autogluon}, \
                                       'ordinalclassification'  : {'train'   : self.train_autogluon, \
                                                                   'predict' : self.predict_autogluon}, \
                                       'onehotclassification'   : {'train'   : self.train_autogluon, \
                                                                   'predict' : self.predict_autogluon}, \
                                       'regression'             : {'train'   : self.train_autogluon, \
                                                                   'predict' : self.predict_autogluon}}})
    
    return autoMLer

  def train_randomforest_classifier(self, ML_cmnd, df_train_filltrain, df_train_filllabel, randomseed, printstatus):
    """
    #performs tuning if appropriate based on ML_cmnd
    #initializes model
    #trains model
    
    #uses scikit-learn random forest models
    
    #where tuning is activated by passing parameters to the model as lists or distributions instead of distinct values
    #and uses grid search or random search based on ML_cmnd
    #see also ML_cmnd documentation in read me
    
    #determination of whether parameters passed as targets for tuning is by inspect_ML_cmnd function
    #and if tuning applied aggregation of entries distinguishing tuning targets use assemble_param_sets function
    
    #in short, in addition to parameters passed to model
    #ML_cmnd also accepts arguments for hyperparam_tuner and randomCV_n_iter
    #where hyperparam_tuner can be one of {False, 'gridCV', 'randomCV'}
    #and randomCV_n_iter can be passed as an integer when hyperparam_tuner passed as randomCV
    
    #model initialization makes use of initRandomForestClassifier function
    #and default values for Random Forest Classifer are initialized with populateMLinfilldefaults
    """
    
    df_train_filltrain = df_train_filltrain.values
    df_train_filllabel = df_train_filllabel.values
    
    #sometimes may be one column, then need ravel flattening
    if df_train_filllabel.shape[1] == 1:
      df_train_filllabel = np.ravel(df_train_filllabel)
    
    #initialize defaults dictionary, these are the default parameters for random forest model initialization
    MLinfilldefaults = \
    self.populateMLinfilldefaults(randomseed)
    
    #ML_cmnd accepts specification for type of tuner when hyperparaemter tuning applied, else defaults to gridCV
    if 'hyperparam_tuner' in ML_cmnd:
      MLinfill_tuner = ML_cmnd['hyperparam_tuner']
    else:
      ML_cmnd.update({'hyperparam_tuner' : 'gridCV'})
      MLinfill_tuner = 'gridCV'
      
    #if randomCV tuner applied, number of iterations accepted as randomCV_n_iter, else defaults to 100
    if MLinfill_tuner == 'randomCV':
      if 'randomCV_n_iter' not in ML_cmnd:
        ML_cmnd.update({'randomCV_n_iter' : 100})
      randomCV_n_iter = ML_cmnd['randomCV_n_iter']
    
    autoML_type = ML_cmnd['autoML_type']
    MLinfill_alg = 'RandomForestClassifier'
    
    #tune marker tells us if user passed some parameters as a list for hyperparameter tuning
    tune_marker = self.inspect_ML_cmnd(ML_cmnd, autoML_type, MLinfill_alg)
    
    if tune_marker is True:
    
      #static_params are user passed parameters that won't be tuned, 
      #tune_params are user passed params (passed as list or range) that will be tuned
      static_params, tune_params = self.assemble_param_sets(ML_cmnd, autoML_type, MLinfill_alg)
    
      #we'll create a temp ML_cmnd to initialize a tuning model
      temp_ML_cmnd = {'MLinfill_cmnd':{MLinfill_alg : static_params}}
      
      #then we'll initialize a tuning model
      #note that this populates the parameters to be tuned with defaults
      #my understanding is that scikit gridsearch still allows tuning for parameters
      #that were previously initialized in the model
      tuning_model = self.initRandomForestClassifier(temp_ML_cmnd, MLinfilldefaults)
    
      #for now we'll default to grid scoring of ‘accuracy’
      #I've heard that F1 score is a better general default, but not sure how it handles edge cases
      #need to do a little more investsigation on this point
      grid_scoring = 'accuracy'
      
      #now we'll initialize a grid search
      if MLinfill_tuner == 'gridCV':
        tune_search = GridSearchCV(tuning_model, cv=5, iid='deprecated', \
                                   param_grid = tune_params, scoring = grid_scoring)
      elif MLinfill_tuner == 'randomCV':
        tune_search = RandomizedSearchCV(tuning_model, cv=5, iid='deprecated', \
                                     param_distributions = tune_params, scoring = grid_scoring, \
                                     n_iter = randomCV_n_iter)
      else:
        print("error: hyperparam_tuner currently only supports 'gridCV' or 'randomCV'.")      
      
      #now we'll run a fit on the grid search
      #for now won't pass any fit parameters
      fit_params = {}
      tune_search.fit(df_train_filltrain, df_train_filllabel, **fit_params)    
    
      #acess the tuned parameters based on the tuning operation
      tuned_params = tune_search.best_params_    

      if printstatus is True:

        #print("")
        print("tuned parameters:")
        print(tuned_params)
        print("")

      #now assemble final static params by incorporating the tuned params
      static_params.update(tuned_params)

      #now initialize our tuned model
      #first create another temp_ML_cmnd for the tuned set
      temp_ML_cmnd_two = {'MLinfill_cmnd':{MLinfill_alg : static_params}}

      model = self.initRandomForestClassifier(temp_ML_cmnd_two, MLinfilldefaults)

    else:
      
      model = self.initRandomForestClassifier(ML_cmnd, MLinfilldefaults)

    model.fit(df_train_filltrain, df_train_filllabel)

    return model

  def predict_randomforest_classifier(self, ML_cmnd, model, fillfeatures, printstatus, categorylist=[]):
    """
    #runs and inference operation
    #on corresponding model trained in train_randomforest_classifier
    #for random forest
    #returns infill predictions

    #the categorylist parameter is used to handle an edge case for when predict_autogluon is called
    """
    
    fillfeatures = fillfeatures.values
    
    infill = model.predict(fillfeatures)
    
    return infill

  def train_randomforest_regressor(self, ML_cmnd, df_train_filltrain, df_train_filllabel, randomseed, printstatus):
    """
    #performs tuning if appropriate based on ML_cmnd
    #initializes model
    #trains model
    
    #uses scikit-learn random forest models
    
    #where tuning is activated by passing parameters to the model as lists or distributions instead of distinct values
    #and uses grid search or random search based on ML_cmnd
    #see also ML_cmnd documentation in read me
    
    #determination of whether parameters passed as targets for tuning is by inspect_ML_cmnd function
    #and if tuning applied aggregation of entries distinguishing tuning targets use assemble_param_sets function
    
    #in short, in addition to parameters passed to model
    #ML_cmnd also accepts arguments for hyperparam_tuner and randomCV_n_iter
    #where hyperparam_tuner can be one of {False, 'gridCV', 'randomCV'}
    #and randomCV_n_iter can be passed as an integer when hyperparam_tuner passed as randomCV
    
    #model initialization makes use of initRandomForestRegressor function
    #and default values for Random Forest Regressor are initialized with populateMLinfilldefaults
    """
    
    df_train_filltrain = df_train_filltrain.values
    df_train_filllabel = df_train_filllabel.values
    
    #single label column needs to be flattened from [[#,...]] to [#,...] with ravel
    df_train_filllabel = np.ravel(df_train_filllabel)
    
    #initialize defaults dictionary, these are the default parameters for random forest model initialization
    MLinfilldefaults = \
    self.populateMLinfilldefaults(randomseed)
    
    #ML_cmnd accepts specification for type of tuner when hyperparaemter tuning applied, else defaults to gridCV
    if 'hyperparam_tuner' in ML_cmnd:
      MLinfill_tuner = ML_cmnd['hyperparam_tuner']
    else:
      ML_cmnd.update({'hyperparam_tuner' : 'gridCV'})
      MLinfill_tuner = 'gridCV'
      
    #if randomCV tuner applied, number of iterations accepted as randomCV_n_iter, else defaults to 100
    if MLinfill_tuner == 'randomCV':
      if 'randomCV_n_iter' not in ML_cmnd:
        ML_cmnd.update({'randomCV_n_iter' : 100})
      randomCV_n_iter = ML_cmnd['randomCV_n_iter']
    
    autoML_type = ML_cmnd['autoML_type']
    MLinfill_alg = 'RandomForestRegressor'
    
    #tune marker tells us if user passed some parameters as a list for hyperparameter tuning
    tune_marker = self.inspect_ML_cmnd(ML_cmnd, autoML_type, MLinfill_alg)
    
    if tune_marker is True:
    
      #static_params are user passed parameters that won't be tuned, 
      #tune_params are user passed params (passed as list or range) that will be tuned
      static_params, tune_params = self.assemble_param_sets(ML_cmnd, autoML_type, MLinfill_alg)
    
      #we'll create a temp ML_cmnd to initialize a tuning model
      temp_ML_cmnd = {'MLinfill_cmnd':{MLinfill_alg : static_params}}
      
      #then we'll initialize a tuning model
      #note that this populates the parameters to be tuned with defaults
      #my understanding is that scikit gridsearch still allows tuning for parameters
      #that were previously initialized in the model
      tuning_model = self.initRandomForestRegressor(temp_ML_cmnd, MLinfilldefaults)
    
      #for now we'll default to grid scoring of ‘neg_mean_squared_error’
      #am not positive this is best default this is worth some further investigation when get a chance
      grid_scoring = 'neg_mean_squared_error'
      
      #now we'll initialize a grid search
      if MLinfill_tuner == 'gridCV':
        tune_search = GridSearchCV(tuning_model, cv=5, iid='deprecated', \
                                   param_grid = tune_params, scoring = grid_scoring)
      elif MLinfill_tuner == 'randomCV':
        tune_search = RandomizedSearchCV(tuning_model, cv=5, iid='deprecated', \
                                     param_distributions = tune_params, scoring = grid_scoring, \
                                     n_iter = randomCV_n_iter)
      else:
        print("error: hyperparam_tuner currently only supports 'gridCV' or 'randomCV'.")      
      
      #now we'll run a fit on the grid search
      #for now won't pass any fit parameters
      fit_params = {}
      tune_search.fit(df_train_filltrain, df_train_filllabel, **fit_params)    
    
      #acess the tuned parameters based on the tuning operation
      tuned_params = tune_search.best_params_    

      if printstatus is True:

        #print("")
        print("tuned parameters:")
        print(tuned_params)
        print("")

      #now assemble final static params by incorporating the tuned params
      static_params.update(tuned_params)

      #now initialize our tuned model
      #first create another temp_ML_cmnd for the tuned set
      temp_ML_cmnd_two = {'MLinfill_cmnd':{MLinfill_alg : static_params}}

      model = self.initRandomForestRegressor(temp_ML_cmnd_two, MLinfilldefaults)

    else:
      
      model = self.initRandomForestRegressor(ML_cmnd, MLinfilldefaults)

    model.fit(df_train_filltrain, df_train_filllabel)

    return model

  def predict_randomforest_regressor(self, ML_cmnd, model, fillfeatures, printstatus, categorylist=[]):
    """
    #runs and inference operation
    #on corresponding model trained in train_randomforest_classifier
    #for random forest
    #returns infill predictions

    #the categorylist parameter is used to handle an edge case for when predict_autogluon is called
    """
    
    fillfeatures = fillfeatures.values
    
    infill = model.predict(fillfeatures)
    
    return infill

  def train_autogluon(self, ML_cmnd, df_train_filltrain, df_train_filllabel, randomseed, printstatus):
    """
    #Trains a model for ML infill using AutoGluon library
    #assumes that AutoGluon is imported external to the automunge(.) function call as
    
    import autogluon.core as ag
    from autogluon.tabular import TabularPrediction as task
    
    #currently applies default parameters to training operation, extended parameter support pending
    
    #same function used for both classification and regression relying on AutoGluon to infer label type
    """

    # import autogluon.core as ag
    from autogluon.tabular import TabularPrediction as task

    try:
      
      #column headers matter for convert_onehot_to_singlecolumn methods, reset as integers
      #I'm not sure why simply renaming columns to integers doesn't work here
      # df_train_filltrain.columns = list(range(len(list(df_train_filltrain.columns))))
      # df_train_filllabel.columns = list(range(len(list(df_train_filllabel.columns))))
      df_train_filltrain = pd.DataFrame(df_train_filltrain.values)
      df_train_filllabel = pd.DataFrame(df_train_filllabel.values)

      df_train_filltrain.columns = ['train_' + str(x) for x in list(df_train_filltrain.columns)]
      
      ag_label_column = list(df_train_filllabel.columns)

      if len(ag_label_column) == 1:
        ag_label_column = ag_label_column[0]
      else:
        df_train_filllabel = self.convert_onehot_to_singlecolumn(df_train_filllabel)
        ag_label_column = list(df_train_filllabel.columns)[0]

      #autogluon accepts labels as part of training set
      df_train_filltrain = pd.concat([df_train_filltrain, df_train_filllabel], axis=1)

      # #now get name of columns, ag_label_column is the label, ag_trainset_columns is the other columns
      # ag_trainset_columns = list(df_train_filltrain.columns)
      # ag_label_column = ag_trainset_columns[-1]
      # ag_trainset_columns.remove(ag_label_column)

      #apply the autogluon data set loader
      df_train_filltrain = task.Dataset(df_train_filltrain)

      #train the model
      model = task.fit(train_data=df_train_filltrain, label=ag_label_column)
      
      return model
        
    except ValueError:
      return False

  def predict_autogluon(self, ML_cmnd, model, fillfeatures, printstatus, categorylist=[]):
    """
    #runs and inference operation
    #on corresponding model trained in train_AutoGluon_classifier
    #for AutoGluon
    #returns infill predictions

    #the categorylist parameter is used to handle an edge case
    #note that in some cases the passed categorylist may be a proxy list of equivalent length
    #such as a range of integers
    """

    # import autogluon.core as ag
    from autogluon.tabular import TabularPrediction as task
    
    if model is not False:
      
      # fillfeatures.columns = list(range(len(list(fillfeatures.columns))))
      fillfeatures = pd.DataFrame(fillfeatures.values)

      fillfeatures.columns = ['train_' + str(x) for x in list(fillfeatures.columns)]

      #load dataset
      fillfeatures = task.Dataset(fillfeatures)
      
      try:
        infill = model.predict(fillfeatures)
        
        if len(categorylist) > 1:
          
          infill = self.convert_singlecolumn_to_onehot(infill, categorylist)
        
    #     infill = np.array(infill)
        
        return infill
      
      except ValueError:

        return np.zeros(shape=(fillfeatures.shape[0],len(categorylist)))
    
    else:

      infill = np.zeros(shape=(1,len(categorylist)))
      
      return infill

  def convert_onehot_to_singlecolumn(self, df):
    """
    #support function for autoML libraries that don't accept multicolumn labels
    #converts onehot encoded sets to single column
    #with entries corresponding to the column header
    #for cases where a row did not have an entry (such as all zeros)
    #we'll populate with -1
    #which since these are dervied from a numpy set won't overlap with headers
    """
    
    df[-1] = -1
    
    for column in df:
      if column != -1:
        df[-1] = np.where(df[column]==1, column, df[-1])
      
    df2 = pd.DataFrame(df[-1].copy())
    df2 = df2.rename(columns = {-1:'labels'})
    
    df2['labels'] = df2['labels'].astype(str)
        
    return df2

  def convert_singlecolumn_to_onehot(self, df, columnslist):
    """
    #support function for autoML libraries that don't accept multicolumn labels
    #converts single column encoded sets back to onehot
    #with entries corresponding to the column header
    #where the entries will be
    #for cases where a row did not have an entry (such as all zeros)
    #we'll populate with -1
    #which since these are dervied from a numpy set won't overlap with headers
    """
    
    df = pd.DataFrame(df)

    df[0] = df[0].astype(int)
    df = df.rename(columns = {0:'labels'})
    
    df2 = pd.DataFrame(np.zeros(shape = (df.shape[0], len(columnslist))))
    
    df2.columns = list(range(len(columnslist)))
    
    df = pd.concat([df, df2], axis=1)
    
    del df2
    
    for entry in list(range(len(columnslist))):
      df[entry] = np.where(df['labels'] == entry, 1, 0)
      
    del df['labels']

    df = df.values
    
    return df

  def convert_1010_to_onehot(self, df_array):  
    """
    takes as input dataframe encoded in 1010 format
    and translates to a one-hot encoding equivalent
    with number of columns based on 2^n where n is number of 1010 columns
    and potentially with columns with all 0
    """

    received_column_count = df_array.shape[1]

    #initialize a column to store encodings
    #this relies on convention that received columns with suffix appenders have '_' included to ensure no overlap
    df_array['-1'] = ''

    #populate column to store encodings 
    for column in df_array.columns:
      if column != '-1':
        df_array['-1'] = \
        df_array['-1'] + df_array[column].astype(int).astype(str)

    #discard other columns
    df_array = pd.DataFrame(df_array['-1'])

    #create list of columns for the encoding with binary encodings
    #this will be full list of range of values based on number of 1010 columns
    #postprocess_textsupport_class support function needs string headers
    #this relies on convention that received columns with suffix appenders have '_' included to ensure no overlap
    textcolumns = list(range(2**received_column_count))
    textcolumns = ['-1' + str(format(item, f"0{received_column_count}b")) for item in textcolumns]

    df_onehot = \
    self.postprocess_textsupport_class(df_array, '-1', {}, 'tempkey', {'textcolumns':textcolumns})

    del df_onehot['-1']

    return df_onehot
  
  def convert_onehot_to_1010(self, np_onehot):
    """
    takes as input numpy array encoded in one-hot format
    and translates to a 1010 encoding equivalent
    based on assumption that order of columns consistent per 
    convention of convert_1010_to_onehot(.)
    """
    
#     #if not all zeros (all zeros is an edge case)
#     if np_onehot.any():

    #create list of binary encodings corresponding to the onehot array
    #assumes consistent order of columns from convert_1010_to_onehot basis
    columnslist = list(range(np_onehot.shape[1]))
    columnslist = \
    [str(format(item, f"0{int(np.log2(np_onehot.shape[1]))}b")) for item in columnslist]

    #convert to dataframe with columnslist as column headers
    df_array = pd.DataFrame(np_onehot, columns = columnslist)

    #create new column to store encodings
    df_array['1010'] = 0

    #copy columns headers to activated cells, others are 0
    for column in df_array:

      if column != '1010':

        df_array[column].replace(1, column, inplace=True)

        df_array['1010'] = \
        np.where(df_array[column] != 0, df_array[column], df_array['1010'])

        del df_array[column]

#       uniquevalues = df_array['1010'].unique()
#       uniquevalues.sort()
#       uniquevalues = list(uniquevalues)

#       #get number of 1010 columns
#       nbrcolumns = len(str(uniquevalues[0]))

    nbrcolumns = int(np.ceil(np.log2(np_onehot.shape[1])))
  
    #replace zeros with infill partition (a string of zeros of lenth nmbrcolumns)
    #note this corresponds to the default infill encoding for '1010'
    infill_plug = '0' * nbrcolumns
    df_array['1010'] = np.where(df_array.eq(0).all(1), infill_plug, df_array['1010'])

    _1010_columns = []
    for i in range(nbrcolumns):
      _1010_columns.append('1010_'+str(i))

    df_array['1010'] = df_array['1010'].astype(str)

    #now let's store the encoding
    i=0
    for _1010_column in _1010_columns:

      df_array[_1010_column] = df_array['1010'].str.slice(i,i+1).astype(np.int8)

      i+=1

    del df_array['1010']

    np_1010 = df_array.values
    
#     #else if np_onehot was all zeros (edge case)
#     else:
      
#       nbrcolumns = int(np.ceil(np.log2(np_onehot.shape[1])))
      
#       np_1010 = \
#       np.zeros((np_onehot.shape[0], nbrcolumns))

    return np_1010

  def LabelSetGenerator(self, df, column, label):
    '''
    #LabelSetGenerator
    #takes as input dataframe for test set, label column name, and label
    #returns a dataframe set of all rows which included that label in the column
    '''
    
    df = df[df[column] == label]

    return df

  def LabelFrequencyLevelizer(self, train_df, labels_df, \
                                postprocess_dict, process_dict, LabelSmoothing):
    """
    #LabelFrequencyLevelizer(.)
    #takes as input dataframes for train set, labels, and label category
    #combines them to single df, then creates sets for each label category
    #such as to add on multiples of each set to achieve near levelized
    #frequency of label occurence in training set (increases the size
    #of the training set by redundant inclusion of rows with lower frequency
    #labels.) Returns train_df, labels_df, trainID_df.
    #for now have convention that MLinfilltypes of 1010 or concurrent_act
    #not yet supported (future extension)
    """
    
    columns_labels = list(labels_df)
    
    #find origcateogry of am_labels from FSpostprocess_dict
    labelcolumnkey = list(labels_df)[0]
    origcolumn = postprocess_dict['column_dict'][labelcolumnkey]['origcolumn']
    origcategory = postprocess_dict['column_dict'][labelcolumnkey]['origcategory']

    #find labelctgy from process_dict based on this origcategory
    labelscategory = process_dict[origcategory]['labelctgy']
    
    MLinfilltype = postprocess_dict['process_dict'][labelscategory]['MLinfilltype']
    
    labels = list(labels_df)
    #labels.sort()
    
    if labels != []:

      setnameslist = []
      setlengthlist = []
      multiplierlist = []

      #if labelscategory == 'bnry':
      if MLinfilltype in ['singlct', 'binary']:
        
        singlctcolumn = False
        
        if len(labels) == 1:
          singlctcolumn = labels[0]
        else:
          for labelcolumn in labels:
            if postprocess_dict['column_dict'][labelcolumn]['category'] == labelscategory:
              singlctcolumn = labelcolumn
          #if the label category is custom processdict entry with improperly specced labelctgy just apply this heuristic (remote edge case)
          if singlctcolumn is False:
            print("label category processdict entry contained a labelctgy not found in the transformdict entry, applying heuristic")
            print()
            singlctcolumn = labels[0]
        
        uniquevalues = list(labels_df[singlctcolumn].unique())

        #for label in labels:
        #for label in [0,1]:
        for label in uniquevalues:
          
          #value = 
          
          #derive set of labels dataframe for counting length
          df = self.LabelSetGenerator(labels_df, singlctcolumn, label)

          #append length onto list
          setlength = df.shape[0]
          #setlengthlist = setlengthlist.append(setlength)
          setlengthlist.append(setlength)

        #length of biggest label set
        maxlength = max(setlengthlist)
        #set counter to 0
        i = 0
        #for label in labels:
        #for label in [0,1]:
        for label in uniquevalues:
          #derive multiplier to levelize label frequency
          setlength = setlengthlist[i]
          if setlength > 0:
            
            labelmultiplier = int(round(maxlength / setlength)) - 1
          else:
            labelmultiplier = 0
          #append multiplier onto list
          #multiplierlist = multiplierlist.append(labelmultiplier)
          multiplierlist.append(labelmultiplier)
          #increment counter
          i+=1

        #concatinate labels onto train set
        train_df = pd.concat([train_df, labels_df], axis=1)

        #reset counter
        i=0
        #for loop through labels
        #for label in labels:
        #for label in [0,1]:
        for label in uniquevalues:

          #create train subset corresponding to label
          df = self.LabelSetGenerator(train_df, singlctcolumn, label)

          #set j counter to 0
          j = 0
          #concatinate an additional copy of the label set multiplier times
          while j < multiplierlist[i]:
            train_df = pd.concat([train_df, df], axis=0)
            #train_df = train_df.reset_index()
            j+=1
            
          i+=1

        #now seperate the labels df from the train df
        labels_df = pd.DataFrame(train_df[labels].copy())
        #now delete the labels column from train set
        for labelcolumn in labels:
          del train_df[labelcolumn]

      #if labelscategory in ['nmbr', 'bxcx']:
      if MLinfilltype in ['numeric']:

        columns_labels = []
        for label in labels_df.columns:
          #here we're checking if the column is a numneric set aggregated bins
          if postprocess_dict['process_dict'][postprocess_dict['column_dict'][label]['category']]['MLinfilltype'] \
          in ['multirt', 'concurrent_act']:
          
            columns_labels.append(label)
            
      #if labelscategory in ['text', 'nmbr', 'bxcx']:
      if MLinfilltype in ['numeric', 'multirt']:
        if columns_labels != []:
          
          #note for. label smoothing activation values won't be 1
          level_activation = LabelSmoothing
          if level_activation <= 0.0 \
          or level_activation >= 1.0 \
          or str(level_activation) == 'False':
            level_activation = 1
          
          i=0
          #for label in labels:
          for label in columns_labels:
            
            column = columns_labels[i]
            #derive set of labels dataframe for counting length
            df = self.LabelSetGenerator(labels_df, column, level_activation)
            
            #append length onto list
            setlength = df.shape[0]
            
            #setlengthlist = setlengthlist.append(setlength)
            setlengthlist.append(setlength)

            i+=1

          #length of biggest label set
          maxlength = max(setlengthlist)

          #set counter to 0
          i = 0
          #for label in labels:
          for label in columns_labels:

            #derive multiplier to levelize label frequency
            setlength = setlengthlist[i]
            if setlength > 0:
              labelmultiplier = int(round(maxlength / setlength)) - 1
            else:
              labelmultiplier = 0
            #append multiplier onto list
            #multiplierlist = multiplierlist.append(labelmultiplier)
            multiplierlist.append(labelmultiplier)
            #increment counter
            i+=1

          #concatinate labels onto train set
          train_df = pd.concat([train_df, labels_df], axis=1)

          #reset counter
          i=0
          #for loop through labels
          
          #for label in labels:
          for label in columns_labels:

            #create train subset corresponding to label
            column = columns_labels[i]
            df = self.LabelSetGenerator(train_df, column, level_activation)

            #set j counter to 0
            j = 0
            #concatinate an additional copy of the label set multiplier times
            while j < multiplierlist[i]:
              train_df = pd.concat([train_df, df], axis=0)
              #train_df = train_df.reset_index()
              j+=1

            i+=1

          columns_labels = list(labels_df)

          #now seperate the labels df from the train df
          labels_df = train_df[columns_labels]
          #now delete the labels column from train set
          train_df = train_df.drop(columns_labels, axis=1)

    return train_df, labels_df
  
  def trainFSmodel(self, am_subset, am_labels, randomseed, \
                   process_dict, postprocess_dict, labelctgy, ML_cmnd, printstatus):
    
    if len(list(am_labels)) > 0:

      df_train_fillfeatures_plug = pd.DataFrame(am_subset[:][:1].copy())
      df_test_fillfeatures_plug = pd.DataFrame(am_subset[:][:1].copy())
      categorylist = postprocess_dict['column_dict'][list(am_labels)[0]]['categorylist']

      _infilla, _infillb, FSmodel, postprocess_dict = \
      self.predictinfill(labelctgy, am_subset, am_labels, \
                         df_train_fillfeatures_plug, df_test_fillfeatures_plug, \
                         randomseed, postprocess_dict, ML_cmnd, postprocess_dict['autoMLer'], printstatus, \
                         categorylist = categorylist)

      del _infilla, _infillb
      
    else:
      
      FSmodel = False
    
    return FSmodel
  
  def createFSsets(self, am_subset, column, columnslist, randomseed):
    '''
    very simply shuffles rows of columns from columnslist with randomseed
    then returns the resulting dataframe
    
    hat tip for permutation method from "Beware Default Random Forest Importances"
    by Terence Parr, Kerem Turgutlu, Christopher Csiszar, and Jeremy Howard
    '''
    
    shuffleset = am_subset.copy()
    
    for clcolumn in columnslist:
      
      #uses support function
      shuffleset = self.df_shuffle_series(shuffleset, clcolumn, randomseed)
      
    return shuffleset

  def createFSsets2(self, am_subset, column, columnslist, randomseed):
    '''
    similar to createFSsets except performed such as to only leave one column from
    the columnslist untouched and shuffle the rest 
    '''
    shuffleset2 = am_subset.copy()
    
    for clcolumn in columnslist:
        
      if clcolumn != column:
            
        #uses support function
        shuffleset2 = self.df_shuffle_series(shuffleset2, clcolumn, randomseed)
    
    return shuffleset2

  def shuffleaccuracy(self, np_shuffleset, np_labels, FSmodel, randomseed, label_categorylist, \
                      process_dict, labelctgy, postprocess_dict):
    '''
    measures accuracy of predictions of shuffleset (which had permutation method)
    against the model trained on the unshuffled set

    np_shuffleset and np_labels are now recast as pandas dataframe, leaving the "np" in place for convenience
    '''

    ML_cmnd = postprocess_dict['ML_cmnd']

    autoMLer = postprocess_dict['autoMLer']
    
    categorylist_for_predict = label_categorylist

    printstatus_for_predict = postprocess_dict['printstatus']

    #if autoML_type not specified than we'll apply default (randomforest)
    #note this is only a temporary update to ML_cmnd and is not returned from function call
    if 'autoML_type' not in postprocess_dict['ML_cmnd']:
      postprocess_dict['ML_cmnd'].update({'autoML_type' : 'randomforest'})
    #grab autoML_type from ML_cmnd, this will be one of our keys for autoMLer dictionary
    autoML_type = postprocess_dict['ML_cmnd']['autoML_type']

    labelscategory = labelctgy
    MLinfilltype = process_dict[labelscategory]['MLinfilltype']
    
    if MLinfilltype in ['numeric', 'concurrent_nmbr']:
      ML_application = 'regression'
    elif MLinfilltype in ['singlct']:
      ML_application = 'ordinalclassification'
    elif MLinfilltype in ['binary', 'concurrent_act']:
      ML_application = 'booleanclassification'
    elif MLinfilltype in ['multirt', '1010']:
      ML_application = 'onehotclassification'
    
    #if labelscategory in ['nmbr']:
    if MLinfilltype in ['numeric', 'concurrent_nmbr']:
      
      #generate predictions
      np_predictions = autoMLer[autoML_type][ML_application]['predict'](ML_cmnd, FSmodel, np_shuffleset, printstatus_for_predict, categorylist_for_predict)
      #np_predictions = FSmodel.predict(np_shuffleset)
      
      #just in case this returned any negative predictions
      np_predictions = np.absolute(np_predictions)
      #and we're trying to generalize here so will go ahead and apply to labels
      np_labels = np.absolute(np_labels)
      
      #evaluate accuracy metric
      #columnaccuracy = accuracy_score(np_labels, np_predictions)
      #columnaccuracy = mean_squared_error(np_labels, np_predictions)
      #columnaccuracy = mean_squared_log_error(np_labels, np_predictions)
      columnaccuracy = 1 - mean_squared_log_error(np_labels, np_predictions)
      
    #if labelscategory in ['bnry']:
    if MLinfilltype in ['singlct', 'binary', 'concurrent_act']:
      
      #generate predictions
      np_predictions = autoMLer[autoML_type][ML_application]['predict'](ML_cmnd, FSmodel, np_shuffleset, printstatus_for_predict, categorylist_for_predict)
      #np_predictions = FSmodel.predict(np_shuffleset)
      
      #evaluate accuracy metric
      columnaccuracy = accuracy_score(np_labels, np_predictions)
      
    #if labelscategory in ['text']:
    if MLinfilltype in ['multirt']:
      
      #generate predictions
      np_predictions = autoMLer[autoML_type][ML_application]['predict'](ML_cmnd, FSmodel, np_shuffleset, printstatus_for_predict, categorylist_for_predict)
      #np_predictions = FSmodel.predict(np_shuffleset)
      
      #evaluate accuracy metric
      #columnaccuracy = accuracy_score(np_labels, np_predictions)
      columnaccuracy = accuracy_score(np_labels, np_predictions)

    if MLinfilltype in ['1010']:
      
      np_labels = \
      self.convert_1010_to_onehot(np_labels)
      
      #generate predictions
      np_predictions = autoMLer[autoML_type][ML_application]['predict'](ML_cmnd, FSmodel, np_shuffleset, printstatus_for_predict, categorylist_for_predict)
      #np_predictions = FSmodel.predict(np_shuffleset)
      
      #evaluate accuracy metric
      #columnaccuracy = accuracy_score(np_labels, np_predictions)
      columnaccuracy = accuracy_score(np_labels, np_predictions)
        
    #I think this will clear some memory
    del np_labels, np_shuffleset
    
    return columnaccuracy
  
  def assemblemadethecut(self, FScolumn_dict, featurepct, featuremetric, featuremethod, \
                         am_subset_columns, FSprocess_dict):
    '''
    takes as input the FScolumn_dict and the passed automunge argument featurepct
    and a list of the columns from automunge application in featureselect
    and uses to assemble a list of columns that made it through the feature
    selection process
    
    returns list madethecut
    '''
    
    #create empty dataframe for sorting purposes
    FSsupport_df = pd.DataFrame(columns=['FS_column', 'metric', 'category'])
    
    #Future extension:
    #FSsupport_df = pd.DataFrame(columns=['FS_column', 'metric', 'metric2', 'category'])
    
    #add rows to the dataframe for each column
    for key in FScolumn_dict:
      
      column_df = pd.DataFrame([[key, FScolumn_dict[key]['metric'], FScolumn_dict[key]['category']]], \
                               columns=['FS_column', 'metric', 'category'])
  
      FSsupport_df = pd.concat([FSsupport_df, column_df], axis=0)
    
    #sort the rows by metric (from large to small, not that higher metric implies
    #more predictive power associated with that column's feature)
    #(note that NaN rows will have NaN values at bottom of list)
    FSsupport_df = FSsupport_df.sort_values(['metric'], ascending=False)
    
    #create list of candidate entries for madethecut
    candidates = list(FSsupport_df['FS_column'])
    
    #count the total number of rows
    totalrowcount =  FSsupport_df.shape[0]
    #count ranked rows
    metriccount = totalrowcount
    
    #create list of NArws
    #candidateNArws = candidates[-NaNcount:]
#     candidateNArws = list(FSsupport_df[FSsupport_df['category']=='NArw']['FS_column'])
    candidateNArws = list()
    
    #create list of feature rows
    #candidatefeaturerows = candidates[:-NaNcount]
#     candidatefeaturerows = list(FSsupport_df[FSsupport_df['category']!='NArw']['FS_column'])
    candidatefeaturerows = list(FSsupport_df['FS_column'])
    
#     #calculate the number of features we'll keep using the ratio passed from automunge
#     numbermakingcut = int(metriccount * featurepct)
    
    if featuremethod not in ['default', 'pct', 'metric', 'report']:
      print("error featuremethod object must be one of ['default', 'pct', 'metric', 'report']")
      
    if featuremethod == 'default':

      #calculate the number of features we'll keep using the ratio passed from automunge
      numbermakingcut = len(FSsupport_df)
    
    if featuremethod == 'pct':

      #calculate the number of features we'll keep using the ratio passed from automunge
      numbermakingcut = int(metriccount * featurepct)
      
    if featuremethod == 'metric':
      
      #calculate the number of features we'll keep using the ratio passed from automunge
      numbermakingcut = len(FSsupport_df[FSsupport_df['metric'] >= featuremetric])
      
    if featuremethod == 'report':
      #just a plug vlaue
      numbermakingcut = 1
      
    #generate list of rows making the cut
    madethecut = candidatefeaturerows[:numbermakingcut]
    
    #this is to retain full sets if any 1010 sets returned
    madethecut_copy = madethecut.copy()
    for entry in madethecut_copy:
      if FSprocess_dict[FScolumn_dict[entry]['category']]['MLinfilltype'] == '1010':
        if not set(FScolumn_dict[entry]['categorylist']).issubset(set(madethecut)):
          for entry2 in FScolumn_dict[entry]['categorylist']:
            if entry2 not in madethecut:
              madethecut.append(entry2)
    
    return madethecut

  def featureselect(self, df_train, labels_column, trainID_column, \
                    powertransform, binstransform, randomseed, \
                    numbercategoryheuristic, assigncat, transformdict, \
                    processdict, featurepct, featuremetric, featuremethod, \
                    ML_cmnd, process_dict, valpercent1, valpercent2, printstatus, \
                    NArw_marker, assignparam):
    """
    featureselect is a function called within automunge() that applies methods
    to evaluate predictive power of derived features towards a downstream model
    such as to trim the branches of the transform tree.
    
    The function returns a list of column names that "made the cut" so that
    automunge() can then remove extraneous branches.
    """
    
    #now we'll use automunge() to prepare the subset for feature evaluation
    #note the passed arguments, these are all intentional (no MLinfill applied,
    #primary goal here is to produce a processed dataframe for df_subset
    #with corresponding labels)
    
    #printout display progress
    if printstatus is True:
      print("_______________")
      print("Begin Feature Importance evaluation")
      print("")
    
    if labels_column is False:
      
      FSmodel = False

      baseaccuracy = False
      
      #printout display progress
      if printstatus is True:
        print("_______________")
        print("No labels_column passed, Feature Importance halted")
        print("")
    
    elif labels_column is not False:

      #but first real quick we'll just deal with PCA default functionality for FS
      FSML_cmnd = deepcopy(ML_cmnd)
      FSML_cmnd['PCA_type'] = 'off'

      FS_LabelSmoothing = False

  #     #FUTURE EXTENSION
  #     #user has option to turn off LabelSmoothing for feature importance evaluation such as by passing False
  #     FS_LabelSmoothing = 0.9
  #     if 'LabelSmoothing' in ML_cmnd['MLinfill_cmnd']['RandomForestClassifier']:
  #       FS_LabelSmoothing = ML_cmnd['MLinfill_cmnd']['RandomForestClassifier']['LabelSmoothing']

      FS_assignparam = deepcopy(assignparam)

      totalvalidation = valpercent1 + valpercent2

      if totalvalidation == 0:
        totalvalidation = 0.2

      am_train, _1, am_labels, \
      am_validation1, _3, am_validationlabels1, \
      _5, _6, _7, \
      _8, _9, _10, \
      labelsencoding_dict, finalcolumns_train, _10,  \
      _11, FSpostprocess_dict = \
      self.automunge(df_train, df_test = False, labels_column = labels_column, trainID_column = trainID_column, \
                    testID_column = False, valpercent1 = totalvalidation, valpercent2 = 0.0, \
                    shuffletrain = True, TrainLabelFreqLevel = False, powertransform = powertransform, \
                    binstransform = binstransform, MLinfill = False, infilliterate=1, randomseed = randomseed, \
                    LabelSmoothing_train = FS_LabelSmoothing, excl_suffix = True, \
                    numbercategoryheuristic = numbercategoryheuristic, pandasoutput = True, NArw_marker = NArw_marker, \
                    featureselection = False, featurepct = 1.00, featuremetric = featuremetric, \
                    featuremethod = 'pct', ML_cmnd = FSML_cmnd, assigncat = assigncat, \
                    assigninfill = {'stdrdinfill':[], 'MLinfill':[], 'zeroinfill':[], 'oneinfill':[], \
                                   'adjinfill':[], 'meaninfill':[], 'medianinfill':[]}, \
                    assignparam = FS_assignparam, \
                    transformdict = transformdict, processdict = processdict, printstatus=printstatus)

      #this is the returned process_dict
      #(remember "processdict" is what we pass to automunge() call, "process_dict" is what is 
      #assembled inside automunge, there is a difference)
      FSprocess_dict = FSpostprocess_dict['process_dict']

      if am_labels.empty is True:
        FSmodel = False

        baseaccuracy = False
        
        #printout display progress
        if printstatus is True:
          print("_______________")
          print("No labels returned from automunge(.), Feature Importance halted")
          print("")
    
      #if am_labels is not an empty set
      if am_labels.empty is False:

        #find origcateogry of am_labels from FSpostprocess_dict
        labelcolumnkey = list(am_labels)[0]
        origcolumn = FSpostprocess_dict['column_dict'][labelcolumnkey]['origcolumn']
        origcategory = FSpostprocess_dict['column_dict'][labelcolumnkey]['origcategory']

        #find labelctgy from process_dict based on this origcategory
        labelctgy = process_dict[origcategory]['labelctgy']

        am_categorylist = []

        for am_label_column in am_labels.columns:

          if FSpostprocess_dict['column_dict'][am_label_column]['category'] == labelctgy:

            am_categorylist = FSpostprocess_dict['column_dict'][am_label_column]['categorylist']
            
            #we'll follow convention that if target label category MLinfilltype is concurrent
            #we'll arbitrarily take the first column and use that as target
            if FSpostprocess_dict['process_dict'][labelctgy]['MLinfilltype'] \
            in ['concurrent_act', 'concurrent_nmbr']:
              
              am_categorylist = [am_categorylist[0]]
              
            break

        if len(am_categorylist) == 0:
          if printstatus is True:
            #this is a remote edge case, printout added for troubleshooting support
            print("Label category processdict entry contained a labelctgy entry not found in transformdict entry")
            print("Feature Seclection model training will not run without valid labelgctgy processdict entry")
            print()

        elif len(am_categorylist) == 1:
          am_labels = pd.DataFrame(am_labels[am_categorylist[0]])
          am_validationlabels1 = pd.DataFrame(am_validationlabels1[am_categorylist[0]])

        else:
          am_labels = am_labels[am_categorylist]
          am_validationlabels1 = am_validationlabels1[am_categorylist]

        #if there's a bug occuring after this point it might mean the labelctgy wasn't
        #properly populated in the process_dict for the root category assigned to the labels
        #again the labelctgy entry to process_dict represents for labels returned in 
        #multiple configurations the trasnofrmation category whose returned set will be
        #used to train the feature selection model

        #printout display progress
        if printstatus is True:
          print("_______________")
          print("Training feature importance evaluation model")
          print("")

        #apply function trainFSmodel
        #FSmodel, baseaccuracy = \
        FSmodel = \
        self.trainFSmodel(am_train, am_labels, randomseed, \
                          FSprocess_dict, FSpostprocess_dict, labelctgy, ML_cmnd, \
                          printstatus)
        
        if FSmodel is False:
          
          FScolumn_dict = {}
          
          FS_origcolumns = list(FSpostprocess_dict['origcolumn'])

          baseaccuracy = False
          
          #printout display progress
          if printstatus is True:
            print("_______________")
            print("No model returned from training, Feature Importance halted")
            print("")
          
        
        elif FSmodel is not False:

          #update v2.11 baseaccuracy should be based on validation set
          baseaccuracy = self.shuffleaccuracy(am_validation1, am_validationlabels1, \
                                              FSmodel, randomseed, am_categorylist, \
                                              FSprocess_dict, labelctgy, FSpostprocess_dict)

          if printstatus is True:
            print("Base Accuracy of feature importance model:")
            print(baseaccuracy)
            print()

          #get list of columns
          am_train_columns = list(am_train)

          #initialize dictionary FScolumn_dict = {}
          FScolumn_dict = {}
          
          FS_origcolumns = list(FSpostprocess_dict['origcolumn'])
          
          #assemble FScolumn_dict to support the feature evaluation
          for column in am_train_columns:

            #pull categorylist, category, columnslist
            categorylist = FSpostprocess_dict['column_dict'][column]['categorylist']
            category = FSpostprocess_dict['column_dict'][column]['category']
            columnslist = FSpostprocess_dict['column_dict'][column]['columnslist']
            origcolumn = FSpostprocess_dict['column_dict'][column]['origcolumn']

            #create entry to FScolumn_dict
            FScolumn_dict.update({column : {'categorylist' : categorylist, \
                                            'category' : category, \
                                            'columnslist' : columnslist, \
                                            'origcolumn' : origcolumn, \
                                            'FScomplete' : False, \
                                            'shuffleaccuracy' : None, \
                                            'shuffleaccuracy2' : None, \
                                            'baseaccuracy' : baseaccuracy, \
                                            'metric' : None, \
                                            'metric2' : None}})
            
          #this is for assemblemadethecut
          FSprocess_dict = FSpostprocess_dict['process_dict']

          #printout display progress
          if printstatus is True:
            print("_______________")
            print("Evaluating feature importances")
            print("")

          #perform feature evaluation on each column
          for column in am_train_columns:

            if FScolumn_dict[column]['FScomplete'] is False:

              #categorylist = FScolumn_dict[column]['categorylist']
              #update version 1.80, let's perform FS on columnslist instead of categorylist
              columnslist = FScolumn_dict[column]['columnslist']

              #create set with columns shuffle from columnslist
              #shuffleset = self.createFSsets(am_train, column, categorylist, randomseed)
              #shuffleset = self.createFSsets(am_train, column, columnslist, randomseed)
              shuffleset = self.createFSsets(am_validation1, column, columnslist, randomseed)

              #determine resulting accuracy after shuffle
              columnaccuracy = self.shuffleaccuracy(shuffleset, am_validationlabels1, \
                                                    FSmodel, randomseed, am_categorylist, \
                                                    FSprocess_dict, labelctgy, FSpostprocess_dict)

              #I think this will clear some memory
              del shuffleset

              #category accuracy penalty metric
              metric = baseaccuracy - columnaccuracy
              #metric2 = baseaccuracy - columnaccuracy2

              #save accuracy to FScolumn_dict and set FScomplete to True
              #(for each column in the categorylist)
              #for categorycolumn in FSpostprocess_dict['column_dict'][column]['categorylist']:
              for categorycolumn in FSpostprocess_dict['column_dict'][column]['columnslist']:

                FScolumn_dict[categorycolumn]['FScomplete'] = True
                FScolumn_dict[categorycolumn]['shuffleaccuracy'] = columnaccuracy
                FScolumn_dict[categorycolumn]['metric'] = metric
                #FScolumn_dict[categorycolumn]['shuffleaccuracy2'] = columnaccuracy2
                #FScolumn_dict[categorycolumn]['metric2'] = metric2

            columnslist = FScolumn_dict[column]['columnslist']

            #create second set with all but one columns shuffled from columnslist
            #this will allow us to compare the relative importance between columns
            #derived from the same parent
            #shuffleset2 = self.createFSsets2(am_train, column, columnslist, randomseed)
            shuffleset2 = self.createFSsets2(am_validation1, column, columnslist, randomseed)

            #determine resulting accuracy after shuffle
            columnaccuracy2 = self.shuffleaccuracy(shuffleset2, am_validationlabels1, \
                                                  FSmodel, randomseed, am_categorylist, \
                                                  FSprocess_dict, labelctgy, FSpostprocess_dict)

            metric2 = baseaccuracy - columnaccuracy2

            FScolumn_dict[column]['shuffleaccuracy2'] = columnaccuracy2
            FScolumn_dict[column]['metric2'] = metric2
          
          madethecut = self.assemblemadethecut(FScolumn_dict, featurepct, featuremetric, \
                                           featuremethod, am_train_columns, FSprocess_dict)
    
    #if the only column left in madethecut from origin column is a NArw, delete from the set
    #(this is going to lean on the column ID string naming conventions)
    #couldn't get this to work, this functionality a future extension
#     trimfrommtc = []
#     for traincolumn in list(df_train):
#       if (traincolumn + '_') not in [checkmtc[:(len(traincolumn)+1)] for checkmtc in madethecut]:
#         for mtc in madethecut:
#           #if mtc originated from traincolumn
#           if mtc[:(len(traincolumn)+1)] == traincolumn + '_':
#             #count the number of same instance in madethecut set
#             madethecut_trim = [mdc_trim[:(len(traincolumn)+1)] for mdc_trim in madethecut]
#             if madethecut_trim.count(mtc[:(len(traincolumn)+1)]) == 1 \
#             and mtc[-5:] == '_NArw':
#               trimfrommtc = trimfrommtc + [mtc]
#     madethecut = list(set(madethecut).difference(set(trimfrommtc)))
       
    #apply function madethecut(FScolumn_dict, featurepct)
    #return madethecut
    #where featurepct is the percent of features that we intend to keep
    #(might want to make this a passed argument from automunge)
    
        #I think this will clear some memory
        del am_train, _1, am_labels, am_validation1, _3, \
        am_validationlabels1, _5, _6, _7, \
        _8, _9, labelsencoding_dict, finalcolumns_train, _10,  \
        FSpostprocess_dict

        if printstatus is True:
          print("_______________")
          print("Feature Importance results:")
          print("")

        #to inspect values returned in featureimportance object one could run
        if printstatus is True:
          for keys,values in FScolumn_dict.items():
            print(keys)
            print('metric = ', values['metric'])
            print('metric2 = ', values['metric2'])
            print("")

    FS_sorted = {'baseaccuracy' : baseaccuracy, \
                 'metric_key':{}, \
                 'column_key':{}, \
                 'metric2_key':{}, \
                 'metric2_column_key':{}}
    
    #first we'll handle first metric based on source column
    for FS_origcolumn in FS_origcolumns:
      for key in FScolumn_dict:
        if FScolumn_dict[key]['origcolumn'] == FS_origcolumn:
          if FScolumn_dict[key]['metric'] in FS_sorted['metric_key']:
            if isinstance(FS_sorted['metric_key'][FScolumn_dict[key]['metric']], list):
              FS_sorted['metric_key'][FScolumn_dict[key]['metric']].append(FS_origcolumn)
            else:
              FS_sorted['metric_key'][FScolumn_dict[key]['metric']] = \
              [FS_sorted['metric_key'][FScolumn_dict[key]['metric']]]
              FS_sorted['metric_key'][FScolumn_dict[key]['metric']].append(FS_origcolumn)
          else:
            FS_sorted['metric_key'].update({FScolumn_dict[key]['metric'] : [FS_origcolumn]})
          break
      
    FS_sorted['metric_key'] = dict(sorted(FS_sorted['metric_key'].items(), reverse=True))
    
    for key in FS_sorted['metric_key']:
      for entry in FS_sorted['metric_key'][key]:
        entry_index = FS_sorted['metric_key'][key].index(entry)
        FS_sorted['column_key'].update({FS_sorted['metric_key'][key][entry_index] : key})
    
    #now for metric2 based on derived columns relative importance, note sorted in other order
    for FS_origcolumn in FS_origcolumns:
      FS_sorted['metric2_key'].update({FS_origcolumn : {}})
      for key in FScolumn_dict:
        if FScolumn_dict[key]['origcolumn'] == FS_origcolumn:
          if FScolumn_dict[key]['metric2'] in FS_sorted['metric2_key'][FS_origcolumn]:
            if isinstance(FS_sorted['metric2_key'][FS_origcolumn][FScolumn_dict[key]['metric2']], list):
              FS_sorted['metric2_key'][FS_origcolumn][FScolumn_dict[key]['metric2']].append(key)
            else:
              FS_sorted['metric2_key'][FS_origcolumn][FScolumn_dict[key]['metric2']] = \
              [FS_sorted['metric2_key'][FS_origcolumn][FScolumn_dict[key]['metric2']]]
              FS_sorted['metric2_key'][FS_origcolumn][FScolumn_dict[key]['metric2']].append(key)
          else:
            FS_sorted['metric2_key'][FS_origcolumn].update({FScolumn_dict[key]['metric2'] : [key]})
    
    for key in FS_sorted['metric2_key']:
      FS_sorted['metric2_key'][key] = dict(sorted(FS_sorted['metric2_key'][key].items(), reverse=False))
    
    for key1 in FS_sorted['metric2_key']:
      FS_sorted['metric2_column_key'].update({key1 : {}})
      for key2 in FS_sorted['metric2_key'][key1]:
        for entry in FS_sorted['metric2_key'][key1][key2]:
          entry_index = FS_sorted['metric2_key'][key1][key2].index(entry)
          FS_sorted['metric2_column_key'][key1].update({FS_sorted['metric2_key'][key1][key2][entry_index] : key2})
    
    if printstatus is True:
      print()
      print("______________________")
      print("sorted metric results:")
      print()
      for keys,values in FS_sorted['metric_key'].items():
        for entry in values:
          print(entry)
          print(keys)
          print()
      print("______________________")
      print("sorted metric2 results:")
      print()
      for key in FS_sorted['metric2_key']:
        print("for source column: ", key)
        for keys,values in FS_sorted['metric2_key'][key].items():
          for entry in values:
            print(entry)
            print(keys)
            print()
        print()
    
    if FSmodel is False:
      
      madethecut = []
      FScolumn_dict = {}
    
    #printout display progress
    if printstatus is True:
      print("_______________")
      print("Feature Importance evaluation complete")
      print("")
    
    return madethecut, FSmodel, FScolumn_dict, FS_sorted
  
  def assemblepostprocess_assigninfill(self, assigninfill, infillcolumns_list, 
                                       columns_train, postprocess_dict, MLinfill):
    """
    #this function converts user passed assigninfill
    #into a collection of post-transform column assignments
    #to various infill methods for application in apply infill functions
    
    #assigninfill is as passed by user
    #(note assigninfill previously had validations performed in check_assigninfill)
    #infillcolumns_list is all dervied columns in train set
    #columns_train is all source columns in train set
    #postprocess_dict is how data shared between functions
    #MLinfill is boolean marker for default MLinfill applciation
    
    #The convention is that unspecified columns are cast to
    #stndrdinfill or MLinfill based on MLinfill parameter
    #The other convention is that user may assign column headers
    #both with and without suffix appenders
    #the source column headers are converted to set of dervied column headers
    #and then any user specified derived columns w/ suffix take precendence over the converted ones
    
    #so workflow is as follows
    #- received spec'd assigninfill (which may include both pre-suf and w/-suf)
    #- aggregate just spec'd w/-suf into mirror spec assigninfill_withsuffix
    #- aggregate spec'd pre-suf into mirror spec assigninfill_sourcecolumn
    #- identify unspecified source columns missing from assigninfill_sourcecolumn,
    #add as new category assigninfill_sourcecolumn['unspecified']
    #- convert source columns to dervied columns from assigninfill_sourcecolumn
    #to assigninfill_sourcecolumn_converted
    #- if assigninfill_sourcecolumn_converted doesn't yet have entries for stdrdinfill or MLinfill, create
    #- based on MLinfill, copy entries from assigninfill_sourcecolumn_converted['unspecified']
    #into either 'stdrdinfill' or 'MLinfill'
    #(we'll keep 'unspecified' entry in case might be of use down the road)
    #- for duplicates between entries to assigninfill_withsuffix and assigninfill_sourcecolumn_converted
    #assigninfill_withsuffix takes precedence
    #so cycle through and if duplicates found remove from assigninfill_sourcecolumn_converted
    #- combine assigninfill_withsuffix and assigninfill_sourcecolumn_converted into
    #the returned set postprocess_assigninfill_dict
    #- insert any missing keys needed for apply_am_infill
    #- return postprocess_assigninfill_dict
    """
    
    #- received spec'd assigninfill (which may include both pre-suf and w/-suf)
    
    #- aggregate just spec'd w/-suf into mirror spec assigninfill_withsuffix
    assigninfill_withsuffix = {}
    for key in assigninfill:
      assigninfill_withsuffix.update({key:[]})
      for entry in assigninfill[key]:
        if entry in infillcolumns_list:
          assigninfill_withsuffix[key].append(entry)
          
    #- aggregate spec'd pre-suf into mirror spec assigninfill_sourcecolumn
    assigninfill_sourcecolumn = {}
    for key in assigninfill:
      assigninfill_sourcecolumn.update({key:[]})
      for entry in assigninfill[key]:
        if entry in columns_train:
          assigninfill_sourcecolumn[key].append(entry)
          
    #- identify unspecified source columns missing from assigninfill_sourcecolumn,
    #add as new category assigninfill_sourcecolumn['unspecified']
    specd_sourcecolumns = []
    for key in assigninfill_sourcecolumn:
      specd_sourcecolumns += assigninfill_sourcecolumn[key]
    unspecd_sourcecolumns = list(set(columns_train) - set(specd_sourcecolumns))
    assigninfill_sourcecolumn.update({'unspecified':unspecd_sourcecolumns})
    
    #- convert source columns to dervied columns from assigninfill_sourcecolumn
    #to assigninfill_sourcecolumn_converted
    assigninfill_sourcecolumn_converted = {}
    for key in assigninfill_sourcecolumn:
      assigninfill_sourcecolumn_converted.update({key:[]})
      for entry in assigninfill_sourcecolumn[key]:
        #accessing dervied columns from source column, 
        #adding as entries to assigninfill_sourcecolumn_converted[key]
        assigninfill_sourcecolumn_converted[key] += postprocess_dict['origcolumn'][entry]['columnkeylist']
        
    #- if assigninfill_sourcecolumn_converted doesn't yet have entries for stdrdinfill or MLinfill, create
    if 'stdrdinfill' not in assigninfill_sourcecolumn_converted:
      assigninfill_sourcecolumn_converted.update({'stdrdinfill':[]})
    if 'MLinfill' not in assigninfill_sourcecolumn_converted:
      assigninfill_sourcecolumn_converted.update({'MLinfill':[]})
    
    #- based on MLinfill, copy entries from assigninfill_sourcecolumn_converted['unspecified']
    #into either 'stdrdinfill' or 'MLinfill'
    #(we'll keep 'unspecified' entry in case might be of use down the road)
    if MLinfill is True:
      assigninfill_sourcecolumn_converted['MLinfill'] += assigninfill_sourcecolumn_converted['unspecified']
    else:
      assigninfill_sourcecolumn_converted['stdrdinfill'] += assigninfill_sourcecolumn_converted['unspecified']
      
    #- for duplicates between entries to assigninfill_withsuffix and assigninfill_sourcecolumn_converted
    #assigninfill_withsuffix takes precedence
    #so cycle through and if duplicates found remove from assigninfill_sourcecolumn_converted
    all_specd_withsuffix = []
    for key in assigninfill_withsuffix:
      all_specd_withsuffix += assigninfill_withsuffix[key]
    for key in assigninfill_sourcecolumn_converted:
      for entry in assigninfill_sourcecolumn_converted[key]:
        if entry in all_specd_withsuffix:
          assigninfill_sourcecolumn_converted[key].remove(entry)
          
    #- combine assigninfill_withsuffix and assigninfill_sourcecolumn_converted into
    #the returned set postprocess_assigninfill_dict
    
    #first let's make sure they have equivalent keys
    for key1 in assigninfill_withsuffix:
      if key1 not in assigninfill_sourcecolumn_converted:
        assigninfill_sourcecolumn_converted.update({key1:[]})
    for key2 in assigninfill_sourcecolumn_converted:
      if key2 not in assigninfill_withsuffix:
        assigninfill_withsuffix.update({key2:[]})
    
    #ok now populate 
    postprocess_assigninfill_dict = {}
    
    for key in assigninfill_sourcecolumn_converted:
      postprocess_assigninfill_dict.update({key: assigninfill_withsuffix[key] + assigninfill_sourcecolumn_converted[key]})
    
    #- insert any missing keys needed for apply_am_infill
    if 'stdrdinfill' not in postprocess_assigninfill_dict:
      postprocess_assigninfill_dict['stdrdinfill'] = []
    
    if 'zeroinfill' not in postprocess_assigninfill_dict:
      postprocess_assigninfill_dict['zeroinfill'] = []

    if 'oneinfill' not in postprocess_assigninfill_dict:
      postprocess_assigninfill_dict['oneinfill'] = []
      
    if 'naninfill' not in postprocess_assigninfill_dict:
      postprocess_assigninfill_dict['naninfill'] = []

    if 'adjinfill' not in postprocess_assigninfill_dict:
      postprocess_assigninfill_dict['adjinfill'] = []

    if 'medianinfill' not in postprocess_assigninfill_dict: 
      postprocess_assigninfill_dict['medianinfill'] = []

    if 'meaninfill' not in postprocess_assigninfill_dict: 
      postprocess_assigninfill_dict['meaninfill'] = []

    if 'modeinfill' not in postprocess_assigninfill_dict: 
      postprocess_assigninfill_dict['modeinfill'] = []
      
    if 'lcinfill' not in postprocess_assigninfill_dict:
      postprocess_assigninfill_dict['lcinfill'] = []
      
    if 'MLinfill' not in postprocess_assigninfill_dict:
      postprocess_assigninfill_dict['MLinfill'] = []
    
    #- return postprocess_assigninfill_dict
    return postprocess_assigninfill_dict
  
  def apply_am_infill(self, df_train, df_test, postprocess_assigninfill_dict, \
                      postprocess_dict, infilliterate, printstatus, infillcolumns_list, \
                      masterNArows_train, masterNArows_test, process_dict, randomseed, ML_cmnd):
    """
    #Modularizes the application of infill to train and test sets
    """

    #infilliterate allows ML infill sets to run multiple times
    #as may be beneficial if set had a high proportion of infill for instance
    iteration = 0
    if infilliterate == 0:
      infilliterate = 1
      
    #if we're uysing this method we'll have some extra printouts
    if infilliterate > 1:
      print_infilliterate = True
    else:
      print_infilliterate = False
      
    #initialize validation results
    infill_validations = {}
      
    while iteration < infilliterate:
      
      #resent MLinfill infillcomplete markers to False
      if iteration > 0:
        for key in postprocess_assigninfill_dict['MLinfill']:
          postprocess_dict['column_dict'][key]['infillcomplete'] = False
      
      if printstatus is True:
        if print_infilliterate is True:
          print("______")
          print("ML infill infilliterate iteration: ", iteration + 1)
          print(" ")
          
      for column in infillcolumns_list:
          
        if column in postprocess_dict['column_dict']:
          
          if process_dict[postprocess_dict['column_dict'][column]['category']]['MLinfilltype'] \
          not in ['boolexclude', 'totalexclude']:

            if iteration == 0:
              
              #stndrdinfill (just prinouts, this was done in processing funcitons)
              if column in postprocess_assigninfill_dict['stdrdinfill']:

                #printout display progress
                if printstatus is True:
                  print("infill to column: ", column)
                  print("     infill type: stdrdinfill")
                  print("")
                  
              #zeroinfill
              if column in postprocess_assigninfill_dict['zeroinfill']:

                #printout display progress
                if printstatus is True:
                  print("infill to column: ", column)
                  print("     infill type: zeroinfill")
                  print("")

                categorylistlength = len(postprocess_dict['column_dict'][column]['categorylist'])

                df_train = \
                self.zeroinfillfunction(df_train, column, postprocess_dict, \
                                        masterNArows_train)

                df_test = \
                self.zeroinfillfunction(df_test, column, postprocess_dict, \
                                        masterNArows_test)

              #oneinfill
              if column in postprocess_assigninfill_dict['oneinfill']:

                #printout display progress
                if printstatus is True:
                  print("infill to column: ", column)
                  print("     infill type: oneinfill")
                  print("")

                categorylistlength = len(postprocess_dict['column_dict'][column]['categorylist'])

                df_train = \
                self.oneinfillfunction(df_train, column, postprocess_dict, \
                                       masterNArows_train)

                df_test = \
                self.oneinfillfunction(df_test, column, postprocess_dict, \
                                       masterNArows_test)
                
              #naninfill
              if column in postprocess_assigninfill_dict['naninfill']:

                #printout display progress
                if printstatus is True:
                  print("infill to column: ", column)
                  print("     infill type: naninfill")
                  print("")

                categorylistlength = len(postprocess_dict['column_dict'][column]['categorylist'])

                df_train = \
                self.naninfillfunction(df_train, column, postprocess_dict, \
                                       masterNArows_train)

                df_test = \
                self.naninfillfunction(df_test, column, postprocess_dict, \
                                       masterNArows_test)

              #adjinfill
              if column in postprocess_assigninfill_dict['adjinfill']:

                #printout display progress
                if printstatus is True:
                  print("infill to column: ", column)
                  print("     infill type: adjinfill")
                  print("")

                df_train = \
                self.adjinfillfunction(df_train, column, postprocess_dict, \
                                       masterNArows_train)

                df_test = \
                self.adjinfillfunction(df_test, column, postprocess_dict, \
                                       masterNArows_test)

              #medianinfill
              if column in postprocess_assigninfill_dict['medianinfill']:

                #printout display progress
                if printstatus is True:
                  print("infill to column: ", column)
                  print("     infill type: medianinfill")
                  print("")

                #check if column is boolean
                boolcolumn = False
                #exclude boolean and ordinal from this infill method
                if postprocess_dict['process_dict'][postprocess_dict['column_dict'][column]['category']]['MLinfilltype'] \
                in ['multirt', 'singlct', 'binary', '1010', 'boolexclude', 'concurrent_act']:
                  boolcolumn = True

                categorylistlength = len(postprocess_dict['column_dict'][column]['categorylist'])

                if (categorylistlength == 1) \
                and boolcolumn is False:
                  #noting that currently we're only going to infill 0 for single column categorylists
                  #some comparable address for multi-column categories is a future extension

                  df_train, infillvalue = \
                  self.train_medianinfillfunction(df_train, column, postprocess_dict, \
                                                  masterNArows_train)

                  postprocess_dict['column_dict'][column]['normalization_dict'][column].update({'infillvalue':infillvalue})

                  df_test = \
                  self.test_medianinfillfunction(df_test, column, postprocess_dict, \
                                                 masterNArows_test, infillvalue)
              
              #meaninfill
              if column in postprocess_assigninfill_dict['meaninfill']:

                #printout display progress
                if printstatus is True:
                  print("infill to column: ", column)
                  print("     infill type: meaninfill")
                  print("")

                #check if column is boolean
                boolcolumn = False
                #exclude boolean and ordinal from this infill method
                if postprocess_dict['process_dict'][postprocess_dict['column_dict'][column]['category']]['MLinfilltype'] \
                in ['multirt', 'singlct', 'binary', '1010', 'boolexclude', 'concurrent_act']:
                  boolcolumn = True

                categorylistlength = len(postprocess_dict['column_dict'][column]['categorylist'])

                #if (column not in excludetransformscolumns) \
                if (categorylistlength == 1) \
                and boolcolumn is False:
                  #noting that currently we're only going to infill 0 for single column categorylists
                  #some comparable address for multi-column categories is a future extension

                  df_train, infillvalue = \
                  self.train_meaninfillfunction(df_train, column, postprocess_dict, \
                                                masterNArows_train)

                  postprocess_dict['column_dict'][column]['normalization_dict'][column].update({'infillvalue':infillvalue})

                  df_test = \
                  self.test_meaninfillfunction(df_test, column, postprocess_dict, \
                                               masterNArows_test, infillvalue)
              
              #modeinfill
              if column in postprocess_assigninfill_dict['modeinfill']:

                #printout display progress
                if printstatus is True:
                  print("infill to column: ", column)
                  print("     infill type: modeinfill")
                  print("")

                #check if column is excluded (variable poorly named, interpret boolcolumn here as excluded)
                boolcolumn = False
                
                #seems reasonable to exclude concurrent_nmbr from mode
                if postprocess_dict['process_dict'][postprocess_dict['column_dict'][column]['category']]['MLinfilltype'] \
                in ['boolexclude', 'concurrent_nmbr']:
                  boolcolumn = True

                if boolcolumn is False:

                  df_train, infillvalue = \
                  self.train_modeinfillfunction(df_train, column, postprocess_dict, \
                                                masterNArows_train)

                  postprocess_dict['column_dict'][column]['normalization_dict'][column].update({'infillvalue':infillvalue})

                  df_test = \
                  self.test_modeinfillfunction(df_test, column, postprocess_dict, \
                                               masterNArows_test, infillvalue)
              
              #lcinfill:
              if column in postprocess_assigninfill_dict['lcinfill']:

                #printout display progress
                if printstatus is True:
                  print("infill to column: ", column)
                  print("     infill type: lcinfill")
                  print("")

                #check if column is excluded (variable poorly named, interpret boolcolumn here as excluded)
                boolcolumn = False

                #seems reasonable to exclude concurrent_nmbr from mode
                if postprocess_dict['process_dict'][postprocess_dict['column_dict'][column]['category']]['MLinfilltype'] \
                in ['boolexclude', 'concurrent_nmbr']:
                  boolcolumn = True

                if boolcolumn is False:

                  df_train, infillvalue = \
                  self.train_lcinfillfunction(df_train, column, postprocess_dict, \
                                                masterNArows_train)

                  postprocess_dict['column_dict'][column]['normalization_dict'][column].update({'infillvalue':infillvalue})

                  #repurpose modeinfillfunction for test, only difference is the passed infillvalue
                  df_test = \
                  self.test_modeinfillfunction(df_test, column, postprocess_dict, \
                                               masterNArows_test, infillvalue)

            #MLinfill:
            if column in postprocess_assigninfill_dict['MLinfill']:

              #printout display progress
              if printstatus is True:
                print("infill to column: ", column)
                print("     infill type: MLinfill")
                print("")

              infill_validations = \
              self.check_ML_infill(df_train, column, postprocess_dict, infill_validations)

              df_train, df_test, postprocess_dict = \
              self.MLinfillfunction(df_train, df_test, column, postprocess_dict, \
                                    masterNArows_train, masterNArows_test, randomseed, ML_cmnd, \
                                    printstatus)

      for columnname in df_train.columns:
        postprocess_dict['column_dict'][columnname]['infillcomplete'] = False
      
      iteration += 1
    
    return df_train, df_test, postprocess_dict, infill_validations
  
  def apply_pm_infill(self, df_test, postprocess_assigninfill_dict, \
                      postprocess_dict, printstatus, infillcolumns_list, \
                      masterNArows_test, process_dict):
    """
    #Modularizes the application of infill to test sets
    """
    
    #infilliterate allows ML infill sets to run multiple times
    #as may be bneficial if set had a high number of infill for instance
    iteration = 0
    
    #if we're uysing this method we'll have some extra printouts
    if postprocess_dict['infilliterate'] > 1:
      print_infilliterate = True
    else:
      print_infilliterate = False
    
    #just the convention
    if postprocess_dict['infilliterate'] == 0:
      postprocess_dict['infilliterate'] = 1
    
    while iteration < postprocess_dict['infilliterate']:
      
      #resent MLinfill infillcomplete markers to False
#       if iteration > 0:
      for key in postprocess_assigninfill_dict['MLinfill']:
        postprocess_dict['column_dict'][key]['infillcomplete'] = False
      
      if printstatus is True:
        if print_infilliterate is True:
          print("______")
          print("ML infill infilliterate iteration: ", iteration + 1)
          print(" ")
    
      for column in infillcolumns_list:
        
        if process_dict[postprocess_dict['column_dict'][column]['category']]['MLinfilltype'] \
        not in ['boolexclude', 'totalexclude']:

          if iteration == 0:
            
            #stndrdinfill (just prinouts, this was done in processing funcitons)
            if column in postprocess_assigninfill_dict['stdrdinfill']:

              #printout display progress
              if printstatus is True:
                print("infill to column: ", column)
                print("     infill type: stdrdinfill")
                print("")

            #zeroinfill:
            if column in postprocess_assigninfill_dict['zeroinfill']:

              #printout display progress
              if printstatus is True:
                print("infill to column: ", column)
                print("     infill type: zeroinfill")
                print("")

              df_test = \
              self.zeroinfillfunction(df_test, column, postprocess_dict, \
                                      masterNArows_test)
              
            #oneinfill:
            if column in postprocess_assigninfill_dict['oneinfill']:

              #printout display progress
              if printstatus is True:
                print("infill to column: ", column)
                print("     infill type: oneinfill")
                print("")

              df_test = \
              self.oneinfillfunction(df_test, column, postprocess_dict, \
                                     masterNArows_test)
              
            #naninfill
            if column in postprocess_assigninfill_dict['naninfill']:

              #printout display progress
              if printstatus is True:
                print("infill to column: ", column)
                print("     infill type: naninfill")
                print("")

              df_test = \
              self.naninfillfunction(df_test, column, postprocess_dict, \
                                     masterNArows_test)
              
            #adjinfill:
            if column in postprocess_assigninfill_dict['adjinfill']:

              #printout display progress
              if printstatus is True:
                print("infill to column: ", column)
                print("     infill type: adjinfill")
                print("")

              df_test = \
              self.adjinfillfunction(df_test, column, postprocess_dict, \
                                     masterNArows_test)
              
            #medianinfill
            if column in postprocess_assigninfill_dict['medianinfill']:

              #printout display progress
              if printstatus is True:
                print("infill to column: ", column)
                print("     infill type: medianinfill")
                print("")

              #check if column is boolean
              boolcolumn = False
              #exclude boolean and ordinal from this infill method
              if postprocess_dict['process_dict'][postprocess_dict['column_dict'][column]['category']]['MLinfilltype'] \
              in ['multirt', 'singlct', 'binary', '1010', 'boolexclude', 'concurrent_act']:
                boolcolumn = True

              categorylistlength = len(postprocess_dict['column_dict'][column]['categorylist'])

              #if (column not in excludetransformscolumns) \
              if (categorylistlength == 1) \
              and boolcolumn is False:
                #noting that currently we're only going to infill 0 for single column categorylists
                #some comparable address for multi-column categories is a future extension

                infillvalue = postprocess_dict['column_dict'][column]['normalization_dict'][column]['infillvalue']

                df_test = \
                self.test_medianinfillfunction(df_test, column, postprocess_dict, \
                                               masterNArows_test, infillvalue)

            #meaninfill:
            if column in postprocess_assigninfill_dict['meaninfill']:

              #printout display progress
              if printstatus is True:
                print("infill to column: ", column)
                print("     infill type: meaninfill")
                print("")

              #check if column is boolean
              boolcolumn = False
              #exclude boolean and ordinal from this infill method
              if postprocess_dict['process_dict'][postprocess_dict['column_dict'][column]['category']]['MLinfilltype'] \
              in ['multirt', 'singlct', 'binary', '1010', 'boolexclude', 'concurrent_act']:
                boolcolumn = True

              categorylistlength = len(postprocess_dict['column_dict'][column]['categorylist'])

              #if (column not in excludetransformscolumns) \
              if (categorylistlength == 1) \
              and boolcolumn is False:
                #noting that currently we're only going to infill 0 for single column categorylists
                #some comparable address for multi-column categories is a future extension

                infillvalue = postprocess_dict['column_dict'][column]['normalization_dict'][column]['infillvalue']

                df_test = \
                self.test_meaninfillfunction(df_test, column, postprocess_dict, \
                                             masterNArows_test, infillvalue)

            #modeinfill:
            if column in postprocess_assigninfill_dict['modeinfill']:

              #printout display progress
              if printstatus is True:
                print("infill to column: ", column)
                print("     infill type: modeinfill")
                print("")

              #check if column is excluded (variable poorly named, interpret boolcolumn here as excluded)
              boolcolumn = False

              if postprocess_dict['process_dict'][postprocess_dict['column_dict'][column]['category']]['MLinfilltype'] \
              in ['boolexclude', 'concurrent_nmbr']:
                boolcolumn = True

              if boolcolumn is False:

                infillvalue = postprocess_dict['column_dict'][column]['normalization_dict'][column]['infillvalue']

                df_test = \
                self.test_modeinfillfunction(df_test, column, postprocess_dict, \
                                               masterNArows_test, infillvalue)
                
            #lcinfill:
            if column in postprocess_assigninfill_dict['lcinfill']:

              #printout display progress
              if printstatus is True:
                print("infill to column: ", column)
                print("     infill type: lcinfill")
                print("")

              #check if column is excluded (variable poorly named, interpret boolcolumn here as excluded)
              boolcolumn = False

              if postprocess_dict['process_dict'][postprocess_dict['column_dict'][column]['category']]['MLinfilltype'] \
              in ['boolexclude', 'concurrent_nmbr']:
                boolcolumn = True

              if boolcolumn is False:

                #noting that currently we're only going to infill 0 for single column categorylists
                #some comparable address for multi-column categories is a future extension

                infillvalue = postprocess_dict['column_dict'][column]['normalization_dict'][column]['infillvalue']

                #repurpose modeinfillfunction for test, only difference is the passed infillvalue
                df_test = \
                self.test_modeinfillfunction(df_test, column, postprocess_dict, \
                                               masterNArows_test, infillvalue)

          #MLinfill:
          if column in postprocess_assigninfill_dict['MLinfill']:

            #printout display progress
            if printstatus is True:
              print("infill to column: ", column)
              print("     infill type: MLinfill")
              print("")

            df_test, postprocess_dict = \
            self.postMLinfillfunction (df_test, column, postprocess_dict, \
                                       masterNArows_test, printstatus)

      for columnname in df_test.columns:
        postprocess_dict['column_dict'][columnname]['infillcomplete'] = False
      
      iteration += 1
      
    return df_test

  def zeroinfillfunction(self, df, column, postprocess_dict, \
                        masterNArows):
    
    #copy the datatype to ensure returned set is consistent
    df_temp_dtype = pd.DataFrame(df[column][:1]).copy()

    #create infill dataframe of all zeros with number of rows corepsonding to the
    #number of 1's found in masterNArows
    NArw_columnname = \
    postprocess_dict['column_dict'][column]['origcolumn'] + '_NArows'

    NAcount = len(masterNArows[masterNArows[NArw_columnname] == 1])

    infill = pd.DataFrame(np.zeros((NAcount, 1)), columns=[column])
    
    category = postprocess_dict['column_dict'][column]['category']
    columnslist = postprocess_dict['column_dict'][column]['columnslist']
    categorylist = postprocess_dict['column_dict'][column]['categorylist']

    #insert infill
    df = self.insertinfill(df, column, infill, category, \
                           pd.DataFrame(masterNArows[NArw_columnname]), \
                           postprocess_dict, columnslist = columnslist, \
                           categorylist = categorylist, singlecolumncase=True)
    
    #reset data type to ensure returned data is consistent with what was passed
    df[column] = \
    df[column].astype({column:df_temp_dtype[column].dtypes})
    
    return df

  def oneinfillfunction(self, df, column, postprocess_dict, \
                        masterNArows):

    #copy the datatype to ensure returned set is consistent
    df_temp_dtype = pd.DataFrame(df[column][:1]).copy()
    

    #create infill dataframe of all zeros with number of rows corepsonding to the
    #number of 1's found in masterNArows
    NArw_columnname = \
    postprocess_dict['column_dict'][column]['origcolumn'] + '_NArows'

    NAcount = len(masterNArows[masterNArows[NArw_columnname] == 1])

    infill = pd.DataFrame(np.ones((NAcount, 1)), columns=[column])
    
    category = postprocess_dict['column_dict'][column]['category']
    columnslist = postprocess_dict['column_dict'][column]['columnslist']
    categorylist = postprocess_dict['column_dict'][column]['categorylist']

    #insert infill
    df = self.insertinfill(df, column, infill, category, \
                           pd.DataFrame(masterNArows[NArw_columnname]), \
                           postprocess_dict, columnslist = columnslist, \
                           categorylist = categorylist, singlecolumncase=True)

    #reset data type to ensure returned data is consistent with what was passed
    df[column] = \
    df[column].astype({column:df_temp_dtype[column].dtypes})
    
    return df
    
  def naninfillfunction(self, df, column, postprocess_dict, \
                        masterNArows):

    #copy the datatype to ensure returned set is consistent
    df_temp_dtype = pd.DataFrame(df[column][:1]).copy()
    

    #create infill dataframe of all zeros with number of rows corepsonding to the
    #number of 1's found in masterNArows
    NArw_columnname = \
    postprocess_dict['column_dict'][column]['origcolumn'] + '_NArows'

    NAcount = len(masterNArows[masterNArows[NArw_columnname] == 1])

    infill = pd.DataFrame(np.ones((NAcount, 1)), columns=[column])
    
    infill = pd.DataFrame(np.where(infill[column] == 1, np.nan, infill[column]))
    
    category = postprocess_dict['column_dict'][column]['category']
    columnslist = postprocess_dict['column_dict'][column]['columnslist']
    categorylist = postprocess_dict['column_dict'][column]['categorylist']

    #insert infill
    df = self.insertinfill(df, column, infill, category, \
                           pd.DataFrame(masterNArows[NArw_columnname]), \
                           postprocess_dict, columnslist = columnslist, \
                           categorylist = categorylist, singlecolumncase=True)

    #reset data type to ensure returned data is consistent with what was passed
    #(we won't do this for naninfill since nan may be different data type)
    # df[column] = \
    # df[column].astype({column:df_temp_dtype[column].dtypes})
    
    return df

  def adjinfillfunction(self, df, column, postprocess_dict, \
                        masterNArows):

    #copy the datatype to ensure returned set is consistent
    df_temp_dtype = pd.DataFrame(df[column][:1]).copy()
    
    #create infill dataframe of all nan with number of rows corepsonding to the
    #number of 1's found in masterNArows
    NArw_columnname = \
    postprocess_dict['column_dict'][column]['origcolumn'] + '_NArows'

    NAcount = len(masterNArows[masterNArows[NArw_columnname] == 1])
    
    infill = pd.DataFrame(np.zeros((NAcount, 1)), columns=[column])
    infill = infill.replace(0, np.nan)
    
    category = postprocess_dict['column_dict'][column]['category']
    columnslist = postprocess_dict['column_dict'][column]['columnslist']
    categorylist = postprocess_dict['column_dict'][column]['categorylist']

    #insert infill
    df = self.insertinfill(df, column, infill, category, \
                           pd.DataFrame(masterNArows[NArw_columnname]), \
                           postprocess_dict, columnslist = columnslist, \
                           categorylist = categorylist, singlecolumncase=True)
    
    #this is hack
    df[column] = df[column].replace('nan', np.nan)
    
    #apply ffill to replace NArows with value from adjacent cell in pre4ceding row
    df[column] = df[column].fillna(method='ffill')
    
    #we'll follow with a bfill just in case first row had a nan
    df[column] = df[column].fillna(method='bfill')
    
    #and final edge case if all cells were subject to infill we'll just insert 0
    df[column] = df[column].fillna(value=0)
    
    #reset data type to ensure returned data is consistent with what was passed
    df[column] = \
    df[column].astype({column:df_temp_dtype[column].dtypes})

    return df

  def train_medianinfillfunction(self, df, column, postprocess_dict, \
                                 masterNArows):

    #copy the datatype to ensure returned set is consistent
    df_temp_dtype = pd.DataFrame(df[column][:1]).copy()

    #create infill dataframe of all zeros with number of rows corepsonding to the
    #number of 1's found in masterNArows
    NArw_columnname = \
    postprocess_dict['column_dict'][column]['origcolumn'] + '_NArows'

    NAcount = len(masterNArows[masterNArows[NArw_columnname] == 1])
    
    #create df without rows that were subject to infill to dervie median
    tempdf = pd.concat([df[column], masterNArows[NArw_columnname]], axis=1)
    #remove rows that were subject to infill
    tempdf = tempdf[tempdf[NArw_columnname] != 1]
    #calculate median of remaining rows
    median = tempdf[column].median()
    
    #edge case
    if median != median:
      median = 0
    
    del tempdf

    infill = pd.DataFrame(np.zeros((NAcount, 1)))
    infill = infill.replace(0, median)

    category = postprocess_dict['column_dict'][column]['category']
    columnslist = postprocess_dict['column_dict'][column]['columnslist']
    categorylist = postprocess_dict['column_dict'][column]['categorylist']

    #insert infill
    df = self.insertinfill(df, column, infill, category, \
                           pd.DataFrame(masterNArows[NArw_columnname]), \
                           postprocess_dict, columnslist = columnslist, \
                           categorylist = categorylist)

    #reset data type to ensure returned data is consistent with what was passed
    df[column] = \
    df[column].astype({column:df_temp_dtype[column].dtypes})
    
    return df, median

  def test_medianinfillfunction(self, df, column, postprocess_dict, \
                                 masterNArows, median):

    #copy the datatype to ensure returned set is consistent
    df_temp_dtype = pd.DataFrame(df[column][:1]).copy()
    
    #create infill dataframe of all zeros with number of rows corepsonding to the
    #number of 1's found in masterNArows
    NArw_columnname = \
    postprocess_dict['column_dict'][column]['origcolumn'] + '_NArows'

    NAcount = len(masterNArows[masterNArows[NArw_columnname] == 1])
    
    median = median

    infill = pd.DataFrame(np.zeros((NAcount, 1)))
    infill = infill.replace(0, median)

    category = postprocess_dict['column_dict'][column]['category']
    columnslist = postprocess_dict['column_dict'][column]['columnslist']
    categorylist = postprocess_dict['column_dict'][column]['categorylist']

    #insert infill
    df = self.insertinfill(df, column, infill, category, \
                           pd.DataFrame(masterNArows[NArw_columnname]), \
                           postprocess_dict, columnslist = columnslist, \
                           categorylist = categorylist)

    #reset data type to ensure returned data is consistent with what was passed
    df[column] = \
    df[column].astype({column:df_temp_dtype[column].dtypes})
    
    return df

  def train_meaninfillfunction(self, df, column, postprocess_dict, \
                                 masterNArows):

    #copy the datatype to ensure returned set is consistent
    df_temp_dtype = pd.DataFrame(df[column][:1]).copy()
    
    #create infill dataframe of all zeros with number of rows corepsonding to the
    #number of 1's found in masterNArows
    NArw_columnname = \
    postprocess_dict['column_dict'][column]['origcolumn'] + '_NArows'

    NAcount = len(masterNArows[masterNArows[NArw_columnname] == 1])
    
    #create df without rows that were subject to infill to dervie median
    tempdf = pd.concat([df[column], masterNArows[NArw_columnname]], axis=1)
    #remove rows that were subject to infill
    tempdf = tempdf[tempdf[NArw_columnname] != 1]
    #calculate median of remaining rows
    mean = tempdf[column].mean()
    
    #edge case
    if mean != mean:
      mean = 0
    
    del tempdf

    infill = pd.DataFrame(np.zeros((NAcount, 1)))
    infill = infill.replace(0, mean)

    category = postprocess_dict['column_dict'][column]['category']
    columnslist = postprocess_dict['column_dict'][column]['columnslist']
    categorylist = postprocess_dict['column_dict'][column]['categorylist']

    #insert infill
    df = self.insertinfill(df, column, infill, category, \
                           pd.DataFrame(masterNArows[NArw_columnname]), \
                           postprocess_dict, columnslist = columnslist, \
                           categorylist = categorylist)

    #reset data type to ensure returned data is consistent with what was passed
    df[column] = \
    df[column].astype({column:df_temp_dtype[column].dtypes})
    
    return df, mean

  def test_meaninfillfunction(self, df, column, postprocess_dict, \
                                 masterNArows, mean):

    #copy the datatype to ensure returned set is consistent
    df_temp_dtype = pd.DataFrame(df[column][:1]).copy()

    #create infill dataframe of all zeros with number of rows corepsonding to the
    #number of 1's found in masterNArows
    NArw_columnname = \
    postprocess_dict['column_dict'][column]['origcolumn'] + '_NArows'

    NAcount = len(masterNArows[masterNArows[NArw_columnname] == 1])
    
    mean = mean

    infill = pd.DataFrame(np.zeros((NAcount, 1)))
    infill = infill.replace(0, mean)

    category = postprocess_dict['column_dict'][column]['category']
    columnslist = postprocess_dict['column_dict'][column]['columnslist']
    categorylist = postprocess_dict['column_dict'][column]['categorylist']

    #insert infill
    df = self.insertinfill(df, column, infill, category, \
                           pd.DataFrame(masterNArows[NArw_columnname]), \
                           postprocess_dict, columnslist = columnslist, \
                           categorylist = categorylist)

    #reset data type to ensure returned data is consistent with what was passed
    df[column] = \
    df[column].astype({column:df_temp_dtype[column].dtypes})
    
    return df

  def train_modeinfillfunction(self, df, column, postprocess_dict, \
                               masterNArows):

    #copy the datatype to ensure returned set is consistent
    df_temp_dtype = pd.DataFrame(df[column][:1]).copy()

    #create infill dataframe of all zeros with number of rows corepsonding to the
    #number of 1's found in masterNArows
    NArw_columnname = \
    postprocess_dict['column_dict'][column]['origcolumn'] + '_NArows'

    NAcount = len(masterNArows[masterNArows[NArw_columnname] == 1])
    
    #deriving a mode for one-hot encoded sets requires a different approach
    if len(postprocess_dict['column_dict'][column]['categorylist']) > 1 \
    and postprocess_dict['process_dict'][postprocess_dict['column_dict'][column]['category']]['MLinfilltype'] \
    in ['multirt']:
      
      categorylist = postprocess_dict['column_dict'][column]['categorylist']
      
      tempdf = pd.concat([df[categorylist], masterNArows[NArw_columnname]], axis=1)
      #remove rows that were subject to infill
      tempdf = tempdf[tempdf[NArw_columnname] != 1]
      
      del tempdf[NArw_columnname]
      
      tempdf_mode_dict = {}
      
      #since this is one hot encoded we can count activations in a column with sum
      for tempcolumn in tempdf.columns:
        tempdf_mode_dict.update({tempdf[tempcolumn].sum() : tempcolumn})
      
      #create a list of those sums then sort to grab the mode column
      tempdf_mode_dict_keys = list(tempdf_mode_dict)
      tempdf_mode_dict_keys = sorted(tempdf_mode_dict_keys)
      mode_column = tempdf_mode_dict[tempdf_mode_dict_keys[-1]]
      
      if mode_column == column:
        mode = 1
      else:
        mode = 0
        
      del tempdf
      del tempdf_mode_dict
      del tempdf_mode_dict_keys
      del mode_column
      
    #deriving a mode for multi-column binary encoded sets requires a different approach
    elif len(postprocess_dict['column_dict'][column]['categorylist']) > 1 \
    and postprocess_dict['process_dict'][postprocess_dict['column_dict'][column]['category']]['MLinfilltype'] \
    in ['1010']:
      
      categorylist = postprocess_dict['column_dict'][column]['categorylist']
      
      tempdf = pd.concat([df[categorylist], masterNArows[NArw_columnname]], axis=1)
      #remove rows that were subject to infill
      tempdf = tempdf[tempdf[NArw_columnname] != 1]
      
      del tempdf[NArw_columnname]
      
      #initialize a column to store encodings
      tempdf['onehot'] = ''

      #populate column to store aggregated encodings 
      for tempdf_column in tempdf.columns:
        if tempdf_column != 'onehot':
          tempdf['onehot'] = \
          tempdf['onehot'] + tempdf[tempdf_column].astype(int).astype(str)

      #find mode of the aggregation
      binary_mode = tempdf['onehot'].mode()
      
      if len(binary_mode) > 0:
        binary_mode = binary_mode[0]
      else:
        binary_mode = tempdf['onehot'].values[0]
      
#       if len(binary_mode) < 1:
#         binary_mode = 0
      
      #remove rows other than mode
      tempdf = tempdf[tempdf['onehot'] == binary_mode]
      
      #mode is the current columns value associated with that mode
      mode = tempdf[column].values[0]
      
      del tempdf
      del binary_mode
    
    #else if columns were not multi-column
    #note this scenario also includes 'concurrent_act' MLinfilltype
    else:
    
      #create df without rows that were subject to infill to dervie mode
      tempdf = pd.concat([df[column], masterNArows[NArw_columnname]], axis=1)
      #remove rows that were subject to infill
      tempdf = tempdf[tempdf[NArw_columnname] != 1]
      
      #calculate mode of remaining rows
      mode = tempdf[column].mode()
      
      if len(mode) > 0:
        mode = mode[0]
      else:
        mode = 0

      del tempdf

    infill = pd.DataFrame(np.zeros((NAcount, 1)))
    infill = infill.replace(0, mode)

    category = postprocess_dict['column_dict'][column]['category']
    columnslist = postprocess_dict['column_dict'][column]['columnslist']
    categorylist = postprocess_dict['column_dict'][column]['categorylist']

    #insert infill
    df = self.insertinfill(df, column, infill, category, \
                           pd.DataFrame(masterNArows[NArw_columnname]), \
                           postprocess_dict, columnslist = columnslist, \
                           categorylist = categorylist, singlecolumncase = True)

    #reset data type to ensure returned data is consistent with what was passed
    df[column] = \
    df[column].astype({column:df_temp_dtype[column].dtypes})

    return df, mode

  def test_modeinfillfunction(self, df, column, postprocess_dict, \
                              masterNArows, mode):

    #copy the datatype to ensure returned set is consistent
    df_temp_dtype = pd.DataFrame(df[column][:1]).copy()

    #create infill dataframe of all zeros with number of rows corepsonding to the
    #number of 1's found in masterNArows
    NArw_columnname = \
    postprocess_dict['column_dict'][column]['origcolumn'] + '_NArows'

    NAcount = len(masterNArows[masterNArows[NArw_columnname] == 1])
    
    mode = mode

    infill = pd.DataFrame(np.zeros((NAcount, 1)))
    infill = infill.replace(0, mode)

    category = postprocess_dict['column_dict'][column]['category']
    columnslist = postprocess_dict['column_dict'][column]['columnslist']
    categorylist = postprocess_dict['column_dict'][column]['categorylist']

#     #insert infill
#     df = self.insertinfill(df, column, infill, category, \
#                            pd.DataFrame(masterNArows[NArw_columnname]), \
#                            postprocess_dict, columnslist = columnslist, \
#                            categorylist = categorylist)

    #insert infill
    df = self.insertinfill(df, column, infill, category, \
                           pd.DataFrame(masterNArows[NArw_columnname]), \
                           postprocess_dict, columnslist = columnslist, \
                           categorylist = categorylist, singlecolumncase = True)

    #reset data type to ensure returned data is consistent with what was passed
    df[column] = \
    df[column].astype({column:df_temp_dtype[column].dtypes})

    return df

  def train_lcinfillfunction(self, df, column, postprocess_dict, \
                             masterNArows):
    """
    #comparable to modeinfill function but uses least common value instead of most common value
    """

    #copy the datatype to ensure returned set is consistent
    df_temp_dtype = pd.DataFrame(df[column][:1]).copy()

    #create infill dataframe of all zeros with number of rows corepsonding to the
    #number of 1's found in masterNArows
    NArw_columnname = \
    postprocess_dict['column_dict'][column]['origcolumn'] + '_NArows'

    NAcount = len(masterNArows[masterNArows[NArw_columnname] == 1])
    
    #deriving a mode for one-hot encoded sets requires a different approach
    if len(postprocess_dict['column_dict'][column]['categorylist']) > 1 \
    and postprocess_dict['process_dict'][postprocess_dict['column_dict'][column]['category']]['MLinfilltype'] \
    in ['multirt']:
      
      categorylist = postprocess_dict['column_dict'][column]['categorylist']
      
      tempdf = pd.concat([df[categorylist], masterNArows[NArw_columnname]], axis=1)
      #remove rows that were subject to infill
      tempdf = tempdf[tempdf[NArw_columnname] != 1]
      
      del tempdf[NArw_columnname]
      
      tempdf_mode_dict = {}
      
      #since this is one hot encoded we can count activations in a column with sum
      for tempcolumn in tempdf.columns:
        tempdf_mode_dict.update({tempdf[tempcolumn].sum() : tempcolumn})
      
      #create a list of those sums then sort to grab the mode column
      tempdf_mode_dict_keys = list(tempdf_mode_dict)
      tempdf_mode_dict_keys = sorted(tempdf_mode_dict_keys)
      #mode_column = tempdf_mode_dict[tempdf_mode_dict_keys[-1]]
      mode_column = tempdf_mode_dict[tempdf_mode_dict_keys[0]]
      
      if mode_column == column:
        mode = 1
      else:
        mode = 0
        
      del tempdf
      del tempdf_mode_dict
      del tempdf_mode_dict_keys
      del mode_column
      
    #deriving a mode for multi-column binary encoded sets requires a different approach
    elif len(postprocess_dict['column_dict'][column]['categorylist']) > 1 \
    and postprocess_dict['process_dict'][postprocess_dict['column_dict'][column]['category']]['MLinfilltype'] \
    in ['1010']:
      
      categorylist = postprocess_dict['column_dict'][column]['categorylist']
      
      tempdf = pd.concat([df[categorylist], masterNArows[NArw_columnname]], axis=1)
      #remove rows that were subject to infill
      tempdf = tempdf[tempdf[NArw_columnname] != 1]
      
      del tempdf[NArw_columnname]
      
      #initialize a column to store encodings
      #note this arbitrary column won't overlap with any
      #because categorylist all have suffixes with '_' character
      tempdf['onehot'] = ''

      #populate column to store aggregated encodings 
      for tempdf_column in tempdf.columns:
        if tempdf_column != 'onehot':
          tempdf['onehot'] = \
          tempdf['onehot'] + tempdf[tempdf_column].astype(int).astype(str)

      #find mode of the aggregation
      #binary_mode = tempdf['onehot'].mode()
      mode_valuecounts_list = pd.DataFrame(tempdf['onehot'].value_counts())
      mode_valuecounts_list = mode_valuecounts_list.rename_axis('zzzinfill').sort_values(by = ['onehot', 'zzzinfill'], ascending = [False, True])
      mode_valuecounts_list = list(mode_valuecounts_list.index)
      if len(mode_valuecounts_list) > 0:
        binary_mode = mode_valuecounts_list[-1]
      else:
        binary_mode = 0
      
#       if len(binary_mode) > 0:
#         binary_mode = binary_mode[0]
#       else:
#         binary_mode = tempdf['onehot'].values[0]

      if binary_mode != binary_mode:
        binary_mode = 0
      
#       if len(binary_mode) < 1:
#         binary_mode = 0
      
      #remove rows other than mode
      tempdf = tempdf[tempdf['onehot'] == binary_mode]
      
      #mode is the current columns value associated with that mode
      mode = tempdf[column].values[0]
      
      del tempdf
      del binary_mode
    
    #else if columns were not multi-column
    #note this scenario also includes 'concurrent_act' MLinfilltype
    else:
    
      #create df without rows that were subject to infill to dervie mode
      tempdf = pd.concat([df[column], masterNArows[NArw_columnname]], axis=1)
      #remove rows that were subject to infill
      tempdf = tempdf[tempdf[NArw_columnname] != 1]

      #calculate mode of remaining rows
      mode_valuecounts_list = pd.DataFrame(tempdf[column].value_counts())
      mode_valuecounts_list = mode_valuecounts_list.rename_axis('zzzinfill').sort_values(by = [column, 'zzzinfill'], ascending = [False, True])
      mode_valuecounts_list = list(mode_valuecounts_list.index)
      if len(mode_valuecounts_list) > 0:
        mode = mode_valuecounts_list[-1]
      else:
        mode = 0
      
      if mode != mode:
        mode = 0
      
#       if len(mode) > 0:
#         mode = mode[0]
#       else:
#         mode = 0

      del tempdf

    infill = pd.DataFrame(np.zeros((NAcount, 1)))
    infill = infill.replace(0, mode)

    category = postprocess_dict['column_dict'][column]['category']
    columnslist = postprocess_dict['column_dict'][column]['columnslist']
    categorylist = postprocess_dict['column_dict'][column]['categorylist']

    #insert infill
    df = self.insertinfill(df, column, infill, category, \
                           pd.DataFrame(masterNArows[NArw_columnname]), \
                           postprocess_dict, columnslist = columnslist, \
                           categorylist = categorylist, singlecolumncase = True)

    #reset data type to ensure returned data is consistent with what was passed
    df[column] = \
    df[column].astype({column:df_temp_dtype[column].dtypes})

    return df, mode

  def populatePCAdefaults(self, randomseed):
    '''
    populates sa dictionary with default values for PCA methods PCA, 
    SparsePCA, and KernelPCA. (Each based on ScikitLearn default values)
    #note that for SparsePCA the 'normalize_components' is not passed 
    #since will be depreciated
    '''

    PCAdefaults = {'PCA':{}, 'SparsePCA':{}, 'KernelPCA':{}}

    PCAdefaults['PCA'].update({'copy':True, \
                               'whiten':False, \
                               'svd_solver':'auto', \
                               'tol':0.0, \
                               'iterated_power':'auto', \
                               'random_state':randomseed})

    PCAdefaults['SparsePCA'].update({'alpha':1, \
                                     'ridge_alpha':0.01, \
                                     'max_iter':1000, \
                                     'tol':1e-08, \
                                     'method':'lars', \
                                     'n_jobs':None, \
                                     'U_init':None, \
                                     'V_init':None, \
                                     'verbose':False, \
                                     'random_state':randomseed})
#                                       , \
#                                      'normalize_components':True})

    PCAdefaults['KernelPCA'].update({'kernel':'linear', \
                                     'gamma':None, \
                                     'degree':3, \
                                     'coef0':1, \
                                     'kernel_params':None, \
                                     'alpha':1.0, \
                                     'fit_inverse_transform':False, \
                                     'eigen_solver':'auto', \
                                     'tol':0, \
                                     'max_iter':None, \
                                     'remove_zero_eig':False, \
                                     'random_state':randomseed, \
                                     'copy_X':True, \
                                     'n_jobs':None})

    return PCAdefaults

  def evalPCA(self, df_train, PCAn_components, ML_cmnd):
    '''
    function serves to evaluate properties of dataframe to determine 
    if an automated application of PCA is appropriate, and if so 
    what kind of PCA to apply
    returns PCActgy as
    'noPCA' -> self explanatory, this is the default when number of features 
                is less than 15% of number of rows
                Please note this is a somewhat arbitrary ratio and some more
                research is needed to validate methods for this rule
                A future iteration may perform addition kinds of evaluations
                such as distribuytions and correlations of data for this method.
    'KernelPCA' -> dataset suitable for automated KernelPCA application
                    (preffered method when data is all non-negative)
    'SparsePCA' -> dataset suitable for automated SparsePCA application
                    (prefered method when data is not all non-negative)
    'PCA' -> not currently used as a default
    also returns a n_components value which is based on the user passed 
    value to PCAn_components or if user passes None (the default) then
    one is assigned based on properties of the data set
    also returns a value for n_components based on that same 15% rule
    where PCA application will default to user passed n_components but if
    none passed will apply this returned value
    '''

    number_rows = df_train.shape[0]
    number_columns = df_train.shape[1]
    
    #ok this is to allow user to set the default columns/rows ratio for automated PCA
    if 'col_row_ratio' in ML_cmnd['PCA_cmnd']:
      col_row_ratio = ML_cmnd['PCA_cmnd']['col_row_ratio']
    else:
      col_row_ratio = 0.50

    if ML_cmnd['PCA_type'] == 'default':

      #if number_columns / number_rows < 0.15:
      if number_columns / number_rows < col_row_ratio:

        if PCAn_components == None:

          PCActgy = 'noPCA'

          n_components = PCAn_components

        if PCAn_components != None:

          #if df_train[df_train < 0.0].count() == 0:
          if any(df_train < 0.0):

            PCActgy = 'SparsePCA'

          #else if there were negative values in the dataframe
          else:

            PCActgy = 'KernelPCA'

          n_components = PCAn_components

      #else if number_columns / number_rows > 0.15
      #else:
      if number_columns / number_rows > col_row_ratio:

        #if df_train[df_train < 0.0].count() == 0:
        #if df_train[df_train < 0.0].sum() == 0:
        if any(df_train < 0.0):

          PCActgy = 'SparsePCA'

        #else if there were negative values in the dataframe
        else:

          PCActgy = 'KernelPCA'

        #if user did not pass a PCAn_component then we'll create one
        if PCAn_components == None:

          #this is a somewhat arbitrary figure, some
          #additional research needs to be performed
          #a future expansion may base this on properties
          #of the data
          #n_components = int(round(0.15 * number_rows))
          n_components = int(round(col_row_ratio * number_rows))

        else:

          n_components = PCAn_components
    
    if isinstance(PCAn_components, (int, float)):
    
      if PCAn_components > 0.0 and PCAn_components < 1.0:
        
        PCActgy = 'PCA'
    
        n_components = PCAn_components

    if ML_cmnd['PCA_type'] != 'default':

      PCActgy = ML_cmnd['PCA_type']

      n_components = PCAn_components
    
    return PCActgy, n_components

  def initSparsePCA(self, ML_cmnd, PCAdefaults, PCAn_components):
    '''
    function that assigns appropriate parameters based on defaults and user inputs
    and then initializes a SparsePCA model
    '''

    #if user passed values use those, otherwise pass scikit defaults
    if 'alpha' in ML_cmnd['PCA_cmnd']:
      alpha = ML_cmnd['PCA_cmnd']['alpha']
    else:
      alpha = PCAdefaults['SparsePCA']['alpha']

    if 'ridge_alpha' in ML_cmnd['PCA_cmnd']:
      ridge_alpha = ML_cmnd['PCA_cmnd']['ridge_alpha']
    else:
      ridge_alpha = PCAdefaults['SparsePCA']['ridge_alpha']

    if 'max_iter' in ML_cmnd['PCA_cmnd']:
      max_iter = ML_cmnd['PCA_cmnd']['max_iter']
    else:
      max_iter = PCAdefaults['SparsePCA']['max_iter']

    if 'tol' in ML_cmnd['PCA_cmnd']:
      tol = ML_cmnd['PCA_cmnd']['tol']
    else:
      tol = PCAdefaults['SparsePCA']['tol']

    if 'method' in ML_cmnd['PCA_cmnd']:
      method = ML_cmnd['PCA_cmnd']['method']
    else:
      method = PCAdefaults['SparsePCA']['method']

    if 'n_jobs' in ML_cmnd['PCA_cmnd']:
      n_jobs = ML_cmnd['PCA_cmnd']['n_jobs']
    else:
      n_jobs = PCAdefaults['SparsePCA']['n_jobs']

    if 'U_init' in ML_cmnd['PCA_cmnd']:
      U_init = ML_cmnd['PCA_cmnd']['U_init']
    else:
      U_init = PCAdefaults['SparsePCA']['U_init']

    if 'V_init' in ML_cmnd['PCA_cmnd']:
      V_init = ML_cmnd['PCA_cmnd']['V_init']
    else:
      V_init = PCAdefaults['SparsePCA']['V_init']

    if 'verbose' in ML_cmnd['PCA_cmnd']:
      verbose = ML_cmnd['PCA_cmnd']['verbose']
    else:
      verbose = PCAdefaults['SparsePCA']['verbose']

    if 'random_state' in ML_cmnd['PCA_cmnd']:
      random_state = ML_cmnd['PCA_cmnd']['random_state']
    else:
      random_state = PCAdefaults['SparsePCA']['random_state']

#     if 'normalize_components' in ML_cmnd['PCA_cmnd']:
#       normalize_components = ML_cmnd['PCA_cmnd']['normalize_components']
#     else:
#       normalize_components = PCAdefaults['SparsePCA']['normalize_components']

    #do other stuff?

    #then train PCA model 
    PCAmodel = SparsePCA(n_components = PCAn_components, \
                         alpha = alpha, \
                         ridge_alpha = ridge_alpha, \
                         max_iter = max_iter, \
                         tol = tol, \
                         method = method, \
                         n_jobs = n_jobs, \
                         U_init = U_init, \
                         V_init = V_init, \
                         verbose = verbose, \
                         random_state = random_state)
#                          , \
#                          normalize_components = normalize_components)

    return PCAmodel

  def initKernelPCA(self, ML_cmnd, PCAdefaults, PCAn_components):
    '''
    function that assigns approrpiate parameters based on defaults and user inputs
    and then initializes a KernelPCA model
    '''

    #if user passed values use those, otherwise pass scikit defaults
    if 'kernel' in ML_cmnd['PCA_cmnd']:
      kernel = ML_cmnd['PCA_cmnd']['kernel']
    else:
      kernel = PCAdefaults['KernelPCA']['kernel']

    if 'gamma' in ML_cmnd['PCA_cmnd']:
      gamma = ML_cmnd['PCA_cmnd']['gamma']
    else:
      gamma = PCAdefaults['KernelPCA']['gamma']

    if 'degree' in ML_cmnd['PCA_cmnd']:
      degree = ML_cmnd['PCA_cmnd']['degree']
    else:
      degree = PCAdefaults['KernelPCA']['degree']

    if 'coef0' in ML_cmnd['PCA_cmnd']:
      coef0 = ML_cmnd['PCA_cmnd']['coef0']
    else:
      coef0 = PCAdefaults['KernelPCA']['coef0']

    if 'kernel_params' in ML_cmnd['PCA_cmnd']:
      kernel_params = ML_cmnd['PCA_cmnd']['kernel_params']
    else:
      kernel_params = PCAdefaults['KernelPCA']['kernel_params']

    if 'alpha' in ML_cmnd['PCA_cmnd']:
      alpha = ML_cmnd['PCA_cmnd']['alpha']
    else:
      alpha = PCAdefaults['KernelPCA']['alpha']

    if 'fit_inverse_transform' in ML_cmnd['PCA_cmnd']:
      fit_inverse_transform = ML_cmnd['PCA_cmnd']['fit_inverse_transform']
    else:
      fit_inverse_transform = PCAdefaults['KernelPCA']['fit_inverse_transform']

    if 'eigen_solver' in ML_cmnd['PCA_cmnd']:
      eigen_solver = ML_cmnd['PCA_cmnd']['eigen_solver']
    else:
      eigen_solver = PCAdefaults['KernelPCA']['eigen_solver']

    if 'tol' in ML_cmnd['PCA_cmnd']:
      tol = ML_cmnd['PCA_cmnd']['tol']
    else:
      tol = PCAdefaults['KernelPCA']['tol']

    if 'max_iter' in ML_cmnd['PCA_cmnd']:
      max_iter = ML_cmnd['PCA_cmnd']['max_iter']
    else:
      max_iter = PCAdefaults['KernelPCA']['max_iter']

    if 'remove_zero_eig' in ML_cmnd['PCA_cmnd']:
      remove_zero_eig = ML_cmnd['PCA_cmnd']['remove_zero_eig']
    else:
      remove_zero_eig = PCAdefaults['KernelPCA']['remove_zero_eig']

    if 'random_state' in ML_cmnd['PCA_cmnd']:
      random_state = ML_cmnd['PCA_cmnd']['random_state']
    else:
      random_state = PCAdefaults['KernelPCA']['random_state']

    if 'copy_X' in ML_cmnd['PCA_cmnd']:
      copy_X = ML_cmnd['PCA_cmnd']['copy_X']
    else:
      copy_X = PCAdefaults['KernelPCA']['copy_X']

    if 'n_jobs' in ML_cmnd['PCA_cmnd']:
      n_jobs = ML_cmnd['PCA_cmnd']['n_jobs']
    else:
      n_jobs = PCAdefaults['KernelPCA']['n_jobs']

    #do other stuff?

    #then train PCA model 
    PCAmodel = KernelPCA(n_components = PCAn_components, \
                         kernel = kernel, \
                         gamma = gamma, \
                         degree = degree, \
                         coef0 = coef0, \
                         kernel_params = kernel_params, \
                         alpha = alpha, \
                         fit_inverse_transform = fit_inverse_transform, \
                         eigen_solver = eigen_solver, \
                         tol = tol, \
                         max_iter = max_iter, \
                         remove_zero_eig = remove_zero_eig, \
                         random_state = random_state, \
                         copy_X = copy_X, \
                         n_jobs = n_jobs)

    return PCAmodel

  def initPCA(self, ML_cmnd, PCAdefaults, PCAn_components):
    '''
    function that assigns approrpiate parameters based on defaults and user inputs
    and then initializes a basic PCA model
    '''

    #run PCA version

    #if user passed values use those, otherwise pass scikit defaults
    if 'copy' in ML_cmnd['PCA_cmnd']:
      copy = ML_cmnd['PCA_cmnd']['copy']
    else:
      copy = PCAdefaults['PCA']['copy']

    if 'whiten' in ML_cmnd['PCA_cmnd']:
      whiten = ML_cmnd['PCA_cmnd']['whiten']
    else:
      whiten = PCAdefaults['PCA']['whiten']

    if 'svd_solver' in ML_cmnd['PCA_cmnd']:
      svd_solver = ML_cmnd['PCA_cmnd']['svd_solver']
    else:
      svd_solver = PCAdefaults['PCA']['svd_solver']

    if 'tol' in ML_cmnd['PCA_cmnd']:
      tol = ML_cmnd['PCA_cmnd']['tol']
    else:
      tol = PCAdefaults['PCA']['tol']

    if 'iterated_power' in ML_cmnd['PCA_cmnd']:
      iterated_power = ML_cmnd['PCA_cmnd']['iterated_power']
    else:
      iterated_power = PCAdefaults['PCA']['iterated_power']

    if 'random_state' in ML_cmnd['PCA_cmnd']:
      random_state = ML_cmnd['PCA_cmnd']['random_state']
    else:
      random_state = PCAdefaults['PCA']['random_state']

    #do other stuff?

    #then train PCA model 
    PCAmodel = PCA(n_components = PCAn_components, \
                   copy = copy, \
                   whiten = whiten, \
                   svd_solver = svd_solver, \
                   tol = tol, \
                   iterated_power = iterated_power, \
                   random_state = random_state)

    return PCAmodel

  def boolexcl(self, ML_cmnd, df, PCAexcl, postprocess_dict):
    """
    If user passed bool_PCA_excl as True in ML_cmnd['PCA_cmnd']
    {'PCA_cmnd':{'bool_PCA_excl': True}}
    Then add boolean columns to the PCAexcl list of columns
    to be carved out from PCA application
    If user passed bool_ordl_PCAexcl as True in ML_cmnd['PCA_cmnd']
    Then add ordinal columns (recognized becayuse they are catehgorical)
    to the PCAexcl list of columns
    to be carved out from PCA application
    
    Note that PCAexcl may alreadyn be populated with user-passed
    columns to 4exclude from PCA. The returned bool_PCAexcl list
    seperately tracks just those columns that were added as part 
    of this function, in case may be of later use
    """
    
    bool_PCAexcl = []
    if 'bool_PCA_excl' in ML_cmnd['PCA_cmnd']:
        
      #if user passed the bool_PCA_excl as True in ML_cmnd['PCA_cmnd'] 
      if ML_cmnd['PCA_cmnd']['bool_PCA_excl'] is True:
        for checkcolumn in df:
          #if column is boolean then add to lists
        #   if set(df[checkcolumn].unique()) == {0,1} \
        #   or set(df[checkcolumn].unique()) == {0} \
        #   or set(df[checkcolumn].unique()) == {1}:
          if postprocess_dict['process_dict'][postprocess_dict['column_dict'][checkcolumn]['category']]['MLinfilltype'] \
          in ['multirt', 'binary', '1010', 'boolexclude', 'concurrent_act', 'totalexclude']:
            if checkcolumn not in PCAexcl:
              PCAexcl.append(checkcolumn)
            bool_PCAexcl.append(checkcolumn)
    
    if 'bool_ordl_PCAexcl' in ML_cmnd['PCA_cmnd']:
      #if user passed the bool_ordl_PCAexcl as True in ML_cmnd['PCA_cmnd'] 
      if ML_cmnd['PCA_cmnd']['bool_ordl_PCAexcl'] is True:
        for checkcolumn in df:
          #if column is boolean then add to lists
        #   if set(df[checkcolumn].unique()) == {0,1} \
        #   or set(df[checkcolumn].unique()) == {0} \
        #   or set(df[checkcolumn].unique()) == {1} \
        #   or checkcolumn[-5:] == '_ordl':
          if postprocess_dict['process_dict'][postprocess_dict['column_dict'][checkcolumn]['category']]['MLinfilltype'] \
          in ['singlct', 'binary', 'multirt', '1010', 'boolexclude', 'concurrent_act', 'totalexclude']:
            #or isinstance(df[checkcolumn].dtype, pd.api.types.CategoricalDtype):
            if checkcolumn not in PCAexcl:
              PCAexcl.append(checkcolumn)
            bool_PCAexcl.append(checkcolumn)
            
    return PCAexcl, bool_PCAexcl

  def createPCAsets(self, df_train, df_test, PCAexcl, postprocess_dict):
    '''
    Function that takes as input the dataframes df_train and df_test 
    Removes those columns associated with the PCAexcl (which are the original 
    columns passed to automunge which are to be exlcuded from PCA), and returns 
    those sets as PCAset_trian, PCAset_test, and the list of columns extracted as
    PCAexcl_posttransform.
    '''

    #initiate list PCAexcl_postransform
    PCAexcl_posttransform = []

    #derive the excluded columns post-transform using postprocess_dict
    for exclcolumn in PCAexcl:
      
      #if this is one of the original columns (pre-transform)
      if exclcolumn in postprocess_dict['origcolumn']:
      
        #get a column key for this column (used to access stuff in postprofcess_dict)
        exclcolumnkey = postprocess_dict['origcolumn'][exclcolumn]['columnkey']

        #get the columnslist from this columnkey
        exclcolumnslist = postprocess_dict['column_dict'][exclcolumnkey]['columnslist']

        #add these items to PCAexcl_posttransform
        PCAexcl_posttransform.extend(exclcolumnslist)
        
      #if this is a post-transformation column
      elif exclcolumn in postprocess_dict['column_dict']:
        
        #if we hadn't already done another column from the same source
        if exclcolumn not in PCAexcl_posttransform:
          
          #add these items to PCAexcl_posttransform
          PCAexcl_posttransform.extend([exclcolumn])
          
    #assemble the sets by dropping the columns excluded
    PCAset_train = df_train.drop(PCAexcl_posttransform, axis=1)
    PCAset_test = df_test.drop(PCAexcl_posttransform, axis=1)

    return PCAset_train, PCAset_test, PCAexcl_posttransform

  def PCAfunction(self, PCAset_train, PCAset_test, PCAn_components, postprocess_dict, \
                  randomseed, ML_cmnd):
    '''
    Function that takes as input the train and test sets intended for PCA
    dimensionality reduction. Returns a trained PCA model saved in postprocess_dict
    and trasnformed sets.
    '''
    
    #initialize ML_cmnd
    #ML_cmnd = postprocess_dict['ML_cmnd']
    ML_cmnd = ML_cmnd
    
    #Find PCA type
    PCActgy, n_components = \
    self.evalPCA(PCAset_train, PCAn_components, ML_cmnd)
    
    #Save the PCActgy to the postprocess_dict
    postprocess_dict.update({'PCActgy' : PCActgy})
    
    #initialize PCA defaults dictionary
    PCAdefaults = \
    self.populatePCAdefaults(randomseed)
    
    #convert PCAsets to numpy arrays
    PCAset_train = PCAset_train.values
    PCAset_test = PCAset_test.values
    
    #initialize a PCA model
    #PCAmodel = PCA(n_components = PCAn_components, random_state = randomseed)
    if PCActgy == 'default' or PCActgy == 'SparsePCA':
  
      #PCAmodel = self.initSparsePCA(ML_cmnd, PCAdefaults, PCAn_components)
      PCAmodel = self.initSparsePCA(ML_cmnd, PCAdefaults, n_components)

    if PCActgy == 'KernelPCA':
  
      #PCAmodel = self.initKernelPCA(ML_cmnd, PCAdefaults, PCAn_components)
      PCAmodel = self.initKernelPCA(ML_cmnd, PCAdefaults, n_components)
    
    if PCActgy == 'PCA':
  
      #PCAmodel = self.initPCA(ML_cmnd, PCAdefaults, PCAn_components)
      PCAmodel = self.initPCA(ML_cmnd, PCAdefaults, n_components)
    
    #derive the PCA model (note htis is unsupervised training, no labels)
    PCAmodel.fit(PCAset_train)

    #Save the trained PCA model to the postprocess_dict
    postprocess_dict.update({'PCAmodel' : PCAmodel})

    #apply the transform
    PCAset_train = PCAmodel.transform(PCAset_train)
    PCAset_test = PCAmodel.transform(PCAset_test)

    #get new number of columns
    newcolumncount = np.size(PCAset_train,1)

    #generate a list of column names for the conversion to pandas
    columnnames = ['PCAcol'+str(y) for y in range(newcolumncount)]

    #convert output to pandas
    PCAset_train = pd.DataFrame(PCAset_train, columns = columnnames)
    PCAset_test = pd.DataFrame(PCAset_test, columns = columnnames)

    return PCAset_train, PCAset_test, postprocess_dict, PCActgy
  
  def check_am_miscparameters(self, valpercent1, valpercent2, floatprecision, shuffletrain, \
                             TrainLabelFreqLevel, powertransform, binstransform, MLinfill, \
                             infilliterate, randomseed, eval_ratio, LabelSmoothing_train, LabelSmoothing_test, \
                             LabelSmoothing_val, LSfit, numbercategoryheuristic, pandasoutput, \
                             NArw_marker, featureselection, featurepct, featuremetric, \
                             featuremethod, Binary, PCAn_components, PCAexcl, printstatus, excl_suffix):
    """
    #Performs validation to confirm valid entries of passed automunge(.) parameters
    #Note that this function is intended specifically for non-dictionary parameters
    #eg assigncat, assigninfill, assignparam, transformdict, processdict validated elsewhere
    #also note that labels_column, trainID_column, testID_column are checked inside automunge function
    #also note that df_train, df_test, evalcat parameters validation methods still pending
    #returns a dictionary of results
    #False is good
    """
    
    miscparameters_results = {'suffixoverlap_results':{}}
    
    #check valpercent1
    valpercent1_valresult = False
    if isinstance(valpercent1, (int, float)) and not isinstance(valpercent1, bool):
      if valpercent1 < 0 or valpercent1 >= 1:
        valpercent1_valresult = True
        print("Error: invalid entry passed for valpercent1")
        print("Acceptable values are numbers in range 0 <= valpercent1 < 1.")
        print()
    else:
      valpercent1_valresult = True
      print("Error: invalid entry passed for valpercent1")
      print("Acceptable values are numbers in range 0 <= valpercent1 < 1.")
      print()
      
    miscparameters_results.update({'valpercent1_valresult' : valpercent1_valresult})
    
    #check valpercent2
    valpercent2_valresult = False
    if isinstance(valpercent2, (int, float)) and not isinstance(valpercent2, bool):
      if valpercent2 < 0 or valpercent2 >= 1:
        valpercent2_valresult = True
        print("Error: invalid entry passed for valpercent2")
        print("Acceptable values are numbers in range 0 <= valpercent2 < 1")
        print()
    else:
      valpercent2_valresult = True
      print("Error: invalid entry passed for valpercent2")
      print("Acceptable values are numbers in range 0 <= valpercent2 < 1")
      print()
      
    miscparameters_results.update({'valpercent2_valresult' : valpercent2_valresult})
    
    #check valpercent_sum
    valpercent_sum_valresult = False
    if (isinstance(valpercent1, (int, float)) and not isinstance(valpercent1, bool)) \
    and isinstance(valpercent2, (int, float)) and not isinstance(valpercent2, bool):
      valpercent_sum = valpercent1 + valpercent2
      if valpercent_sum >= 1.0:
        print("Error: invalid entries passed for valpercent1 &/or valpercent2.")
        print("Valid entries for these two paraemters are numbers subject to constraint")
        print("valpercent1 + valpercent2 < 1.0")
        print()
        valpercent_sum_valresult = True
        
    miscparameters_results.update({'valpercent_sum_valresult' : valpercent_sum_valresult})
    
    #check floatprecision
    floatprecision_valresult = False
    if floatprecision not in [16, 32, 64]:
      floatprecision_valresult = True
      print("Error: invalid entry passed for floatprecision parameter.")
      print("Acceptable values are one of {16, 32, 64}")
      print()
      
    miscparameters_results.update({'floatprecision_valresult' : floatprecision_valresult})
    
    #check shuffletrain
    shuffletrain_valresult = False
    if shuffletrain not in [True, False, 'traintest']:
      shuffletrain_valresult = True
      print("Error: invalid entry passed for shuffletrain parameter.")
      print("Acceptable values are one of {True, False, 'traintest'}")
      print()
    elif shuffletrain not in ['traintest'] \
    and not isinstance(shuffletrain, bool):
      shuffletrain_valresult = True
      print("Error: invalid entry passed for shuffletrain parameter.")
      print("Acceptable values are one of {True, False, 'traintest'}")
      print()
      
    miscparameters_results.update({'shuffletrain_valresult' : shuffletrain_valresult})
    
    #check TrainLabelFreqLevel
    TrainLabelFreqLevel_valresult = False
    if TrainLabelFreqLevel not in [True, False, 'test', 'traintest']:
      TrainLabelFreqLevel_valresult = True
      print("Error: invalid entry passed for TrainLabelFreqLevel parameter.")
      print("Acceptable values are one of {True, False, 'test', 'traintest'}")
      print()
    elif TrainLabelFreqLevel not in ['test', 'traintest'] and not isinstance(TrainLabelFreqLevel, bool):
      TrainLabelFreqLevel_valresult = True
      print("Error: invalid entry passed for TrainLabelFreqLevel parameter.")
      print("Acceptable values are one of {True, False, 'test', 'traintest'}")
      print()
      
    miscparameters_results.update({'TrainLabelFreqLevel_valresult' : TrainLabelFreqLevel_valresult})
    
    #check powertransform
    powertransform_valresult = False
    if powertransform not in [True, False, 'excl', 'exc2']:
      powertransform_valresult = True
      print("Error: invalid entry passed for powertransform parameter.")
      print("Acceptable values are one of {True, False, 'excl', 'exc2'}")
      print()
    elif powertransform not in ['excl', 'exc2'] \
    and not isinstance(powertransform, bool):
      powertransform_valresult = True
      print("Error: invalid entry passed for powertransform parameter.")
      print("Acceptable values are one of {True, False, 'excl', 'exc2'}")
      print()
      
    miscparameters_results.update({'powertransform_valresult' : powertransform_valresult})
    
    #check binstransform
    binstransform_valresult = False
    if binstransform not in [True, False] or not isinstance(binstransform, bool):
      binstransform_valresult = True
      print("Error: invalid entry passed for binstransform parameter.")
      print("Acceptable values are one of {True, False}")
      print()
      
    miscparameters_results.update({'binstransform_valresult' : binstransform_valresult})
    
    #check MLinfill
    MLinfill_valresult = False
    if MLinfill not in [True, False] or not isinstance(MLinfill, bool):
      MLinfill_valresult = True
      print("Error: invalid entry passed for MLinfill parameter.")
      print("Acceptable values are one of {True, False}")
      print()
      
    miscparameters_results.update({'MLinfill_valresult' : MLinfill_valresult})
    
    #check infilliterate
    infilliterate_valresult = False
    if not isinstance(infilliterate, (int)) \
    or isinstance(infilliterate, bool):
      infilliterate_valresult = True
      print("Error: invalid entry passed for infilliterate parameter.")
      print("Acceptable values are integers >= 0")
      print()
    elif infilliterate < 0:
      infilliterate_valresult = True
      print("Error: invalid entry passed for infilliterate parameter.")
      print("Acceptable values are integers >= 0")
      print()
      
    miscparameters_results.update({'infilliterate_valresult' : infilliterate_valresult})
    
    #check randomseed
    randomseed_valresult = False
    if not isinstance(randomseed, (int)) \
    or isinstance(randomseed, bool):
      randomseed_valresult = True
      print("Error: invalid entry passed for randomseed parameter.")
      print("Acceptable values are integers >= 0")
      print()
    elif randomseed < 0:
      randomseed_valresult = True
      print("Error: invalid entry passed for randomseed parameter.")
      print("Acceptable values are integers >= 0")
      print()
      
    miscparameters_results.update({'randomseed_valresult' : randomseed_valresult})
    
    #check eval_ratio
    eval_ratio_valresult = False
    if not (isinstance(eval_ratio, (int)) \
    or isinstance(eval_ratio, (float))):
      eval_ratio_valresult = True
      print("Error: invalid entry passed for eval_ratio parameter.")
      print("Acceptable values are floats 0-1 or integers >1")
      print()
    elif eval_ratio < 0:
      eval_ratio_valresult = True
      print("Error: invalid entry passed for eval_ratio parameter.")
      print("Acceptable values are floats 0-1 or integers >1")
      print()
      
    miscparameters_results.update({'eval_ratio_valresult' : eval_ratio_valresult})
    
    #check LabelSmoothing_train
    LabelSmoothing_train_valresult = False
    if not isinstance(LabelSmoothing_train, (float, bool)):
      
      LabelSmoothing_train_valresult = True
      print("Error: invalid entry passed for LabelSmoothing_train parameter.")
      print("Acceptable values are floats within range 0.0 < LabelSmoothing_train < 1.0")
      print("Or boolean value of False")
      print()
      
    elif isinstance(LabelSmoothing_train, float) \
    and (LabelSmoothing_train <= 0.0 or LabelSmoothing_train >= 1.0):
      
      LabelSmoothing_train_valresult = True
      print("Error: invalid entry passed for LabelSmoothing_train parameter.")
      print("Acceptable values are floats within range 0.0 < LabelSmoothing_train < 1.0")
      print("Or boolean value of False")
      print()
      
    elif isinstance(LabelSmoothing_train, bool) \
    and LabelSmoothing_train is not False:
      
      LabelSmoothing_train_valresult = True
      print("Error: invalid entry passed for LabelSmoothing_train parameter.")
      print("Acceptable values are floats within range 0.0 < LabelSmoothing_train < 1.0")
      print("Or boolean value of False")
      print()
      
    miscparameters_results.update({'LabelSmoothing_train_valresult' : LabelSmoothing_train_valresult})
    
    #check LabelSmoothing_test
    LabelSmoothing_test_valresult = False
    if not isinstance(LabelSmoothing_test, (float, bool)):
      
      LabelSmoothing_test_valresult = True
      print("Error: invalid entry passed for LabelSmoothing_test parameter.")
      print("Acceptable values are floats within range 0.0 < LabelSmoothing_test < 1.0")
      print("Or boolean value.")
      print()
      
    elif isinstance(LabelSmoothing_test, float) \
    and (LabelSmoothing_test <= 0.0 or LabelSmoothing_test >= 1.0):
      
      LabelSmoothing_test_valresult = True
      print("Error: invalid entry passed for LabelSmoothing_test parameter.")
      print("Acceptable values are floats within range 0.0 < LabelSmoothing_test < 1.0")
      print("Or boolean value.")
      print()
      
    miscparameters_results.update({'LabelSmoothing_test_valresult' : LabelSmoothing_test_valresult})
    
    #check LabelSmoothing_val
    LabelSmoothing_val_valresult = False
    if not isinstance(LabelSmoothing_val, (float, bool)):
      
      LabelSmoothing_val_valresult = True
      print("Error: invalid entry passed for LabelSmoothing_val parameter.")
      print("Acceptable values are floats within range 0.0 < LabelSmoothing_val < 1.0")
      print("Or boolean value.")
      print()
      
    elif isinstance(LabelSmoothing_val, float) \
    and (LabelSmoothing_val <= 0.0 or LabelSmoothing_val >= 1.0):
      
      LabelSmoothing_val_valresult = True
      print("Error: invalid entry passed for LabelSmoothing_val parameter.")
      print("Acceptable values are floats within range 0.0 < LabelSmoothing_val < 1.0")
      print("Or boolean value.")
      print()
      
    miscparameters_results.update({'LabelSmoothing_val_valresult' : LabelSmoothing_val_valresult})
    
    #check LSfit
    LSfit_valresult = False
    if LSfit not in [True, False] or not isinstance(LSfit, bool):
      LSfit_valresult = True
      print("Error: invalid entry passed for LSfit parameter.")
      print("Acceptable values are one of {True, False}")
      print()
      
    miscparameters_results.update({'LSfit_valresult' : LSfit_valresult})
    
    #check numbercategoryheuristic
    numbercategoryheuristic_valresult = False
    if not isinstance(numbercategoryheuristic, int):
      numbercategoryheuristic_valresult = True
      print("Error: invalid entry passed for numbercategoryheuristic parameter.")
      print("Acceptable values are integers >= 1")
      print()
    elif numbercategoryheuristic < 1:
      numbercategoryheuristic_valresult = True
      print("Error: invalid entry passed for numbercategoryheuristic parameter.")
      print("Acceptable values are integers >= 1")
      print()
      
    #check pandasoutput
    pandasoutput_valresult = False
    if pandasoutput not in [True, False] or not isinstance(pandasoutput, bool):
      pandasoutput_valresult = True
      print("Error: invalid entry passed for pandasoutput parameter.")
      print("Acceptable values are one of {True, False}")
      print()
      
    miscparameters_results.update({'pandasoutput_valresult' : pandasoutput_valresult})
    
    #check NArw_marker
    NArw_marker_valresult = False
    if NArw_marker not in [True, False] or not isinstance(NArw_marker, bool):
      NArw_marker_valresult = True
      print("Error: invalid entry passed for NArw_marker parameter.")
      print("Acceptable values are one of {True, False}")
      print()
      
    miscparameters_results.update({'NArw_marker_valresult' : NArw_marker_valresult})

    #check featureselection
    featureselection_valresult = False
    if featureselection not in [True, False] or not isinstance(featureselection, bool):
      featureselection_valresult = True
      print("Error: invalid entry passed for featureselection parameter.")
      print("Acceptable values are one of {True, False}")
      print()
      
    miscparameters_results.update({'featureselection_valresult' : featureselection_valresult})
    
    #check featurepct
    featurepct_valresult = False
    if not isinstance(featurepct, float):
      featurepct_valresult = True
      print("Error: invalid entry passed for featurepct parameter.")
      print("Acceptable values are floats within range 0.0 < featurepct < 1.0")
      print()
    elif (featurepct <= 0.0 or featurepct > 1.0):
      featurepct_valresult = True
      print("Error: invalid entry passed for featurepct parameter.")
      print("Acceptable values are floats within range 0.0 < featurepct <= 1.0")
      print()
      
    miscparameters_results.update({'featurepct_valresult' : featurepct_valresult})
    
    #check featuremetric
    featuremetric_valresult = False
    if not isinstance(featuremetric, float):
      featuremetric_valresult = True
      print("Error: invalid entry passed for featuremetric parameter.")
      print("Acceptable values are floats within range 0.0 <= featuremetric < 100.0")
      print()
    elif (featuremetric < 0.0 or featuremetric >= 100.0):
      featuremetric_valresult = True
      print("Error: invalid entry passed for featuremetric parameter.")
      print("Acceptable values are floats within range 0.0 <= featuremetric < 100.0")
      print()
      
    miscparameters_results.update({'featuremetric_valresult' : featuremetric_valresult})
    
    #check featuremethod
    featuremethod_valresult = False
    if featuremethod not in ['pct', 'metric', 'default', 'report']:
      featuremethod_valresult = True
      print("Error: invalid entry passed for featuremethod parameter.")
      print("Acceptable values are one of {'pct', 'metric', 'default', 'report'}")
      print()
      
    miscparameters_results.update({'featuremethod_valresult' : featuremethod_valresult})
  
    #check Binary
    Binary_valresult = False
    if Binary not in [True, False, 'retain'] and not isinstance(Binary, list):
      Binary_valresult = True
      print("Error: invalid entry passed for Binary parameter.")
      print("Acceptable values are one of {True, False, 'retain', [list]}")
      print()
    elif Binary not in ['retain'] \
    and not isinstance(Binary, bool) \
    and not isinstance(Binary, list):
      Binary_valresult = True
      print("Error: invalid entry passed for Binary parameter.")
      print("Acceptable values are one of {True, False, 'retain', [list]}")
      
    miscparameters_results.update({'Binary_valresult' : Binary_valresult})
    
    #check PCAn_components
    #accepts integers >1 or floats between 0-1 or None
    PCAn_components_valresult = False
    if isinstance(PCAn_components, (int, float)) \
    or PCAn_components is False \
    or PCAn_components == None:
      
      if PCAn_components is not False:

        if isinstance(PCAn_components, int):
          if PCAn_components < 1:
            PCAn_components_valresult = True
            print("Error: invalid entry passed for PCAn_components")
            print("Acceptable values are integers > 1, floats between 0-1, False, or None.")
            print()
          
        if isinstance(PCAn_components, float):
          if (PCAn_components > 1.0 or PCAn_components < 0.0):
            PCAn_components_valresult = True
            print("Error: invalid entry passed for PCAn_components")
            print("Acceptable values are integers > 1, floats between 0-1, False, or None.")
            print()

        if PCAn_components is True:
          PCAn_components_valresult = True
          print("Error: invalid entry passed for PCAn_components")
          print("Acceptable values are integers > 1, floats between 0-1, False, or None.")
          print()

    else:
      PCAn_components_valresult = True
      print("Error: invalid entry passed for PCAn_components")
      print("Acceptable values are integers > 1, floats between 0-1, False, or None.")
      print()
      
    miscparameters_results.update({'PCAn_components_valresult' : PCAn_components_valresult})
    
    #check PCAexcl
    #defer on this one for now, this is just a list of column headers to exclude from PCA
    #to validate owuld need to pass list of column headers to this funciton
    
    #check printstatus
    printstatus_valresult = False
    if printstatus not in [True, False] or not isinstance(printstatus, bool):
      printstatus_valresult = True
      print("Error: invalid entry passed for printstatus parameter.")
      print("Acceptable values are one of {True, False}")
      print()
      
    miscparameters_results.update({'printstatus_valresult' : printstatus_valresult})
    
    #check excl_suffix
    excl_suffix_valresult = False
    if excl_suffix not in [True, False] or not isinstance(excl_suffix, bool):
      excl_suffix_valresult = True
      print("Error: invalid entry passed for excl_suffix parameter.")
      print("Acceptable values are one of {True, False}")
      print()
      
    miscparameters_results.update({'excl_suffix_valresult' : excl_suffix_valresult})
    
    return miscparameters_results
    
  def check_pm_miscparameters(self, pandasoutput, printstatus, TrainLabelFreqLevel, \
                              featureeval, driftreport, LabelSmoothing, LSfit, \
                              returnedsets, shuffletrain, inversion, traindata):
    """
    #Performs validation to confirm valid entries of passed postmunge(.) parameters
    #note one parameter not directly passed is df_test, just pass a list of the columns
    #returns a dictionary of results
    #False is good
    """
    
    pm_miscparameters_results = {}
    
    #check pandasoutput
    pandasoutput_valresult = False
    if pandasoutput not in [True, False] or not isinstance(pandasoutput, bool):
      pandasoutput_valresult = True
      print("Error: invalid entry passed for pandasoutput parameter.")
      print("Acceptable values are one of {True, False}")
      print()
      
    pm_miscparameters_results.update({'pandasoutput_valresult' : pandasoutput_valresult})
    
    #check printstatus
    printstatus_valresult = False
    if printstatus not in [True, False] or not isinstance(printstatus, bool):
      printstatus_valresult = True
      print("Error: invalid entry passed for printstatus parameter.")
      print("Acceptable values are one of {True, False}")
      print()
      
    pm_miscparameters_results.update({'printstatus_valresult' : printstatus_valresult})
    
    #check inversion
    inversion_valresult = False
    if inversion not in [False, 'test', 'labels'] and not isinstance(inversion, list):
      inversion_valresult = True
      print("Error: invalid entry passed for inversion parameter.")
      print("Acceptable values are one of {False, 'test', 'labels', or a list of columns}")
      print()
    elif inversion not in ['test', 'labels'] and not isinstance(inversion, list) \
    and not isinstance(inversion, bool):
      inversion_valresult = True
      print("Error: invalid entry passed for inversion parameter.")
      print("Acceptable values are one of {False, 'test', 'labels', or a list of columns}")
      print()
      
    pm_miscparameters_results.update({'inversion_valresult' : inversion_valresult})
    
    #check TrainLabelFreqLevel
    TrainLabelFreqLevel_valresult = False
    if TrainLabelFreqLevel not in [True, False] or not isinstance(TrainLabelFreqLevel, bool):
      TrainLabelFreqLevel_valresult = True
      print("Error: invalid entry passed for TrainLabelFreqLevel parameter.")
      print("Acceptable values are one of {True, False}")
      print()
      
    pm_miscparameters_results.update({'TrainLabelFreqLevel_valresult' : TrainLabelFreqLevel_valresult})
    
    #check featureeval
    featureeval_valresult = False
    if featureeval not in [True, False] or not isinstance(featureeval, bool):
      featureeval_valresult = True
      print("Error: invalid entry passed for featureeval parameter.")
      print("Acceptable values are one of {True, False}")
      print()
      
    pm_miscparameters_results.update({'featureeval_valresult' : featureeval_valresult})
    
    #check driftreport
    driftreport_valresult = False
    if driftreport not in [True, False, 'efficient', 'report_effic', 'report_full']:
      driftreport_valresult = True
      print("Error: invalid entry passed for driftreport parameter.")
      print("Acceptable values are one of {True, False, 'efficient', 'report_effic', 'report_full'}")
      print()
    elif driftreport not in ['efficient', 'report_effic', 'report_full'] \
    and not isinstance(driftreport, bool):
      driftreport_valresult = True
      print("Error: invalid entry passed for driftreport parameter.")
      print("Acceptable values are one of {True, False, 'efficient', 'report_effic', 'report_full'}")
      print()
      
    pm_miscparameters_results.update({'driftreport_valresult' : driftreport_valresult})
    
    #check LabelSmoothing
    LabelSmoothing_valresult = False
    if not isinstance(LabelSmoothing, (float, bool)):
      
      LabelSmoothing_valresult = True
      print("Error: invalid entry passed for LabelSmoothing parameter.")
      print("Acceptable values are floats within range 0.0 < LabelSmoothing < 1.0")
      print("Or boolean value.")
      print()
      
    elif isinstance(LabelSmoothing, float) \
    and (LabelSmoothing <= 0.0 or LabelSmoothing >= 1.0):
      
      LabelSmoothing_valresult = True
      print("Error: invalid entry passed for LabelSmoothing parameter.")
      print("Acceptable values are floats within range 0.0 < LabelSmoothing < 1.0")
      print("Or boolean value.")
      print()
      
    pm_miscparameters_results.update({'LabelSmoothing_valresult' : LabelSmoothing_valresult})
    
    #check LSfit
    LSfit_valresult = False
    if LSfit not in [True, False] or not isinstance(LSfit, bool):
      LSfit_valresult = True
      print("Error: invalid entry passed for LSfit parameter.")
      print("Acceptable values are one of {True, False}")
      print()
      
    pm_miscparameters_results.update({'LSfit_valresult' : LSfit_valresult})
    
    #check returnedsets
    returnedsets_valresult = False
    if returnedsets not in [True, False, 'test_ID', 'test_labels', 'test_ID_labels']:
      returnedsets_valresult = True
      print("Error: invalid entry passed for returnedsets parameter.")
      print("Acceptable values are one of {True, False, 'test_ID', 'test_labels', 'test_ID_labels'}")
      print()
    elif returnedsets not in ['test_ID', 'test_labels', 'test_ID_labels'] \
    and not isinstance(returnedsets, bool):
      returnedsets_valresult = True
      print("Error: invalid entry passed for returnedsets parameter.")
      print("Acceptable values are one of {True, False, 'test_ID', 'test_labels', 'test_ID_labels'}")
      print()
      
    pm_miscparameters_results.update({'returnedsets_valresult' : returnedsets_valresult})
    
    #check shuffletrain
    shuffletrain_valresult = False
    if shuffletrain not in [True, False] or not isinstance(shuffletrain, bool):
      shuffletrain_valresult = True
      print("Error: invalid entry passed for shuffletrain parameter.")
      print("Acceptable values are one of {True, False}")
      print()
      
    pm_miscparameters_results.update({'shuffletrain_valresult' : shuffletrain_valresult})

    #check traindata
    traindata_valresult = False
    if traindata not in [True, False] or not isinstance(pandasoutput, bool):
      traindata_valresult = True
      print("Error: invalid entry passed for traindata parameter.")
      print("Acceptable values are one of {True, False}")
      print()
      
    pm_miscparameters_results.update({'traindata_valresult' : traindata_valresult})
    
    return pm_miscparameters_results

  def check_FSmodel(self, featureselection, FSmodel):
    """
    If feature importance applied confirms that a model was successfully trained
    """
    
    check_FSmodel_result = False
    
    if featureselection is True:
      if FSmodel is False:
        check_FSmodel_result = True
        
        print("error: Feature importance model was not successfully trained")
        print()
        
    return check_FSmodel_result

  def check_np_shape(self, df_train, df_test):
    """
    Validates any passed numpy arrays are tabular (1D or 2D)
    """
    
    check_np_shape_train_result = False
    check_np_shape_test_result = False
    
    checknp = np.array([])
    
    if isinstance(checknp, type(df_train)):
      if len(df_train.shape) > 2:
        check_np_shape_train_result = True
        print("error: numpy array passed to df_train is not tabular (>2D dimensions)")
        print()
        
    if isinstance(checknp, type(df_test)):
      if len(df_test.shape) > 2:
        check_np_shape_test_result = True
        print("error: numpy array passed to df_test is not tabular (>2D dimensions)")
        print()
        
    return check_np_shape_train_result, check_np_shape_test_result

  def check_ML_infill(self, df_train, column, postprocess_dict, infill_validations = {}):
    """
    #Perform validations that train set is suitable for MLinfill
    #For example ML infill requires >1 source columns in df_train
    """
    
    columnslist = postprocess_dict['column_dict'][column]['columnslist']
    
    if 'MLinfill_validations' not in infill_validations:
      infill_validations.update({'MLinfill_validations':{}})
    
      if len(columnslist) == len(list(df_train)):
        
        print("Error: ML infill requires > 1 source features in df_train")
        print()
        
        infill_validations.update({'MLinfill_validations': True})
          
      else:
        
        infill_validations.update({'MLinfill_validations': False})
      
    return infill_validations

  def check_assigncat(self, assigncat):
    """
    #Here we'll do a quick check for any redundant column assignments in the
    #assigncat, if any found return an error message
    """

    assigncat_redundant_dict = {}
    result = False

    for assigncatkey1 in sorted(assigncat):
      #assigncat_list.append(set(assigncat[key]))
      current_set = set(assigncat[assigncatkey1])
      redundant_items = {}
      for assigncatkey2 in assigncat:
        if assigncatkey2 != assigncatkey1:
          second_set = set(assigncat[assigncatkey2])
          common = current_set & second_set
          if len(common) > 0:
            for common_item in common:
              if common_item not in assigncat_redundant_dict:
                assigncat_redundant_dict.update({common_item:[assigncatkey1, assigncatkey2]})
              else:
                if assigncatkey1 not in assigncat_redundant_dict[common_item]:
                  assigncat_redundant_dict[common_item].append(assigncatkey1)
                  #assigncat_redundant_dict[common_item] += key1
                if assigncatkey2 not in assigncat_redundant_dict[common_item]:
                  assigncat_redundant_dict[common_item].append(assigncatkey2)
                  #assigncat_redundant_dict[common_item] += key2

    #assigncat_redundant_dict

    if len(assigncat_redundant_dict) > 0:
      result = True
      print("Error, the following columns assigned to multiple root categories in assigncat:")
      for assigncatkey3 in sorted(assigncat_redundant_dict):
        print("")
        print("Column: ", assigncatkey3)
        print("Found in following assigncat entries:")
        print(assigncat_redundant_dict[assigncatkey3])
        print("")

    return result
  
  def check_assigncat2(self, assigncat, transform_dict):
    """
    #Here we'll do a quick check to ensure all of the keys of passed assigncat
    #have corresponding entries in transform_dict, (which may include user
    #passed entries in transformdict parameter)
    
    #Note that in many cases a root category may be passed as a family tree primitive 
    #entry to it's own family tree set. The root category is used to access the family tree,
    #and the family tree primitive entries are used to access the transform functions and etc.
    """
    
    #False is good
    result = False
    
    for assigncat_key in assigncat:
      
      #eval is a special case, it triggers the application of evalcategory
      #which may be neccesary when automated inference turned off with powertransform
      #so it doesn't need a process_dit entry
      if assigncat_key not in transform_dict and assigncat_key not in ['eval', 'ptfm']:
        
        result = True
        
        print("Error, the following entry to user passed assigncat was not found")
        print("to have a corresponding entry in transform_dict.")
        print("")
        print("assigncat key missing transform_dict entry: ", assigncat_key)
        print("")

    return result
  
  def check_assigncat3(self, assigncat, process_dict, transform_dict):
    """
    #Here's we'll do a third check on assigncat
    #to ensure that for any listed root categories, 
    #any category entries to corresponding family tree primitives in transform_dict 
    #have a corresponding entry in the process_dict
    #note that transformdict entries not required for root categories, 
    #unless they are also entries to a family tree
    """
    
    #False is good
    result = False
    
    for assigncat_key in assigncat:
      
      if assigncat_key in transform_dict:
        
        familytree_entries = []
        
        for familytree_key in transform_dict[assigncat_key]:
          
          familytree_entries += transform_dict[assigncat_key][familytree_key]
          
        for familytree_entry in familytree_entries:
          
          if familytree_entry != None:

            if familytree_entry not in process_dict:

              print("Error, the following category was found as an entry")
              print("in a family tree without a corresponding entry ")
              print("in the process_dict.")
              print("")
              print("family tree entry missing process_dict entry: ", familytree_entry)
              print("this entry was passed in the family tree of root category: ", assigncat_key)

              result = True
        
    return result

  def check_assigninfill(self, assigninfill):
    """
    #Here we'll do a quick check for any redundant column assignments in the
    #assigninfill, if any found return an error message
    """

    assigninfill_redundant_dict = {}
    result = False

    for assigninfill_key1 in sorted(assigninfill):
      #assigncat_list.append(set(assigncat[key]))
      current_set = set(assigninfill[assigninfill_key1])
      redundant_items = {}
      for assigninfill_key2 in assigninfill:
        if assigninfill_key2 != assigninfill_key1:
          second_set = set(assigninfill[assigninfill_key2])
          common = current_set & second_set
          if len(common) > 0:
            for common_item in common:
              if common_item not in assigninfill_redundant_dict:
                assigninfill_redundant_dict.update({common_item:[assigninfill_key1, assigninfill_key2]})
              else:
                if assigninfill_key1 not in assigninfill_redundant_dict[common_item]:
                  assigninfill_redundant_dict[common_item].append(assigninfill_key1)
                  #assigncat_redundant_dict[common_item] += key1
                if assigninfill_key2 not in assigninfill_redundant_dict[common_item]:
                  assigninfill_redundant_dict[common_item].append(assigninfill_key2)
                  #assigncat_redundant_dict[common_item] += key2

    #assigncat_redundant_dict

    if len(assigninfill_redundant_dict) > 0:
      result = True
      print("Error, the following columns assigned to multiple root categories in assigninfill:")
      for assigninfill_key3 in sorted(assigninfill_redundant_dict):
        print("")
        print("Column: ", assigninfill_key3)
        print("Found in following assigninfill entries:")
        print(assigninfill_redundant_dict[assigninfill_key3])
        print("")
    
    return result

  def check_transformdict(self, transformdict):
    """
    #Here we'll do a quick check for any entries in the user passed
    #transformdict which don't have at least one replacement column specified
    #and if not found apply a cousins excl transform
    
    #we'll also do a test for excl tranfsorms as replacement primitives
    #and if found move to a corresponding supplement primitive
    """
    
    result1 = False
    result2 = False
    
    for transformkey in sorted(transformdict):
      replacements = len(transformdict[transformkey]['parents']) \
                     + len(transformdict[transformkey]['auntsuncles'])

      if replacements == 0:
        
        transformdict[transformkey]['auntsuncles'].append('excl')

        result1 = True
          
      #this ensures 'excl' is final transform in the auntsuncles list if included
      #such as to prioritize applying inplace on excl
      transformdict[transformkey]['auntsuncles'].append('excl')
      transformdict[transformkey]['auntsuncles'].remove('excl')

    return result1, result2, transformdict
  
  def check_transformdict2(self, transformdict):
    """
    #Here we'll do an additional check on transformdict to ensure
    #no redundant specifications in adjacent primitives
    """
    
    result1 = False
    result2 = False
    
    upstream_entries = []
    downstream_entries = []
    
    for transformkey in sorted(transformdict):
      
      for primitive in transformdict[transformkey]:
        
        if primitive in ['parents', 'siblings', 'auntsuncles', 'cousins']:  
          
          for entry in transformdict[transformkey][primitive]:
            
            if entry in upstream_entries:
              
              result1 = True
              
              print("error warning: ")
              print("redundant entries found in the upstream primitives ")
              print("for user-passed transformdict key: ", transformkey)
              
            else:
          
              upstream_entries += [entry]
          
        if primitive in ['children', 'niecesnephews', 'coworkers', 'friends']:  
          
          for entry in transformdict[transformkey][primitive]:
            
            if entry in downstream_entries:
              
              result2 = True
              
              print("error warning: ")
              print("redundant entries found in the downstream primitives ")
              print("for user-passed transformdict key: ", transformkey)
              
            else:
          
              downstream_entries += [entry]

      upstream_entries = []
      downstream_entries = []

    return result1, result2

  def check_transform_dict_roots(self, transform_dict, process_dict):
    """
    #validates that transform_dict root categories after consolidation
    #have corresponding entries in process_dict after consolidation
    """
    
    check_transform_dict_roots_result = False
    
    for entry in transform_dict:
      
      if entry not in process_dict:
        
        check_transform_dict_roots_result = True
        
        print("error: a root category was found in transformdict")
        print("without a corresponding entry in processdict")
        print("for transformdict root category: ", entry)
        print()
        
    return check_transform_dict_roots_result
  
  def check_haltingproblem(self, transformdict, transform_dict, max_check_count = 111):
    """
    #evaluates user passed transformdict entries to check for infinite loops
    #we'll arbitrarily check for a max depth of 111 offspring to keep things manageable
    #we'll assume this takes place after user passed entries have been merged into transform_dict
    #such that transformdict is just user passed entries
    #and transform_dict is both user-passed and internal library
    """
    
    haltingproblem_result = False
    
    for root_category in transformdict:
      
      check_count = 0
      
      offspring_list = []
      
      parents_list = \
      transform_dict[root_category]['parents'] + transform_dict[root_category]['siblings']
      
      for parent in parents_list:
        
        upstream_list = []
        upstream_list = [parent]
        
        offspring_list = \
        transform_dict[parent]['children'] + transform_dict[parent]['niecesnephews']
      
        if len(offspring_list) > 0:

          for offspring in offspring_list:

            check_count += 1

            if offspring in upstream_list:

              haltingproblem_result = True

              print("Error, infinite loop detected in transformdict for root category ", root_category)
              print()

              break

            upstream_list += [offspring]

            if check_count < max_check_count:

              offspring_result, check_count = \
              self.check_offspring(transform_dict, offspring, root_category, \
                                   upstream_list, check_count, max_check_count)
#               offspring_result, check_count = \
#               check_offspring(transform_dict, offspring, root_category, \
#                               upstream_list, check_count, max_check_count)

              if offspring_result is True:

                haltingproblem_result = True

                break

            else:

              print("Number of offspring generations for root category ", root_category)
              print("exceeded 111, infinite loop check halted.")
              print()
              
              break
    
    return haltingproblem_result

  def check_offspring(self, transform_dict, root_category, orig_root_category, \
                      upstream_list, check_count, max_check_count):
    """
    #support function for check_haltingproblem
    """
    
    offspring_result = False

    offspring_list = \
    transform_dict[root_category]['children'] + transform_dict[root_category]['niecesnephews']

    if len(offspring_list) > 0:

      for offspring in offspring_list:
        
        check_count += 1

        if offspring in upstream_list:

          offspring_result = True

          print("Error, infinite loop detected in transformdict for root category ", orig_root_category)
          print()

          break

        else:
            
          upstream_list2 = upstream_list.copy() + [offspring]

          if check_count < max_check_count:

            offspring_result2, check_count = \
            self.check_offspring(transform_dict, offspring, orig_root_category, \
                                 upstream_list, check_count, max_check_count)
#             offspring_result2, check_count = \
#             check_offspring(transform_dict, offspring, orig_root_category, \
#                             upstream_list2, check_count, max_check_count)

            if offspring_result2 is True:

              offspring_result = True

              break

          else:

            print("Number of offspring generations for root category ", root_category)
            print("exceeded 111, infinite loop check halted.")
            print()
            
            break
    
    return offspring_result, check_count

  def check_assignnan(self, assignnan, transform_dict, df_train_list):
    """
    #validates automunge parameter assignnan
    #which accepts form:
    #assignnan = {'categories':{'cat1':[], 'cat2':[]}, 'columns':{'col1':[], 'col2':[]}, 'global':[]}
    
    #where 'cat1' / 'cat2' are examples of root categories
    #and 'col1' / 'col2' are examples of recieved source columns
    #and the lists contain entries for those source columns or root categories
    #which are to be converted to nan for purposes of infill
    
    #this function confirms only entries for 'cat' or 'col' in first tier
    #and that entries within cat are valid root categories from transform_dict
    #and that entries within col are valid column headers from df_train
    """
    
    check_assignnan_toplevelentries_result = False
    check_assignnan_categories_result = False
    check_assignnan_columns_result = False
    
    for entry1 in assignnan:
      
      if entry1 not in ['categories', 'columns', 'global', 'injections']:
        
        check_assignnan_toplevelentries_result = True
        print("error: assignnan parameter valid entries for first tier are 'categories', 'columns', 'global', and 'injections'")
        print()
        
    if 'categories' in assignnan:
      
      for entry2 in assignnan['categories']:        
        
        if entry2 not in transform_dict:
          
          check_assignnan_categories_result = True
          print("error: assignnan parameter valid entries under 'categories' must be root categories defined in transform_dict")
          print()
          
    if 'columns' in assignnan:
      
      for entry2 in assignnan['columns']:        
        
        if entry2 not in df_train_list:
          
          check_assignnan_columns_result = True
          print("error: assignnan parameter valid entries under 'columns' must be source columns from passed df_train")
          print()

    if 'injections' in assignnan:
      
      for entry3 in assignnan['injections']:        
        
        if entry3 not in df_train_list:
          
          check_assignnan_columns_result = True
          print("error: assignnan parameter valid entries under 'injections' must be source columns from passed df_train")
          print()

    return check_assignnan_toplevelentries_result, check_assignnan_categories_result, check_assignnan_columns_result

  def check_ML_cmnd(self, ML_cmnd):
    """
    #Here we'll do a quick check for any entries in the user passed
    #ML_cmnd and add any missing entries with default values
    #a future extension should validate any entries
    """
    
    result = False
    
    if 'MLinfill_type' not in ML_cmnd:
      ML_cmnd.update({'MLinfill_type':'default'})
    if 'MLinfill_cmnd' not in ML_cmnd:
      ML_cmnd.update({'MLinfill_cmnd':{'RandomForestClassifier':{}, 'RandomForestRegressor':{}}})
    if 'PCA_type' not in ML_cmnd:
      ML_cmnd.update({'PCA_type':'default'})
    if 'PCA_cmnd' not in ML_cmnd:
      ML_cmnd.update({'PCA_cmnd':{}})

    return result
  
  def check_assignparam(self, assignparam, process_dict):
    """
    #Here we'll do a quick check to validate the passed assign param.
    
    #reminder the format of assign_param is e.g.
    #assignparam = {'splt' : {'column1' : {'minsplit' : 4}}, \
    #               'spl2' : {'column2' : {'minsplit' : 3}}}
    
    #we'll validate that any category entries are valid
    #we won't validate column entries since they may be dervied columns which we don't know yet
    """
    
    result = False
    
    for key in assignparam:
      
      if key not in ['global_assignparam', 'default_assignparam', '(category)'] \
      and key not in process_dict:
        
        result = True
        print("error, assignparam category key ", key)
        print("was not found in process_dict")
        
      elif key == 'default_assignparam':
        
        for key2 in assignparam['default_assignparam']:
          
          if key2 != '(category)' and key2 not in process_dict:
            
            result = True
            print("error, assignparam['default_assignparam'] category key ", key2)
            print("was not found in process_dict")
    
    return result
  
  def check_normalization_dict(self, postprocess_dict):
    """
    #Double checks that any postprocess_dict['column_dict'] entries for
    #normalization_dict don't have overlaps for those entries where
    #postprocess functions use normalization_dict keys to 
    #retrieve a column key

    #in other words, for set of required unique normalizaiton_dict entry identifiers
    #ensures they are only used in correct transformation categories
    """

    result = False

    required_unique_normalization_dict_entries = \
    {'textlabelsdict_text'  : 'text', \
     'textlabelsdict_onht'  : 'onht', \
     'powerlabelsdict_pwrs' : 'pwrs', \
     'splt_newcolumns_splt' : 'splt', \
     'splt_newcolumns_sp19' : 'sp19', \
     'splt_newcolumns_sbst' : 'sbst', \
     'splt_newcolumns_sbs3' : 'sbs3', \
     'bn_width_bnwd'        : 'bnwd', \
     'bincount_bnep'        : 'bnep', \
     'buckets_bkt1'         : 'bkt1', \
     'buckets_bkt2'         : 'bkt2'}

    for column_dict_entry in postprocess_dict['column_dict']:

      if column_dict_entry in postprocess_dict['column_dict'][column_dict_entry]['normalization_dict']:

        for normalization_dict_entry in postprocess_dict['column_dict'][column_dict_entry]['normalization_dict'][column_dict_entry]:

          if normalization_dict_entry in required_unique_normalization_dict_entries:

            if required_unique_normalization_dict_entries[normalization_dict_entry] != \
            postprocess_dict['column_dict'][column_dict_entry]['category']:

              result = True

              print("Warning of potential error for multi-generation family trees ")
              print("from overlap in normalization_dict entry identifiers")
              print("In a few cases for transforms with multicolumn derived sets")
              print("The postprocess functions rely on uniquity of this item to dervie a key to access parameters")
              print("potential overlap found for category ", postprocess_dict['column_dict'][column_dict_entry]['category'])
              print("In regards to normalization_dict entry ", normalization_dict_entry)
              print("Which overlaps with the category ", required_unique_normalization_dict_entries[normalization_dict_entry])
              print("Overlap found for column ", column_dict_entry)

    return result
  
  def check_columnheaders(self, columnheaders_list):
    """
    #Performs a validation that all of the column headers are unique
    """
    
    result = False
    
    if len(columnheaders_list) > len(set(columnheaders_list)):
      
      result = True
      
      print("Warning of potential error from duplicate column headers.")
      print("")
      
    return result

  def check_processdict(self, processdict):
    """
    #runs validations on user passed processdict
    #assumes any conversion from functionpointer already taken place
    
    #checks that NArowtype and MLinfilltype have valid entries
    #checks that labelctgy is present
    #checks that dualprocess postprocess and singleprocess have entries
    """
    
    check_processdict_result = False

    if 'global_assignparam' in processdict:
      check_processdict_result = True
      print("error: processdict has entry for 'global_assignparam'")
      print("which is a reserved category string for use in assignparam")

    if 'default_assignparam' in processdict:
      check_processdict_result = True
      print("error: processdict has entry for 'default_assignparam'")
      print("which is a reserved category string for use in assignparam")
    
    for entry in processdict:
      
      if 'NArowtype' not in processdict[entry]:
        check_processdict_result = True
        print("error: processdict missing 'NArowtype' entry for category: ", entry)
        print()
      else:
        if processdict[entry]['NArowtype'] not in \
        ['numeric', 'integer', 'justNaN', 'exclude', 'positivenumeric', 'nonnegativenumeric', \
        'nonzeronumeric', 'parsenumeric', 'parsenumeric_commas', 'parsenumeric_EU', 'datetime']:
          check_processdict_result = True
          print("error: invalid 'NArowtype' processdict entry for category: ", entry)
          print()
        
      if 'MLinfilltype' not in processdict[entry]:
        check_processdict_result = True
        print("error: processdict missing 'MLinfilltype' entry for category: ", entry)
        print()
      else:
        if processdict[entry]['MLinfilltype'] not in \
        ['numeric', 'singlct', 'binary', 'multirt', 'concurrent_act', 'concurrent_nmbr', '1010', \
        'exclude', 'boolexclude', 'totalexclude']:
          check_processdict_result = True
          print("error: invalid 'MLinfilltype' processdict entry for category: ", entry)
          print()
        
      if 'labelctgy' not in processdict[entry]:
        check_processdict_result = True
        print("error: processdict missing 'labelctgy' entry for category: ", entry)
        print()
      # else:
      #   #this isn't a full validation, just checking that labelctgy is a valid entry in processdict
      #   if processdict[entry]['labelctgy'] not in processdict:
      #     check_processdict_result = True
      #     print("error: invalid 'labelctgy' processdict entry for category: ", entry)
      #     print()
      
      #we'll have convention that at least one entry for processing funtions required
      #even thought there is scenario where no corresponding function populated
      #such as when processdict is for a root category not used as a transformation category
      if ('singleprocess' not in processdict[entry]) and ('dualprocess' not in processdict[entry] or 'postprocess' not in processdict[entry]):
        check_processdict_result = True
        print("error: processdict entry missing processing function entrys for categery: ", entry)
        print("requires entries for (both 'dualprocess' and 'postprocess') or (entry for 'singleprocess')")
        print("(alternately a valid 'functionpointer' entry can be included)")
        print()
      else:
        pass
        #for now won't validate the transformation function entries
      
    return check_processdict_result

  def grab_processdict_functions_support(self, targetcategory, pointercategory, processdict, process_dict, \
                                         i, check_functionpointer_result):
    """
    #support function for grab_processdict_functions
    #takes as input the targetcategory that has a pointer entry
    #and the associated pointercategory
    #where if pointercategory itself has a pointer, calls this function recursively
    #to find the associated functions
    #and assign them to targetcategory entry in processdict
    #where first checks processdict and if not present checks process_dict
    #where processdcit is user passed data strcuture
    #and process_dcit is internal library prior to consolidation
    #we'll have convention that only processdict entries can have functionpointers, not proces_dict entries
    #ie only externally defined processdict can have functionpointers
    #tracks a counter i to ensure don't get caught in infinite loop, defaults to 111 cycles

    #where targetcategory is the processdict category entry that has a functionpointer entry
    #pointercategory is the corresponding functionpointer entry
    #and for chains of functionpointer entries the targetcategory remains same and pointercategory is updated
    """
    
    #counter i is here to ensure if we're recursively following chains of pointers we don't get caught in loop
    if i > 1111:
      
      print("error: functionpointer cycled through 1111 entries without finding a stopping point")
      print("for processdict category entry: ", targetcategory)
      print("likely infinite loop")
      
      check_functionpointer_result = True
      
    else:
      
      i+=1
      
      if pointercategory in processdict:
        
        #if function poitner points to a category that itself has a functionpointer
        if 'functionpointer' in processdict[pointercategory]:
          
          if 'inverseprocess' in processdict[pointercategory] \
          and 'inverseprocess' not in processdict[targetcategory]:
            processdict[targetcategory]['inverseprocess'] = processdict[pointercategory]['inverseprocess']

          if 'info_retention' in processdict[pointercategory] \
          and 'info_retention' not in processdict[targetcategory]:
            processdict[targetcategory]['info_retention'] = processdict[pointercategory]['info_retention']
          
          #for chains of functionpointers, we'll still update defaultparams for each link
          if 'defaultparams' in processdict[pointercategory]:
            if 'defaultparams' in processdict[targetcategory]:
              defaultparams = deepcopy(processdict[pointercategory]['defaultparams'])
              defaultparams.update(processdict[targetcategory]['defaultparams'])
              processdict[targetcategory]['defaultparams'] = defaultparams
            else:
              processdict[targetcategory]['defaultparams'] = processdict[pointercategory]['defaultparams']
              
          if 'inplace_option' in processdict[pointercategory] \
          and 'inplace_option' not in processdict[targetcategory]:
            processdict[targetcategory]['inplace_option'] = processdict[pointercategory]['inplace_option']

          if 'NArowtype' in processdict[pointercategory] \
          and 'NArowtype' not in processdict[targetcategory]:
            processdict[targetcategory]['NArowtype'] = processdict[pointercategory]['NArowtype']

          if 'MLinfilltype' in processdict[pointercategory] \
          and 'MLinfilltype' not in processdict[targetcategory]:
            processdict[targetcategory]['MLinfilltype'] = processdict[pointercategory]['MLinfilltype']

          if 'labelctgy' in processdict[pointercategory] \
          and 'labelctgy' not in processdict[targetcategory]:
            processdict[targetcategory]['labelctgy'] = processdict[pointercategory]['labelctgy']
          
          #now new pointer category is the functionpointer entry of the prior functionpointer entry
          pointercategory = processdict[pointercategory]['functionpointer']
          
          #follow through recursion
          processdict, i, check_functionpointer_result = \
          self.grab_processdict_functions_support(targetcategory, pointercategory, processdict, process_dict, \
                                                  i, check_functionpointer_result)
            
        else:
          
          #function pointers have to point to a category with either a funcitonpointer or processing function entries
          if ('singleprocess' not in processdict[pointercategory]) and \
              ('dualprocess' not in processdict[pointercategory] or 'postprocess' not in processdict[pointercategory]):

            check_functionpointer_result = True
            print("error: processdict entry found without functionpointer or (dualprocess / postprocess) or singleprocess")
            print("for processdict entry ", pointercategory)
            print()

          #so if processing function entries were present, we can grab them and pass to targetcategory
          else:
            
            if 'dualprocess' in processdict[pointercategory]:
              processdict[targetcategory]['dualprocess'] = processdict[pointercategory]['dualprocess']
            if 'singleprocess' in processdict[pointercategory]:
              processdict[targetcategory]['singleprocess'] = processdict[pointercategory]['singleprocess']
            if 'postprocess' in processdict[pointercategory]['postprocess']:
              processdict[targetcategory]['postprocess'] = processdict[pointercategory]['postprocess']
            
            if 'inverseprocess' in processdict[pointercategory] \
            and 'inverseprocess' not in processdict[targetcategory]:
              processdict[targetcategory]['inverseprocess'] = processdict[pointercategory]['inverseprocess']
              
            if 'info_retention' in processdict[pointercategory] \
            and 'info_retention' not in processdict[targetcategory]:
              processdict[targetcategory]['info_retention'] = processdict[pointercategory]['info_retention']
              
            if 'defaultparams' in processdict[pointercategory]:
              if 'defaultparams' in processdict[targetcategory]:
                defaultparams = deepcopy(processdict[pointercategory]['defaultparams'])
                defaultparams.update(processdict[targetcategory]['defaultparams'])
                processdict[targetcategory]['defaultparams'] = defaultparams
              else:
                processdict[targetcategory]['defaultparams'] = processdict[pointercategory]['defaultparams']
                
            if 'inplace_option' in processdict[pointercategory] \
            and 'inplace_option' not in processdict[targetcategory]:
              processdict[targetcategory]['inplace_option'] = processdict[pointercategory]['inplace_option']
              
            if 'NArowtype' in processdict[pointercategory] \
            and 'NArowtype' not in processdict[targetcategory]:
              processdict[targetcategory]['NArowtype'] = processdict[pointercategory]['NArowtype']

            if 'MLinfilltype' in processdict[pointercategory] \
            and 'MLinfilltype' not in processdict[targetcategory]:
              processdict[targetcategory]['MLinfilltype'] = processdict[pointercategory]['MLinfilltype']

            if 'labelctgy' in processdict[pointercategory] \
            and 'labelctgy' not in processdict[targetcategory]:
              processdict[targetcategory]['labelctgy'] = processdict[pointercategory]['labelctgy']
      
      #if pointercategory wasn't in user passed processdict, we'll next check the internal library process_dict
      elif pointercategory in process_dict:

        #we'll have convention that only processdict entries can have functionpointers, not proces_dict entries
        #so we don't need as many steps as above, can just assume processing functions are present
        if 'dualprocess' in process_dict[pointercategory]:
          processdict[targetcategory]['dualprocess'] = process_dict[pointercategory]['dualprocess']
        if 'singleprocess' in process_dict[pointercategory]:
          processdict[targetcategory]['singleprocess'] = process_dict[pointercategory]['singleprocess']
        if 'postprocess' in process_dict[pointercategory]:
          processdict[targetcategory]['postprocess'] = process_dict[pointercategory]['postprocess']
        
        if 'inverseprocess' in process_dict[pointercategory] \
        and 'inverseprocess' not in processdict[targetcategory]:
          processdict[targetcategory]['inverseprocess'] = process_dict[pointercategory]['inverseprocess']
        
        if 'info_retention' in process_dict[pointercategory] \
        and 'info_retention' not in processdict[targetcategory]:
          processdict[targetcategory]['info_retention'] = process_dict[pointercategory]['info_retention']
          
        if 'defaultparams' in process_dict[pointercategory]:
          if 'defaultparams' in processdict[targetcategory]:
            defaultparams = deepcopy(process_dict[pointercategory]['defaultparams'])
            defaultparams.update(processdict[targetcategory]['defaultparams'])
            processdict[targetcategory]['defaultparams'] = defaultparams
          else:
            processdict[targetcategory]['defaultparams'] = process_dict[pointercategory]['defaultparams']
            
        if 'inplace_option' in process_dict[pointercategory] \
        and 'inplace_option' not in processdict[targetcategory]:
          processdict[targetcategory]['inplace_option'] = process_dict[pointercategory]['inplace_option']
          
        if 'NArowtype' in process_dict[pointercategory] \
        and 'NArowtype' not in processdict[targetcategory]:
          processdict[targetcategory]['NArowtype'] = process_dict[pointercategory]['NArowtype']
          
        if 'MLinfilltype' in process_dict[pointercategory] \
        and 'MLinfilltype' not in processdict[targetcategory]:
          processdict[targetcategory]['MLinfilltype'] = process_dict[pointercategory]['MLinfilltype']
          
        if 'labelctgy' in process_dict[pointercategory] \
        and 'labelctgy' not in processdict[targetcategory]:
          processdict[targetcategory]['labelctgy'] = process_dict[pointercategory]['labelctgy']

      #if pointercategory wasn't found in either of processdict or process_dict
      else:
        
        check_functionpointer_result = True
        
        print("error: user passed processdict entry for category ", targetcategory)
        print("contained a functionpointer that did not point to a category with function definitions")
        print()

    return processdict, i, check_functionpointer_result
  
  def grab_processdict_functions(self, processdict, process_dict):
    """
    #checks for functionpointer entries in user passed processdict
    #when present populates that category with associated functions
    #where processdict is user passed data structure
    #and process_dict is internal library prior to consolidation
    #and functionpointer refers to an option to populate processdict
    #with an entry that grabs processing functions from another prcoess_dict entry
    #i.e. singleprocess, dualprocess, postprocess, 
    #and if present inverseprocess, info_retention
    #functionpointer first checks the processdict, and if pointer not present checks the process_dict
    #note that functionpointer also grabs defaultparams
    #although if any defaultparam entries in the set with functionpointer they will override corresponding entries
    #for example if category with pointer has defaultparam entires for suffix
    #which points to anotehr category with differnt defaultparam entries for suffix
    #the entry in the set with the pointer takes precedence
    #note that in cases where processing functions already present in a set with pointer, they will be overwritten
    #we'll have convention that only processdict entries can have functionpointers, not proces_dict entries
    """
    
    check_functionpointer_result = False
    
    for entry in processdict:
      
      if 'functionpointer' in processdict[entry]:
        
        i = 0
        targetcategory = entry
        pointercategory = processdict[entry]['functionpointer']
        
        processdict, i, check_functionpointer_result = \
        self.grab_processdict_functions_support(targetcategory, pointercategory, processdict, process_dict, \
                                               i, check_functionpointer_result)

    return processdict, check_functionpointer_result
  
  def assigncat_str_convert(self, assigncat):
    """
    #Converts all assigncat entries to string (just in case user passed integer)
    #
    #also converts any single string/integer entries passed without list brackets 
    #into a single entry list
    """
    
    #ignore edge case where user passes empty dictionary
    if assigncat != {}:
    
      for assigncatkey in sorted(assigncat):
        current_list = assigncat[assigncatkey]
        
        #check if current_list is a list, if not populate it as single entry in a list
        if type(current_list) != type([]):
          assigncat[assigncatkey] = [current_list]
          current_list = assigncat[assigncatkey]
        
        #then convert any entries in the list to string type
        assigncat[assigncatkey] = [str(i) for i in current_list]

      del current_list

    return assigncat

  def assigninfill_str_convert(self, assigninfill):
    """
    #Converts all assigninfill entries to string (just in case user passed integer)
    #
    #also converts any single string/integer entries passed without list brackets 
    #into a single entry list
    """
    
    #ignore edge case where user passes empty dictionary
    if assigninfill != {}:

      for assigninfillkey in sorted(assigninfill):
        current_list = assigninfill[assigninfillkey]
          
        #check if current_list is a list, if not populate it as single entry in a list
        if type(current_list) != type([]):
          assigninfill[assigninfillkey] = [current_list]
          current_list = assigninfill[assigninfillkey]
        
        #then convert any entries in the list to string type
        assigninfill[assigninfillkey] = [str(i) for i in current_list]

      del current_list

    return assigninfill
  
  def parameter_str_convert(self, parameter):
    """
    #Converts parameter, such as one that might be either list or int or str, to a str or list of str
    #where True or False left unchanged
    """

    if isinstance(parameter, int) and str(parameter) != 'False' and str(parameter) != 'True':
      parameter = str(parameter)
    if isinstance(parameter, float):
      parameter = str(parameter)
    if isinstance(parameter, list):
      parameter = [str(i) for i in parameter]

    return parameter
  
  def assignparam_str_convert(self, assignparam):
    """
    #Converts all column entries to assignparam to string in case user passed integer
    #such as for numpy arrays
    #reminder the format of assign_param is e.g.
    #assignparam = {'splt' : {'column1' : {'minsplit' : 4}}, \
    #               'spl2' : {'column2' : {'minsplit' : 3}}}
    """

    assignparam_copy = deepcopy(assignparam)
  
    #ignore edge case where user passes empty dictionary
    if assignparam_copy != {}:
      
      for categorykey in assignparam_copy:
        
        for columnkey in assignparam_copy[categorykey]:
          
          assignparam[categorykey][str(columnkey)] = assignparam[categorykey].pop(columnkey)

    del assignparam_copy
          
    return assignparam

  def assignnan_str_convert(self, assignnan):
    """
    #convention is that user can pass integers to column assignments
    #and they are converted to strings
    """
    
    if 'columns' in assignnan:
      
      columns_copy = deepcopy(assignnan['columns'])
      
      for entry in columns_copy:
        
        if isinstance(entry, int):
          
          assignnan['columns'].update({str(entry) : assignnan['columns'][entry]})
          
          del assignnan['columns'][entry]
          
    return assignnan

  def assignnan_list_convert(self, assignnan):
    """
    #converts any passed infill values to lists in case passed as single value
    """
    
    if 'categories' in assignnan:
      
      #convert any entries to lists
      for entry1 in assignnan['categories']:
        
        if not isinstance(assignnan['categories'][entry1], list):
          
          assignnan['categories'][entry1] = [assignnan['categories'][entry1]]
          
    if 'columns' in assignnan:
      
      #convert any entries to lists
      for entry2 in assignnan['columns']:
        
        if not isinstance(assignnan['columns'][entry2], list):
          
          assignnan['columns'][entry2] = [assignnan['columns'][entry2]]
          
    if 'global' in assignnan:
      
      if not isinstance(assignnan['global'], list):
        
        assignnan['global'] = [assignnan['global']]
          
    return assignnan
  
  def floatprecision_transform(self, df, columnkeylist, floatprecision):
    """
    #floatprecision is a parameter user passed to automunge
    #allowable values are 16/32/64
    #if 64 do nothing (we'll assume our transofrm functions default to 64)
    #if 16 or 32 then check each column in df for columnkeylist and if
    #float convert to this precision
    """
    
    if isinstance(columnkeylist, str):
      columnkeylist = [columnkeylist]
    
    #if floatprecision in [16, 32, 64]:
    if floatprecision in [16, 32]:
      
      for columnkey in columnkeylist:
        
        if pd.api.types.is_float_dtype(df[columnkey]):
          
          if floatprecision == 32:
            df[columnkey] = df[columnkey].astype(np.float32)
            
          elif floatprecision == 16:
            df[columnkey] = df[columnkey].astype(np.float16)
            
#           elif floatprecision == 64:
#             df[columnkey] = df[columnkey].astype(np.float64)

    return df

  def grab_params(self, assign_param, category, column, processdict_entry, postprocess_dict):
    """
    #In order of precendence, parameters assigned to distinct 
    #category/column configurations take precedence 
    #to default_assignparam assigned to categories which take precendence 
    #to global_assignparam assigned to all transformations which take precendence 
    #to parameters set as defaultparams in processdict definition.
    """
    
    params = {}
    
    if 'defaultparams' in processdict_entry:
      
      #key are parameters
      for key in processdict_entry['defaultparams']:
        
        params.update({key : processdict_entry['defaultparams'][key]})

    #if assign_param is not empty
    if bool(assign_param):
      
      if 'global_assignparam' in assign_param:
        
        #key are parameters
        for key in assign_param['global_assignparam']:
          
          params.update({key : assign_param['global_assignparam'][key]})
          
      if 'default_assignparam' in assign_param:
        
        if category in assign_param['default_assignparam']:
            
          #key are parameters
          for key in assign_param['default_assignparam'][category]:

            params.update({key : assign_param['default_assignparam'][category][key]})
                
      if category in assign_param:
        
        #distinct category/column configurations can either be assigned
        #using the source column or the derived column serving as input to the transform
        #in case both are present the dervied column specifciation takes precedence
          
        if column in assign_param[category]:

          #key are parameters
          for key in assign_param[category][column]:

            params.update({key : assign_param[category][column][key]})

        #we won't use the source column entry if a derived column entry was specfied
        elif column in postprocess_dict['column_dict'] \
        and postprocess_dict['column_dict'][column]['origcolumn'] in assign_param[category]:

          #key are parameters
          for key in assign_param[category][postprocess_dict['column_dict'][column]['origcolumn']]:

            params.update({key : assign_param[category][postprocess_dict['column_dict'][column]['origcolumn']][key]})
    
    return params

  def apply_LabelSmoothing(self, df, targetcolumn, epsilon, label_categorylist, label_category, categorycomplete_dict, LSfit, LSfitparams_dict):
    """
    #applies label smoothing based on user passed epsilon 
    
    #if LSfit is False
    #based on method described in "Rethinking the Inception Architecture for Computer Vision" by Szegedy et al
    #hat tip to Stack Overflow user lejlot for some implementaiton suggestions
    # https://stackoverflow.com/questions/39335535/label-smoothing-soft-targets-in-pandas
    
    #if LSfit is True
    #based on extension wherein the Smoothing factor K for each column is fit to the
    #distribution of the set is is a function of the activation column and target column
    
    #we'll follow convention that in both cases label smoothing applied to all columns in categorylist
    #and return a diciotnary indicating which columns have recieved
    #(dictionary categorycomplete_dict initialized external to function)
    
    #we'll also return a dictionary containing any dervied parameters for LSfit such as for
    #subsequent consistent encoding
    
    #Note that categorylist is the list of columns originating from same transformation
    #and we currently exlcude '1010' binary encoded sets from the method
    #a future extension will address 1010 for MLinfill by smoothing after conversion to one-hot encoding 
    #such as for MLinfill
    """
    
#     unique_set = set(pd.unique(df[label_categorylist].values.ravel('K')))
    
#     if (unique_set == {0,1} \
#     or unique_set == {0} \
#     or unique_set == {1}) \
#     and label_category != '1010' \
#     and len(label_categorylist) > 1:
    
#     LSfitparams_dict = {}
  
    #initialize store of derived parameters
    for column1 in label_categorylist:
      
      LSfitparams_dict.update({column1 : {'LSfit' : LSfit, \
                                          'epsilon' : epsilon, \
                                          'label_categorylist' : label_categorylist, \
                                          'label_category' : label_category}})
    
    if LSfit is True:
      
      unique_set = set(pd.unique(df[label_categorylist].values.ravel('K')))
      
      #plus let's check if this is one-hot encoded (vs like binary encoded or something)
      #this is a proxy for testing if category is '1010'
      onehot = True
      #if df[label_categorylist].sum().sum() != df.shape[0]:
      if df[label_categorylist].sum().sum() > df.shape[0]:
        onehot = False

      if (unique_set == {0,1} \
      or unique_set == {0} \
      or unique_set == {1}) \
      and onehot is True \
      and len(label_categorylist) > 1:
        
        #if LSfit is True we'll apply LS to all columns in categorylist

        #activation_dcit will track the count of activations for each column in the categorylist
        activation_dict = {}

        for column1 in label_categorylist:

          activations = df[column1].sum()

          activation_dict.update({column1 : activations})
        
        #populate this in params dictionary
        for column1 in label_categorylist:
          
          LSfitparams_dict[column1].update({'activation_dict' : activation_dict})

        #LS_dict will be where we keep track of the activation distributions associated with each column
        #note we'll need activation ratios for each column as a function of activated column for a row
        LS_dict = {}
        for column1 in label_categorylist:

          LS_dict.update({column1 : {}})

          total_activations = 0

          for column2 in label_categorylist:

            if column1 != column2:

              total_activations += activation_dict[column2]

          for column2 in label_categorylist:

            if column1 != column2:

              LS_dict[column1].update({column2 : activation_dict[column2] / total_activations})
           
        #populate this in params dictionary
        for column1 in label_categorylist:
          
          LSfitparams_dict[column1].update({'LS_dict' : LS_dict})

        for column1 in label_categorylist:

          for column2 in label_categorylist:

            if column2 != column1:
              
              Smoothing_K = LS_dict[column2][column1]

              df[column1] = \
              np.where(df[column2] == 1, (1 - epsilon) * Smoothing_K, df[column1].values)

        for column1 in label_categorylist:

          df[column1] = \
          np.where(df[column1]==1, df[column1] * (epsilon), df[column1].values)

          categorycomplete_dict[column1] = True
    
    #if LSfit is not True:
    #else we'll only apply LS to the passed column with assumption of level distribution for K
    else:
      
      unique_set = set(pd.unique(df[targetcolumn]))
      
      #plus let's check if this is one-hot encoded (vs like binary encoded or something)
      #this is a proxy for testing if category is '1010'
      onehot = True
      #if df[label_categorylist].sum().sum() != df.shape[0]:
      if df[label_categorylist].sum().sum() > df.shape[0]:
        onehot = False

      if (unique_set == {0,1} \
      or unique_set == {0} \
      or unique_set == {1}) \
      and onehot is True \
      and len(label_categorylist) > 1:
        
        for column1 in label_categorylist:
        
          Smoothing_K = len(label_categorylist)

          #==1 value corresponds to 'bnry' encoding, >1 corresonds to one-hot encodings eg 'text'
          if Smoothing_K > 1:
            Smoothing_K -= 1

          df[column1] = \
          df[column1] * (epsilon) + (1 - df[column1]) * (1 - epsilon) / Smoothing_K

          categorycomplete_dict[column1] = True

    return df, categorycomplete_dict, LSfitparams_dict
                
  def postapply_LabelSmoothing(self, df, targetcolumn, categorycomplete_dict, LSfitparams_dict):
    """
    #applies label smoothing based on user passed LSfitparams_dict
    #consiostently to label smoothing from corresponding train data
    
    #if LSfit is False
    #based on method described in "Rethinking the Inception Architecture for Computer Vision" by Szegedy et al
    #hat tip to Stack Overflow user lejlot for some implementaiton suggestions
    # https://stackoverflow.com/questions/39335535/label-smoothing-soft-targets-in-pandas
    
    #if LSfit is True
    #based on extension wherein the Smoothing factor K for each column is fit to the
    #distribution of the set is is a function of the activation column and target column
    
    #we'll follow convention that in both cases label smoothing applied to all columns in categorylist
    #and return a diciotnary indicating which columns have recieved
    #(dictionary categorycomplete_dict initialized external to function)
    
    #we'll also return a dictionary containing any dervied parameters for LSfit such as for
    #subsequent consistent encoding
    
    #Note that categorylist is the list of columns originating from same transformation
    #and we currently exlcude '1010' binary encoded sets from the method
    #a future extension will address 1010 for MLinfill by smoothing after conversion to one-hot encoding 
    #such as for MLinfill
    """

    #grab passed parameters from LSfitparams_dict
    LSfit = LSfitparams_dict[targetcolumn]['LSfit']
    epsilon = LSfitparams_dict[targetcolumn]['epsilon']
    label_categorylist = LSfitparams_dict[targetcolumn]['label_categorylist']
    label_category = LSfitparams_dict[targetcolumn]['label_category']
    
    
    if LSfit is True:
      
      unique_set = set(pd.unique(df[label_categorylist].values.ravel('K')))
      
      #plus let's check if this is one-hot encoded (vs like binary encoded or something)
      #this is a proxy for testing if category is '1010'
      onehot = True
      #if df[label_categorylist].sum().sum() != df.shape[0]:
      if df[label_categorylist].sum().sum() > df.shape[0]:
        onehot = False

      if (unique_set == {0,1} \
      or unique_set == {0} \
      or unique_set == {1}) \
      and onehot is True \
      and len(label_categorylist) > 1:
        
#         activation_dict = LSfitparams_dict[targetcolumn]['activation_dict']
        LS_dict = LSfitparams_dict[targetcolumn]['LS_dict']
        
        #if LSfit is True we'll apply LS to all columns in categorylist

#         #activation_dcit will track the count of activations for each column in the categorylist
#         activation_dict = {}

#         for column1 in label_categorylist:

#           activations = df[column1].sum()

#           activation_dict.update({column1 : activations})
        
#         #populate this in params dictionary
#         for column1 in label_categorylist:
          
#           LSfitparams_dict.update({column1 : {'activation_dict' : activation_dict}})

#         #LS_dict will be where we keep track of the activation distributions associated with each column
#         #note we'll need activation ratios for each column as a function of activated column for a row
#         LS_dict = {}
#         for column1 in label_categorylist:

#           LS_dict.update({column1 : {}})

#           total_activations = 0

#           for column2 in label_categorylist:

#             if column1 != column2:

#               total_activations += activation_dict[column2]

#           for column2 in label_categorylist:

#             if column1 != column2:

#               LS_dict[column1].update({column2 : activation_dict[column2] / total_activations})
                 
#         #populate this in params dictionary
#         for column1 in label_categorylist:
          
#           LSfitparams_dict[column1].update({'LS_dict' : LS_dict})

        for column1 in label_categorylist:

          for column2 in label_categorylist:

            if column2 != column1:
              
              Smoothing_K = LS_dict[column2][column1]

              df[column1] = \
              np.where(df[column2] == 1, (1 - epsilon) * Smoothing_K, df[column1].values)

        for column1 in label_categorylist:

          df[column1] = \
          np.where(df[column1]==1, df[column1] * (epsilon), df[column1].values)

          categorycomplete_dict[column1] = True
    
    #if LSfit is not True:
    #else we'll only apply LS to the passed column with assumption of level distribution for K
    else:
      
      unique_set = set(pd.unique(df[targetcolumn]))
      
      #plus let's check if this is one-hot encoded (vs like binary encoded or something)
      #this is a proxy for testing if category is '1010'
      onehot = True
      #if df[label_categorylist].sum().sum() != df.shape[0]:
      if df[label_categorylist].sum().sum() > df.shape[0]:
        onehot = False

      if (unique_set == {0,1} \
      or unique_set == {0} \
      or unique_set == {1}) \
      and onehot is True \
      and len(label_categorylist) > 1:
        
        for column1 in label_categorylist:
        
          Smoothing_K = len(label_categorylist)

          #==1 value corresponds to 'bnry' encoding, >1 corresonds to one-hot encodings eg 'text'
          if Smoothing_K > 1:
            Smoothing_K -= 1

          df[column1] = \
          df[column1] * (epsilon) + (1 - df[column1]) * (1 - epsilon) / Smoothing_K

          categorycomplete_dict[column1] = True
  
    return df, categorycomplete_dict
  
  def Binary_convert(self, df_train, df_test, bool_column_list, Binary):
    """
    #Binary_convert takes as input a processed dataframe and a list of boolean encoded columns
    #and applies a dimensionality reduction on the boolean set as a binary encodiong
    
    #returns the set wtih a reduced number of columns and a dictionary containing the
    #parameters of conversion
    
    #note that infill has already been applied on these columns so no need for infill
    #on train set (but yes on test set)
    """
    
    if 'Binary' in df_train.columns:
      #(this will only happen when a column with header 'Binary' was passed to 'excl')
      print("error: column header 'Binary' present in set")
      print("Binary is a reserved column header when applying Binary transform")
      
      Binary_present = {'Binary':True}
    
    else:
      Binary_present = {'Binary':False}
    
    df_train['Binary'] = ''
    df_test['Binary'] = ''
  
    for column in bool_column_list:
  
      df_train['Binary'] = df_train['Binary'] + df_train[column].astype(str)
      df_test['Binary'] = df_test['Binary'] + df_test[column].astype(str)
      
#     #this step ensures thqt infill is all zeros since we rely on sort
#     df_train['Binary'] = 'B_' + df_train['Binary']
#     df_test['Binary'] = 'B_' + df_test['Binary']
    
    #now we'll apply process_1010_class
    df_train, df_test, Binary_column_dict_list = \
    self.process_1010_class(df_train, df_test, 'Binary', '1010', {}, {})
    
    Binary_dict = {'column_dict' : {}}
    
    for column_dict in Binary_column_dict_list:
      
      Binary_dict['column_dict'].update(column_dict)
    
    #add suffix overlap results for 'Binary' initialization
    Binary_dict['column_dict'].update({'Binary':{'suffixoverlap_results':Binary_present}})
      
    Binary_dict.update({'bool_column_list' : bool_column_list})
    
    del df_train['Binary']
    del df_test['Binary']
    
    #we won't delete the origin columns if Binary passed as 'retain'
    #(such that the binary encoding is a supplement instead of a replacement)
    if Binary not in ['retain']:

      for column in bool_column_list:

        del df_train[column]
        del df_test[column]
    
    return df_train, df_test, Binary_dict
  
  def postBinary_convert(self, df_test, Binary_dict, Binary):
    """
    #Binary_convert takes as input a processed dataframe and a list of boolean encoded columns
    #and applies a dimensionality reduction on the boolean set as a binary encodiong
    
    #returns the set wtih a reduced number of columns and a dictionary containing the
    #parameters of conversion
    
    #note that infill has already been applied on these columns so no need for infill
    #on train set (but yes on test set)
    """
    
    bool_column_list = Binary_dict['bool_column_list']    
    
    df_test['Binary'] = ''
    
    for column in bool_column_list:
  
      df_test['Binary'] = df_test['Binary'] + df_test[column].astype(str)
    
#     #this step ensures thqt infill is all zeros since we rely on sort
#     df_test['Binary'] = 'B_' + df_test['Binary']
    
    #now we'll apply postprocess_1010_class
    df_test = self.postprocess_1010_class(df_test, 'Binary', Binary_dict, 'columnkey', {})
    
    del df_test['Binary']
    
    #we won't delete the origin columns if Binary passed as 'retain'
    #(such that the binary encoding is a supplement instead of a replacement)
    if Binary not in ['retain']:
      
      for column in bool_column_list:

        del df_test[column]
    
    return df_test
  
  def meta_inverseprocess_Binary(self, df, postprocess_dict):
    """
    #converts string boolean integers returned from inverseprocess_Binary to int
    #cleans up columns
    """
    
    Binary_returned_columns = list(postprocess_dict['Binary_dict']['column_dict'])
    
    df, inputcolumn = \
    self.inverseprocess_Binary(df, Binary_returned_columns, postprocess_dict)
    
    Binary_source_columns = postprocess_dict['Binary_dict']['bool_column_list']
    
    i=0
    for Binary_source_column in Binary_source_columns:

      df[Binary_source_column] = df[inputcolumn].str.slice(start=i, stop=i+1)

      df[Binary_source_column] = df[Binary_source_column].astype(np.int8)
      i += 1
      
    #we'll have convention that the Binary columns not returned from inversion
    for Binary_returned_column in Binary_returned_columns:
      del df[Binary_returned_column]
      
    # del df[inputcolumn]
    
    return df
    
  def inverseprocess_Binary(self, df, Binary_columns, postprocess_dict):
    """
    #inverse transform similar to process_1010_class
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    #note that this will return numeric entries as str
    """
    
    normkey = Binary_columns[0]
    
    _1010_binary_encoding_dict = \
    postprocess_dict['Binary_dict']['column_dict'][normkey]['normalization_dict'][normkey]['_1010_binary_encoding_dict']
    
    inverse_binary_encoding_dict = {value:key for key,value in _1010_binary_encoding_dict.items()}
    
    inputcolumn = 'Binary'
    
    for Binary_column in Binary_columns:

      if Binary_column != 'Binary':
        
        if Binary_column == Binary_columns[0]:
          
          df[inputcolumn] = df[Binary_column].astype(int).astype(str)
          
        else:
          
          df[inputcolumn] = df[inputcolumn] + df[Binary_column].astype(int).astype(str)
        
    df[inputcolumn] = df[inputcolumn].replace(inverse_binary_encoding_dict)
      
    return df, inputcolumn
  
  def convert_inf_to_nan(self, df, column, category, postprocess_dict):
    """
    #converts all np.inf values in a dataframe to np.nan
    #similar to pandas pd.options.mode.use_inf_as_na = True
    #except that it works
    """
    
    #don't apply to totalexclude MLinfilltype
    if postprocess_dict['process_dict'][category]['MLinfilltype'] not in ['totalexclude']:

      df.loc[df[column] == np.inf, column] = np.nan
      df.loc[df[column] == -np.inf, column] = np.nan
    
    return df

  def assignnan_convert(self, df, column, category, assignnan, postprocess_dict):
    """
    #assignnan is automunge(.) parameter that allows user to designate values that will
    #be given infill treatment for a given root category or source column
    #such as to supplement processdict NArowtype entries with values that may be 
    #special for a data set
    #as an example, in some cases datasets may not be recieved with NaN for infill, 
    #and may instead be a designated value such as a number or string such as 'unknown'
    #assignnan_convert addresses this scenario by simply converting those designations to nan

    #where values are passed in automunge(.) parameter assignnan
    #assignnan = {'categories':{'cat1':[], 'cat2':[]}, 'columns':{'col1':[], 'col2':[]}, 'global':[]}
    
    #where in case of specification redundancy column designation takes precedence
    #and where category is reffering to the root category associated with a column
    
    #some further options for nan injections are documented in the assignnan_inject
    #including stochastic and range injections
    #which is called at the conclusion of this one
    """
    
    nanpoints = []

    cat_process = False

    if 'categories' in assignnan:

      if category in assignnan['categories']:

        cat_process = True

        if 'columns' in assignnan:

          if column in assignnan['columns']:

            cat_process = False

        if cat_process is True:

          nanpoints = assignnan['categories'][category]

    if cat_process is False:

      if 'columns' in assignnan:

        if column in assignnan['columns']:

          nanpoints = assignnan['columns'][column]
          
    #we'll have convention that for complete passthrough columns without infill (like: excl, exc6)
    #global assignnan assignments don't apply and must be assigned explicitly
    #either in assignnan categories or columns entries
    if postprocess_dict['process_dict'][category]['MLinfilltype'] not in ['totalexclude']:

      if 'global' in assignnan:

        nanpoints += assignnan['global']
          
    #great we've got our designated infill values, now just convert to nan
    for entry in nanpoints:
      
      df.loc[df[column] == entry, column] = np.nan
      
    #finally some further options for nan injections are supported 
    #including stochastic and range injections
    #as documented further in assignnan_inject function
    if 'injections' in assignnan:
      df = self.assignnan_inject(df, column, assignnan, postprocess_dict['randomseed'])
    
    return df

  def assignnan_inject(self, df, column, assignnan, randomseed):
    """
    #allows custom range or stochastic nan injections to distinct source columns
    #assignnan now accepts entries as
    
    {'injections' : {'(column)' : {'inject_ratio' : (float), \
                                   'range' : {'ratio'  : (float), \
                                              'ranges' : [[min1, max1], [min2, max2]]}, \
                                   'minmax_range' : {'ratio'  : (float), \
                                                     'ranges' : [[min1, max1], [min2, max2]]}, \
                                   'entries' : ['(entry1)', '(entry2)'], \
                                   'entry_ratio' : {'(entry1)' : float, \
                                                    '(entry2)' : float}
                                  }
                    }
    }
    
    #where injections may be specified for each source column passed to automunge(.)
    #- inject_ratio is uniform randomly injected nan points to ratio of entries
    #- range is injection within a specified range based on ratio float defaulting to 1.0
    #- minmax_range is injection within scaled range (accepting floats 0-1 based on received 
    #column max and min (returned column is not scaled
    #- entries are full replacement of specific entries to a categoric set
    #- entry_ratio are partial injection to specific entries to a categoric set, 
    #per specified float ratio 
    
    #the purposes of these options are to support some experiments on missing data infill
    
    #currently assignnan_inject only supports dataframes with range integer index
    #this is deemed acceptable since this functionality not intended for mainstream use
    """
    
    if 'injections' in assignnan:
      
      #for injections functinoality we'll reset index to range integer
      #non-range indexes will have already been copied into ID set
      if type(df.index) != pd.RangeIndex:
        df = df.reset_index(drop=True)
      
      for columnkey in assignnan['injections']:
        if columnkey in df:
          for actionkey in assignnan['injections'][columnkey]:
            if actionkey not in ['inject_ratio', 'range', 'minmax_range', 'entries', 'entry_ratio']:
              print("assignnan['injections'] has an invalid action entry")
              print("for column: ", columnkey)
              print("and action: ", actionkey)
              print("accepted form of injetion specifications are documented in read me")
              
            elif actionkey == 'inject_ratio':
              #inject_ratio is uniform randomly injected nan points to ratio of entries
              ratio = assignnan['injections'][columnkey][actionkey]
              index = list(pd.DataFrame(df.index).sample(frac=ratio, replace=False, random_state=randomseed).values.ravel())
              df.loc[index, columnkey] = np.nan
              
            elif actionkey == 'range':
              #range inserts nan in designated ranges of numeric set
              #accepts parameter for injection ratio, we'll actually use for our method 1-ratio
              if 'ratio' in assignnan['injections'][columnkey][actionkey]:
                ratio = 1 - assignnan['injections'][columnkey][actionkey]['ratio']
              else:
                ratio = 1 - 1
              if 'ranges' in assignnan['injections'][columnkey][actionkey]:
                #ranges is a list of list, each sub list with two entries for min and max inclusive
                for range_list in assignnan['injections'][columnkey][actionkey]['ranges']:
                  rangemin = range_list[0]
                  rangemax = range_list[-1]
                  #we'll create a support dataframe to populate masks
                  df_mask = pd.DataFrame()
                  df_mask['lessthan'] = np.where(df[columnkey]<=rangemax, 1, 0)
                  df_mask['greaterthan'] = np.where(df[columnkey]>=rangemin, 1, 0)
                  #this populates mask with 1's for candidates for injection, 0's elsewhere
                  df_mask['mask'] = df_mask['lessthan'] * df_mask['greaterthan']
                  #this populates mask with nan for injection candidates and 1 elsewhere
                  df_mask['mask'] = np.where(df_mask['mask'] == 1, np.nan, 1)
                  if ratio > 0:
                    #this get's index list of mask nan entries for use with .loc
                    index = list(pd.DataFrame(df_mask[df_mask['mask'] != df_mask['mask']].index).sample(frac=ratio, replace=False, random_state=randomseed).values.ravel())
                    #this reverts some of the nans if ratio was passed as <1
                    df_mask.loc[index, 'mask'] = 1
                  #now inject the nan to the target column in df
                  df[columnkey] = df[columnkey] * df_mask['mask']
                  del df_mask
                  
            elif actionkey == 'minmax_range':
              #minmax_range is similar to range but the max and min are recieved in range 0-1
              #and applied corresponding to a minmax scaling of recieved set
              #(set is returned without scaling)
              #range inserts nan in designated ranges of numeric set
              #accepts parameter for injection ratio, we'll actually use for our method 1-ratio
              if 'ratio' in assignnan['injections'][columnkey][actionkey]:
                ratio = 1 - assignnan['injections'][columnkey][actionkey]['ratio']
              else:
                ratio = 1 - 1
              if 'ranges' in assignnan['injections'][columnkey][actionkey]:
                #this might return nan if target column does not contain numeric entries
                setmin = df[columnkey].min()
                setmax = df[columnkey].max()
                #ranges is a list of list, each sub list with two entries for min and max inclusive
                for range_list in assignnan['injections'][columnkey][actionkey]['ranges']:
                  rangemin = range_list[0]
                  rangemax = range_list[-1]
                  #now scale these ranges based on inverse of min-max scaling formula
                  #minmax = (xi - min)/(max - min)
                  # => xi = minmax * (max - min) + min
                  rangemin = rangemin * (setmax - setmin) + setmin
                  rangemax = rangemax * (setmax - setmin) + setmin
                  #the rest is comparable to unscaled range option
                  #we'll create a support dataframe to populate masks
                  df_mask = pd.DataFrame()
                  df_mask['lessthan'] = np.where(df[columnkey]<=rangemax, 1, 0)
                  df_mask['greaterthan'] = np.where(df[columnkey]>=rangemin, 1, 0)
                  #this populates mask with 1's for candidates for injection, 0's elsewhere
                  df_mask['mask'] = df_mask['lessthan'] * df_mask['greaterthan']
                  #this populates mask with nan for injection candidates and 1 elsewhere
                  df_mask['mask'] = np.where(df_mask['mask'] == 1, np.nan, 1)
                  if ratio > 0:
                    #this get's index list of mask nan entries for use with .loc
                    index = list(pd.DataFrame(df_mask[df_mask['mask'] != df_mask['mask']].index).sample(frac=ratio, replace=False, random_state=randomseed).values.ravel())
                    #this reverts some of the nans if ratio was passed as <1
                    df_mask.loc[index, 'mask'] = 1
                  #now inject the nan to the target column in df
                  df[columnkey] = df[columnkey] * df_mask['mask']
                  del df_mask
                  
            elif actionkey == 'entries':
              #entries are full replacement of specific entries to a categoric set
              for targetentry in assignnan['injections'][columnkey][actionkey]:
                df[columnkey] = np.where(df[columnkey] == targetentry, np.nan, df[columnkey])
                
            elif actionkey == 'entry_ratio':
              #entry_ratio are partial injection to specific entries to a categoric set, 
              #per specified float ratio in range 0-1
              for targetentry in assignnan['injections'][columnkey][actionkey]:
                ratio = 1 - assignnan['injections'][columnkey][actionkey][targetentry]
                df_mask = pd.DataFrame()
                #here we'll use convention that 1 is a target for injection 0 remains static
                df_mask['mask'] = np.where(df[columnkey] == targetentry, 1, 0)
                if ratio > 0:
                  #this get's index list of mask 1 entries for use with .loc
                  index = list(pd.DataFrame(df_mask[df_mask['mask'] == 1].index).sample(frac=ratio, replace=False, random_state=randomseed).values.ravel())
                  #this reverts some of the 1's if ratio was passed as <1
                  df_mask.loc[index, 'mask'] = 0
                df[columnkey] = np.where(df_mask['mask'] == 1, np.nan, df[columnkey])
                
    return df
  
  def df_split(self, df, ratio, shuffle_param, randomseed):
    """
    #performs a split of passed dataframe df
    #based on proportions of ratio where 0<ratio<1
    #bool shuffle False means rows taken from bottom of set sequentially
    #bool shuffle True means randomly selected rows 
    #per seeding of randomseed
    #(if run on two df's with same number of rows, will return consistent partitioning)
    #returns two dataframes df1 and df2
    """

    if ratio > 0 and ratio < 1:

      start = int(df.shape[0] * (1-ratio))
      end = df.shape[0]

      if shuffle_param is True:
        df = self.df_shuffle(df, randomseed)

      df1 = df[0:start]
      df2 = df[start:end]

    else:

      df1 = df
      df2 = pd.DataFrame()

    return df1, df2
    
  
  def df_shuffle(self, df, randomseed):
    """
    #Shuffles the rows of a dataframe
    #per seeding of randomseed
    """
    
    df = df.sample(frac=1, random_state=randomseed)
    
    return df  
  
  def df_shuffle_series(self, df, column, randomseed):
    """
    #Shuffles single column in a dataframe
    """
    
    df_temp = df[column].copy()
    df_temp = self.df_shuffle(df_temp, randomseed)
    df[column] = df_temp.values
    
    del df_temp
    
    return df

  def populate_columntype_report(self, postprocess_dict, target_columns):
    """
    #populates a report for types of returned columns
    #such as to distingiush between continous, categoric, categoric sets, etc
    """
    
    columntype_report = {'continuous' : [], \
                         'boolean' : [], \
                         'ordinal' : [], \
                         'onehot' : [], \
                         'onehot_sets' : [], \
                         'binary' : [], \
                         'binary_sets' : [], \
                         'passthrough' : []}
    
    populated_columns = []
    
    #for column in postprocess_dict['finalcolumns_train']:
    for column in target_columns:
      
      if column not in populated_columns:
        
        if 'returned_PCA_columns' in postprocess_dict and \
        column in postprocess_dict['returned_PCA_columns']:
          
          #add to numeric
          columntype_report['continuous'].append(column)
          
          populated_columns.append(column)
          
        elif 'returned_Binary_columns' in postprocess_dict and \
        column in postprocess_dict['returned_Binary_columns']:
          
          #initialize binary_sets
          columntype_report['binary_sets'].append([])

          for entry in postprocess_dict['returned_Binary_columns']:

            columntype_report['binary_sets'][-1] = \
            columntype_report['binary_sets'][-1] + [entry]

            #add to binary
            columntype_report['binary'].append(entry)

            populated_columns.append(entry)
          
        elif 'excl_suffix' in postprocess_dict and \
        'excl_columns_without_suffix' in postprocess_dict and \
        postprocess_dict['excl_suffix'] is False and \
        column in postprocess_dict['excl_columns_without_suffix']:
            
          #add column to passthrough
          columntype_report['passthrough'].append(column)

          populated_columns.append(column)
            
        elif column in postprocess_dict['column_dict']:
          
          MLinfilltype = \
          postprocess_dict['process_dict'][postprocess_dict['column_dict'][column]['category']]['MLinfilltype']
          
          if MLinfilltype in ['numeric', 'concurrent_nmbr']:
            
            #add to numeric
            columntype_report['continuous'].append(column)
            
            populated_columns.append(column)
            
          elif MLinfilltype in ['binary', 'concurrent_act', 'boolexclude']:
            
            #add to boolean
            columntype_report['boolean'].append(column)
            
            populated_columns.append(column)
          
          elif MLinfilltype in ['singlct']:
            
            #add to ordinal
            columntype_report['ordinal'].append(column)
            
            populated_columns.append(column)
            
          elif MLinfilltype in ['multirt']:
            
            #initialize onehot_sets
            columntype_report['onehot_sets'].append([])
            
            for entry in postprocess_dict['column_dict'][column]['categorylist']:
              
              columntype_report['onehot_sets'][-1] = \
              columntype_report['onehot_sets'][-1] + [entry]
              
              #add to onehot
              columntype_report['onehot'].append(entry)
              
              populated_columns.append(entry)
              
          elif MLinfilltype in ['1010']:
              
            #initialize binary_sets
            columntype_report['binary_sets'].append([])

            for entry in postprocess_dict['column_dict'][column]['categorylist']:

              columntype_report['binary_sets'][-1] = \
              columntype_report['binary_sets'][-1] + [entry]

              #add to binary
              columntype_report['binary'].append(entry)

              populated_columns.append(entry)
              
          elif MLinfilltype in ['exclude', 'totalexclude']:
            
            #add to ordinal
            columntype_report['passthrough'].append(column)
            
            populated_columns.append(column)
            
    return columntype_report

  def populate_column_map_report(self, postprocess_dict):
    """
    #populates a report that maps input columns to their corresponding set of output columns
    #as a dictionary with keys of input columns and values of a list of associated output columns
    #includes label columns
    #returned in postprocess_dict as postprocess_dict['column_map']
    #excludes any returned columns that were part of a dimensionality reduction consolidation
    #those are available in postprocess_dict as returned_Binary_columns and returned_PCA_columns
    """
    
    column_map = {}
    
    allfinalcolumns = postprocess_dict['finalcolumns_train'] + postprocess_dict['finalcolumns_labels']
    
    for finalcolumn in allfinalcolumns:
      
      finalcolumn2 = finalcolumn
      
      if finalcolumn2 not in postprocess_dict['returned_PCA_columns'] and \
      finalcolumn2 not in postprocess_dict['returned_Binary_columns']:
      
        if postprocess_dict['excl_suffix'] is False:

          if finalcolumn in postprocess_dict['excl_columns_without_suffix']:

            finalcolumn2 += '_excl'

        columnkeylist = \
        postprocess_dict['origcolumn'][postprocess_dict['column_dict'][finalcolumn2]['origcolumn']]['columnkeylist']

        #if entry was already populated for multiple returned columns it overwrites it with same info
        column_map.update({postprocess_dict['column_dict'][finalcolumn2]['origcolumn'] : columnkeylist})
      
    return column_map
  
  def automunge(self, df_train, df_test = False, \
                labels_column = False, trainID_column = False, testID_column = False, \
                valpercent1=0.0, valpercent2 = 0.0, floatprecision = 32, shuffletrain = True, \
                TrainLabelFreqLevel = False, powertransform = False, binstransform = False, \
                MLinfill = False, infilliterate=1, randomseed = 42, eval_ratio = .5, \
                LabelSmoothing_train = False, LabelSmoothing_test = False, LabelSmoothing_val = False, LSfit = False, \
                numbercategoryheuristic = 63, pandasoutput = False, NArw_marker = False, \
                featureselection = False, featurepct = 1.0, featuremetric = 0.0, featuremethod = 'default', \
                Binary = False, PCAn_components = False, PCAexcl = [], excl_suffix = False, \
                ML_cmnd = {'MLinfill_type':'default', \
                           'MLinfill_cmnd':{'RandomForestClassifier':{}, 'RandomForestRegressor':{}}, \
                           'PCA_type':'default', \
                           'PCA_cmnd':{}}, \
                assigncat = {'nmbr':[], 'retn':[], 'mnmx':[], 'mean':[], 'MAD3':[], 'lgnm':[], \
                             'bins':[], 'bsor':[], 'pwrs':[], 'pwr2':[], 'por2':[], 'bxcx':[], \
                             'addd':[], 'sbtr':[], 'mltp':[], 'divd':[], \
                             'log0':[], 'log1':[], 'logn':[], 'sqrt':[], 'rais':[], 'absl':[], \
                             'bnwd':[], 'bnwK':[], 'bnwM':[], 'bnwo':[], 'bnKo':[], 'bnMo':[], \
                             'bnep':[], 'bne7':[], 'bne9':[], 'bneo':[], 'bn7o':[], 'bn9o':[], \
                             'bkt1':[], 'bkt2':[], 'bkt3':[], 'bkt4':[], \
                             'nbr2':[], 'nbr3':[], 'MADn':[], 'MAD2':[], 'tlbn':[], \
                             'mnm2':[], 'mnm3':[], 'mnm4':[], 'mnm5':[], 'mnm6':[], \
                             'ntgr':[], 'ntg2':[], 'ntg3':[], 'mea2':[], 'mea3':[], 'bxc2':[], \
                             'dxdt':[], 'd2dt':[], 'd3dt':[], 'dxd2':[], 'd2d2':[], 'd3d2':[], \
                             'nmdx':[], 'nmd2':[], 'nmd3':[], 'mmdx':[], 'mmd2':[], 'mmd3':[], \
                             'shft':[], 'shf2':[], 'shf3':[], 'shf4':[], 'shf7':[], 'shf8':[], \
                             'bnry':[], 'onht':[], 'text':[], 'txt2':[], '1010':[], 'or10':[], \
                             'ordl':[], 'ord2':[], 'ord3':[], 'ord4':[], 'om10':[], 'mmor':[], \
                             'Unht':[], 'Utxt':[], 'Utx2':[], 'Uor3':[], 'Uor6':[], 'U101':[], \
                             'splt':[], 'spl2':[], 'spl5':[], 'sp15':[], 'sp19':[], 'sbst':[], \
                             'spl8':[], 'spl9':[], 'sp10':[], 'sp16':[], 'sp20':[], 'sbs2':[], \
                             'srch':[], 'src2':[], 'src4':[], 'strn':[], 'lngt':[], 'aggt':[], \
                             'nmrc':[], 'nmr2':[], 'nmcm':[], 'nmc2':[], 'nmEU':[], 'nmE2':[], \
                             'nmr7':[], 'nmr8':[], 'nmc7':[], 'nmc8':[], 'nmE7':[], 'nmE8':[], \
                             'ors2':[], 'ors5':[], 'ors6':[], 'ors7':[], 'ucct':[], 'Ucct':[], \
                             'or15':[], 'or17':[], 'or19':[], 'or20':[], 'or21':[], 'or22':[], \
                             'date':[], 'dat2':[], 'dat6':[], 'wkdy':[], 'bshr':[], 'hldy':[], \
                             'wkds':[], 'wkdo':[], 'mnts':[], 'mnto':[], \
                             'yea2':[], 'mnt2':[], 'mnt6':[], 'day2':[], 'day5':[], \
                             'hrs2':[], 'hrs4':[], 'min2':[], 'min4':[], 'scn2':[], 'DPrt':[], \
                             'DPnb':[], 'DPmm':[], 'DPbn':[], 'DPod':[], 'DP10':[], 'DPoh':[], \
                             'excl':[], 'exc2':[], 'exc3':[], 'exc4':[], 'exc5':[], \
                             'null':[], 'copy':[], 'shfl':[], 'eval':[], 'ptfm':[]}, \
                assignparam = {'default_assignparam' : {'(category)' : {'(parameter)' : 42}}, \
                                        '(category)' : {'(column)'   : {'(parameter)' : 42}}}, \
                assigninfill = {'stdrdinfill':[], 'MLinfill':[], 'zeroinfill':[], 'oneinfill':[], \
                                'adjinfill':[], 'meaninfill':[], 'medianinfill':[], \
                                'modeinfill':[], 'lcinfill':[], 'naninfill':[]}, \
                assignnan = {'categories':{}, 'columns':{}, 'global':[]}, \
                transformdict = {}, processdict = {}, evalcat = False, \
                privacy_encode = False, printstatus = True):
    """
    #This function documented in READ ME, available online at:
    # (anonymized)
    """
    
    application_timestamp = dt.datetime.now().strftime("%Y-%m-%dT%H:%M:%S.%f")
    application_number = random.randint(100000000000,999999999999)
    indexcolumn = 'Automunge_index_' + str(application_number)
    trainID_column_orig = trainID_column
    testID_column_orig = testID_column

    #quick conversion of any assigncat and assigninfill entries to str (such as for cases if user passed integers)
    assigncat = self.assigncat_str_convert(assigncat)
    assigninfill = self.assigninfill_str_convert(assigninfill)
    assignparam = self.assignparam_str_convert(assignparam)
    assignnan = self.assignnan_str_convert(assignnan)

    #similarily, quick conversion of any passed column idenitfiers to str
    labels_column = self.parameter_str_convert(labels_column)
    trainID_column = self.parameter_str_convert(trainID_column)
    testID_column = self.parameter_str_convert(testID_column)

    #convert assignnan if not recieved as lists in bottom tiers
    assignnan = self.assignnan_list_convert(assignnan)

    #quick check to ensure each column only assigned once in assigncat and assigninfill
    check_assigncat_result = self.check_assigncat(assigncat)
    check_assigninfill_result = self.check_assigninfill(assigninfill)
    check_ML_cmnd_result = self.check_ML_cmnd(ML_cmnd)

    #check the range of parameters 
    #(generally speaking other than passed dictionaries, dataframes, or column identifiers)
    miscparameters_results = \
    self.check_am_miscparameters(valpercent1, valpercent2, floatprecision, shuffletrain, \
                                 TrainLabelFreqLevel, powertransform, binstransform, MLinfill, \
                                 infilliterate, randomseed, eval_ratio, LabelSmoothing_train, LabelSmoothing_test, \
                                 LabelSmoothing_val, LSfit, numbercategoryheuristic, pandasoutput, \
                                 NArw_marker, featureselection, featurepct, featuremetric, \
                                 featuremethod, Binary, PCAn_components, PCAexcl, printstatus, excl_suffix)

    miscparameters_results.update({'check_assigncat_result' : check_assigncat_result, \
                                   'check_assigninfill_result' : check_assigninfill_result, \
                                   'check_ML_cmnd_result' : check_ML_cmnd_result})
    
    #initialize processing dicitonaries
    transform_dict = self.assembletransformdict(binstransform, NArw_marker)

    #transformdict is user passed data structure
    #vs transform_dict which is the internal library
    if bool(transformdict) is not False:
      
      #perform some validaitons on transformdict
      check_transformdict_result1, check_transformdict_result2, transformdict = \
      self.check_transformdict(transformdict)

      miscparameters_results.update({'check_transformdict_result1' : check_transformdict_result1, \
                                     'check_transformdict_result2' : check_transformdict_result2})
      
      check_transformdict2_result1, check_transformdict2_result2 = \
      self.check_transformdict2(transformdict)
      
      miscparameters_results.update({'check_transformdict2_result1' : check_transformdict2_result1, \
                                     'check_transformdict2_result2' : check_transformdict2_result2})

      #now consolidate the transform_dict and transformdict into single dictionary
      transform_dict.update(transformdict)
      
    #check for infinite loops in user passed transformdict
    check_haltingproblem_result = \
    self.check_haltingproblem(transformdict, transform_dict, max_check_count = 111)
    
    miscparameters_results.update({'check_haltingproblem_result' : check_haltingproblem_result})

    #initialize process_dict
    process_dict = self.assembleprocessdict()
    
    #Special case if we are running Binary dimensionality reduction for boolean sets
    if Binary is True:
      #transform_dict['1010'] = transform_dict['text']
      
      #we'll also have default that if running Binary transform boolean columns 
      #excluded from any PCA unless otherwise specified
      if 'PCA_cmnd' not in ML_cmnd:
        ML_cmnd.update({'PCA_cmnd':{'bool_PCA_excl':True}})
      else:
        if 'bool_PCA_excl' not in ML_cmnd['PCA_cmnd']:
          ML_cmnd['PCA_cmnd'].update({'bool_PCA_excl':True})

    #processdict is user passed data strucure, vs process_dcit which is the internal library
    if bool(processdict) is not False:
      
      #this function checks if any category entries in user passed processdict 
      #have their processing functions assigned by way of a functionpointer entry
      #and if so populate the entry with the associated processing functions
      processdict, check_functionpointer_result = \
      self.grab_processdict_functions(processdict, process_dict)
      miscparameters_results.update({'check_functionpointer_result' : check_functionpointer_result})
      
      #this funcion applies some misc validations on processdict
      check_processdict_result = \
      self.check_processdict(processdict)
      miscparameters_results.update({'check_processdict_result' : check_processdict_result})

      #now consolidate user passed entries from processdict and internal library in process_dict
      process_dict.update(processdict)
      
    else:
      miscparameters_results.update({'check_functionpointer_result' : False})
      miscparameters_results.update({'check_processdict_result' : False})

    #now that both transform_dict and process_dict are consolidated, validate transformdict roots have processdict entries
    check_transform_dict_roots_result = \
    self.check_transform_dict_roots(transform_dict, process_dict)
    miscparameters_results.update({'check_transform_dict_roots_result' : check_transform_dict_roots_result})
      
    #here we confirm that all of the keys of assigncat have corresponding entries in process_dict
    check_assigncat_result2 = self.check_assigncat2(assigncat, transform_dict)
    
    #now double check that any category entries in the assigncat have populated family trees
    #with categories that all have entries in the transform_dict
    check_assigncat_result3 = self.check_assigncat3(assigncat, process_dict, transform_dict)
    
    miscparameters_results.update({'check_assigncat_result2' : check_assigncat_result2, \
                                   'check_assigncat_result3' : check_assigncat_result3})

    check_assignparam = self.check_assignparam(assignparam, process_dict)
    miscparameters_results.update({'check_assignparam' : check_assignparam})

    #initialize autoMLer which is data structure to support ML infill
    #a future extension may allow user to pass custom entries
    autoMLer = self.assemble_autoMLer()

    #feature selection analysis performed here if elected
    if featureselection is True:

      if labels_column is False:
        print("featureselection not available without labels_column in training set")
        
        madethecut = []
        FSmodel = False
        FScolumn_dict = {}
        FS_sorted = {}

      else:
        madethecut, FSmodel, FScolumn_dict, FS_sorted = \
        self.featureselect(df_train, labels_column, trainID_column, \
                          powertransform, binstransform, randomseed, \
                          numbercategoryheuristic, assigncat, transformdict, \
                          processdict, featurepct, featuremetric, featuremethod, \
                          ML_cmnd, process_dict, valpercent1, valpercent2, printstatus, NArw_marker, \
                          assignparam)

      #if featuremethod is report then no further processing just return the results
      if featuremethod == 'report':

        #printout display progress
        if printstatus is True:
          print("_______________")
          print("Feature Importance results returned")
          print("")
          print("_______________")
          print("Automunge Complete")
          print("")

        return [], [], [], \
        [], [], [], \
        [], [], [], \
        [], [], [], \
        [], [], [],  \
        FScolumn_dict, FS_sorted

    else:

      madethecut = []
      FSmodel = False
      FScolumn_dict = {}
      FS_sorted = {}

    #validate that a model was trained
    check_FSmodel_result = self.check_FSmodel(featureselection, FSmodel)
    miscparameters_results.update({'check_FSmodel_result' : check_FSmodel_result})

    #printout display progress
    if printstatus is True:
      print("_______________")
      print("Begin Automunge processing")
      print("")

    #functionality to support passed numpy arrays
    #if passed object was a numpy array, convert to pandas dataframe
    checknp = np.array([])

    #first validate numpy data is tabular
    if isinstance(checknp, type(df_train)):
      check_np_shape_train_result, check_np_shape_test_result = \
      self.check_np_shape(df_train, df_test)
    else:
      check_np_shape_train_result, check_np_shape_test_result = False, False
    miscparameters_results.update({'check_np_shape_train_result' : check_np_shape_train_result, \
                                   'check_np_shape_test_result' : check_np_shape_test_result})

    if isinstance(checknp, type(df_train)):
      df_train = pd.DataFrame(df_train)
    if isinstance(checknp, type(df_test)):
      df_test = pd.DataFrame(df_test)

    #this converts any numeric columns labels, such as from a passed numpy array, to strings
    trainlabels=[]
    for column in df_train.columns:
      trainlabels.append(str(column))
    df_train.columns = trainlabels
        
    #if user passes as True labels_column passed based on final column (including single column scenario)
    #this is here so it can get carried through to featureselect function
    if labels_column is True:
      labels_column = trainlabels[-1]
    
    #confirm all unique column headers
    check_columnheaders_result = \
    self.check_columnheaders(list(df_train))

    miscparameters_results.update({'check_columnheaders_result' : check_columnheaders_result})

    origcolumns_all = list(df_train)

    #validate assignnan has valid root categories and source columns
    #note this takes place before any label column split from df_train
    check_assignnan_toplevelentries_result, check_assignnan_categories_result, check_assignnan_columns_result \
    = self.check_assignnan(assignnan, transform_dict, list(df_train))
  
    miscparameters_results.update({'check_assignnan_toplevelentries_result' : check_assignnan_toplevelentries_result, \
                                   'check_assignnan_categories_result'      : check_assignnan_categories_result, \
                                   'check_assignnan_columns_result'         : check_assignnan_columns_result})
    
    #we originally had the convention that some preprocessing was done on assignparam here
    #to create assign_param
    #keeping the assign_param convention in cases we later reintroduce
    assign_param = assignparam

    #we'll introduce convention that if df_test provided as False then we'll create
    #a dummy set derived from df_train's first rows
    #test_plug_marker used to identify that this step was taken
    test_plug_marker = False
    if not isinstance(df_test, pd.DataFrame):
#       df_test = df_train[0:10].copy()
      df_test = df_train[0:1].copy()
      testID_column = trainID_column
      test_plug_marker = True
      if labels_column is not False:
        del df_test[labels_column]

    #this converts any numeric columns labels, such as from a passed numpy array, to strings
    testlabels=[]
    for column in df_test.columns:
      testlabels.append(str(column))
    df_test.columns = testlabels

    #copy input dataframes to internal state so as not to edit exterior objects
    #this helps with recursion in feature importance
#     if inplace is False:
    df_train = df_train.copy()
    df_test = df_test.copy()

#     elif inplace is True:
#       pass
    
    #we'll have convention that if testID_column=False, if trainID_column in df_test etc
    trainID_columns_in_df_test = False
    if testID_column is False or testID_column is True:
      if trainID_column is not False:
        trainID_columns_in_df_test = True
        if isinstance(trainID_column, list):
          for trainIDcolumn in trainID_column:
            if trainIDcolumn not in df_test.columns:
              trainID_columns_in_df_test = False
              break
        elif isinstance(trainID_column, str):
          if trainID_column not in df_test.columns:
            trainID_columns_in_df_test = False
    if trainID_columns_in_df_test is True:
      testID_column = trainID_column

    if trainID_column is False:
      trainID_column = []
    elif isinstance(trainID_column, str):
      trainID_column = [trainID_column]
    elif not isinstance(trainID_column, list):
      print("error, trainID_column allowable values are False, string, or list")

    #non-range indexes we'll move into the ID sets for consistent shuffling and validation splits
    if type(df_train.index) != pd.RangeIndex:
      #if df_train.index.names == [None]:
      if None in df_train.index.names:
        df_train = df_train.rename_axis('Orig_index_' +  str(application_number))
      trainID_column = trainID_column + list(df_train.index.names)
      df_train = df_train.reset_index(drop=False)

    if testID_column is False:
      testID_column = []
    elif isinstance(testID_column, str):
      testID_column = [testID_column]
    elif not isinstance(testID_column, list):
      print("error, testID_column allowable values are False, string, or list")

    if type(df_test.index) != pd.RangeIndex:
      #if df_train.index.names == [None]:
      if None in df_test.index.names:
        df_test = df_test.rename_axis('Orig_index_' +  str(application_number))
      testID_column = testID_column + list(df_test.index.names)
      df_test = df_test.reset_index(drop=False)

    #here we derive a range integer index for inclusion in the ID sets
    df_train_tempID = pd.DataFrame({indexcolumn:range(0,df_train.shape[0])})
    tempIDlist = []
    
    #extract the ID columns from train set
    if trainID_column is not False:
      df_trainID = pd.DataFrame(df_train[trainID_column])

      # if isinstance(trainID_column, str):
      #   trainID_column = [trainID_column]
      # elif isinstance(trainID_column, list):
      #   trainID_column = trainID_column
      # else:
      #   print("error, trainID_column value must be False, str, or list")
      
      df_train_tempID.index = df_trainID.index
        
      df_trainID = pd.concat([df_trainID, df_train_tempID], axis=1)
    
      for IDcolumn in trainID_column:
        del df_train[IDcolumn]
      
      #then append the indexcolumn to trainID_column list for use in later methods
      trainID_column.append(indexcolumn)
      
    else:
      df_train_tempID.index = df_train.index
      df_trainID = df_train_tempID.copy()
      trainID_column = [indexcolumn]
      
    del df_train_tempID

    #here we derive a range integer index for inclusion in the test ID sets
    tempIDlist = []
    df_test_tempID = pd.DataFrame({indexcolumn:range(0,df_test.shape[0])})
    
    #extract the ID columns from test set
    #decided to do this stuff even if there's a dummy set for df_test 
    #to ensure downstream stuff works
    if testID_column is not False:
      df_testID = pd.DataFrame(df_test[testID_column])

      # if isinstance(testID_column, str):
      #   testID_column = [testID_column]
      # elif isinstance(testID_column, list):
      #   testID_column = testID_column
      # else:
      #   print("error, testID_column value must be False, str, or list")
      
      df_test_tempID.index = df_testID.index
      
      df_testID = pd.concat([df_testID, df_test_tempID], axis=1)

      for IDcolumn in testID_column:
        del df_test[IDcolumn]
      
      #then append the indexcolumn to testID_column list for use in later methods
      testID_column.append(indexcolumn)
      
    else:
      df_test_tempID.index = df_test.index
      df_testID = df_test_tempID.copy()
      testID_column = [indexcolumn]

    del df_test_tempID

    #carve out the validation rows

    #set randomness seed number (sorry I was trying to be cute here)
    answer = randomseed

    #ok now carve out the validation rows. We'll process these later
    #(we're processing train data from validation data seperately to
    #ensure no leakage)

    totalvalidationratio = valpercent1 + valpercent2

    if totalvalidationratio > 0.0:

      val2ratio = valpercent2 / totalvalidationratio
      
      if shuffletrain in [True, 'traintest']:
        shuffle_param=True
      else:
        shuffle_param=False

      #we'll wait to split out the validation labels
      df_train, df_validation1 = \
      self.df_split(df_train, totalvalidationratio, shuffle_param, randomseed)

      if trainID_column is not False:
        df_trainID, df_validationID1 = \
        self.df_split(df_trainID, totalvalidationratio, shuffle_param, randomseed)

      else:
        df_trainID = pd.DataFrame()
        df_validationID1 = pd.DataFrame()

      df_train = df_train.reset_index(drop=True)
      df_validation1 = df_validation1.reset_index(drop=True)
      df_trainID = df_trainID.reset_index(drop=True)
      df_validationID1 = df_validationID1.reset_index(drop=True)

    #else if total validation was <= 0.0
    else:
      df_validation1 = pd.DataFrame()
      df_validationID1 = pd.DataFrame()

    #extract labels from train set
    #an extension to this function could be to delete the training set rows\
    #where the labels are missing or improperly formatted prior to performing\
    #this step
    #initialize a helper 
    labelspresenttrain = False
    labelspresenttest = False
    single_train_column_labels_case = False

    #wasn't sure where to put this seems as a good place as any
    if labels_column is False:
      labelsencoding_dict = {}

    if labels_column is not False:
      df_labels = pd.DataFrame(df_train[labels_column])

      del df_train[labels_column]
      labelspresenttrain = True
      
      #if we only had one (label) column to begin with we'll create a dummy train set
      if df_train.shape[1] == 0:
#         df_train = df_labels[0:10].copy()
        df_train = df_labels[0:1].copy()
        single_train_column_labels_case = True

      #if the labels column is present in test set too
      if labels_column in df_test.columns:
        df_testlabels = pd.DataFrame(df_test[labels_column])
        del df_test[labels_column]
        labelspresenttest = True
        
#         #if we only had one (label) column to begin with we'll create a dummy test set
#         if df_test.shape[1] == 0:
#           df_test = df_testlabels[0:10].copy()

    if labelspresenttrain is False:
      df_labels = pd.DataFrame()
    if labelspresenttest is False:

      #we'll introduce convention that if no df_testlabels we'll create
      #a dummy set derived from df_label's first rows
#       df_testlabels = df_labels[0:10].copy()
      df_testlabels = df_labels[0:1].copy()
        
    #if we only had one (label) column to begin with we'll create a dummy test set
    if df_test.shape[1] == 0:
#       df_test = df_testlabels[0:10].copy()
      df_test = df_testlabels[0:1].copy()

    #confirm consistency of train an test sets

    #check number of columns is consistent
    if df_train.shape[1] != df_test.shape[1]:
      print("error, different number of columns in train and test sets")
      print("(This assesment excludes labels and ID columns.)")
      return

    #check column headers are consistent (this works independent of order)
    columns_train = set(list(df_train))
    columns_test = set(list(df_test))
    if columns_train != columns_test:
      print("error, different column labels in the train and test set")
      print("(This assesment excludes labels and ID columns.)")
      return

    column_labels_count = len(list(df_train))
    unique_column_labels_count = len(set(list(df_train)))
    if unique_column_labels_count < column_labels_count:
      print("error, redundant column labels found, each column requires unique label")
      return

    columns_train = list(df_train)
    columns_test = list(df_test)
    if columns_train != columns_test:
      print("error, different order of column labels in the train and test set")
      print("(This assesment excludes labels and ID columns.)")
      return

    #extract column lists again but this time as a list
    columns_train = list(df_train)
    columns_test = list(df_test)

    #create an empty dataframe to serve as a store for each column's NArows
    #the column id's for this df will follow convention from NArows of 
    #column+'_NArows' for each column in columns_train
    #these are used in the ML infill methods
    masterNArows_train = pd.DataFrame()
    masterNArows_test = pd.DataFrame()

    #create an empty dictionary to serve as store for categorical transforms lists
    #of associated columns
    multicolumntransform_dict = {}
    
    #create empty dictionary for cases where otherwise not populated with labels
    LSfitparams_dict = {}
    
    #create an empty dictionary to serve as a store of processing variables from \
    #processing that were specific to the train dataset. These can be used for \
    #future processing of a later test set without the need to reprocess the \
    #original train. The dictionary will be populated with an entry for each \
    #column post processing, and will contain a column specific and category \
    #specific (i.e. nmbr, bnry, text, date) set of variable.
    postprocess_dict = {'column_dict' : {}, 'origcolumn' : {}, \
                        'orig_noinplace' : [], \
                        'process_dict' : process_dict, \
                        'printstatus' : printstatus, \
                        'randomseed' : randomseed, \
                        'autoMLer' : autoMLer }
    
    #mirror assigncat which will populate the returned categories from eval function
    final_assigncat = deepcopy(assigncat)
    
    #create empty dictionary to serve as store for drift metrics
    drift_dict = {}
    
    #For each column, determine appropriate processing function
    #processing function will be based on evaluation of train set
    for column in columns_train:

      #re-initialize the column specific dictionary for later insertion into
      #our postprocess_dict
      column_dict = {}

      #
      categorycomplete = False

      if bool(assigncat) is True:

        for key in assigncat:
          if column in assigncat[key]:
            category = key
            category_test = key
            categorycomplete = True

            #printout display progress
            if printstatus is True:
              print("evaluating column: ", column)

            #special case, if user assigned column to 'eval' then we'll run evalcategory
            #passing a False for powertransform parameter
            if key in ['eval']:
              if evalcat is False:
                category = self.evalcategory(df_train, column, randomseed, eval_ratio, \
                                             numbercategoryheuristic, False, False)
              elif type(evalcat) == types.FunctionType:
                category = evalcat(df_train, column, randomseed, eval_ratio, \
                                   numbercategoryheuristic, False, False)
              else:
                print("error: evalcat must be passed as either False or as a defined function per READ ME")
              
              category_test = category

            #or for 'ptfm' passing a True for powertransform parameter
            if key in ['ptfm']:
              if evalcat is False:
                category = self.evalcategory(df_train, column, randomseed, eval_ratio, \
                                             numbercategoryheuristic, True, False)
              elif type(evalcat) == types.FunctionType:
                category = evalcat(df_train, column, randomseed, eval_ratio, \
                                   numbercategoryheuristic, True, False)
              else:
                print("error: evalcat must be passed as either False or as a defined function per READ ME")

              category_test = category

      #
      if categorycomplete is False:

        #printout display progress
        if printstatus is True:
          print("evaluating column: ", column)

        if evalcat is False:
          category = self.evalcategory(df_train, column, randomseed, eval_ratio, \
                                       numbercategoryheuristic, powertransform, False)
        elif type(evalcat) == types.FunctionType:
          category = evalcat(df_train, column, randomseed, eval_ratio, \
                             numbercategoryheuristic, powertransform, False)
        else:
          print("error: evalcat must be passed as either False or as a defined function per READ ME")
          
        #populate the result in the final_assigncat as informational resource
        if category in final_assigncat:
          final_assigncat[category].append(column)
        else:
          final_assigncat.update({category:[column]})

      #Previously had a few methods here to validate consistensy of data between train
      #and test sets. Found it was introducing too much complexity and was having trouble
      #keeping track of all the edge cases. So let's just make outright assumption that
      #test data if passed is consistently formatted as train data (for now)
      #added benefit that this reduces running time

      ##
      #to support the postprocess_dict entry below, let's first create a temp
      #list of columns
      templist1 = list(df_train)

      #Before calling getNArows, we'll allow user to designate either by category or column 
      #designated source column values that will be treated as infill
      #where in case of specification redundancy column designation takes precedence
      #and where category is reffering to the root category associated with a column
      #and global just means this value treated universally as nan
      #where values are passed in automunge(.) parameter assignnan
      #assignnan = {'categories':{'cat1':[], 'cat2':[]}, 'columns':{'col1':[], 'col2':[]}, 'global':[]}

      df_train = self.assignnan_convert(df_train, column, category, assignnan, postprocess_dict)
      df_test = self.assignnan_convert(df_test, column, category, assignnan, postprocess_dict)

      #we also have convention that infinity values are by default subjected to infill
      #based on understanding that ML libraries in general do not accept thesae kind of values
      df_train = self.convert_inf_to_nan(df_train, column, category, postprocess_dict)
      df_test = self.convert_inf_to_nan(df_test, column, category, postprocess_dict)

      #create NArows (column of True/False where True coresponds to missing data)
      trainNArows, drift_dict = self.getNArows(df_train, column, category, postprocess_dict, drift_dict=drift_dict, driftassess=True)
      testNArows = self.getNArows(df_test, column, category, postprocess_dict)

      #now append that NArows onto a master NA rows df
      masterNArows_train = pd.concat([masterNArows_train, trainNArows], axis=1)
      masterNArows_test = pd.concat([masterNArows_test, testNArows], axis=1)

      #printout display progress
      if printstatus is True:
        print("processing column: ", column)
        print("    root category: ", category)

      ##
      #now process family
      df_train, df_test, postprocess_dict = \
      self.processfamily(df_train, df_test, column, category, category, process_dict, \
                        transform_dict, postprocess_dict, assign_param)

      ##
      #now delete columns that were subject to replacement
      df_train, df_test, postprocess_dict = \
      self.circleoflife(df_train, df_test, column, category, category, process_dict, \
                        transform_dict, postprocess_dict, templist1)
      ##
      #here's another templist to support the postprocess_dict entry below
      templist2 = list(df_train)

      #ok now we're going to pick one of the new entries in templist2 to serve 
      #as a "columnkey" for pulling datas from the postprocess_dict down the road
      #columnkeylist = list(set(templist2) - set(templist1))[0]
      #columnkeylist = list(set(templist2) - set(templist1))
      columnkeylist = []
      for templist2_entry in templist2:
        if templist2_entry not in templist1:
          columnkeylist.append(templist2_entry)

#       #now we'll apply the floatprecision transformation
#       df_train = self.floatprecision_transform(df_train, columnkeylist, floatprecision)
#       df_test = self.floatprecision_transform(df_test, columnkeylist, floatprecision)
      #(floatprocsiion is now done at conclusion of transforms below)

      ##
      #so last line I believe returns string if only one entry, so let's run a test
      if isinstance(columnkeylist, str):
        columnkey = columnkeylist
      else:
        #if list is empty
        if len(columnkeylist) == 0:
          columnkey = column
        else:
          columnkey = columnkeylist[0]

      ##
      #we're going to create an entry to postprocess_dict to
      #store a columnkey for each of the original columns
      postprocess_dict['origcolumn'].update({column : {'category' : category, \
                                                       'columnkeylist' : columnkeylist, \
                                                       'columnkey' : columnkey}})

      ##
      #printout display progress
      if printstatus is True:
        print(" returned columns:")
        print(postprocess_dict['origcolumn'][column]['columnkeylist'])
        print("")

    #ok here's where we address labels

    if labels_column is not False:        

      #for now we'll just assume consistent processing approach for labels as for data
      #a future extension may segregate this approach

      #initialize processing dicitonaries (we'll use same as for train set)
      #a future extension may allow custom address for labels
      labelstransform_dict = transform_dict

      labelsprocess_dict = process_dict

      #we'll allow user to assign category to labels as well via assigncat call
      categorycomplete = False

      if bool(assigncat) is True:

        for key in assigncat:
          if labels_column in assigncat[key]:
            labelscategory = key
            categorycomplete = True

            #printout display progress
            if printstatus is True:
              print("______")
              print("")
              print("evaluating label column: ", labels_column)
              
            #special case, if user assigned column to 'eval' then we'll run evalcategory
            #passing a False for powertransform parameter
            if key in ['eval']:
              if evalcat is False:
                category = self.evalcategory(df_labels, labels_column, randomseed, eval_ratio, \
                                             numbercategoryheuristic, False, True)
              elif type(evalcat) == types.FunctionType:
                category = evalcat(df_labels, labels_column, randomseed, eval_ratio, \
                                   numbercategoryheuristic, False, True)
              else:
                print("error: evalcat must be passed as either False or as a defined function per READ ME")

              labelscategory = category
              
            #or for 'ptfm' passing a True for powertransform parameter
            if key in ['ptfm']:
              if evalcat is False:
                category = self.evalcategory(df_labels, labels_column, randomseed, eval_ratio, \
                                             numbercategoryheuristic, True, True)
              elif type(evalcat) == types.FunctionType:
                category = evalcat(df_labels, labels_column, randomseed, eval_ratio, \
                                   numbercategoryheuristic, True, True)
              else:
                print("error: evalcat must be passed as either False or as a defined function per READ ME")

              labelscategory = category

      if categorycomplete is False:
        
        #ok this isnt; exactly elegant, but going to add a special case for LabelSmoothing
        #first we[ll determine if labelsmoothing present
        labelsmoothingpresent = False
        if (LabelSmoothing_train > 0.0 and LabelSmoothing_train < 1.0 and str(LabelSmoothing_train) != 'False') \
        or (LabelSmoothing_test > 0.0 and LabelSmoothing_test < 1.0 and str(LabelSmoothing_test) != 'False') \
        or (LabelSmoothing_val > 0.0 and LabelSmoothing_val < 1.0 and str(LabelSmoothing_val) != 'False'):
          labelsmoothingpresent = True

        #printout display progress
        if printstatus is True:
          print("______")
          print("")
          print("evaluating label column: ", labels_column)

        #determine labels category and apply appropriate function
        #labelscategory = self.evalcategory(df_labels, labels_column, numbercategoryheuristic, powertransform)

        #we'll follow convention that default powertransform option not applied to labels
        #user can apply instead by passing column to ptfm in assigncat
        if evalcat is False:
          labelscategory = self.evalcategory(df_labels, labels_column, randomseed, eval_ratio, \
                                             numbercategoryheuristic, False, True)
        elif type(evalcat) == types.FunctionType:
          labelscategory = evalcat(df_labels, labels_column, randomseed, eval_ratio, \
                                   numbercategoryheuristic, False, True)
        else:
          print("error: evalcat must be passed as either False or as a defined function per READ ME")
          
        #special case for label smoothing we'll replace evaluated category bnry with text
        #label smoothing doesn't work with bnry, needs one hot encoding
        if labelsmoothingpresent and labelscategory == 'bnry':
          labelscategory = 'text'
          
        #populate the result in the final_assigncat as informational resource
        if labelscategory in final_assigncat:
          final_assigncat[labelscategory].append(labels_column)
        else:
          final_assigncat.update({labelscategory:[labels_column]})

      #apply assignnan_convert
      df_labels = self.assignnan_convert(df_labels, labels_column, labelscategory, assignnan, postprocess_dict)
      df_testlabels = self.assignnan_convert(df_testlabels, labels_column, labelscategory, assignnan, postprocess_dict)
      
      #apply convert_inf_to_nan
      df_labels = self.convert_inf_to_nan(df_labels, labels_column, labelscategory, postprocess_dict)
      df_testlabels = self.convert_inf_to_nan(df_testlabels, labels_column, labelscategory, postprocess_dict)

      #printout display progress
      if printstatus is True:

        print("processing label column: ", labels_column)
        print("    root label category: ", labelscategory)
        print("")

      #initialize a dictionary to serve as the store between labels and their \
      #associated encoding
      labelsencoding_dict = {labelscategory:{}}

      #to support the postprocess_dict entry below, let's first create a temp
      #list of columns
      templist1 = list(df_labels)

      #now process family
      df_labels, df_testlabels, postprocess_dict = \
      self.processfamily(df_labels, df_testlabels, labels_column, labelscategory, labelscategory, \
                        labelsprocess_dict, labelstransform_dict, postprocess_dict, assign_param)
      
      #now delete columns subject to replacement
      df_labels, df_testlabels, postprocess_dict = \
      self.circleoflife(df_labels, df_testlabels, labels_column, labelscategory, labelscategory, \
                        labelsprocess_dict, labelstransform_dict, postprocess_dict, templist1)

      #here's another templist to support the postprocess_dict entry below
      templist2 = list(df_labels)

      #ok now we're going to pick one of the new entries in templist2 to serve 
      #as a "columnkey" for pulling datas from the postprocess_dict down the road
      #columnkeylist = list(set(templist2) - set(templist1))[0]
      #columnkeylist = list(set(templist2) - set(templist1))
      columnkeylist = []
      for templist2_entry in templist2:
        if templist2_entry not in templist1:
          columnkeylist.append(templist2_entry)
      
#       #now we'll apply the floatprecision transformation
#       df_labels = self.floatprecision_transform(df_labels, columnkeylist, floatprecision)
#       df_testlabels = self.floatprecision_transform(df_testlabels, columnkeylist, floatprecision)

      if isinstance(columnkeylist, str):
        columnkey = columnkeylist
      else:
        #if list is empty
        if len(columnkeylist) == 0:
          columnkey = labels_column
        else:
          columnkey = columnkeylist[0]

      finalcolumns_labels = list(df_labels)

      #we're going to create an entry to postprocess_dict to
      #store a columnkey for each of the original columns
      postprocess_dict['origcolumn'].update({labels_column : {'category' : labelscategory, \
                                                              'columnkeylist' : finalcolumns_labels, \
                                                              'columnkey' : columnkey}})
      
      #labelsencoding_dict is returned from automunge(.) and supports the reverse encoding of labels after predictions
      #note that postmunge inversion parameter is a cleaner method for label inversion now available
      labelsencoding_dict[labelscategory] = {}
      for finalcolumn_labels in finalcolumns_labels:
        labelsnormalization_dict = postprocess_dict['column_dict'][finalcolumn_labels]['normalization_dict']
        labelsencoding_dict[labelscategory].update(labelsnormalization_dict)

      #markers for label smoothing printouts
      trainsmoothing = False
      testsmoothing = False
        
      #apply label smoothing to train set if elected
      if LabelSmoothing_train > 0.0 and LabelSmoothing_train < 1.0 and str(LabelSmoothing_train) != 'False':
        
        trainsmoothing = True
        
        LSfitparams_dict = {}
        
        #this will be our marker to indicate if labelsmoothing is already conducted
        categorycomplete_dict = dict(zip(postprocess_dict['origcolumn'][labels_column]['columnkeylist'], [False]*len(postprocess_dict['origcolumn'][labels_column]['columnkeylist'])))
        
        #for column in label columns
        for labelsmoothingcolumn in postprocess_dict['origcolumn'][labels_column]['columnkeylist']:
          
          if categorycomplete_dict[labelsmoothingcolumn] is False:

            label_categorylist = postprocess_dict['column_dict'][labelsmoothingcolumn]['categorylist']
            label_category = postprocess_dict['column_dict'][labelsmoothingcolumn]['category']
            
            df_labels, categorycomplete_dict, LSfitparams_dict = \
            self.apply_LabelSmoothing(df_labels, labelsmoothingcolumn, LabelSmoothing_train, label_categorylist, label_category, categorycomplete_dict, LSfit, LSfitparams_dict)
  
      #else prepare empty LSfitparams_dict
      else:
        
        LSfitparams_dict = {}
        
        for labelsmoothingcolumn in postprocess_dict['origcolumn'][labels_column]['columnkeylist']:
          
          LSfitparams_dict.update({labelsmoothingcolumn : {'LSfit' : LSfit, \
                                                          'epsilon' : LabelSmoothing_train, \
                                                          'label_categorylist' : postprocess_dict['column_dict'][labelsmoothingcolumn]['categorylist'], \
                                                          'label_category' : postprocess_dict['column_dict'][labelsmoothingcolumn]['category']}})
          
      #marker
      match_testLS_to_train = False
      
      #apply label smoothing to test set if elected
      if str(LabelSmoothing_test) == 'True':
        LabelSmoothing_test = LabelSmoothing_train
        match_testLS_to_train = True
      
      if LabelSmoothing_test > 0.0 and LabelSmoothing_test < 1.0 and str(LabelSmoothing_test) != 'False':
        
        testsmoothing = True
        
        #this will be our marker to indicate if labelsmoothing is already conducted
        categorycomplete_dict = dict(zip(postprocess_dict['origcolumn'][labels_column]['columnkeylist'], [False]*len(postprocess_dict['origcolumn'][labels_column]['columnkeylist'])))                                   
        
        #for column in label columns
        for labelsmoothingcolumn in postprocess_dict['origcolumn'][labels_column]['columnkeylist']:
          
          if categorycomplete_dict[labelsmoothingcolumn] is False:
            
            if match_testLS_to_train is False:
                                     
              label_categorylist = postprocess_dict['column_dict'][labelsmoothingcolumn]['categorylist']
              label_category = postprocess_dict['column_dict'][labelsmoothingcolumn]['category']

              df_testlabels, categorycomplete_dict, _1 = \
              self.apply_LabelSmoothing(df_testlabels, labelsmoothingcolumn, LabelSmoothing_test, label_categorylist, label_category, categorycomplete_dict, LSfit, {})
              
              del _1
              
            else:
              
              df_testlabels, categorycomplete_dict = \
              self.postapply_LabelSmoothing(df_testlabels, labelsmoothingcolumn, categorycomplete_dict, LSfitparams_dict)

      #printout display progress
      if printstatus is True:
        print(" returned columns:")
        print(postprocess_dict['origcolumn'][labels_column]['columnkeylist'])
        print("")
        
        if trainsmoothing or testsmoothing:
          
          if trainsmoothing and testsmoothing:
            
            print("Label Smoothing applied to both train and test set labels")
            print("")
          
          elif trainsmoothing:
            
            print("Label Smoothing applied to train set labels")
            print("")
            
          elif testsmoothing:
            
            print("Label Smoothing applied to test set labels")
            print("")

    #now that all transforms are applied on training set and labels
    #we'll do quick validation of normalization_dict entries
    
    check_normalization_dict_result = \
    self.check_normalization_dict(postprocess_dict)

    miscparameters_results.update({'check_normalization_dict_result' : check_normalization_dict_result})

    #now that we've pre-processed all of the columns, let's run through them again\
    #using infill to derive plug values for the previously missing cells
    
    #printout display progress
    if printstatus is True:
      print("______")
      print("")

    infillcolumns_list = list(df_train)

    #Here is the application of assemblepostprocess_assigninfill
    #This assembles the posttransform column headers for each infill type
    postprocess_assigninfill_dict = \
    self.assemblepostprocess_assigninfill(assigninfill, infillcolumns_list, \
                                          columns_train, postprocess_dict, MLinfill)

    #now apply infill
    df_train, df_test, postprocess_dict, infill_validations = \
    self.apply_am_infill(df_train, df_test, postprocess_assigninfill_dict, \
                        postprocess_dict, infilliterate, printstatus, infillcolumns_list, \
                        masterNArows_train, masterNArows_test, process_dict, randomseed, ML_cmnd)

    miscparameters_results.update(infill_validations)

    #quickly gather a list of columns before any dimensionalioty reductions for populating mirror trees
    pre_dimred_finalcolumns_train = list(df_train)
    
    #Here's where we'll trim the columns that were stricken as part of featureselection method

    #trim branches here associated with featureselect

    if featureselection is True:

      #get list of columns currently included
      currentcolumns = list(df_train)
      
      #this is to address an edge case for featuremethod == 'default'
      if featuremethod in ['default', 'report'] or FSmodel is False \
      or (featuremethod in ['pct'] and featurepct == 1.0):
        madethecut = currentcolumns

      #get list of columns to trim
      trimcolumns = [b for b in currentcolumns if b not in madethecut]

      if len(trimcolumns) > 0:
        #printout display progress
        if printstatus is True:
          print("_______________")
          print("Begin feature importance dimensionality reduction")
          print("")
          print("   method: ", featuremethod)
  #           if featuremethod == 'default':
  #             print("no feature importance dimensionality reductions")          
          if featuremethod == 'pct':
            print("threshold: ", featurepct)
          if featuremethod == 'metric':
            print("threshold: ", featuremetric)
          if featuremethod == 'report':
            print("only returning results")
          print("")
          print("trimmed columns: ")
          print(trimcolumns)
          print("")

      #trim columns
      for trimmee in trimcolumns:

        del df_train[trimmee]
        del df_test[trimmee]
        
      if len(trimcolumns) > 0:
        if printstatus is True:
          print("returned columns: ")
          print(list(df_train))
          print("")

    prePCAcolumns = list(df_train)
    
    #marker if PCA applied
    PCA_applied = False

    #if user passed anything to automunbge argument PCAn_components 
    #(either the number of columns integer or a float between 0-1)

    #ok this isn't the cleanest implementation, fixing that we may want to 
    #assign a new n_components

    n_components = PCAn_components

    if n_components is not False:
      if ML_cmnd['PCA_type'] == 'default':

        _1, n_components = \
        self.evalPCA(df_train, PCAn_components, ML_cmnd)

    else:
      n_components = None
      PCAn_components = None

    #if PCAn_components != None:
    if n_components != None:

      #this is for cases where automated PCA methods performed and we want to carry through 
      #results to postmunge through postprocess_dict
      PCAn_components = n_components

      #If user passed bool_PCA_excl as True in ML_cmnd['PCA_cmnd']
      #Then add boolean columns to the PCAexcl list of columns
      #and bool_PCAexcl just tracks what columns were added

      #PCAexcl, bool_PCAexcl = self.boolexcl(ML_cmnd, df_train, PCAexcl)
      if 'bool_PCA_excl' in ML_cmnd['PCA_cmnd'] \
      or 'bool_ordl_PCAexcl' in ML_cmnd['PCA_cmnd']:
        PCAexcl, bool_PCAexcl = self.boolexcl(ML_cmnd, df_train, PCAexcl, postprocess_dict)
      else:
        bool_PCAexcl = []

      #only perform PCA if the specified/defrived number of columns < the number of
      #columns after removing the PCAexcl columns
      #if (n_components < len(list(df_train)) - len(PCAexcl) and n_components >= 1.0):
      if n_components < (len(list(df_train)) - len(PCAexcl)) \
      and (n_components != 0) \
      and (n_components != None) \
      and (n_components is not False):

        #printout display progress
        if printstatus is True:
          print("_______________")
          print("Applying PCA dimensionality reduction")
          print("")
          print("Before PCA train set column count:")
          print(len(df_train.columns))
          print()
          if len(bool_PCAexcl) > 0:
            print("columns excluded from PCA: ")
            print(bool_PCAexcl)
            print("")
        
        #PCA applied marker set to true
        PCA_applied = True

        #this is to carve the excluded columns out from the set
        PCAset_train, PCAset_test, PCAexcl_posttransform = \
        self.createPCAsets(df_train, df_test, PCAexcl, postprocess_dict)

        #this is to train the PCA model and perform transforms on train and test set
        PCAset_train, PCAset_test, postprocess_dict, PCActgy = \
        self.PCAfunction(PCAset_train, PCAset_test, PCAn_components, postprocess_dict, \
                         randomseed, ML_cmnd)

        #printout display progress
        if printstatus is True:
          print("PCA model applied: ")
          print(PCActgy)
          print("")

        PCA_suffixoverlap_results = \
        self.df_check_suffixoverlap(df_train, list(PCAset_train), suffixoverlap_results = {})

        miscparameters_results.update({'PCA_suffixoverlap_results':PCA_suffixoverlap_results})

        #reattach the excluded columns to PCA set
        df_train = pd.concat([PCAset_train.set_index(df_train.index), df_train[PCAexcl_posttransform]], axis=1)
        df_test = pd.concat([PCAset_test.set_index(df_test.index), df_test[PCAexcl_posttransform]], axis=1)

        returned_PCA_columns = list(PCAset_train)

        del PCAset_train
        del PCAset_test

        #printout display progress
        if printstatus is True:
          print("returned PCA columns: ")
          print(returned_PCA_columns)
          print("")
          print("After PCA train set column count:")
          print(len(df_train.columns))
          print()

      else:
        #else we'll just populate the PCAmodel slot in postprocess_dict with a placeholder
        postprocess_dict.update({'PCAmodel' : None})
        returned_PCA_columns = []

    else:
      #else we'll just populate the PCAmodel slot in postprocess_dict with a placeholder
      postprocess_dict.update({'PCAmodel' : None})
      returned_PCA_columns = []

      miscparameters_results.update({'PCA_suffixoverlap_results':{}})

    #Binary dimensionality reduction goes here
    
    #we'll only apply to training and test data not labels
    #making an executive decvision for now that ordinal encoded columns will be excluded
    if Binary in [True, 'retain'] or isinstance(Binary, list):
      
      #printout display progress
      if printstatus is True:
        print("_______________")
        print("Begin Binary dimensionality reduction")
        print("")
        print("Before Binary train set column count = ")
        print(df_train.shape[1])
        print("")
      
      if isinstance(Binary, list):
        
        temp_Binary = True
        
        #check for any first entry signaling replace vs retain, 
        #True is replace, False is retain, no boolean first entry defaults to replace
        if isinstance(Binary[0], bool):
          if Binary[0] is True:  
            temp_Binary = True
          else:
            temp_Binary = 'retain'
          del Binary[0]
        
        #for Binary need returned columns
        Binary_copy = Binary.copy()
        for entry in Binary_copy:
          if entry in columns_train:
            Binary.remove(entry)
            Binary += postprocess_dict['origcolumn'][entry]['columnkeylist']
        
        #consolidate any redundancies
        Binary_copy = Binary.copy()
        Binary = []
        for entry in Binary_copy:
          if entry not in Binary:
            Binary.append(entry)
        
        Binary_target_columns = Binary.copy()
        Binary = temp_Binary
      
      else:
        
        Binary_target_columns = list(df_train)
        
      bool_column_list = []
      
      for column in Binary_target_columns:
        
        if column in postprocess_dict['column_dict']:

          column_category = postprocess_dict['column_dict'][column]['category']

          if process_dict[column_category]['MLinfilltype'] in \
          ['multirt', 'binary', '1010', 'boolexclude', 'concurrent_act']:

            bool_column_list.append(column)
            
      if printstatus is True:
        print("Consolidating boolean columns:")
        print(bool_column_list)
        print()
          
      df_train, df_test, Binary_dict = self.Binary_convert(df_train, df_test, bool_column_list, Binary)

      returned_Binary_columns = list(Binary_dict['column_dict'])
      returned_Binary_columns.remove('Binary')

      #aggregate suffix overlap validations
      Binary_suffixoverlap_results = {}
      for entry in Binary_dict['column_dict']:
        Binary_suffixoverlap_results.update(Binary_dict['column_dict'][entry]['suffixoverlap_results'])
        
      miscparameters_results.update({'Binary_suffixoverlap_results' : Binary_suffixoverlap_results})
      
      if printstatus is True:
        print("Boolean column count = ")
        print(len(bool_column_list))
        print("")
        print("Returned Binary columns:")
        print(returned_Binary_columns)
        print()
        print("Returned Binary column count = ")
        print(len(returned_Binary_columns))
        print("")
        print("After Binary train set column count = ")
        print(df_train.shape[1])
        print("")
      
    else:
      
      Binary_dict = {'bool_column_list' : [], 'column_dict' : {}}
      miscparameters_results.update({'Binary_suffixoverlap_results' : {}})
      returned_Binary_columns = []

    #here is the process to levelize the frequency of label rows in train data
    #currently only label categories of 'bnry' or 'text' are considered
    #a future extension will include numerical labels by adding supplemental 
    #label columns to designate inclusion in some fractional bucket of the distribution
    #e.g. such as quintiles for instance
    if TrainLabelFreqLevel in [True, 'traintest'] \
    and labels_column is not False:

      #printout display progress
      if printstatus is True:
        print("_______________")
        print("Begin label rebalancing")
        print("")
        print("Before rebalancing train set row count = ")
        print(df_labels.shape[0])
        print("")

      if trainID_column is not False:
        df_train = pd.concat([df_train, df_trainID], axis=1)                        
        
      #apply LabelFrequencyLevelizer defined function
      df_train, df_labels = \
      self.LabelFrequencyLevelizer(df_train, df_labels, \
                                   postprocess_dict, process_dict, LabelSmoothing_train)

      #extract trainID
      if trainID_column is not False:

        df_trainID = pd.DataFrame(df_train[trainID_column])

        if isinstance(trainID_column, str):
          tempIDlist = [trainID_column]
        elif isinstance(trainID_column, list):
          tempIDlist = trainID_column
        for IDcolumn in tempIDlist:
          del df_train[IDcolumn]

      #printout display progress
      if printstatus is True:

        print("")
        print("After rebalancing train set row count = ")
        print(df_labels.shape[0])
        print("")

    if TrainLabelFreqLevel in ['test', 'traintest'] \
    and labelspresenttest is True:

      #printout display progress
      if printstatus is True:
        print("_______________")
        print("Begin test set label rebalancing")
        print("")
        print("Before rebalancing test set row count = ")
        print(df_testlabels.shape[0])
        print("")

      if testID_column is not False:
        df_test = pd.concat([df_test, df_testID], axis=1)                        
        
      if LabelSmoothing_test is True:
        LabelSmoothing_test = LabelSmoothing_train

      #apply LabelFrequencyLevelizer defined function
      df_test, df_testlabels = \
      self.LabelFrequencyLevelizer(df_test, df_testlabels, \
                                   postprocess_dict, process_dict, LabelSmoothing_test)
        
      #extract testID
      if testID_column is not False:

        df_testID = pd.DataFrame(df_test[testID_column])

        if isinstance(testID_column, str):
          tempIDlist = [testID_column]
        elif isinstance(testID_column, list):
          tempIDlist = testID_column
        for IDcolumn in tempIDlist:
          del df_test[IDcolumn]
          
      #printout display progress
      if printstatus is True:

        print("")
        print("After rebalancing test set row count = ")
        print(df_testlabels.shape[0])
        print("")

    #then if shuffle was elected perform here    
    if shuffletrain is True or shuffletrain == 'traintest':
      
      #shuffle training set and labels
      df_train = self.df_shuffle(df_train, answer)
      
      if labels_column is not False:
        df_labels = self.df_shuffle(df_labels, answer)

      if trainID_column is not False:
        df_trainID = self.df_shuffle(df_trainID, answer)
      
    if shuffletrain == 'traintest':
      
      df_test = self.df_shuffle(df_test, answer)
      
      if labelspresenttest is True:
        df_testlabels = self.df_shuffle(df_testlabels, answer)

      if testID_column is not False:
        df_testID = self.df_shuffle(df_testID, answer)

    #great the data is processed now let's do a few moore global training preps

    #a special case, those columns that we completely excluded from processing via excl
    #we'll scrub the suffix appender
    
    #first let's create a list of excl columns with and without suffix, just in case might come in handy
    postprocess_dict.update({'excl_columns_with_suffix':[], 'excl_columns_without_suffix':[]})
    for cd_column in postprocess_dict['column_dict']:
      if postprocess_dict['column_dict'][cd_column]['category'] == 'excl':
        postprocess_dict['excl_columns_with_suffix'].append(cd_column)
        postprocess_dict['excl_columns_without_suffix'].append(cd_column[:-5])
        
    if excl_suffix is False:
      #we'll duplicate excl columns in postprocess_dict['column_dict'] to list both with and without suffix as keys
      #I don't think this redundant entry is used anywhere, just thought might prove helpful downstream
      for excl_column_with_suffix in postprocess_dict['excl_columns_with_suffix']:
        excl_index = postprocess_dict['excl_columns_with_suffix'].index(excl_column_with_suffix)
        excl_column_without_suffix = postprocess_dict['excl_columns_without_suffix'][excl_index]
        postprocess_dict['column_dict'].update({excl_column_without_suffix : \
                                                deepcopy(postprocess_dict['column_dict'][excl_column_with_suffix])})

    if excl_suffix is False:
      #run a quick suffix overlap validation before changing excl headers
      excl_suffixoverlap_results = \
      self.df_check_suffixoverlap(df_train, postprocess_dict['excl_columns_without_suffix'], suffixoverlap_results = {})
      miscparameters_results.update({'excl_suffixoverlap_results' : excl_suffixoverlap_results})
    else:
      miscparameters_results.update({'excl_suffixoverlap_results' : {}})

    if excl_suffix is False:
      df_train.columns = [column[:-5] if column in postprocess_dict['column_dict'] and \
                          postprocess_dict['column_dict'][column]['category'] == 'excl' \
                          else column for column in df_train.columns]
      df_test.columns = [column[:-5] if column in postprocess_dict['column_dict'] and \
                         postprocess_dict['column_dict'][column]['category'] == 'excl' \
                         else column for column in df_test.columns]
      
    if labels_column is not False and excl_suffix is False:
      df_labels.columns = [column[:-5] if column in postprocess_dict['column_dict'] and \
                          postprocess_dict['column_dict'][column]['category'] == 'excl' \
                          else column for column in df_labels.columns]

    if labelspresenttest is True and excl_suffix is False:
      df_testlabels.columns = [column[:-5] if column in postprocess_dict['column_dict'] and \
                              postprocess_dict['column_dict'][column]['category'] == 'excl' \
                              else column for column in df_testlabels.columns]

    #this is admiuttedly kind of a weird spot to put this, we had introduced earlier
    #convention of a dummy testlabels set, here we'll delete if no labels in test set
    #(seems as good a place as any)
    elif labelspresenttest is False:
      df_testlabels = pd.DataFrame()

    #here's a list of final column names saving here since the translation to \
    #numpy arrays scrubs the column names
    finalcolumns_train = list(df_train)
    finalcolumns_test = list(df_test)

    #we'll create some tags specific to the application to support postprocess_dict versioning
    automungeversion = '5.22'
#     application_number = random.randint(100000000000,999999999999)
#     application_timestamp = dt.datetime.now().strftime("%Y-%m-%dT%H:%M:%S.%f")
    version_combined = '_' + str(automungeversion) + '_' + str(application_number) + '_' \
                       + str(application_timestamp)

    #here we'll populate the postprocess_dictt that is returned from automunge
    #as it will be used in the postmunge call below to process validation sets
    postprocess_dict.update({'origtraincolumns' : columns_train, \
                             'origcolumns_all' : origcolumns_all, \
                             'finalcolumns_train' : finalcolumns_train, \
                             'pre_dimred_finalcolumns_train' : pre_dimred_finalcolumns_train, \
                             'labels_column' : labels_column, \
                             'finalcolumns_labels' : list(df_labels), \
                             'single_train_column_labels_case' : single_train_column_labels_case, \
                             'trainID_column_orig' : trainID_column_orig, \
                             'trainID_column' : trainID_column, \
                             'finalcolumns_trainID' : list(df_trainID), \
                             'testID_column_orig' : testID_column_orig, \
                             'testID_column' : testID_column, \
                             'indexcolumn' : indexcolumn, \
                             'valpercent1' : valpercent1, \
                             'valpercent2' : valpercent2, \
                             'floatprecision' : floatprecision, \
                             'shuffletrain' : shuffletrain, \
                             'TrainLabelFreqLevel' : TrainLabelFreqLevel, \
                             'MLinfill' : MLinfill, \
                             'infilliterate' : infilliterate, \
                             'eval_ratio' : eval_ratio, \
                             'powertransform' : powertransform, \
                             'binstransform' : binstransform, \
                             'LabelSmoothing_train' : LabelSmoothing_train, \
                             'LabelSmoothing_test' : LabelSmoothing_test, \
                             'LabelSmoothing_val' : LabelSmoothing_val, \
                             'LSfit' : LSfit, \
                             'LSfitparams_dict' : LSfitparams_dict, \
                             'numbercategoryheuristic' : numbercategoryheuristic, \
                             'pandasoutput' : pandasoutput, \
                             'NArw_marker' : NArw_marker, \
                             'labelsencoding_dict' : labelsencoding_dict, \
                             'featureselection' : featureselection, \
                             'featurepct' : featurepct, \
                             'featuremetric' : featuremetric, \
                             'featuremethod' : featuremethod, \
                             'FSmodel' : FSmodel, \
                             'FScolumn_dict' : FScolumn_dict, \
                             'FS_sorted' : FS_sorted, \
                             'drift_dict' : drift_dict, \
                             'Binary' : Binary, \
                             'Binary_dict' : Binary_dict, \
                             'returned_Binary_columns' : returned_Binary_columns, \
                             'PCA_applied' : PCA_applied, \
                             'PCAn_components' : PCAn_components, \
                             'PCAexcl' : PCAexcl, \
                             'prePCAcolumns' : prePCAcolumns, \
                             'returned_PCA_columns' : returned_PCA_columns, \
                             'madethecut' : madethecut, \
                             'excl_suffix' : excl_suffix, \
                             'traindata' : False, \
                             'assigncat' : assigncat, \
                             'final_assigncat' : final_assigncat, \
                             'assigninfill' : assigninfill, \
                             'transformdict' : transformdict, \
                             'transform_dict' : transform_dict, \
                             'processdict' : processdict, \
                             'process_dict' : process_dict, \
                             'postprocess_assigninfill_dict' : postprocess_assigninfill_dict, \
                             'assignparam' : assignparam, \
                             'assign_param' : assign_param, \
                             'assignnan' : assignnan, \
                             'ML_cmnd' : ML_cmnd, \
                             'miscparameters_results' : miscparameters_results, \
                             'printstatus' : printstatus, \
                             'automungeversion' : automungeversion, \
                             'application_number' : application_number, \
                             'application_timestamp' : application_timestamp, \
                             'version_combined' : version_combined})
    
    #mirror tree assembly functions go here, these mirror the progression of transformation functions
    #where categorytree is forward pass and inverse_categorytree is backward pass
    
    #don't currently use categorytree explicitly but populating in case downstream users find use
    categorytree = self.populate_categorytree(postprocess_dict)
    
    #the inverse tree supports inversion in postmunge
    inverse_categorytree = self.populate_inverse_categorytree(postprocess_dict)
    
    #we'll create another structure, this one flatted, similar to origcolumn or column_dict
    #this one as inputcolumn_dict
    inputcolumn_dict = self.populate_inputcolumn_dict(postprocess_dict)
    
    #the trees are returned in postprocess_dict
    postprocess_dict.update({'categorytree' : categorytree, \
                             'inverse_categorytree' : inverse_categorytree, \
                             'inputcolumn_dict' : inputcolumn_dict})

    #populate a report for column types of returned set
    columntype_report = \
    self.populate_columntype_report(postprocess_dict, postprocess_dict['finalcolumns_train'])
    postprocess_dict.update({'columntype_report' : columntype_report})

    label_columntype_report = \
    self.populate_columntype_report(postprocess_dict, postprocess_dict['finalcolumns_labels'])
    postprocess_dict.update({'label_columntype_report' : label_columntype_report})

    column_map = \
    self.populate_column_map_report(postprocess_dict)
    postprocess_dict.update({'column_map' : column_map})

    finalcolumns_labels = list(df_labels)
    
    finalcolumns_trainID = list(df_trainID)
    finalcolumns_testID = list(df_testID)    
    
    #now if user selects privacy preserving encodings, we'll update the headers
    #since any received integers will have been converted to string, we'll use integers
    #for privacy headers to avoid overlap
    
    privacy_headers_train = list(range(len(finalcolumns_train)))
    
    privacy_headers_train_dict = dict(zip(finalcolumns_train, privacy_headers_train))
    inverse_privacy_headers_train_dict = {value:key for key,value in privacy_headers_train_dict.items()}
    
    len_privacy_headers_train = len(privacy_headers_train)
    
    if postprocess_dict['labels_column'] is not False:

      privacy_headers_labels = list(range(len(finalcolumns_labels)))
      len_privacy_headers_labels = len(finalcolumns_trainID)
      privacy_headers_labels = [x + len_privacy_headers_train for x in privacy_headers_labels]

      privacy_headers_labels_dict = dict(zip(finalcolumns_labels, privacy_headers_labels))
      inverse_privacy_headers_labels_dict = {value:key for key,value in privacy_headers_labels_dict.items()}
    
    else:
      
      privacy_headers_labels = []
      len_privacy_headers_labels = 0
      privacy_headers_labels_dict = {}
      inverse_privacy_headers_labels_dict = {}
    
    #at a minimum ID sets will contain automunge index column
    privacy_headers_trainID = list(range(len(finalcolumns_trainID)))
    len_privacy_headers_trainID = len(privacy_headers_trainID)
    privacy_headers_trainID = \
    [x + len_privacy_headers_train + len_privacy_headers_labels for x in privacy_headers_trainID]
    
    privacy_headers_trainID_dict = dict(zip(finalcolumns_trainID, privacy_headers_trainID))
    inverse_privacy_headers_trainID_dict = {value:key for key,value in privacy_headers_trainID_dict.items()}
    
    if finalcolumns_testID == finalcolumns_trainID:
      
      privacy_headers_testID = privacy_headers_trainID
      privacy_headers_testID_dict = deepcopy(privacy_headers_trainID_dict)
      inverse_privacy_headers_testID_dict = deepcopy(inverse_privacy_headers_trainID_dict)
      
    else:
      
      privacy_headers_testID = list(range(len(finalcolumns_testID)))
      privacy_headers_testID = \
      [x + len_privacy_headers_train + len_privacy_headers_labels + len_privacy_headers_trainID \
       for x in privacy_headers_testID]
      
      privacy_headers_testID_dict = dict(zip(finalcolumns_testID, privacy_headers_testID))
      inverse_privacy_headers_testID_dict = {value:key for key,value in privacy_headers_testID_dict.items()}
    
    del len_privacy_headers_train, len_privacy_headers_labels, len_privacy_headers_trainID
    
    if privacy_encode is True:
      
      df_train = df_train.rename(columns = privacy_headers_train_dict)
      df_test = df_test.rename(columns = privacy_headers_train_dict)
      
      df_labels = df_labels.rename(columns = privacy_headers_labels_dict)
      df_testlabels = df_testlabels.rename(columns = privacy_headers_labels_dict)
      
      df_trainID = df_trainID.rename(columns = privacy_headers_trainID_dict)
      df_testID = df_testID.rename(columns = privacy_headers_testID_dict)
      
    #now add entries to postprocess_dict
    postprocess_dict.update({'privacy_encode' : privacy_encode})
    postprocess_dict.update({'privacy_headers_train' : privacy_headers_train})
    postprocess_dict.update({'privacy_headers_train_dict' : privacy_headers_train_dict})
    postprocess_dict.update({'inverse_privacy_headers_train_dict' : inverse_privacy_headers_train_dict})
    postprocess_dict.update({'privacy_headers_labels' : privacy_headers_labels})
    postprocess_dict.update({'privacy_headers_labels_dict' : privacy_headers_labels_dict})
    postprocess_dict.update({'inverse_privacy_headers_labels_dict' : inverse_privacy_headers_labels_dict})
    postprocess_dict.update({'privacy_headers_trainID' : privacy_headers_trainID})
    postprocess_dict.update({'privacy_headers_trainID_dict' : privacy_headers_trainID_dict})
    postprocess_dict.update({'inverse_privacy_headers_trainID_dict' : inverse_privacy_headers_trainID_dict})
    postprocess_dict.update({'privacy_headers_testID' : privacy_headers_testID})
    postprocess_dict.update({'privacy_headers_testID_dict' : privacy_headers_testID_dict})
    postprocess_dict.update({'inverse_privacy_headers_testID_dict' : inverse_privacy_headers_testID_dict})

    if totalvalidationratio > 0:

      #printout display progress
      if printstatus is True:
        print("_______________")
        print("Begin Validation set processing with Postmunge")
        print("")

      #(postmunge shuffletrain not needed since data was already shuffled)

      #process validation set consistent to train set with postmunge here
      #df_validation1, _2, _3, _4, _5 = \
      df_validation1, _2, df_validationlabels1, _4, _5 = \
      self.postmunge(postprocess_dict, df_validation1, testID_column = False, \
                    labelscolumn = labels_column, pandasoutput = True, printstatus = printstatus, \
                    LabelSmoothing = LabelSmoothing_val, LSfit = LSfit, \
                    shuffletrain = False)

    #ok now that validation is processed, carve out second validation set if one was elected

    if totalvalidationratio > 0.0:

      if val2ratio > 0.0:

        if labels_column is not False:
          #split validation2 sets from training and labels
          
          df_validation1, df_validation2 = \
          self.df_split(df_validation1, val2ratio, False, answer)
          
          df_validationlabels1, df_validationlabels2 = \
          self.df_split(df_validationlabels1, val2ratio, False, answer)

        else:

          df_validation1, df_validation2 = \
          self.df_split(df_validation1, val2ratio, False, answer)

          df_validationlabels2 = pd.DataFrame()

        if trainID_column is not False:
          df_validationID1, df_validationID2 = \
          self.df_split(df_validationID1, val2ratio, False, answer)
          
        else:
          df_trainID = pd.DataFrame()
          df_validationID2 = pd.DataFrame()

      else:
        df_validation2 = pd.DataFrame()
        df_validationlabels2 = pd.DataFrame()
        df_validationID2 = pd.DataFrame()

    #else if totalvalidationratio <= 0.0
    else:
      df_validation1 = pd.DataFrame()
      df_validationlabels1 = pd.DataFrame()
      df_validationID1 = pd.DataFrame()
      df_validation2 = pd.DataFrame()
      df_validationlabels2 = pd.DataFrame()
      df_validationID2 = pd.DataFrame()

    if testID_column is not False:
      df_testID = df_testID
    else:
      df_testID = pd.DataFrame()

    #now if user never passed a test set and we just created a dummy set 
    #then reset returned test sets to empty
    if test_plug_marker is True:
      df_test = pd.DataFrame()
      df_testID = pd.DataFrame()    

    #now we'll apply the floatprecision transformation
    floatcolumns_train = finalcolumns_train
    floatcolumns_labels = finalcolumns_labels
    if privacy_encode is True:
      floatcolumns_train = privacy_headers_train
      floatcolumns_labels = privacy_headers_labels

    df_train = self.floatprecision_transform(df_train, floatcolumns_train, floatprecision)
    if test_plug_marker is False:
      df_test = self.floatprecision_transform(df_test, floatcolumns_train, floatprecision)
    if labels_column is not False:
      # finalcolumns_labels = list(df_labels)
      df_labels = self.floatprecision_transform(df_labels, floatcolumns_labels, floatprecision)
      if labelspresenttest is True:
        df_testlabels = self.floatprecision_transform(df_testlabels, floatcolumns_labels, floatprecision)

    #printout display progress
    if printstatus is True:

      print("______")
      print("")
      print("versioning serial stamp:")
      print(version_combined)
      print("")

      if df_trainID.empty is False:
        print("Automunge returned ID column set: ")
        print(list(df_trainID))
        print("")

      print("Automunge returned train column set: ")
      print(list(df_train))
      print("")

      if df_labels.empty is False:
        print("Automunge returned label column set: ")
        print(list(df_labels))
        print("")
          
    #else set output to numpy arrays
    if pandasoutput is False:

      df_train = df_train.values
      df_trainID = df_trainID.values
      df_labels = df_labels.values
      df_testlabels = df_testlabels.values
      df_validation1 = df_validation1.values
      df_validationID1 = df_validationID1.values
      df_validationlabels1 = df_validationlabels1.values
      df_validation2 = df_validation2.values
      df_validationID2 = df_validationID2.values
      df_validationlabels2 = df_validationlabels2.values
      df_test = df_test.values
      df_testID = df_testID.values

      #apply ravel to labels if appropriate - converts from eg [[1,2,3]] to [1,2,3]
      if df_labels.ndim == 2 and df_labels.shape[1] == 1:
        df_labels = np.ravel(df_labels)
      if df_validationlabels1.ndim == 2 and df_validationlabels1.shape[1] == 1:
        df_validationlabels1 = np.ravel(df_validationlabels1)
      if df_validationlabels2.ndim == 2 and df_validationlabels2.shape[1] == 1:
        df_validationlabels2 = np.ravel(df_validationlabels2)

    #then at completion of automunge(.), aggregate the suffixoverlap results
    #and do an additional printout if any column overlap error to be sure user sees message
    postprocess_dict = self.suffix_overlap_final_aggregation_and_printouts(postprocess_dict)
        
    #a reasonable extension would be to perform some validation functions on the\
    #sets here (or also prior to transform to numpy arrays) and confirm things \
    #like consistency between format of columns and data between our train and \
    #test sets and if any issues return a coresponding error message to alert user

    #printout display progress
    if printstatus is True:

      print("_______________")
      print("Automunge Complete")
      print("")

    return df_train, df_trainID, df_labels, \
    df_validation1, df_validationID1, df_validationlabels1, \
    df_validation2, df_validationID2, df_validationlabels2, \
    df_test, df_testID, df_testlabels, \
    labelsencoding_dict, finalcolumns_train, finalcolumns_test,  \
    FScolumn_dict, postprocess_dict

  def postprocessfamily(self, df_test, column, category, origcategory, process_dict, \
                        transform_dict, postprocess_dict, columnkey, assign_param):
    '''
    #as postmunge runs a for loop through each column, this is the  
    #first tier processing function applied which runs through the family primitives
    #populated in the transform_dict by assembletransformdict.
    #a little simpler than the processfamily version in automunge
    #since doesn't have to populate data structures, just applied transforms
    
    #we will run in order of
    #siblings, cousins, parents, auntsuncles
    '''

    inplaceperformed = False
    
    #final upstream transform from parents or auntsuncles is elligible for inplace
    #as long as no supplement transforms were applied
    final_upstream = False
    if len(transform_dict[category]['auntsuncles']) == 0:
      if len(transform_dict[category]['parents']) > 0:
        final_upstream = transform_dict[category]['parents'][-1]
    else:
      if len(transform_dict[category]['auntsuncles']) > 0:
        final_upstream = transform_dict[category]['auntsuncles'][-1]
        
    #process the siblings (with downstream, supplemental)
    for sibling in transform_dict[category]['siblings']:

  #       print("sibling = ", sibling)

      if sibling != None:
        #note we use the processparent function here
        df_test = \
        self.postprocessparent(df_test, column, sibling, origcategory, final_upstream, process_dict, \
                              transform_dict, postprocess_dict, columnkey, assign_param)
  
    #process the cousins (no downstream, supplemental)
    for cousin in transform_dict[category]['cousins']:

  #       print("cousin = ", cousin)

      if cousin != None:
        #note we use the processsibling function here
        df_test = \
        self.postprocesscousin(df_test, column, cousin, origcategory, final_upstream, process_dict, \
                                transform_dict, postprocess_dict, columnkey, assign_param)
  
    #process the parents (with downstream, with replacement)
    for parent in transform_dict[category]['parents']:

  #       print("parent = ", parent)

      if parent != None:
        df_test = \
        self.postprocessparent(df_test, column, parent, origcategory, final_upstream, process_dict, \
                              transform_dict, postprocess_dict, columnkey, assign_param)
        
    #process the auntsuncles (no downstream, with replacement)
    for auntuncle in transform_dict[category]['auntsuncles']:

  #       print("auntuncle = ", auntuncle)

      if auntuncle != None:
        df_test = \
        self.postprocesscousin(df_test, column, auntuncle, origcategory, final_upstream, process_dict, \
                                transform_dict, postprocess_dict, columnkey, assign_param)

  #     #if we had replacement transformations performed then delete the original column 
  #     #(circle of life)
  #     if len(transform_dict[category]['auntsuncles']) + len(transform_dict[category]['parents']) > 0:
  #       del df_test[column]

    return df_test

  def postcircleoflife(self, df_test, column, category, origcategory, process_dict, \
                        transform_dict, postprocess_dict, columnkey):
    '''
    This functino deletes source columns for family primitives that included replacement.
    '''

    #if we had replacement transformations performed on first generation \
    #then delete the original column
    if len(transform_dict[category]['auntsuncles']) \
    + len(transform_dict[category]['parents']) > 0:
      if column in postprocess_dict['orig_noinplace']:
        del df_test[column]

    #if we had replacement transformations performed on downstream generation \
    #then delete the associated parent column 
    for columndict_column in postprocess_dict['column_dict']:
      if postprocess_dict['column_dict'][columndict_column]['deletecolumn'] is True:

        # #first we'll remove the column from columnslists 
        # for columnslistcolumn in postprocess_dict['column_dict'][columndict_column]['columnslist']:

        #   if columndict_column in postprocess_dict['column_dict'][columnslistcolumn]['columnslist']:

        #     postprocess_dict['column_dict'][columnslistcolumn]['columnslist'].remove(columndict_column)

        #now we'll delete column
        #note this only worksa on single column  parents, need to incioroprate categorylist
        #for multicolumn parents (future extension)
        if columndict_column in df_test.columns:
          del df_test[columndict_column]

    return df_test

  def postprocesscousin(self, df_test, column, cousin, origcategory, final_upstream, process_dict, \
                       transform_dict, postprocess_dict, columnkey, assign_param):
    """
    #postprocesscousin is comparable to processcousin but applied in postmunge instead of automunge
    #a little simpler in that doesn't have to populate data structures, just applies transform functions
    #for dualprocess case applies postprocess instead of dualprocess
    """

    #for checking type of processdict entries of custom externally defined transformation functions
    def check_function():
      return

    inplaceperformed = False
    inplacecandidate = False
    if final_upstream == cousin:
      inplacecandidate = True
    
    params = self.grab_params(assign_param, cousin, column, process_dict[cousin], postprocess_dict)
    
    #if this is a dual process function
    if 'postprocess' in process_dict[cousin] \
    and (isinstance(process_dict[cousin]['postprocess'], type(self.postprocesscousin)) \
    or isinstance(process_dict[cousin]['postprocess'], type(check_function))):
      
      if inplacecandidate is True:
        if 'inplace_option' in process_dict[cousin]:
          if process_dict[cousin]['inplace_option'] is True:
            if 'inplace' not in params:
              inplaceperformed = True
              params.update({'inplace' : True})
            elif ('inplace' in params and params['inplace'] != False):
              inplaceperformed = True
              params.update({'inplace' : True})
            else:
              inplaceperformed = False
      else:
        #user cannot manually specify inplace by design
        if ('inplace' in params and params['inplace'] is True):
          inplaceperformed = False
          params.update({'inplace' : False})
      
      df_test = \
      process_dict[cousin]['postprocess'](df_test, column, postprocess_dict, \
                                            columnkey, params)

    #else if this is a single process function
    elif 'singleprocess' in process_dict[cousin] \
    and (isinstance(process_dict[cousin]['singleprocess'], type(self.postprocesscousin)) \
    or isinstance(process_dict[cousin]['singleprocess'], type(check_function))):
      
      if inplacecandidate is True:
        if 'inplace_option' in process_dict[cousin]:
          if process_dict[cousin]['inplace_option'] is True:
            if 'inplace' not in params:
              inplaceperformed = True
              params.update({'inplace' : True})
            elif ('inplace' in params and params['inplace'] != False):
              inplaceperformed = True
              params.update({'inplace' : True})
            else:
              inplaceperformed = False
      else:
        #user cannot manually specify inplace by design
        if ('inplace' in params and params['inplace'] is True):
          inplaceperformed = False
          params.update({'inplace' : False})
            
      df_test, _1 = \
      process_dict[cousin]['singleprocess'](df_test, column, origcategory, \
                                            postprocess_dict, params)

    return df_test

  def postprocessparent(self, df_test, column, parent, origcategory, final_upstream, process_dict, \
                      transform_dict, postprocess_dict, columnkey, assign_param):
    """
    #postprocessparent is comparable to processparent but applied in postmunge instead of automunge
    #a little simpler in that doesn't have to populate data structures, just applies transform functions
    #for dualprocess case applies postprocess instead of dualprocess

    #we want to apply in order of
    #upstream process, niecesnephews, friends, children, coworkers
    """

    #for checking type of processdict entries of custom externally defined transformation functions
    def check_function():
      return
    
    #this is used to derive the new columns from the trasform
    origcolumnsset = set(df_test)
    
    inplaceperformed = False
    inplacecandidate = False
    if final_upstream == parent:
      inplacecandidate = True

    params = self.grab_params(assign_param, parent, column, process_dict[parent], postprocess_dict)

    #if this is a dual process function
    if 'postprocess' in process_dict[parent] \
    and (isinstance(process_dict[parent]['postprocess'], type(self.postprocessparent)) \
    or isinstance(process_dict[parent]['postprocess'], type(check_function))):
      
      if inplacecandidate is True:
        if 'inplace_option' in process_dict[parent]:
          if process_dict[parent]['inplace_option'] is True:
            if 'inplace' not in params:
              inplaceperformed = True
              params.update({'inplace' : True})
            elif ('inplace' in params and params['inplace'] != False):
              inplaceperformed = True
              params.update({'inplace' : True})
            else:
              inplaceperformed = False
      else:
        #user cannot manually specify inplace by design
        if ('inplace' in params and params['inplace'] is True):
          inplaceperformed = False
          params.update({'inplace' : False})
            
      df_test = \
      process_dict[parent]['postprocess'](df_test, column, postprocess_dict, \
                                            columnkey, params)

    #else if this is a single process function process train and test seperately
    elif 'singleprocess' in process_dict[parent] \
    and (isinstance(process_dict[parent]['singleprocess'], type(self.postprocessparent)) \
    or isinstance(process_dict[parent]['singleprocess'], type(check_function))):
      
      if inplacecandidate is True:
        if 'inplace_option' in process_dict[parent]:
          if process_dict[parent]['inplace_option'] is True:
            if 'inplace' not in params:
              inplaceperformed = True
              params.update({'inplace' : True})
            elif ('inplace' in params and params['inplace'] != False):
              inplaceperformed = True
              params.update({'inplace' : True})
            else:
              inplaceperformed = False
      else:
        #user cannot manually specify inplace by design
        if ('inplace' in params and params['inplace'] is True):
          inplaceperformed = False
          params.update({'inplace' : False})
      
      df_test, _1 = \
      process_dict[parent]['singleprocess'](df_test, column, origcategory, \
                                            postprocess_dict, params)

    #this is used to derive the new columns from the trasform
    newcolumnsset = set(df_test)

    #derive the new columns from the trasform
    categorylist = list(origcolumnsset^newcolumnsset)

    for entry in categorylist:
      if entry in postprocess_dict['column_dict']:
        if column == postprocess_dict['column_dict'][entry]['inputcolumn']:
          parentcolumn = entry
      
    #final upstream transform from parents or auntsuncles is elligible for inplace
    #as long as no supplement transforms were applied
    final_downstream = False
    if len(transform_dict[parent]['coworkers']) == 0:
      if len(transform_dict[parent]['children']) > 0:
        final_downstream = transform_dict[parent]['children'][-1]
    else:
      if len(transform_dict[parent]['coworkers']) > 0:
        final_downstream = transform_dict[parent]['coworkers'][-1]

    #process any niecesnephews
    for niecenephew in transform_dict[parent]['niecesnephews']:

      if niecenephew != None:

        #process the niecenephew
        #note the function applied is postprocessparent (using recursion)
        df_test = \
        self.postprocessparent(df_test, parentcolumn, niecenephew, origcategory, final_downstream, \
                                process_dict, transform_dict, postprocess_dict, columnkey, assign_param)
        
    #process any friends
    for friend in transform_dict[parent]['friends']:

      if friend != None:

        #process the friend
        #note the function applied is processcousin
        df_test = \
        self.postprocesscousin(df_test, parentcolumn, friend, origcategory, final_downstream, \
                                process_dict, transform_dict, postprocess_dict, columnkey, assign_param)

    #process any children
    for child in transform_dict[parent]['children']:

      if child != None:

        #process the child
        #note the function applied is postprocessparent (using recursion)
        #parent column
        df_test = \
        self.postprocessparent(df_test, parentcolumn, child, origcategory, final_downstream, process_dict, \
                                transform_dict, postprocess_dict, columnkey, assign_param)

    #process any coworkers
    for coworker in transform_dict[parent]['coworkers']:

      if coworker != None:

        #process the coworker
        #note the function applied is processcousin
        df_test = \
        self.postprocesscousin(df_test, parentcolumn, coworker, origcategory, final_downstream, \
                                process_dict, transform_dict, postprocess_dict, columnkey, assign_param)

    return df_test
  
  def postprocess_numerical_class(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #postprocess_numerical_class(mdf_test, column, postprocess_dict, columnkey)
    #function to normalize data to mean of 0 and standard deviation of 1 from training distribution
    #takes as arguement pandas dataframe of training and test data (mdf_train), (mdf_test)\
    #and the name of the column string ('column'), and the mean and std from the train set \
    #stored in postprocess_dict
    #replaces missing or improperly formatted data with mean of remaining values
    #leaves original specified column in dataframe
    #returns transformed dataframe
    #expect this approach works better when the numerical distribution is thin tailed
    #if only have training but not test data handy, use same training data for both dataframe inputs
    '''
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    #retrieve normalizastion parameters from postprocess_dict
    normkey = column + '_nmbr'
    
    mean = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['mean']
    std = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['std']
    offset = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['offset']
    multiplier = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['multiplier']
    cap = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['cap']
    floor = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['floor']
    adjinfill = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['adjinfill']
    
    if inplace is not True:
      #copy source column into new column
      mdf_test[column + '_nmbr'] = mdf_test[column].copy()
    else:
      mdf_test.rename(columns = {column : column + '_nmbr'}, inplace = True)

    #convert all values to either numeric or NaN
    mdf_test[column + '_nmbr'] = pd.to_numeric(mdf_test[column + '_nmbr'], errors='coerce')
    
    if cap is not False:
      #replace values in test > cap with cap
      mdf_test.loc[mdf_test[column + '_nmbr'] > cap, (column + '_nmbr')] \
      = cap
    
    if floor is not False:
      #replace values in test < floor with floor
      mdf_test.loc[mdf_test[column + '_nmbr'] < floor, (column + '_nmbr')] \
      = floor
      
    if adjinfill is True:
      mdf_test[column + '_nmbr'] = mdf_test[column + '_nmbr'].fillna(method='ffill')
      mdf_test[column + '_nmbr'] = mdf_test[column + '_nmbr'].fillna(method='bfill')

    #replace missing data with training set mean
    mdf_test[column + '_nmbr'] = mdf_test[column + '_nmbr'].fillna(mean)

    #subtract mean from column
    mdf_test[column + '_nmbr'] = mdf_test[column + '_nmbr'] - mean

    #get standard deviation of training data
    std = std

    #divide column values by std
    #offset, multiplier are parameters that defaults to zero, one
    mdf_test[column + '_nmbr'] = mdf_test[column + '_nmbr'] / std * multiplier + offset

#     #change data type for memory savings
#     mdf_test[column + '_nmbr'] = mdf_test[column + '_nmbr'].astype(np.float32)

    return mdf_test
    
  def postprocess_MADn_class(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #postprocess_MADn_class(mdf_test, column, postprocess_dict, columnkey)
    #function to normalize data to mean of 0 and mean absolute deviation of 1 from training distribution
    #takes as arguement pandas dataframe of test data (mdf_test)\
    #and the name of the column string ('column'), and the mean and MAD from the train set \
    #stored in postprocess_dict
    #replaces missing or improperly formatted data with mean of remaining values
    #leaves original specified column in dataframe
    #returns transformed dataframe
    #expect this approach works better than z-score for when the numerical distribution isn't thin tailed
    #if only have training but not test data handy, use same training data for both dataframe inputs
    '''
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    #retrieve normalizastion parameters from postprocess_dict
    normkey = column + '_MADn'
    
    mean = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['mean']
    MAD = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['MAD']
    adjinfill = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['adjinfill']

    if inplace is not True:
      mdf_test[column + '_MADn'] = mdf_test[column].copy()
    
    else:
      mdf_test.rename(columns = {column : column + '_MADn'}, inplace = True)

    #convert all values to either numeric or NaN
    mdf_test[column + '_MADn'] = pd.to_numeric(mdf_test[column + '_MADn'], errors='coerce')
    
    if adjinfill is True:
      mdf_test[column + '_MADn'] = mdf_test[column + '_MADn'].fillna(method='ffill')
      mdf_test[column + '_MADn'] = mdf_test[column + '_MADn'].fillna(method='bfill')

    #replace missing data with training set mean
    mdf_test[column + '_MADn'] = mdf_test[column + '_MADn'].fillna(mean)

    #subtract mean from column
    mdf_test[column + '_MADn'] = mdf_test[column + '_MADn'] - mean

    #get mean absolute deviation of training data
    MAD = MAD

    #divide column values by std
    mdf_test[column + '_MADn'] = mdf_test[column + '_MADn'] / MAD

#     #change data type for memory savings
#     mdf_test[column + '_MADn'] = mdf_test[column + '_MADn'].astype(np.float32)

    return mdf_test
    
  def postprocess_MAD3_class(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #postprocess_MADn_class(mdf_test, column, postprocess_dict, columnkey)
    #function to normalize data by subtracting max and dividing by mean absolute deviation from training distribution
    #takes as arguement pandas dataframe of test data (mdf_test)\
    #and the name of the column string ('column'), and the mean and MAD from the train set \
    #stored in postprocess_dict
    #replaces missing or improperly formatted data with mean of remaining values
    #leaves original specified column in dataframe
    #returns transformed dataframe
    #expect this approach works better than z-score for when the numerical distribution isn't thin tailed
    #if only have training but not test data handy, use same training data for both dataframe inputs
    '''
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    #retrieve normalizastion parameters from postprocess_dict
    normkey = column + '_MAD3'
    
    mean = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['mean']
    MAD = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['MAD']
    datamax = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['datamax']
    adjinfill = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['adjinfill']

    if inplace is not True:
      mdf_test[column + '_MAD3'] = mdf_test[column].copy()
    
    else:
      mdf_test.rename(columns = {column : column + '_MAD3'}, inplace = True)

    #convert all values to either numeric or NaN
    mdf_test[column + '_MAD3'] = pd.to_numeric(mdf_test[column + '_MAD3'], errors='coerce')
    
    if adjinfill is True:
      mdf_test[column + '_MAD3'] = mdf_test[column + '_MAD3'].fillna(method='ffill')
      mdf_test[column + '_MAD3'] = mdf_test[column + '_MAD3'].fillna(method='bfill')

    #replace missing data with training set mean
    mdf_test[column + '_MAD3'] = mdf_test[column + '_MAD3'].fillna(mean)

    #subtract datamax from column
    mdf_test[column + '_MAD3'] = mdf_test[column + '_MAD3'] - datamax

    #get mean absolute deviation of training data
    MAD = MAD

    #divide column values by std
    mdf_test[column + '_MAD3'] = mdf_test[column + '_MAD3'] / MAD

#     #change data type for memory savings
#     mdf_test[column + '_MAD3'] = mdf_test[column + '_MAD3'].astype(np.float32)

    return mdf_test
    
  def postprocess_mnmx_class(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #postprocess_mnmx_class(mdf_test, column, postprocess_dict, columnkey)
    #function to scale data to minimum of 0 and maximum of 1 based on training distribution
    #takes as arguement pandas dataframe of training and test data (mdf_train), (mdf_test)\
    #and the name of the column string ('column'), and the normalization parameters \
    #stored in postprocess_dict
    #replaces missing or improperly formatted data with mean of training values
    #leaves original specified column in dataframe
    #returns transformed dataframe
    #expect this approach works better when the numerical distribution is thin tailed
    #if only have training but not test data handy, use same training data for both dataframe inputs
    '''
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    #retrieve normalizastion parameters from postprocess_dict
    normkey = column + '_mnmx'
    
    mean = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['mean']
    minimum = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['minimum']
    maximum = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['maximum']
    cap = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['cap']
    floor = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['floor']
    adjinfill = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['adjinfill']
    
    if inplace is not True:
      #copy source column into new column
      mdf_test[column + '_mnmx'] = mdf_test[column].copy()
    else:
      mdf_test.rename(columns = {column : column + '_mnmx'}, inplace = True)

    #convert all values to either numeric or NaN
    mdf_test[column + '_mnmx'] = pd.to_numeric(mdf_test[column + '_mnmx'], errors='coerce')
    
    if adjinfill is True:
      mdf_test[column + '_mnmx'] = mdf_test[column + '_mnmx'].fillna(method='ffill')
      mdf_test[column + '_mnmx'] = mdf_test[column + '_mnmx'].fillna(method='bfill')

    #replace missing data with training set mean
    mdf_test[column + '_mnmx'] = mdf_test[column + '_mnmx'].fillna(mean)
    
    #avoid outlier div by zero when max = min
    maxminusmin = maximum - minimum
    if maxminusmin == 0:
      maxminusmin = 1

    #perform min-max scaling to test set using values from train
    mdf_test[column + '_mnmx'] = (mdf_test[column + '_mnmx'] - minimum) / \
                                 (maxminusmin)

    if cap is not False:
      #replace values in test > cap with cap
      mdf_test.loc[mdf_test[column + '_mnmx'] > (cap - minimum)/maxminusmin, (column + '_mnmx')] \
      = (cap - minimum)/maxminusmin
    
    if floor is not False:
      #replace values in test < floor with floor
      mdf_test.loc[mdf_test[column + '_mnmx'] < (floor - minimum)/maxminusmin, (column + '_mnmx')] \
      = (floor - minimum)/maxminusmin

    return mdf_test

  def postprocess_mnm3_class(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #postprocess_mnmx_class(mdf_test, column, postprocess_dict, columnkey)
    #function to scale data to minimum of 0 and maximum of 1 based on training distribution
    #quantiles with values exceeding quantiles capped
    #takes as arguement pandas dataframe of training and test data (mdf_train), (mdf_test)\
    #and the name of the column string ('column'), and the normalization parameters \
    #stored in postprocess_dict
    #replaces missing or improperly formatted data with mean of training values
    #leaves original specified column in dataframe
    #returns transformed dataframe
    #expect this approach works better when the numerical distribution is thin tailed
    #if only have training but not test data handy, use same training data for both dataframe inputs
    '''
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False

    #retrieve normalizastion parameters from postprocess_dict
    normkey = column + '_mnm3'

    mean = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['mean']

    quantilemin = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['quantilemin']

    quantilemax = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['quantilemax']
    
    adjinfill = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['adjinfill']
    
    if inplace is not True:
      mdf_test[column + '_mnm3'] = mdf_test[column].copy()
    
    else:
      mdf_test.rename(columns = {column : column + '_mnm3'}, inplace = True)

    #convert all values to either numeric or NaN
    mdf_test[column + '_mnm3'] = pd.to_numeric(mdf_test[column + '_mnm3'], errors='coerce')

#     #get mean of training data
#     mean = mean    
    
    #replace values > quantilemax with quantilemax
    mdf_test.loc[mdf_test[column + '_mnm3'] > quantilemax, (column + '_mnm3')] \
    = quantilemax
    #replace values < quantile10 with quantile10
    mdf_test.loc[mdf_test[column + '_mnm3'] < quantilemin, (column + '_mnm3')] \
    = quantilemin
      
    if adjinfill is True:
      mdf_test[column + '_mnm3'] = mdf_test[column + '_mnm3'].fillna(method='ffill')
      mdf_test[column + '_mnm3'] = mdf_test[column + '_mnm3'].fillna(method='bfill')
    
    #replace missing data with training set mean
    mdf_test[column + '_mnm3'] = mdf_test[column + '_mnm3'].fillna(mean)
    
    #avoid outlier div by zero when max = min
    maxminusmin = quantilemax - quantilemin
    if maxminusmin == 0:
      maxminusmin = 1

    #perform min-max scaling to test set using values from train
    mdf_test[column + '_mnm3'] = (mdf_test[column + '_mnm3'] - quantilemin) / \
                                 (maxminusmin)

#     #change data type for memory savings
#     mdf_test[column + '_mnm3'] = mdf_test[column + '_mnm3'].astype(np.float32)

    return mdf_test

  def postprocess_retn_class(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    """
    #process_retn_class(mdf_train, mdf_test, column, category)
    #function to scale data as follows:
    
    # if max >= 0 and min <= 0:
    #   #scaling based on 
    #   x = x / (max - min)

    # elif max >= 0 and min >= 0:
    #   #traditional min/max
    #   x = (x - min) / (max - min)
    
    # elif max <= 0 and min <= 0:
    #   #max/min (retains negative values)
    #   x = (x - max) / (max - min)
    
    #replaces missing or improperly formatted data with mean of remaining values
    
    #returns same dataframes with new column of name column + '_retn'
    #note this is a "dualprocess" function since is applied to both dataframes
    
    #note with parameters divisor can also be set as standard deviation
    #also aprameters accepted for cap/floor/mulitplier/offset
    #where cap/floor based on pretransform values
    #multiplier/offset based on posttransform values, muoltiplier applied betfore offset
    """
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    #retrieve normalizastion parameters from postprocess_dict
    normkey = column + '_retn'
    
    mean = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['mean']
    minimum = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['minimum']
    maximum = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['maximum']
    scalingapproach = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['scalingapproach']
    offset = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['offset']
    multiplier = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['multiplier']
    cap = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['cap']
    floor = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['floor']
    divisor = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['divisor']
    adjinfill = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['adjinfill']
    
    if inplace is not True:
      #copy source column into new column
      mdf_test[column + '_retn'] = mdf_test[column].copy()
    else:
      mdf_test.rename(columns = {column : column + '_retn'}, inplace = True)

    #convert all values to either numeric or NaN
    mdf_test[column + '_retn'] = pd.to_numeric(mdf_test[column + '_retn'], errors='coerce')
    
    if cap is not False:
      #replace values in test > cap with cap
      mdf_test.loc[mdf_test[column + '_retn'] > cap, (column + '_retn')] \
      = cap
    
    if floor is not False:
      #replace values in test < floor with floor
      mdf_test.loc[mdf_test[column + '_retn'] < floor, (column + '_retn')] \
      = floor
    
    if adjinfill is True:
      mdf_test[column + '_retn'] = mdf_test[column + '_retn'].fillna(method='ffill')
      mdf_test[column + '_retn'] = mdf_test[column + '_retn'].fillna(method='bfill')

    #replace missing data with training set mean
    mdf_test[column + '_retn'] = mdf_test[column + '_retn'].fillna(mean)
    
    #avoid outlier div by zero when max = min
    maxminusmin = maximum - minimum
    if maxminusmin == 0:
      maxminusmin = 1
    
    if scalingapproach == 'retn':
      
      mdf_test[column + '_retn'] = (mdf_test[column + '_retn']) / \
                                    (divisor) * multiplier + offset
      
    elif scalingapproach == 'mnmx':
    
      #perform min-max scaling to test set using values from train
      mdf_test[column + '_retn'] = (mdf_test[column + '_retn'] - minimum) / \
                                   (divisor) * multiplier + offset
      
    elif scalingapproach == 'mxmn':
    
      #perform min-max scaling to test set using values from train
      mdf_test[column + '_retn'] = (mdf_test[column + '_retn'] - maximum) / \
                                   (divisor) * multiplier + offset

    return mdf_test

  def postprocess_mean_class(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #postprocess_mean_class(mdf_test, column, postprocess_dict, columnkey)
    #function to scale data to minimum of 0 and maximum of 1 based on training distribution
    #takes as arguement pandas dataframe of training and test data (mdf_train), (mdf_test)\
    #and the name of the column string ('column'), and the normalization parameters \
    #stored in postprocess_dict
    #replaces missing or improperly formatted data with mean of training values
    #leaves original specified column in dataframe
    #returns transformed dataframe
    #expect this approach works better when the numerical distribution is thin tailed
    #if only have training but not test data handy, use same training data for both dataframe inputs
    '''
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    #retrieve normalizastion parameters from postprocess_dict
    normkey = column + '_mean'
    
    mean = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['mean']
    
    minimum = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['minimum']
    
    maximum = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['maximum']
    
    maxminusmin = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['maxminusmin']
    
    offset = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['offset']
    
    multiplier = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['multiplier']
    
    cap = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['cap']
    
    floor = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['floor']
    
    adjinfill = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['adjinfill']

    if inplace is not True:
      mdf_test[column + '_mean'] = mdf_test[column].copy()
    
    else:
      mdf_test.rename(columns = {column : column + '_mean'}, inplace = True)

    #convert all values to either numeric or NaN
    mdf_test[column + '_mean'] = pd.to_numeric(mdf_test[column + '_mean'], errors='coerce')
    
    if cap is not False:
      #replace values in test > cap with cap
      mdf_test.loc[mdf_test[column + '_mean'] > cap, (column + '_mean')] \
      = cap
    
    if floor is not False:
      #replace values in test < floor with floor
      mdf_test.loc[mdf_test[column + '_mean'] < floor, (column + '_mean')] \
      = floor
      
    if adjinfill is True:
      mdf_test[column + '_mean'] = mdf_test[column + '_mean'].fillna(method='ffill')
      mdf_test[column + '_mean'] = mdf_test[column + '_mean'].fillna(method='bfill')

    #replace missing data with training set mean
    mdf_test[column + '_mean'] = mdf_test[column + '_mean'].fillna(mean)

    #perform min-max scaling to test set using values from train
    mdf_test[column + '_mean'] = (mdf_test[column + '_mean'] - mean) / \
                                 (maxminusmin) * multiplier + offset

#     #change data type for memory savings
#     mdf_test[column + '_mnmx'] = mdf_test[column + '_mnmx'].astype(np.float32)

    return mdf_test

  def postprocess_binary_class(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #postprocess_binary_class(mdf, column, postprocess_dict, columnkey)
    #converts binary classification values to 0 or 1
    #takes as arguement a pandas dataframe (mdf_test), \
    #the name of the column string ('column') \
    #and the string classification to assign to missing data ('missing')
    #saved in the postprocess_dict
    #replaces original specified column in dataframe
    #returns transformed dataframe
    #missing category must be identical to one of the two existing categories
    #returns error message if more than two categories remain
    '''
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    #retrieve normalization parameters
    normkey = column + '_bnry'
    binary_missing_plug = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['missing']
    adjinfill = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['adjinfill']
    str_convert = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['str_convert']
    
#     onevalue = \
#     postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['onevalue']
    
#     zerovalue = \
#     postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['zerovalue']

    onevalue = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey][1]
    
    zerovalue = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey][0]

    if inplace is not True:
      mdf_test[column + '_bnry'] = mdf_test[column].copy()
    
    else:
      mdf_test.rename(columns = {column : column + '_bnry'}, inplace = True)
    
    if str_convert is True:
      mdf_test[column + '_bnry'] = mdf_test[column + '_bnry'].astype(str)
    
    if adjinfill is True:
      mdf_test[column + '_bnry'] = mdf_test[column + '_bnry'].fillna(method='ffill')
      mdf_test[column + '_bnry'] = mdf_test[column + '_bnry'].fillna(method='bfill')

    #replace missing data with specified classification
    mdf_test[column + '_bnry'] = mdf_test[column + '_bnry'].fillna(binary_missing_plug)

    #this addressess issue where nunique for mdftest > than that for mdf_train
    #note is currently an oportunity for improvement that NArows won't identify these poinsts as candiadates
    #for user specified infill, and as currently addressed will default to infill with most common value
    #in the mean time a workaround could be for user to manually replace extra values with nan prior to
    #postmunge application such as if they want to apply ML infill
    #this will only be an issue when nunique for df_train == 2, and nunique for df_test > 2
    #if len(mdf_test[column + '_bnry'].unique()) > 2:
    uniqueintest = mdf_test[column + '_bnry'].unique()
    for unique in uniqueintest:
      if unique not in [onevalue, zerovalue]:
        mdf_test[column + '_bnry'] = \
        np.where(mdf_test[column + '_bnry'] == unique, binary_missing_plug, mdf_test[column + '_bnry'])
    
    #convert column to binary 0/1 classification (replaces scikit LabelBinarizer)
    mdf_test[column + '_bnry'] = np.where(mdf_test[column + '_bnry'] == onevalue, 1, 0)

    #create list of columns
    bnrycolumns = [column + '_bnry']

    #change data types to 8-bit (1 byte) integers for memory savings
    mdf_test[column + '_bnry'] = mdf_test[column + '_bnry'].astype(np.int8)

    return mdf_test
  
  def postprocess_binary2_class(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #postprocess_binary_class(mdf, column, postprocess_dict, columnkey)
    #converts binary classification values to 0 or 1
    #takes as arguement a pandas dataframe (mdf_test), \
    #the name of the column string ('column') \
    #and the string classification to assign to missing data ('missing')
    #saved in the postprocess_dict
    #replaces original specified column in dataframe
    #returns transformed dataframe
    #missing category must be identical to one of the two existing categories
    #returns error message if more than two categories remain
    '''
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    #retrieve normalization parameters
    normkey = column + '_bnr2'
    binary_missing_plug = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['missing']
    adjinfill = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['adjinfill']
    str_convert = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['str_convert']
    
#     onevalue = \
#     postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['onevalue']
    
#     zerovalue = \
#     postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['zerovalue']

    onevalue = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey][1]
    
    zerovalue = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey][0]

    if inplace is not True:
      mdf_test[column + '_bnr2'] = mdf_test[column].copy()
    
    else:
      mdf_test.rename(columns = {column : column + '_bnr2'}, inplace = True)
    
    if str_convert is True:
      mdf_test[column + '_bnr2'] = mdf_test[column + '_bnr2'].astype(str)
    
    if adjinfill is True:
      mdf_test[column + '_bnr2'] = mdf_test[column + '_bnr2'].fillna(method='ffill')
      mdf_test[column + '_bnr2'] = mdf_test[column + '_bnr2'].fillna(method='bfill')

    #replace missing data with specified classification
    mdf_test[column + '_bnr2'] = mdf_test[column + '_bnr2'].fillna(binary_missing_plug)

    #this addressess issue where nunique for mdftest > than that for mdf_train
    #note is currently an oportunity for improvement that NArows won't identify these poinsts as candiadates
    #for user specified infill, and as currently addressed will default to infill with most common value
    #in the mean time a workaround could be for user to manually replace extra values with nan prior to
    #postmunge application such as if they want to apply ML infill
    #this will only be an issue when nunique for df_train == 2, and nunique for df_test > 2
    #if len(mdf_test[column + '_bnry'].unique()) > 2:
    uniqueintest = mdf_test[column + '_bnr2'].unique()
    for unique in uniqueintest:
      if unique not in [onevalue, zerovalue]:
        mdf_test[column + '_bnr2'] = \
        np.where(mdf_test[column + '_bnr2'] == unique, binary_missing_plug, mdf_test[column + '_bnr2'])
    
    #convert column to binary 0/1 classification (replaces scikit LabelBinarizer)
    mdf_test[column + '_bnr2'] = np.where(mdf_test[column + '_bnr2'] == onevalue, 1, 0)

    #create list of columns
    bnrycolumns = [column + '_bnr2']

    #change data types to 8-bit (1 byte) integers for memory savings
    mdf_test[column + '_bnr2'] = mdf_test[column + '_bnr2'].astype(np.int8)

    return mdf_test
  
  def postprocess_onht_class(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #process_onht_class(mdf_train, mdf_test, column, category, postprocess_dict, params = {})
    #preprocess column with one hot encoding
    #same as 'text' transform except labels returned column with integer instead of entry appender
    '''
    
    #retrieve a columnkey
    normkey = False

    if normkey is False:

      if column in postprocess_dict['origcolumn']:

        columnkeylist = postprocess_dict['origcolumn'][column]['columnkeylist']

      else:

        origcolumn = postprocess_dict['column_dict'][column]['origcolumn']

        columnkeylist = postprocess_dict['origcolumn'][origcolumn]['columnkeylist']

      for columnkey in columnkeylist:
        
        if column == postprocess_dict['column_dict'][columnkey]['inputcolumn']:

          if 'textlabelsdict_onht' in postprocess_dict['column_dict'][columnkey]['normalization_dict'][columnkey]:

            normkey = columnkey
          
    if normkey is not False:
      
      adjinfill = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['adjinfill']
      
      str_convert = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['str_convert']
      
      tempcolumn = column + '_onht_'

      #create copy of original column for later retrieval
      mdf_test[tempcolumn] = mdf_test[column].copy()

      #convert column to category
      mdf_test[tempcolumn] = mdf_test[tempcolumn].astype('category')

      #if set is categorical we'll need the plug value for missing values included
      if 'zzzinfill' not in mdf_test[tempcolumn].cat.categories:
        mdf_test[tempcolumn] = mdf_test[tempcolumn].cat.add_categories(['zzzinfill'])
        
      if adjinfill is True:
        mdf_test[tempcolumn] = mdf_test[tempcolumn].fillna(method='ffill')
        mdf_test[tempcolumn] = mdf_test[tempcolumn].fillna(method='bfill')

      #replace NA with a dummy variable
      mdf_test[tempcolumn] = mdf_test[tempcolumn].fillna('zzzinfill')

      if str_convert is True:
        #replace numerical with string equivalent
        mdf_test[tempcolumn] = mdf_test[tempcolumn].astype(str)
      else:
        mdf_test[tempcolumn] = mdf_test[tempcolumn].astype('object')

      #textcolumns = postprocess_dict['column_dict'][columnkey]['columnslist']
      textcolumns = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['text_categorylist']
      
      labels_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['labels_dict']

      #extract categories for column labels
      #note that .unique() extracts the labels as a numpy array

      #we'll get the category names from the textcolumns array by stripping the \
      #prefixes of column name + '_'
#       prefixlength = len(column)+1
      labels_train = textcolumns[:]
#       for textcolumn in labels_train:
#         textcolumn = textcolumn[prefixlength :]
      #labels_train.sort(axis=0)
#       labels_train.sort()
      labels_test = mdf_test[tempcolumn].unique()
#       labels_test.sort(axis=0)
      labels_test = sorted(labels_test, key=str)
      labels_test = list(labels_test)

      #pandas one hot encoding doesn't sort integers and strings properly so using my own
      df_test_cat = pd.DataFrame(mdf_test[tempcolumn])
      for entry in labels_train:
        df_test_cat[entry] = np.where(mdf_test[tempcolumn] == entry, 1, 0)
      del df_test_cat[tempcolumn]

      #convert sparse array to pandas dataframe with column labels
      df_test_cat.columns = labels_train

      #Get missing columns in test set that are present in training set
      missing_cols = set( textcolumns ) - set( labels_test )

      del mdf_test[tempcolumn]
      
      #concatinate the sparse set with the rest of our training data
      mdf_test = pd.concat([mdf_test, df_test_cat], axis=1)

      #delete support NArw2 column
      columnNAr2 = column + '_zzzinfill'
      if columnNAr2 in mdf_test.columns:
        del mdf_test[columnNAr2]

      #change data types to 8-bit (1 byte) integers for memory savings
      for textcolumn in textcolumns:

        mdf_test[textcolumn] = mdf_test[textcolumn].astype(np.int8)
        
      #now convert coloumn headers from text convention to onht convention
      mdf_test  = mdf_test.rename(columns=labels_dict)
    
    return mdf_test
  
  def postprocess_text_class(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #postprocess_text_class(mdf_test, column, postprocess_dict, columnkey)
    #process column with text classifications
    #takes as arguement pandas dataframe containing test data  
    #(mdf_test), and the name of the column string ('column'), and an array of
    #the associated transformed column s from the train set (textcolumns)
    #which is saved in the postprocess_dict
    #note this aligns formatting of transformed columns to the original train set
    #fromt he original treatment with automunge
    #retains the original column from master dataframe and
    #adds onehot encodings
    #with columns named after column_ + text classifications
    #missing data replaced with category label 'missing'+column
    #any categories missing from the training set removed from test set
    #any category present in training but missing from test set given a column of zeros for consistent formatting
    #ensures order of all new columns consistent between both sets
    #returns two transformed dataframe (mdf_train, mdf_test) \
    #and a list of the new column names (textcolumns)
    '''
    
    #retrieve normkey
    normkey = False
    if normkey is False:

      if column in postprocess_dict['origcolumn']:

        columnkeylist = postprocess_dict['origcolumn'][column]['columnkeylist']

      else:

        origcolumn = postprocess_dict['column_dict'][column]['origcolumn']

        columnkeylist = postprocess_dict['origcolumn'][origcolumn]['columnkeylist']

      for columnkey in columnkeylist:
        
        if column == postprocess_dict['column_dict'][columnkey]['inputcolumn']:

          if 'textlabelsdict_text' in postprocess_dict['column_dict'][columnkey]['normalization_dict'][columnkey]:

            normkey = columnkey
          
    if normkey is not False:
      
      adjinfill = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['adjinfill']
    
      tempsuffix = str(mdf_test[column].unique()[0])

      tempcolumn = column + '_' + tempsuffix

      #create copy of original column for later retrieval
      mdf_test[tempcolumn] = mdf_test[column].copy()

      #convert column to category
      mdf_test[tempcolumn] = mdf_test[tempcolumn].astype('category')

      #if set is categorical we'll need the plug value for missing values included
      if 'zzzinfill' not in mdf_test[tempcolumn].cat.categories:
        mdf_test[tempcolumn] = mdf_test[tempcolumn].cat.add_categories(['zzzinfill'])
      
      if adjinfill is True:
        mdf_test[tempcolumn] = mdf_test[tempcolumn].fillna(method='ffill')
        mdf_test[tempcolumn] = mdf_test[tempcolumn].fillna(method='bfill')
      
      #replace NA with a dummy variable
      mdf_test[tempcolumn] = mdf_test[tempcolumn].fillna('zzzinfill')

      #replace numerical with string equivalent
      #mdf_train[column] = mdf_train[column].astype(str)
      mdf_test[tempcolumn] = mdf_test[tempcolumn].astype(str)

      #textcolumns = postprocess_dict['column_dict'][columnkey]['columnslist']
      textcolumns = postprocess_dict['column_dict'][normkey]['categorylist']

      #extract categories for column labels
      #note that .unique() extracts the labels as a numpy array

      #we'll get the category names from the textcolumns array by stripping the \
      #prefixes of column name + '_'
      prefixlength = len(column)+1
      labels_train = textcolumns[:]
      for textcolumn in labels_train:
        textcolumn = textcolumn[prefixlength :]
      #labels_train.sort(axis=0)
      labels_train.sort()
      labels_test = mdf_test[tempcolumn].unique()
      labels_test.sort(axis=0)
      labels_test = list(labels_test)

      #apply onehotencoding
      df_test_cat = pd.get_dummies(mdf_test[tempcolumn])

      #append column header name to each category listing
      labels_test = [column + '_' + entry for entry in labels_test]

      #convert sparse array to pandas dataframe with column labels
      df_test_cat.columns = labels_test

      #Get missing columns in test set that are present in training set
      missing_cols = set( textcolumns ) - set( df_test_cat.columns )

      #Add a missing column in test set with default value equal to 0
      for c in missing_cols:
          df_test_cat[c] = 0

      #Ensure the order of column in the test set is in the same order than in train set
      #Note this also removes categories in test set that aren't present in training set
      df_test_cat = df_test_cat[textcolumns]

      del mdf_test[tempcolumn]
      
      #concatinate the sparse set with the rest of our training data
      mdf_test = pd.concat([mdf_test, df_test_cat], axis=1)

      #delete support NArw2 column
      columnNAr2 = column + '_zzzinfill'
      if columnNAr2 in mdf_test.columns:
        del mdf_test[columnNAr2]

      #change data types to 8-bit (1 byte) integers for memory savings
      for textcolumn in textcolumns:

        mdf_test[textcolumn] = mdf_test[textcolumn].astype(np.int8)
    
    return mdf_test
  
  def postprocess_textsupport_class(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #just like the postprocess_text_class function but uses different approach for
    #normalizaation key (uses passed columnkey). This function supports some of the
    #other methods.
    #accepts parameter textcolumns as a list of columns to return 
    '''
    
    if 'textcolumns' in params:
      textcolumns = params['textcolumns']
    else:
      textcolumns = postprocess_dict['column_dict'][columnkey]['categorylist']
    
    if len(textcolumns) > 0:
      tempcolumn = textcolumns[0]
    else:
      tempcolumn = column + '_onht'
    
    #create copy of original column for later retrieval
    mdf_test[tempcolumn] = mdf_test[column].copy()

    #convert column to category
    mdf_test[tempcolumn] = mdf_test[tempcolumn].astype('category')

#     #if set is categorical we'll need the plug value for missing values included
#     mdf_test[column] = mdf_test[column].cat.add_categories(['NArw'])

#     #replace NA with a dummy variable
#     mdf_test[column] = mdf_test[column].fillna('NArw')
    
    #if set is categorical we'll need the plug value for missing values included
    if 'zzzinfill' not in mdf_test[tempcolumn].cat.categories:
      mdf_test[tempcolumn] = mdf_test[tempcolumn].cat.add_categories(['zzzinfill'])

    #replace NA with a dummy variable
    mdf_test[tempcolumn] = mdf_test[tempcolumn].fillna('zzzinfill')

    #replace numerical with string equivalent
    #mdf_train[column] = mdf_train[column].astype(str)
    mdf_test[tempcolumn] = mdf_test[tempcolumn].astype(str)

    #extract categories for column labels
    #note that .unique() extracts the labels as a numpy array

    #we'll get the category names from the textcolumns array by stripping the \
    #prefixes of column name + '_'
    prefixlength = len(column)+1
    labels_train = textcolumns[:]
    for textcolumn in labels_train:
      textcolumn = textcolumn[prefixlength :]
    #labels_train.sort(axis=0)
    labels_train.sort()
    labels_test = mdf_test[tempcolumn].unique()
    labels_test.sort(axis=0)
    labels_test = list(labels_test)

    #apply onehotencoding
    df_test_cat = pd.get_dummies(mdf_test[tempcolumn])
    
    #append column header name to each category listing
    labels_test = [column + '_' + entry for entry in labels_test]
    
    #convert sparse array to pandas dataframe with column labels
    df_test_cat.columns = labels_test

    #Get missing columns in test set that are present in training set
    missing_cols = set( textcolumns ) - set( df_test_cat.columns )

    #Add a missing column in test set with default value equal to 0
    for c in missing_cols:
        df_test_cat[c] = 0

    #Ensure the order of column in the test set is in the same order than in train set
    #Note this also removes categories in test set that aren't present in training set
    df_test_cat = df_test_cat[textcolumns]
    
    del mdf_test[tempcolumn]

    #concatinate the sparse set with the rest of our training data
    mdf_test = pd.concat([mdf_test, df_test_cat], axis=1)
    
    #delete support NArw2 column
    columnNAr2 = column + '_zzzinfill'
    if columnNAr2 in mdf_test.columns:
      del mdf_test[columnNAr2]
    
    #change data types to 8-bit (1 byte) integers for memory savings
    for textcolumn in textcolumns:
      
      mdf_test[textcolumn] = mdf_test[textcolumn].astype(np.int8)

    return mdf_test

  def postprocess_splt_class(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #postprocess_splt_class(mdf_test, column, postprocess_dict, category)
    #preprocess column with categorical entries as strings
    #identifies overlaps of subsets of those strings and records
    #as a new boolan column
    #for example, if a categoical set consisted of unique values ['west', 'north', 'northeast']
    #then a new column would be created idenitifying cells which included 'north' in their entries
    #(here for north and northeast)
    #returns as column titled origcolumn_splt_entry    
    #missing values are ignored by default
    
    #here in postprocess we only create columns foir those overlaps that were identified 
    #fromthe train set
    '''
    
    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = '_splt'
    
    #to retrieve the normalization dictionary we're going to use new method since we don't yet 
    #know what the returned columns titles are yet
    
    normkey = False
    
    if column in postprocess_dict['origcolumn']:
      
      columnkeylist = postprocess_dict['origcolumn'][column]['columnkeylist']
      
    else:
      
      origcolumn = postprocess_dict['column_dict'][column]['origcolumn']
      
      columnkeylist = postprocess_dict['origcolumn'][origcolumn]['columnkeylist']
    
    for columnkey in columnkeylist:
      
      if column == postprocess_dict['column_dict'][columnkey]['inputcolumn']:

        if 'splt_newcolumns_splt' in postprocess_dict['column_dict'][columnkey]['normalization_dict'][columnkey] \
        and postprocess_dict['column_dict'][columnkey]['normalization_dict'][columnkey]['suffix'] == suffix:

          normkey = columnkey
        
    if normkey is not False:

      #great now we can grab normalization parameters
      overlap_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['overlap_dict']
      newcolumns = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['splt_newcolumns_splt']
      int_headers = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['int_headers']
      int_labels_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['int_labels_dict']
      concurrent_activations = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['concurrent_activations']
      suffix = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
      test_same_as_train = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['test_same_as_train']

      #now for mdf_test we'll only consider those overlaps already identified from train set
      
      if test_same_as_train is True:
        test_overlap_dict = overlap_dict
      
      else:

        unique_list_test = list(mdf_test[column].unique())

        unique_list_test = list(map(str, unique_list_test))

        test_overlap_dict = {}

        train_keys = list(overlap_dict)

        train_keys.sort(key = len, reverse=True)

        for key in train_keys:

          test_overlap_dict.update({key:[]})

        for dict_key in train_keys:

          for unique_test in unique_list_test:

            len_key = len(dict_key)

            if len(unique_test) >= len_key:

              nbr_iterations4 = len(unique_test) - len_key + 1

              for l in range(nbr_iterations4):

                extract4 = unique_test[l:(len_key+l)]

                if extract4 == dict_key:

                  test_overlap_dict[dict_key].append(unique_test)

                  if concurrent_activations is False:

                    break

      newcolumns = []

      for dict_key in overlap_dict:

        newcolumn = column + suffix + '_' + dict_key
        
#         mdf_train[newcolumn] = mdf_train[column].copy()
        mdf_test[newcolumn] = mdf_test[column].copy()

#         mdf_train[newcolumn] = mdf_train[newcolumn].astype(str)
        mdf_test[newcolumn] = mdf_test[newcolumn].astype(str)

        mdf_test[newcolumn] = mdf_test[newcolumn].isin(test_overlap_dict[dict_key])
        mdf_test[newcolumn] = mdf_test[newcolumn].astype(np.int8)

        newcolumns.append(newcolumn)
        
      #now convert coloumn headers from string to int convention
      if int_headers is True:
        mdf_test  = mdf_test.rename(columns=int_labels_dict)
    
    return mdf_test

  def postprocess_spl2_class(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #postprocess_spl2_class(mdf_test, column, postprocess_dict, category)
    #preprocess column with categorical entries as strings
    #identifies overlaps of subsets of those strings and replaces entries with their redecued overlap
    #replaces entries without overlap to 0 (unique to spl5)
    #for example, if a categorical set consisted of unique values ['west', 'north', 'northeast']
    #then a new column would be created in which the entry 'north' replaced cells with north in their entries
    #(here for north and northeast)
    #and cells with west would be set to 0
    #returns as column titled origcolumn_spl2
    #missing values are ignored by default
    #this alternative to splt may be benficial for instance if one wanted to follow with an ordl encoding
    '''
    
    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = '_spl2'
    
    #to retrieve the normalization dictionary we're going to use new method since we don't yet 
    #know what the returned columns titles are yet
    
    normkey = False
    
#     if column in postprocess_dict['origcolumn']:
      
#       columnkeylist = postprocess_dict['origcolumn'][column]['columnkeylist']
      
#     else:
      
#       origcolumn = postprocess_dict['column_dict'][column]['origcolumn']
      
#       columnkeylist = postprocess_dict['origcolumn'][origcolumn]['columnkeylist']
    
#     for columnkey in columnkeylist:
    
#       if 'splt_newcolumns' in postprocess_dict['column_dict'][columnkey]['normalization_dict'][columnkey]:
        
#         normkey = columnkey

    normkey = column + suffix
        
    if normkey is not False:

      #great now we can grab normalization parameters
      overlap_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['overlap_dict']
      test_same_as_train = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['test_same_as_train']
      consolidate_nonoverlaps = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['consolidate_nonoverlaps']

#       newcolumns = \
#       postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['spl2_newcolumns']

      #now for mdf_test we'll only consider those overlaps already identified from train set
  
      if test_same_as_train is True:
        test_overlap_dict = overlap_dict

      elif test_same_as_train is False:

        unique_list_test = list(mdf_test[column].unique())

        unique_list_test = list(map(str, unique_list_test))

        test_overlap_dict = {}

        train_keys = list(overlap_dict)

        train_keys.sort(key = len, reverse=True)

        for key in train_keys:

          test_overlap_dict.update({key:[]})

        for dict_key in train_keys:

          for unique_test in unique_list_test:

            len_key = len(dict_key)

            if len(unique_test) >= len_key:

              nbr_iterations4 = len(unique_test) - len_key + 1

              for l in range(nbr_iterations4):

                extract4 = unique_test[l:(len_key+l)]

                if extract4 == dict_key:

                  test_overlap_dict[dict_key].append(unique_test)

                  break
                
      #then we'll populate the spl2 replacement dict

      spl2_test_overlap_dict = {}

      test_overlap_key_list = list(test_overlap_dict)

      test_overlap_key_list.sort()
      test_overlap_key_list.sort(key = len, reverse=True)

      for overlap_key in test_overlap_key_list:

        for entry in test_overlap_dict[overlap_key]:

          if entry not in spl2_test_overlap_dict:

            spl2_test_overlap_dict.update({entry : overlap_key})
            
      #here's where we identify values to set to 0 for spl5
      spl5_test_zero_dict = {}
      if consolidate_nonoverlaps is True:

        if test_same_as_train is True:
          unique_list_test = list(mdf_test[column].unique())
          unique_list_test = list(map(str, unique_list_test))

        for entry in unique_list_test:
          if entry not in spl2_test_overlap_dict:
            spl5_test_zero_dict.update({entry : 0})

#       newcolumns = []

#       for dict_key in overlap_dict:

      newcolumn = column + suffix

#         mdf_test[newcolumn] = mdf_test[column].isin(test_overlap_dict[dict_key])
#         mdf_test[newcolumn] = mdf_test[newcolumn].astype(np.int8)

#       mdf_train[newcolumn] = mdf_train[column].copy()
      mdf_test[newcolumn] = mdf_test[column].copy()

#       mdf_train[newcolumn] = mdf_train[newcolumn].astype(str)
      mdf_test[newcolumn] = mdf_test[newcolumn].astype(str)

      mdf_test[newcolumn] = mdf_test[newcolumn].replace(spl2_test_overlap_dict)
      mdf_test[newcolumn] = mdf_test[newcolumn].replace(spl5_test_zero_dict)

#       newcolumns.append(newcolumn)
    
    return mdf_test

  def postprocess_sp19_class(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #postprocess_splt_class(mdf_test, column, postprocess_dict, category)
    #preprocess column with categorical entries as strings
    #identifies overlaps of subsets of those strings and records
    #as a new boolan column
    #for example, if a categoical set consisted of unique values ['west', 'north', 'northeast']
    #then a new column would be created idenitifying cells which included 'north' in their entries
    #(here for north and northeast)
    #returns as column titled origcolumn_splt_entry    
    #missing values are ignored by default
    
    #here in postprocess we only create columns foir those overlaps that were identified 
    #fromthe train set
    
    #sp15 is comparable to splt but multiple concurrent activations allowed
    #so requires a different MLinfilltype in processdict
    
    #sp19 is comparable to sp15 but with a returned binary encoding aggregation
    '''
    
    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = '_sp19'
    
    #to retrieve the normalization dictionary we're going to use new method since we don't yet 
    #know what the returned columns titles are yet
    
    normkey = False
    
    if column in postprocess_dict['origcolumn']:
      
      columnkeylist = postprocess_dict['origcolumn'][column]['columnkeylist']
      
    else:
      
      origcolumn = postprocess_dict['column_dict'][column]['origcolumn']
      
      columnkeylist = postprocess_dict['origcolumn'][origcolumn]['columnkeylist']
    
    for columnkey in columnkeylist:
      
      if column == postprocess_dict['column_dict'][columnkey]['inputcolumn']:

        if 'splt_newcolumns_sp19' in postprocess_dict['column_dict'][columnkey]['normalization_dict'][columnkey] \
        and postprocess_dict['column_dict'][columnkey]['normalization_dict'][columnkey]['suffix'] == suffix:

          normkey = columnkey
        
    if normkey is not False:

      #great now we can grab normalization parameters
      overlap_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['overlap_dict']
      newcolumns = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['splt_newcolumns_sp19']
      int_headers = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['int_headers']
      int_labels_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['int_labels_dict']
      concurrent_activations = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['concurrent_activations']
      _1010_binary_encoding_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['_1010_binary_encoding_dict']
      _1010_binary_column_count = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['_1010_binary_column_count']
      _1010_activations_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['_1010_activations_dict']
      categorylist = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['categorylist']
      test_same_as_train = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['test_same_as_train']

      #now for mdf_test we'll only consider those overlaps already identified from train set
      
      if test_same_as_train is True:
        test_overlap_dict = overlap_dict

      elif test_same_as_train is False:

        unique_list_test = list(mdf_test[column].unique())

        unique_list_test = list(map(str, unique_list_test))

        test_overlap_dict = {}

        train_keys = list(overlap_dict)

        train_keys.sort(key = len, reverse=True)

        for key in train_keys:

          test_overlap_dict.update({key:[]})

        for dict_key in train_keys:

          for unique_test in unique_list_test:

            len_key = len(dict_key)

            if len(unique_test) >= len_key:

              nbr_iterations4 = len(unique_test) - len_key + 1

              for l in range(nbr_iterations4):

                extract4 = unique_test[l:(len_key+l)]

                if extract4 == dict_key:

                  test_overlap_dict[dict_key].append(unique_test)

                  if concurrent_activations is False:

                    break

      newcolumns = []

      for dict_key in overlap_dict:

        newcolumn = column + '_sp15_' + dict_key
        
#         mdf_train[newcolumn] = mdf_train[column].copy()
        mdf_test[newcolumn] = mdf_test[column].copy()

#         mdf_train[newcolumn] = mdf_train[newcolumn].astype(str)
        mdf_test[newcolumn] = mdf_test[newcolumn].astype(str)

        mdf_test[newcolumn] = mdf_test[newcolumn].isin(test_overlap_dict[dict_key])
        mdf_test[newcolumn] = mdf_test[newcolumn].astype(np.int8)

        newcolumns.append(newcolumn)
        
      #now convert coloumn headers from string to int convention
      if int_headers is True:
        mdf_test  = mdf_test.rename(columns=int_labels_dict)
        
      
      #begin binary encoding of set, leaving the int_headers here out of convenience, not really needed
      sp19_column = column + suffix
      
      mdf_test[sp19_column] = 'activations_'
      
      for entry in newcolumns:
        mdf_test[sp19_column] = mdf_test[sp19_column] + mdf_test[entry].astype(str)
        
      #extract categories for column labels
      #note that .unique() extracts the labels as a numpy array
      labels_train = list(_1010_binary_encoding_dict.keys())
      labels_train.sort()
      labels_test = list(mdf_test[sp19_column].unique())
      labels_test.sort()
      
      #if infill not present in train set, insert
      if 'zzzinfill' not in labels_test:
        labels_test = labels_test + ['zzzinfill']
        labels_test.sort()
        
      #replace the cateogries in train set via ordinal trasnformation
      mdf_test[sp19_column] = mdf_test[sp19_column].replace(_1010_binary_encoding_dict)
      
      #in test set, we'll need to strike any categories that weren't present in train
      #first let'/s identify what applies
      testspecificcategories = list(set(labels_test)-set(labels_train))
      
      #so we'll just replace those items with our plug value
      testplug_dict = dict(zip(testspecificcategories, ['zzzinfill'] * len(testspecificcategories)))
      mdf_test[sp19_column] = mdf_test[sp19_column].replace(testplug_dict)
      
      #now we'll apply the 1010 transformation to the test set
      mdf_test[sp19_column] = mdf_test[sp19_column].replace(_1010_binary_encoding_dict)
      
      #ok let's create a list of columns to store each entry of the binary encoding
      _1010_columnlist = []
      
      for i in range(_1010_binary_column_count):

        _1010_columnlist.append(column + suffix + '_' + str(i))

      #now let's store the encoding
      i=0
      for _1010_column in _1010_columnlist:

        mdf_test[_1010_column] = mdf_test[sp19_column].str.slice(i,i+1).astype(np.int8)

        i+=1
        
      #now delete the support column
      del mdf_test[sp19_column]
      for entry in newcolumns:
        del mdf_test[entry]
    
    return mdf_test

  def postprocess_sbst_class(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #process_sbst_class(mdf_train, mdf_test, column, category)
    #preprocess column with categorical entries as strings
    #identifies cases where a full unique value is present 
    #as a subset of a longer length unique value
    #and returns one-hot activations to aggregate those cases
    
    #accepts parameters concurrent_activations to allow mulitple activations
    #actually let's make concurrent activations the default
    #and int_headers for privacy preserving headers
    
    #this differs from other string parsing functions in that
    #only complete entries are checked for presence as subsets in other entries
    '''
    
    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = '_sbst'
    
    #to retrieve the normalization dictionary we're going to use new method since we don't yet 
    #know what the returned columns titles are yet
    
    normkey = False
    
    if column in postprocess_dict['origcolumn']:
      
      columnkeylist = postprocess_dict['origcolumn'][column]['columnkeylist']
      
    else:
      
      origcolumn = postprocess_dict['column_dict'][column]['origcolumn']
      
      columnkeylist = postprocess_dict['origcolumn'][origcolumn]['columnkeylist']
    
    for columnkey in columnkeylist:
      
      if column == postprocess_dict['column_dict'][columnkey]['inputcolumn']:

        if 'splt_newcolumns_sbst' in postprocess_dict['column_dict'][columnkey]['normalization_dict'][columnkey] \
        and postprocess_dict['column_dict'][columnkey]['normalization_dict'][columnkey]['suffix'] == suffix:

          normkey = columnkey
        
    if normkey is not False:

      #great now we can grab normalization parameters
      overlap_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['overlap_dict']
      newcolumns = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['splt_newcolumns_sbst']
      int_headers = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['int_headers']
      int_labels_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['int_labels_dict']
      concurrent_activations = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['concurrent_activations']
      test_same_as_train = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['test_same_as_train']

      #now for mdf_test we'll only consider those overlaps already identified from train set
      
      if test_same_as_train is True:
        test_overlap_dict = overlap_dict

      elif test_same_as_train is False:

        unique_list_test = list(mdf_test[column].unique())

        unique_list_test = list(map(str, unique_list_test))

        unique_list_test = sorted(unique_list_test, key=len, reverse=True)

        test_overlap_dict = {}

        train_keys = list(overlap_dict)

        train_keys.sort(key = len, reverse=True)

        for key in train_keys:

          test_overlap_dict.update({key:[]})

        for dict_key in train_keys:

          for unique_test in unique_list_test:

            len_key = len(dict_key)

            if len(unique_test) >= len_key:

              nbr_iterations4 = len(unique_test) - len_key + 1

              for l in range(nbr_iterations4):

                extract4 = unique_test[l:(len_key+l)]

                if extract4 == dict_key:

                  test_overlap_dict[dict_key].append(unique_test)

                  if concurrent_activations is False:

                    break

      newcolumns = []

      for dict_key in overlap_dict:

        newcolumn = column + suffix + '_' + dict_key
        
#         mdf_train[newcolumn] = mdf_train[column].copy()
        mdf_test[newcolumn] = mdf_test[column].copy()

#         mdf_train[newcolumn] = mdf_train[newcolumn].astype(str)
        mdf_test[newcolumn] = mdf_test[newcolumn].astype(str)

        mdf_test[newcolumn] = mdf_test[newcolumn].isin(test_overlap_dict[dict_key])
        mdf_test[newcolumn] = mdf_test[newcolumn].astype(np.int8)

        newcolumns.append(newcolumn)
        
      #now convert coloumn headers from string to int convention
      if int_headers is True:
        mdf_test  = mdf_test.rename(columns=int_labels_dict)
    
    return mdf_test

  def postprocess_sbs3_class(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #process_sbst_class(mdf_train, mdf_test, column, category)
    #preprocess column with categorical entries as strings
    #identifies cases where a full unique value is present 
    #as a subset of a longer length unique value
    #and returns one-hot activations to aggregate those cases
    
    #accepts parameters concurrent_activations to allow mulitple activations
    #actually let's make concurrent activations the default
    #and int_headers for privacy preserving headers
    
    #this differs from other string parsing functions in that
    #only complete entries are checked for presence as subsets in other entries
    
    #sbs3 is comparable to sbst but with a returned binary encoding aggregation
    '''
    
    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = '_sbs3'
    
    #to retrieve the normalization dictionary we're going to use new method since we don't yet 
    #know what the returned columns titles are yet
    
    normkey = False
    
    if column in postprocess_dict['origcolumn']:
      
      columnkeylist = postprocess_dict['origcolumn'][column]['columnkeylist']
      
    else:
      
      origcolumn = postprocess_dict['column_dict'][column]['origcolumn']
      
      columnkeylist = postprocess_dict['origcolumn'][origcolumn]['columnkeylist']
    
    for columnkey in columnkeylist:
      
      if column == postprocess_dict['column_dict'][columnkey]['inputcolumn']:

        if 'splt_newcolumns_sbs3' in postprocess_dict['column_dict'][columnkey]['normalization_dict'][columnkey] \
        and postprocess_dict['column_dict'][columnkey]['normalization_dict'][columnkey]['suffix'] == suffix:

          normkey = columnkey
        
    if normkey is not False:

      #great now we can grab normalization parameters
      overlap_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['overlap_dict']
      newcolumns = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['splt_newcolumns_sbs3']
      int_headers = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['int_headers']
      int_labels_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['int_labels_dict']
      concurrent_activations = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['concurrent_activations']
      _1010_binary_encoding_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['_1010_binary_encoding_dict']
      _1010_binary_column_count = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['_1010_binary_column_count']
      _1010_activations_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['_1010_activations_dict']
      categorylist = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['categorylist']
      test_same_as_train = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['test_same_as_train']
      
      #now for mdf_test we'll only consider those overlaps already identified from train set
      
      if test_same_as_train is True:
        test_overlap_dict = overlap_dict

      elif test_same_as_train is False:

        unique_list_test = list(mdf_test[column].unique())

        unique_list_test = list(map(str, unique_list_test))

        unique_list_test = sorted(unique_list_test, key=len, reverse=True)

        test_overlap_dict = {}

        train_keys = list(overlap_dict)

        train_keys.sort(key = len, reverse=True)

        for key in train_keys:

          test_overlap_dict.update({key:[]})

        for dict_key in train_keys:

          for unique_test in unique_list_test:

            len_key = len(dict_key)

            if len(unique_test) >= len_key:

              nbr_iterations4 = len(unique_test) - len_key + 1

              for l in range(nbr_iterations4):

                extract4 = unique_test[l:(len_key+l)]

                if extract4 == dict_key:

                  test_overlap_dict[dict_key].append(unique_test)

                  if concurrent_activations is False:

                    break

      newcolumns = []

      for dict_key in overlap_dict:

        newcolumn = column + '_sbst_' + dict_key
        
#         mdf_train[newcolumn] = mdf_train[column].copy()
        mdf_test[newcolumn] = mdf_test[column].copy()

#         mdf_train[newcolumn] = mdf_train[newcolumn].astype(str)
        mdf_test[newcolumn] = mdf_test[newcolumn].astype(str)

        mdf_test[newcolumn] = mdf_test[newcolumn].isin(test_overlap_dict[dict_key])
        mdf_test[newcolumn] = mdf_test[newcolumn].astype(np.int8)

        newcolumns.append(newcolumn)
        
      #now convert coloumn headers from string to int convention
      if int_headers is True:
        mdf_test  = mdf_test.rename(columns=int_labels_dict)
        
      
      #begin binary encoding of set, leaving the int_headers here out of convenience, not really needed
      
      sbs3_column = column + suffix
      
      mdf_test[sbs3_column] = 'activations_'
      
      for entry in newcolumns:
        mdf_test[sbs3_column] = mdf_test[sbs3_column] + mdf_test[entry].astype(str)
        
      #extract categories for column labels
      #note that .unique() extracts the labels as a numpy array
      labels_train = list(_1010_binary_encoding_dict.keys())
      labels_train.sort()
      labels_test = list(mdf_test[sbs3_column].unique())
      labels_test.sort()
      
      #if infill not present in train set, insert
      if 'zzzinfill' not in labels_test:
        labels_test = labels_test + ['zzzinfill']
        labels_test.sort()
        
      #replace the cateogries in train set via ordinal trasnformation
      mdf_test[sbs3_column] = mdf_test[sbs3_column].replace(_1010_binary_encoding_dict)
      
      #in test set, we'll need to strike any categories that weren't present in train
      #first let'/s identify what applies
      testspecificcategories = list(set(labels_test)-set(labels_train))
      
      #so we'll just replace those items with our plug value
      testplug_dict = dict(zip(testspecificcategories, ['zzzinfill'] * len(testspecificcategories)))
      mdf_test[sbs3_column] = mdf_test[sbs3_column].replace(testplug_dict)
      
      #now we'll apply the 1010 transformation to the test set
      mdf_test[sbs3_column] = mdf_test[sbs3_column].replace(_1010_binary_encoding_dict)
      
      #ok let's create a list of columns to store each entry of the binary encoding
      _1010_columnlist = []
      
      for i in range(_1010_binary_column_count):

        _1010_columnlist.append(column + suffix + '_' + str(i))

      #now let's store the encoding
      i=0
      for _1010_column in _1010_columnlist:

        mdf_test[_1010_column] = mdf_test[sbs3_column].str.slice(i,i+1).astype(np.int8)

        i+=1
        
      #now delete the support column
      del mdf_test[sbs3_column]
      for entry in newcolumns:
        del mdf_test[entry]
    
    return mdf_test
  
  def postprocess_srch_class(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    """
    #process_srch_class(mdf_train, mdf_test, column, category)
    #preprocess column with categorical entries as strings
    #relies on user passed list of strings in search parameter
    #string parses unique entries to identify overlaps with search strings
    #when overlap found returns a column with boolean activation identifiers
    
    #note this differs from original srch in that makes use of pandas str.contains
    #which is expected to be more efficient for unbounded sets
    
    #for example, if a categoical set consisted of unique values 
    #['west', 'north', 'northwest']
    #and a user passed the search parameter as ['west']
    #then a new column would be returned 
    #with activations corresponding to entries of 'west' and 'northwest'
    
    #note this returns all zeros in a column if search value not found
    
    #note returned coluymns are named by search term, e
    #e.g. column + '_srch_' + str(search)
    
    #note that search terms are converted to strings and compared to columns cast as strings

    #missing values are ignored by default
    """
    
    #to retrieve the normalization dictionary we're going to use new method since we don't yet 
    #know what the returned columns titles are yet
    
    normkey = False
    
    if column in postprocess_dict['origcolumn']:
      
      columnkeylist = postprocess_dict['origcolumn'][column]['columnkeylist']
      
    else:
      
      origcolumn = postprocess_dict['column_dict'][column]['origcolumn']
      
      columnkeylist = postprocess_dict['origcolumn'][origcolumn]['columnkeylist']
    
    for columnkey in columnkeylist:
      
      if column == postprocess_dict['column_dict'][columnkey]['inputcolumn']:

        if 'srch_newcolumns_srch' in postprocess_dict['column_dict'][columnkey]['normalization_dict'][columnkey]:

          normkey = columnkey
        
    if normkey is not False:

      #great now we can grab normalization parameters
      search_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['search_dict']
      inverse_search_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['inverse_search_dict']
      newcolumns = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['srch_newcolumns_srch']
#       newcolumns_before_aggregation = \
#       postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['newcolumns_before_aggregation']
#       search = \
#       postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['search']
#       search_preflattening = \
#       postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['search_preflattening']
      aggregated_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['aggregated_dict']
      case = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['case']
      
      for newcolumn in search_dict:

        mdf_test[newcolumn] = \
        np.where(mdf_test[column].astype(str).str.contains(search_dict[newcolumn], case=case, regex=False), 1, 0)
        
      #now we consolidate activations
      #note that this only runs when aggregated_dict was populated with an embedded list of search terms
      for aggregated_dict_key in aggregated_dict:
        aggregated_dict_key_column = inverse_search_dict[aggregated_dict_key]

        for target_for_aggregation in aggregated_dict[aggregated_dict_key]:
          target_for_aggregation_column = inverse_search_dict[target_for_aggregation]

          mdf_test[aggregated_dict_key_column] = \
          np.where(mdf_test[target_for_aggregation_column] == 1, 1, mdf_test[aggregated_dict_key_column])

          del mdf_test[target_for_aggregation_column]

      for newcolumn in newcolumns:

        mdf_test[newcolumn] = mdf_test[newcolumn].astype(np.int8)
    
    return mdf_test

  def postprocess_src2_class(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    """
    #process_src2_class(mdf_train, mdf_test, column, category)
    #preprocess column with categorical entries as strings
    #relies on user passed list of strings in search parameter
    #string parses unique entries to identify overlaps with search strings
    #when overlap found returns a column with boolean activation identifiers
    
    #for example, if a categoical set consisted of unique values 
    #['west', 'north', 'northwest']
    #and a user passed the search parameter as ['west']
    #then a new column would be returned 
    #with activations corresponding to entries of 'west' and 'northwest'

    #missing values are ignored by default
    
    #assumes that unique values of test set are same or subset of train set
    #for more efficient application in postmunge
    """
    
    #to retrieve the normalization dictionary we're going to use new method since we don't yet 
    #know what the returned columns titles are yet
    
    normkey = False
    
    if column in postprocess_dict['origcolumn']:
      
      columnkeylist = postprocess_dict['origcolumn'][column]['columnkeylist']
      
    else:
      
      origcolumn = postprocess_dict['column_dict'][column]['origcolumn']
      
      columnkeylist = postprocess_dict['origcolumn'][origcolumn]['columnkeylist']
    
    for columnkey in columnkeylist:
      
      if column == postprocess_dict['column_dict'][columnkey]['inputcolumn']:

        if 'src2_newcolumns_src2' in postprocess_dict['column_dict'][columnkey]['normalization_dict'][columnkey]:

          normkey = columnkey
        
    if normkey is not False:

      #great now we can grab normalization parameters
      overlap_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['overlap_dict']

      newcolumns = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['newcolumns_before_aggregation']
      
      search = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['search']
      
      inverse_search_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['inverse_search_dict']
      
      aggregated_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['aggregated_dict']
      
#       #now for mdf_test

#       unique_list_test = list(mdf_test[column].unique())

#       unique_list_test = list(map(str, unique_list_test))

#       test_overlap_dict = {}

#       for search_string in search:

#         test_overlap_dict.update({search_string : []})

#       train_keys = list(overlap_dict)

#       train_keys.sort(key = len, reverse=True)

#       for dict_key in train_keys:

#         for unique_test in unique_list_test:

#           len_key = len(dict_key)

#           if len(unique_test) >= len_key:

#             nbr_iterations4 = len(unique_test) - len_key

#             for l in range(nbr_iterations4 + 1):

#               extract4 = unique_test[l:(len_key+l)]

#               if extract4 == dict_key:

#                 test_overlap_dict[dict_key].append(unique_test)

      newcolumns = []

      for dict_key in overlap_dict:
        
        if len(overlap_dict[dict_key]) > 0:

          newcolumn = column + '_src2_' + dict_key

  #         mdf_train[newcolumn] = mdf_train[column].copy()
          mdf_test[newcolumn] = mdf_test[column].copy()

  #         mdf_train[newcolumn] = mdf_train[newcolumn].astype(str)
          mdf_test[newcolumn] = mdf_test[newcolumn].astype(str)

          mdf_test[newcolumn] = mdf_test[newcolumn].isin(overlap_dict[dict_key])
#           mdf_test[newcolumn] = mdf_test[newcolumn].astype(np.int8)

          newcolumns.append(newcolumn)
          
      #now we consolidate activations
      #note that this only runs when aggregated_dict was populated with an embedded list of search terms
      for aggregated_dict_key in aggregated_dict:
        aggregated_dict_key_column = inverse_search_dict[aggregated_dict_key]

        for target_for_aggregation in aggregated_dict[aggregated_dict_key]:
          target_for_aggregation_column = inverse_search_dict[target_for_aggregation]

          mdf_test[aggregated_dict_key_column] = \
          np.where(mdf_test[target_for_aggregation_column] == 1, 1, mdf_test[aggregated_dict_key_column])

          del mdf_test[target_for_aggregation_column]

          newcolumns.remove(target_for_aggregation_column)

      for newcolumn in newcolumns:

        mdf_test[newcolumn] = mdf_test[newcolumn].astype(np.int8)
    
    return mdf_test

  def postprocess_src3_class(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    """
    #process_src3_class(mdf_train, mdf_test, column, category)
    #preprocess column with categorical entries as strings
    #relies on user passed list of strings in search parameter
    #string parses unique entries to identify overlaps with search strings
    #when overlap found returns a column with boolean activation identifiers
    
    #for example, if a categoical set consisted of unique values 
    #['west', 'north', 'northwest']
    #and a user passed the search parameter as ['west']
    #then a new column would be returned 
    #with activations corresponding to entries of 'west' and 'northwest'

    #missing values are ignored by default
    
    #where srch is preferred for unbounded range of unique values
    
    #and src2 preferred when have bounded range of unique values for both train & test
    
    #and speculation is that src3 may be preferred when have a bounded
    #range of unique values but still want capacity to handle values in 
    #test set not found in train set
    """
    
    #to retrieve the normalization dictionary we're going to use new method since we don't yet 
    #know what the returned columns titles are yet
    
    normkey = False
    
    if column in postprocess_dict['origcolumn']:
      
      columnkeylist = postprocess_dict['origcolumn'][column]['columnkeylist']
      
    else:
      
      origcolumn = postprocess_dict['column_dict'][column]['origcolumn']
      
      columnkeylist = postprocess_dict['origcolumn'][origcolumn]['columnkeylist']
    
    for columnkey in columnkeylist:
      
      if column == postprocess_dict['column_dict'][columnkey]['inputcolumn']:

        if 'srch_newcolumns_src3' in postprocess_dict['column_dict'][columnkey]['normalization_dict'][columnkey]:

          normkey = columnkey
        
    if normkey is not False:

      #great now we can grab normalization parameters
      overlap_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['overlap_dict']

      newcolumns = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['srch_newcolumns_src3']
      
      search = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['search']
      
      #now for mdf_test

      unique_list_test = list(mdf_test[column].unique())

      unique_list_test = list(map(str, unique_list_test))

      test_overlap_dict = {}

      for search_string in search:

        test_overlap_dict.update({search_string : []})

      train_keys = list(overlap_dict)

      train_keys.sort(key = len, reverse=True)

      for dict_key in train_keys:

        for unique_test in unique_list_test:

          len_key = len(dict_key)

          if len(unique_test) >= len_key:

            nbr_iterations4 = len(unique_test) - len_key

            for l in range(nbr_iterations4 + 1):

              extract4 = unique_test[l:(len_key+l)]

              if extract4 == dict_key:

                test_overlap_dict[dict_key].append(unique_test)

      newcolumns = []

      for dict_key in overlap_dict:
        
        if len(overlap_dict[dict_key]) > 0:

          newcolumn = column + '_src3_' + dict_key

  #         mdf_train[newcolumn] = mdf_train[column].copy()
          mdf_test[newcolumn] = mdf_test[column].copy()

  #         mdf_train[newcolumn] = mdf_train[newcolumn].astype(str)
          mdf_test[newcolumn] = mdf_test[newcolumn].astype(str)

          mdf_test[newcolumn] = mdf_test[newcolumn].isin(test_overlap_dict[dict_key])
          mdf_test[newcolumn] = mdf_test[newcolumn].astype(np.int8)

          newcolumns.append(newcolumn)
    
    return mdf_test

  def postprocess_src4_class(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    """
    #process_srch_class(mdf_train, mdf_test, column, category)
    #preprocess column with categorical entries as strings
    #relies on user passed list of strings in search parameter
    #string parses unique entries to identify overlaps with search strings
    #when overlap found returns a column with boolean activation identifiers
    
    #note this differs from original srch in that makes use of pandas str.contains
    #which is expected to be more efficient for unbounded sets
    
    #for example, if a categoical set consisted of unique values 
    #['west', 'north', 'northwest']
    #and a user passed the search parameter as ['west']
    #then a new column would be returned 
    #with activations corresponding to entries of 'west' and 'northwest'
    
    #note this returns all zeros in a column if search value not found
    
    #note returned coluymns are named by search term, e
    #e.g. column + '_srch_' + str(search)
    
    #note that search terms are converted to strings and compared to columns cast as strings

    #missing values are ignored by default
    
    #src4 builds on the srch by converting to an ordinal activation
    #with 0 reserved for no activations
    #note that if an entry was activated for multiple search terms
    #the order of entries in search parameter will dictate the final encoding
    #(e.g. entries at end of list are prioritized over beginning)
    """
    
    #to retrieve the normalization dictionary we're going to use new method since we don't yet 
    #know what the returned columns titles are yet
    
    normkey = column + '_src4'

    #great now we can grab normalization parameters
    search_dict = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['search_dict']

    newcolumns = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['srch_newcolumns_src4']

    search = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['search']

    case = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['case']

    ordl_dict1 = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['ordl_dict1']

    ordl_dict2 = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['ordl_dict2']

    aggregated_dict = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['aggregated_dict']
    
    inverse_search_dict = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['inverse_search_dict']
    
    if len(search_dict) == 0:
      mdf_test[column + '_src4'] = 0
      
    else:

      for newcolumn in search_dict:

        mdf_test[newcolumn] = \
        np.where(mdf_test[column].astype(str).str.contains(search_dict[newcolumn], case=case, regex=False), 1, 0)

  #     for newcolumn in newcolumns:

  #       mdf_test[newcolumn] = mdf_test[newcolumn].astype(np.int8)

      mdf_test[column + '_src4'] = 0

      for newcolumn in newcolumns:
        mdf_test[column + '_src4'] = \
        np.where(mdf_test[newcolumn] == 1, ordl_dict2[newcolumn], mdf_test[column + '_src4'])
        del mdf_test[newcolumn]
        
      #now we consolidate activations
      #note that this only runs when aggregated_dict was populated with an embedded list of search terms
      for aggregated_dict_key in aggregated_dict:
        aggregated_dict_key_column = inverse_search_dict[aggregated_dict_key]
        aggregated_dict_key_encoding = ordl_dict2[aggregated_dict_key_column]

        for target_for_aggregation in aggregated_dict[aggregated_dict_key]:
          target_for_aggregation_column = inverse_search_dict[target_for_aggregation]
          target_for_aggregation_encoding = ordl_dict2[target_for_aggregation_column]

          mdf_test[column + '_src4'] = \
          np.where(mdf_test[column + '_src4'] == target_for_aggregation_encoding, aggregated_dict_key_encoding, mdf_test[column + '_src4'])

      #we'll base the integer type on number of ordinal entries
      if len(ordl_dict1) < 254:
        mdf_test[column + '_src4'] = mdf_test[column + '_src4'].astype(np.uint8)
      elif len(ordl_dict1) < 65530:
        mdf_test[column + '_src4'] = mdf_test[column + '_src4'].astype(np.uint16)
      else:
        mdf_test[column + '_src4'] = mdf_test[column + '_src4'].astype(np.uint32)
    
    return mdf_test

  def postprocess_nmr4_class(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    """
    #extract numeric partitions from categoric entries, test treated differently than train
    #accepts parameters
    #convention as numbers/commas/spaces
    #suffix for column suffix identifier
    #test_same_as_train as True/False
    #where True copiues overlap_dict from train for test, False parses test entries not found in train
    """
    
    if 'suffix' in params:
      #accepts string for suffix appender
      suffix = params['suffix']
    else:
      suffix = '_nmr4'
      
    nmrc_column = column + suffix
    
    mean = \
    postprocess_dict['column_dict'][nmrc_column]['normalization_dict'][nmrc_column]['mean']
    unique_list = \
    postprocess_dict['column_dict'][nmrc_column]['normalization_dict'][nmrc_column]['unique_list']
    overlap_dict = \
    postprocess_dict['column_dict'][nmrc_column]['normalization_dict'][nmrc_column]['overlap_dict']
    convention = \
    postprocess_dict['column_dict'][nmrc_column]['normalization_dict'][nmrc_column]['convention']
    test_same_as_train = \
    postprocess_dict['column_dict'][nmrc_column]['normalization_dict'][nmrc_column]['test_same_as_train']
    
    mdf_test[nmrc_column] = mdf_test[column].copy()
    
    test_unique_list = list(mdf_test[nmrc_column].unique())
    test_unique_list = list(map(str, test_unique_list))
    extra_test_unique = list(set(test_unique_list) - set(unique_list))

    test_overlap_dict = deepcopy(overlap_dict)
    
    if test_same_as_train is True:
      
      for test_unique in extra_test_unique:
        test_overlap_dict.update({str(test_unique) : np.nan})
      
    elif test_same_as_train is False:
      
      testmaxlength = max(len(x) for x in unique_list)

      overlap_lengths = list(range(testmaxlength, 0, -1))

  #     overlap_dict = {}

      for overlap_length in overlap_lengths:

        for unique in extra_test_unique:

          if unique not in test_overlap_dict:

            len_unique = len(unique)

            if len_unique >= overlap_length:

              if overlap_length > 1:

                nbr_iterations = len_unique - overlap_length

                for i in range(nbr_iterations + 1):

                  if unique not in test_overlap_dict:

                    extract = unique[i:(overlap_length+i)]

    #                 extract_already_in_overlap_dict = False
                    
                    if convention == 'numbers':
                    
                      if self.is_number(extract):

                        test_overlap_dict.update({unique : float(extract)})
                  
                    elif convention == 'commas':
                    
                      if self.is_number_comma(extract):

                        test_overlap_dict.update({unique : float(extract.replace(',',''))})
                        
                    elif convention == 'spaces':
                    
                      if self.is_number_EU(extract):

                        test_overlap_dict.update({unique : float(extract[0] + extract[1:-1].replace(' ','').replace('.','').replace(',','.') + extract[-1])})

              #else if overlap_length == 1    
              else:

                nbr_iterations = len_unique - overlap_length

                in_dict = False

                for i in range(nbr_iterations + 1):

                  if unique not in test_overlap_dict:

                    extract = unique[i:(overlap_length+i)]

    #                 extract_already_in_overlap_dict = False
                    
                    if self.is_number(extract):

                      in_dict = True

                      test_overlap_dict.update({unique : float(extract)})

                if in_dict is False:

                  test_overlap_dict.update({unique : np.nan})
    
    #great now that test_overlap_dict is populated
    mdf_test[nmrc_column] = mdf_test[nmrc_column].astype(str)
    mdf_test[nmrc_column] = mdf_test[nmrc_column].replace(test_overlap_dict)

    mdf_test[nmrc_column] = pd.to_numeric(mdf_test[nmrc_column], errors='coerce')
    
    #replace missing data with training set mean as default infill
    mdf_test[nmrc_column] = mdf_test[nmrc_column].fillna(mean)
    
    return mdf_test
  
  def postprocess_ordl_class(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #postprocess_ordl_class(mdf_test, column, postprocess_dict, columnkey)
    #preprocess column with categories into ordinal (sequentuial integer) sets
    #corresponding to (sorted) categories
    #adresses infill with new point which we arbitrarily set as 'zzzinfill'
    #intended to show up as last point in set alphabetically
    #for categories presetn in test set not present in train set use this 'zzz' category
    '''
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    normkey = column + '_ordl'
    
    #grab normalization parameters from postprocess_dict
    ordinal_dict = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['ordinal_dict']
    
    overlap_replace = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['ordinal_overlap_replace']
    
    adjinfill = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['adjinfill']
    
    str_convert = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['str_convert']
    
    if inplace is not True:
      #copy source column into new column
      mdf_test[column + '_ordl'] = mdf_test[column].copy()
    else:
      mdf_test.rename(columns = {column : column + '_ordl'}, inplace = True)
    
    #convert column to category
    mdf_test[column + '_ordl'] = mdf_test[column + '_ordl'].astype('category')
    
    #if set is categorical we'll need the plug value for missing values included
    if 'zzzinfill' not in mdf_test[column + '_ordl'].cat.categories:
      mdf_test[column + '_ordl'] = mdf_test[column + '_ordl'].cat.add_categories(['zzzinfill'])
      
    if adjinfill is True:
      mdf_test[column + '_ordl'] = mdf_test[column + '_ordl'].fillna(method='ffill')
      mdf_test[column + '_ordl'] = mdf_test[column + '_ordl'].fillna(method='bfill')

    #replace NA with a dummy variable
    mdf_test[column + '_ordl'] = mdf_test[column + '_ordl'].fillna('zzzinfill')
    
    if str_convert is True:
      #replace numerical with string equivalent
      mdf_test[column + '_ordl'] = mdf_test[column + '_ordl'].astype(str)
    else:
      mdf_test[column + '_ordl'] = mdf_test[column + '_ordl'].astype('object')
    
    #extract categories for column labels
    #note that .unique() extracts the labels as a numpy array
    #train categories are in the ordinal_dict we p[ulled from normalization_dict
    labels_train = list(ordinal_dict.keys())
    labels_train = sorted(labels_train, key=str)
    labels_test = list(mdf_test[column + '_ordl'].unique())
    labels_test = sorted(labels_test, key=str)
    
    #if infill not present in train set, insert
    if 'zzzinfill' not in labels_train:
      labels_train = labels_train + ['zzzinfill']
      labels_train = sorted(labels_train, key=str)
    if 'zzzinfill' not in labels_test:
      labels_test = labels_test + ['zzzinfill']
      labels_test = sorted(labels_test, key=str)
      
    #here we replace the overlaps with version with jibberish suffix
    if len(overlap_replace) > 0:
      mdf_test[column + '_ordl'] = mdf_test[column + '_ordl'].replace(overlap_replace)
    
    #in test set, we'll need to strike any categories that weren't present in train
    #first let'/s identify what applies
    testspecificcategories = list(set(labels_test)-set(labels_train))
    
    #edge case, replace operation do0esn't work when column dtype is int
    if mdf_test[column + '_ordl'].dtype.name != 'object':
      mdf_test[column + '_ordl'] = mdf_test[column + '_ordl'].astype('object')

    #so we'll just replace those items with our plug value
    testplug_dict = dict(zip(testspecificcategories, ['zzzinfill'] * len(testspecificcategories)))
    mdf_test[column + '_ordl'] = mdf_test[column + '_ordl'].replace(testplug_dict)

    #edge case, replace operation do0esn't work when column dtype is int
    if mdf_test[column + '_ordl'].dtype.name != 'object':
      mdf_test[column + '_ordl'] = mdf_test[column + '_ordl'].astype('object')

    #now we'll apply the ordinal transformation to the test set
    mdf_test[column + '_ordl'] = mdf_test[column + '_ordl'].replace(ordinal_dict)
    
    #just want to make sure these arent' being saved as floats for memory considerations
    if len(ordinal_dict) < 254:
      mdf_test[column + '_ordl'] = mdf_test[column + '_ordl'].astype(np.uint8)
    elif len(ordinal_dict) < 65530:
      mdf_test[column + '_ordl'] = mdf_test[column + '_ordl'].astype(np.uint16)
    else:
      mdf_test[column + '_ordl'] = mdf_test[column + '_ordl'].astype(np.uint32)
    
#     #convert column to category
#     mdf_test[column + '_ordl'] = mdf_test[column + '_ordl'].astype('category')
    
    return mdf_test
  
  def postprocess_ord3_class(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #postprocess_ord3_class(mdf_test, column, postprocess_dict, columnkey)
    #preprocess column with categories into ordinal (sequentuial integer) sets
    #corresponding to categories sorted by frequency of occurance
    #adresses infill with new point which we arbitrarily set as 'zzzinfill'
    #intended to show up as last point in set alphabetically
    #for categories presetn in test set not present in train set use this 'zzz' category
    '''
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    normkey = column + '_ord3'
    
    #grab normalization parameters from postprocess_dict
    ordinal_dict = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['ordinal_dict']
    
    overlap_replace = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['ordinal_overlap_replace']
    
    adjinfill = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['adjinfill']
    
    str_convert = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['str_convert']
    
    if inplace is not True:
      #copy source column into new column
      mdf_test[column + '_ord3'] = mdf_test[column].copy()
    else:
      mdf_test.rename(columns = {column : column + '_ord3'}, inplace = True)
    
    #convert column to category
    mdf_test[column + '_ord3'] = mdf_test[column + '_ord3'].astype('category')
    
    #if set is categorical we'll need the plug value for missing values included
    if 'zzzinfill' not in mdf_test[column + '_ord3'].cat.categories:
      mdf_test[column + '_ord3'] = mdf_test[column + '_ord3'].cat.add_categories(['zzzinfill'])
      
    if adjinfill is True:
      mdf_test[column + '_ord3'] = mdf_test[column + '_ord3'].fillna(method='ffill')
      mdf_test[column + '_ord3'] = mdf_test[column + '_ord3'].fillna(method='bfill')

    #replace NA with a dummy variable
    mdf_test[column + '_ord3'] = mdf_test[column + '_ord3'].fillna('zzzinfill')
    
    if str_convert is True:
      #replace numerical with string equivalent
      mdf_test[column + '_ord3'] = mdf_test[column + '_ord3'].astype(str)  
    else:
      mdf_test[column + '_ord3'] = mdf_test[column + '_ord3'].astype('object')
    
    #extract categories for column labels
    #note that .unique() extracts the labels as a numpy array
    #train categories are in the ordinal_dict we p[ulled from normalization_dict
    labels_train = list(ordinal_dict.keys())
#     labels_train.sort()
    labels_test = list(mdf_test[column + '_ord3'].unique())
    labels_test = sorted(labels_test, key=str)
    
    #if infill not present in train set, insert
    if 'zzzinfill' not in labels_train:
      labels_train = labels_train + ['zzzinfill']
#       labels_train.sort()
    if 'zzzinfill' not in labels_test:
      labels_test = labels_test + ['zzzinfill']
      labels_test = sorted(labels_test, key=str)
      
    #here we replace the overlaps with version with jibberish suffix
    if len(overlap_replace) > 0:
      mdf_test[column + '_ord3'] = mdf_test[column + '_ord3'].replace(overlap_replace)
    
    #in test set, we'll need to strike any categories that weren't present in train
    #first let'/s identify what applies
    testspecificcategories = list(set(labels_test)-set(labels_train))
    
    #so we'll just replace those items with our plug value
    testplug_dict = dict(zip(testspecificcategories, ['zzzinfill'] * len(testspecificcategories)))
    #edge case for replace operation if dtype drifted such as to numeric
    if mdf_test[column + '_ord3'].dtype.name != 'object':
      mdf_test[column + '_ord3'] = mdf_test[column + '_ord3'].astype('object')
    mdf_test[column + '_ord3'] = mdf_test[column + '_ord3'].replace(testplug_dict)
    
    #now we'll apply the ordinal transformation to the test set
    if mdf_test[column + '_ord3'].dtype.name != 'object':
      mdf_test[column + '_ord3'] = mdf_test[column + '_ord3'].astype('object')
    mdf_test[column + '_ord3'] = mdf_test[column + '_ord3'].replace(ordinal_dict)
    
    #just want to make sure these arent' being saved as floats for memory considerations
    if len(ordinal_dict) < 254:
      mdf_test[column + '_ord3'] = mdf_test[column + '_ord3'].astype(np.uint8)
    elif len(ordinal_dict) < 65530:
      mdf_test[column + '_ord3'] = mdf_test[column + '_ord3'].astype(np.uint16)
    else:
      mdf_test[column + '_ord3'] = mdf_test[column + '_ord3'].astype(np.uint32)
        
#     #convert column to category
#     mdf_test[column + '_ordl'] = mdf_test[column + '_ordl'].astype('category')
    
    return mdf_test
  
  def postprocess_ucct_class(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #process_ucct_class(mdf_train, mdf_test, column, category)
    #preprocess column with categories into unique class count sets
    #normalized by total row count
    #e.g. for each class in train set, 
    #counts instances and divides by total train set row count
    #(so values will fall in range 0-1)
    #test sets recive comparable encoding
    '''
    
    normkey = column + '_ucct'
    
    #grab normalization parameters from postprocess_dict
    ordinal_dict = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['ordinal_dict']
    
    overlap_replace = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['ordinal_overlap_replace']
    
    #create new column for trasnformation
    mdf_test[column + '_ucct'] = mdf_test[column].copy()
    
    #convert column to category
    mdf_test[column + '_ucct'] = mdf_test[column + '_ucct'].astype('category')
    
    #if set is categorical we'll need the plug value for missing values included
    if 'zzzinfill' not in mdf_test[column + '_ucct'].cat.categories:
      mdf_test[column + '_ucct'] = mdf_test[column + '_ucct'].cat.add_categories(['zzzinfill'])

    #replace NA with a dummy variable
    mdf_test[column + '_ucct'] = mdf_test[column + '_ucct'].fillna('zzzinfill')
    
    #replace numerical with string equivalent
    mdf_test[column + '_ucct'] = mdf_test[column + '_ucct'].astype(str)    
    
    #extract categories for column labels
    #note that .unique() extracts the labels as a numpy array
    #train categories are in the ordinal_dict we p[ulled from normalization_dict
    labels_train = list(ordinal_dict.keys())
#     labels_train.sort()
    labels_test = list(mdf_test[column + '_ucct'].unique())
    labels_test.sort()
    
    #if infill not present in train set, insert
    if 'zzzinfill' not in labels_train:
      labels_train = labels_train + ['zzzinfill']
#       labels_train.sort()
    if 'zzzinfill' not in labels_test:
      labels_test = labels_test + ['zzzinfill']
      labels_test.sort()
      
    #here we replace the overlaps with version with jibberish suffix
    if len(overlap_replace) > 0:
      mdf_test[column + '_ucct'] = mdf_test[column + '_ucct'].replace(overlap_replace)
    
    #in test set, we'll need to strike any categories that weren't present in train
    #first let'/s identify what applies
    testspecificcategories = list(set(labels_test)-set(labels_train))
    
    #so we'll just replace those items with our plug value
    testplug_dict = dict(zip(testspecificcategories, ['zzzinfill'] * len(testspecificcategories)))
    mdf_test[column + '_ucct'] = mdf_test[column + '_ucct'].replace(testplug_dict)
    
    #now we'll apply the ordinal transformation to the test set
    mdf_test[column + '_ucct'] = mdf_test[column + '_ucct'].replace(ordinal_dict)
    
    return mdf_test
  
  def postprocess_1010_class(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #postprocess_1010_class(mdf_test, column, postprocess_dict, columnkey)
    #preprocess column with categories into binary encoded sets
    #corresponding to (sorted) categories of >2 values
    #adresses infill with new point which we arbitrarily set as 'zzzinfill'
    #intended to show up as last point in set alphabetically
    #for categories presetn in test set not present in train set use this 'zzz' category
    '''
    
    normkey = column + '_1010_0'
    
    if normkey in postprocess_dict['column_dict']:
    
      #grab normalization parameters from postprocess_dict
      binary_encoding_dict = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['_1010_binary_encoding_dict']

      overlap_replace = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['_1010_overlap_replace']

      binary_column_count = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['_1010_binary_column_count']
      
      adjinfill = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['adjinfill']
      
      str_convert = \
      postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['str_convert']

      #create new column for trasnformation
      mdf_test[column + '_1010'] = mdf_test[column].copy()    

      #convert column to category
      mdf_test[column + '_1010'] = mdf_test[column + '_1010'].astype('category')

      #if set is categorical we'll need the plug value for missing values included
      if 'zzzinfill' not in mdf_test[column + '_1010'].cat.categories:
        mdf_test[column + '_1010'] = mdf_test[column + '_1010'].cat.add_categories(['zzzinfill'])
        
      if adjinfill is True:
        mdf_test[column + '_1010'] = mdf_test[column + '_1010'].fillna(method='ffill')
        mdf_test[column + '_1010'] = mdf_test[column + '_1010'].fillna(method='bfill')

      #replace NA with a dummy variable
      mdf_test[column + '_1010'] = mdf_test[column + '_1010'].fillna('zzzinfill')

      if str_convert is True:
        #replace numerical with string equivalent
        mdf_test[column + '_1010'] = mdf_test[column + '_1010'].astype(str)
      else:
        mdf_test[column + '_1010'] = mdf_test[column + '_1010'].astype('object')

      #extract categories for column labels
      #note that .unique() extracts the labels as a numpy array
      #train categories are in the ordinal_dict we p[ulled from normalization_dict
      labels_train = list(binary_encoding_dict.keys())
#       labels_train.sort()
      labels_test = list(mdf_test[column + '_1010'].unique())
#       labels_test.sort()
      labels_test = sorted(labels_test, key=str)

      #if infill not present in train set, insert
      if 'zzzinfill' not in labels_train:
        labels_train = labels_train + ['zzzinfill']
#         labels_train.sort()
        labels_train = sorted(labels_train, key=str)
      if 'zzzinfill' not in labels_test:
        labels_test = labels_test + ['zzzinfill']
#         labels_test.sort()
        labels_test = sorted(labels_test, key=str)

      #here we replace the overlaps with version with jibberish suffix
      if len(overlap_replace) > 0:

        mdf_test[column + '_1010'] = mdf_test[column + '_1010'].replace(overlap_replace)

      #in test set, we'll need to strike any categories that weren't present in train
      #first let'/s identify what applies
      testspecificcategories = list(set(labels_test)-set(labels_train))

      #so we'll just replace those items with our plug value
      testplug_dict = dict(zip(testspecificcategories, ['zzzinfill'] * len(testspecificcategories)))
      if mdf_test[column + '_1010'].dtype.name != 'object':
        mdf_test[column + '_1010'] = mdf_test[column + '_1010'].astype('object')
      mdf_test[column + '_1010'] = mdf_test[column + '_1010'].replace(testplug_dict)    

      #now we'll apply the 1010 transformation to the test set
      if mdf_test[column + '_1010'].dtype.name != 'object':
        mdf_test[column + '_1010'] = mdf_test[column + '_1010'].astype('object')
      mdf_test[column + '_1010'] = mdf_test[column + '_1010'].replace(binary_encoding_dict)   

      #ok let's create a list of columns to store each entry of the binary encoding
      _1010_columnlist = []

      for i in range(binary_column_count):

        _1010_columnlist.append(column + '_1010_' + str(i))

      #now let's store the encoding
      i=0
      for _1010_column in _1010_columnlist:

        mdf_test[_1010_column] = mdf_test[column + '_1010'].str.slice(i,i+1).astype(np.int8)

        i+=1

      #now delete the support column
      del mdf_test[column + '_1010']
    
    return mdf_test

  def postprocess_tmsc_class(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    """
    #time data segregated by time scale
    #with sin or cos applied to address periodicity
    #such as may be useful to return both by seperate transformation categories
    #accepts parameter 'scale' to distinguish between year/month/day/hour/minute/second
    #note that some scales can be returned combined by passing 
    #monthday/dayhourminute/hourminutesecond/minutesecond
    #accepts parameter 'suffix' for returned column header suffix
    #accets parameter 'function' to distinguish between sin/cos
    """
    
    if 'suffix' in params:
      #accepts column header suffix appender
      suffix = params['suffix']
    else:
      suffix = '_mdsn'
      
    time_column = column + suffix
    
    scale = \
    postprocess_dict['column_dict'][time_column]['normalization_dict'][time_column]['scale']
    function = \
    postprocess_dict['column_dict'][time_column]['normalization_dict'][time_column]['function']
    
    mdf_test[time_column] = mdf_test[column].copy()
    
    #apply pd.to_datetime to column, note that the errors = 'coerce' needed for messy data
    mdf_test[time_column] = pd.to_datetime(mdf_test[time_column], errors = 'coerce')
    
    #access time scale from one of year/month/day/hour/minute/second
    #monthday/dayhourminute/hourminutesecond/minutesecond
    if scale == 'year':
      mdf_test[time_column] = mdf_test[time_column].dt.year
      
      #we'll scale periodicity by decade
      mdf_test[time_column] = (mdf_test[time_column]) * 2 * np.pi / 10
      
    elif scale == 'month':
      mdf_test[time_column] = mdf_test[time_column].dt.month
      
      #we'll scale periodicity by year
      mdf_test[time_column] = (mdf_test[time_column]) * 2 * np.pi / 12
      
    elif scale == 'day':
      mdf_test[time_column] = mdf_test[time_column].dt.day
      
      #we'll scale periodicity by week
      mdf_test[time_column] = (mdf_test[time_column]) * 2 * np.pi / 7
      
    elif scale == 'hour':
      mdf_test[time_column] = mdf_test[time_column].dt.hour
      
      #we'll scale periodicity by day
      mdf_test[time_column] = (mdf_test[time_column]) * 2 * np.pi / 24
      
    elif scale == 'minute':
      mdf_test[time_column] = mdf_test[time_column].dt.minute
      
      #we'll scale periodicity by hour
      mdf_test[time_column] = (mdf_test[time_column]) * 2 * np.pi / 60
      
    elif scale == 'second':
      mdf_test[time_column] = mdf_test[time_column].dt.second
      
      #we'll scale periodicity by minute
      mdf_test[time_column] = (mdf_test[time_column]) * 2 * np.pi / 60
      
    elif scale == 'monthday':
      tempcolumn1 = time_column + '_tmp1'
      tempcolumn2 = time_column + '_tmp2'
      
      #temp1 is for number of days in month, temp2 is to handle leap year support
      
      mdf_test[tempcolumn1] = mdf_test[time_column].copy()
      mdf_test[tempcolumn2] = mdf_test[time_column].copy()
      
      mdf_test[tempcolumn1] = mdf_test[tempcolumn1].dt.month
      mdf_test[tempcolumn2] = mdf_test[tempcolumn2].dt.is_leap_year
      
      mdf_test[tempcolumn2] = \
      np.where(mdf_test[tempcolumn2], 29, 28)
      
      mdf_test[tempcolumn1] = \
      np.where(mdf_test[tempcolumn1].isin([1,3,5,7,8,10,12]), 31, mdf_test[tempcolumn1].values)
      
      mdf_test[tempcolumn1] = \
      np.where(mdf_test[tempcolumn1].isin([4,6,9,11]), 30, mdf_test[tempcolumn1].values)
      
      mdf_test[tempcolumn1] = \
      np.where(mdf_test[tempcolumn1].isin([2]), mdf_test[tempcolumn2], \
      mdf_test[tempcolumn1].values)
      
      #combine month and day, scale for trigonomic transform, periodicity by year
      mdf_test[time_column] = (mdf_test[time_column].dt.month + mdf_test[time_column].dt.day / \
      mdf_test[tempcolumn1]) * 2 * np.pi / 12
      
      #delete the support columns
      del mdf_test[tempcolumn1]
      del mdf_test[tempcolumn2]
      
    elif scale == 'dayhourminute':
      #we'll scale periodicity by week
      mdf_test[time_column] = (mdf_test[time_column].dt.day + mdf_test[time_column].dt.hour / 24 + mdf_test[time_column].dt.minute / 24 / 60) * 2 * np.pi / 7

    elif scale == 'hourminutesecond':
      #we'll scale periodicity by day
      mdf_test[time_column] = (mdf_test[time_column].dt.hour + mdf_test[time_column].dt.minute / 60 + mdf_test[time_column].dt.second / 60 / 60) * 2 * np.pi / 24

    elif scale == 'minutesecond':
      #we'll scale periodicity by hour
      mdf_test[time_column] = (mdf_test[time_column].dt.minute + mdf_test[time_column].dt.second / 60) * 2 * np.pi / 60
      
    #default infill is adjacent cell
    mdf_test[time_column] = mdf_test[time_column].fillna(method='ffill')
    mdf_test[time_column] = mdf_test[time_column].fillna(method='bfill')
    
    #backup default infill for cases without valid entries
    mdf_test[time_column] = mdf_test[time_column].fillna(0)
    
    #apply trigometric transform
    
    if function == 'sin':
      mdf_test[time_column] = np.sin(mdf_test[time_column])
      
    if function == 'cos':
      mdf_test[time_column] = np.cos(mdf_test[time_column])
      
    return mdf_test

  def postprocess_time_class(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    """
    #z-score normalized time data segregated by a particular time scale
    #accepts parameter 'scale' to distinguish between year/month/day/hour/minute/second
    #accepts parameter 'suffix' for returned column header suffix
    #accepts parameter 'normalization' to distinguish between zscore/minmax/unscaled
    """
    
    if 'suffix' in params:
      #accepts column header suffix appender
      suffix = params['suffix']
    else:
      suffix = '_year'
    
    time_column = column + suffix
    
    scale = \
    postprocess_dict['column_dict'][time_column]['normalization_dict'][time_column]['scale']
    normalization = \
    postprocess_dict['column_dict'][time_column]['normalization_dict'][time_column]['normalization']
    scaler = \
    postprocess_dict['column_dict'][time_column]['normalization_dict'][time_column]['scaler']
    divisor = \
    postprocess_dict['column_dict'][time_column]['normalization_dict'][time_column]['divisor']
    
    mdf_test[time_column] = mdf_test[column].copy()
    
    #apply pd.to_datetime to column, note that the errors = 'coerce' needed for messy data
    mdf_test[time_column] = pd.to_datetime(mdf_test[time_column], errors = 'coerce')
    
    #access time scale from one of year/month/day/hour/minute/second
    if scale == 'year':
      mdf_test[time_column] = mdf_test[time_column].dt.year
    elif scale == 'month':
      mdf_test[time_column] = mdf_test[time_column].dt.month
    elif scale == 'day':
      mdf_test[time_column] = mdf_test[time_column].dt.day
    elif scale == 'hour':
      mdf_test[time_column] = mdf_test[time_column].dt.hour
    elif scale == 'minute':
      mdf_test[time_column] = mdf_test[time_column].dt.minute
    elif scale == 'second':
      mdf_test[time_column] = mdf_test[time_column].dt.second
      
    #apply default infill
    mdf_test[time_column] = mdf_test[time_column].fillna(method='ffill')
    mdf_test[time_column] = mdf_test[time_column].fillna(method='bfill')
    
    #backup default infill for cases without valid entries
    mdf_test[time_column] = mdf_test[time_column].fillna(0)
    
    #apply normalization
    if normalization != 'unscaled':
      mdf_test[time_column] = (mdf_test[time_column] - scaler) / divisor
      
    return mdf_test
  
  def postprocess_bxcx_class(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    Applies box-cox method within postmunge function.
    '''
  #   #df_train, nmbrcolumns, nmbrnormalization_dict, categorylist = \
  #   mdf_train, column_dict_list = \
  #   self.process_bxcx_support(mdf_train, column, category, 1, bxcx_lmbda = None, \
  #                             trnsfrm_mean = None, trnsfrm_std = None)

  #   #grab the normalization_dict associated with the bxcx category
  #   columnkeybxcx = column + '_bxcx'
  #   for column_dict in column_dict_list:
  #     if columnkeybxcx in column_dict:
  #       bxcxnormalization_dict = column_dict[columnkeybxcx]['normalization_dict'][columnkey]

    #bxcxkey = columnkey[:-5] + '_bxcx'

    #grab the normalization_dict associated with the bxcx category
    normkey = column+'_bxcx'
    bxcxnormalization_dict = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]
    #postprocess_dict['column_dict'][bxcxkey]['normalization_dict'][bxcxkey]
    
    temporigcategoryplug = 'bxcx'

    #df_test, nmbrcolumns, _1, _2 = \
    mdf_test, _1 = \
    self.process_bxcx_support(mdf_test, column, temporigcategoryplug, 1, bxcx_lmbda = \
                             bxcxnormalization_dict['bxcx_lmbda'], \
                             trnsfrm_mean = bxcxnormalization_dict['trnsfrm_mean'])

    return mdf_test

  def postprocess_log0_class(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
        
    '''
    #function to apply logatrithmic transform
    #takes as arguement pandas dataframe of training and test data (mdf_train), (mdf_test)\
    #and the name of the column string ('column') and parent category (category)
    #applies a logarithmic transform (base 10)
    #replaces zeros, negative, and missing or improperly formatted data with post-log mean as default infill
    #returns same dataframes with new column of name column + '_log0'
    '''
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    #retrieve normalizastion parameters from postprocess_dict
    normkey = column + '_log0'
    
    meanlog = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['meanlog']

    if inplace is not True:
      mdf_test[column + '_log0'] = mdf_test[column].copy()
    else:
      mdf_test.rename(columns = {column : column + '_log0'}, inplace = True)

    #convert all values to either numeric or NaN
    mdf_test[column + '_log0'] = pd.to_numeric(mdf_test[column + '_log0'], errors='coerce')
    
    #replace all non-positive with nan for the log operation
    mdf_test.loc[mdf_test[column + '_log0'] <= 0, (column + '_log0')] = np.nan
    
    #log transform column
    #note that this replaces negative values with nan which we will infill with meanlog
    mdf_test[column + '_log0'] = np.log10(mdf_test[column + '_log0'])

    #get mean of training data
    meanlog = meanlog  

    #replace missing data with training set mean
    mdf_test[column + '_log0'] = mdf_test[column + '_log0'].fillna(meanlog)

#     #replace missing data with 0
#     mdf_test[column + '_log0'] = mdf_test[column + '_log0'].fillna(0)

#     #change data type for memory savings
#     mdf_test[column + '_log0'] = mdf_test[column + '_log0'].astype(np.float32)

    return mdf_test

  def postprocess_logn_class(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
        
    '''
    #function to apply natural logatrithmic transform
    #takes as arguement pandas dataframe of training and test data (mdf_train), (mdf_test)\
    #and the name of the column string ('column') and parent category (category)
    #applies a logarithmic transform (base e)
    #replaces zeros, negative, and missing or improperly formatted data with post-log mean as default infill
    #returns same dataframes with new column of name column + '_logn'
    '''
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    #retrieve normalizastion parameters from postprocess_dict
    normkey = column + '_logn'
    
    meanlog = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['meanlog']

    if inplace is not True:
      mdf_test[column + '_logn'] = mdf_test[column].copy()
    else:
      mdf_test.rename(columns = {column : column + '_logn'}, inplace = True)

    #convert all values to either numeric or NaN
    mdf_test[column + '_logn'] = pd.to_numeric(mdf_test[column + '_logn'], errors='coerce')
    
    #replace all non-positive with nan for the log operation
    mdf_test.loc[mdf_test[column + '_logn'] <= 0, (column + '_logn')] = np.nan
    
    #log transform column
    #note that this replaces negative values with nan which we will infill with meanlog
    mdf_test[column + '_logn'] = np.log(mdf_test[column + '_logn'])

    #get mean of training data
    meanlog = meanlog  

    #replace missing data with training set mean
    mdf_test[column + '_logn'] = mdf_test[column + '_logn'].fillna(meanlog)

#     #replace missing data with 0
#     mdf_test[column + '_log0'] = mdf_test[column + '_log0'].fillna(0)

#     #change data type for memory savings
#     mdf_test[column + '_log0'] = mdf_test[column + '_log0'].astype(np.float32)

    return mdf_test

  def postprocess_sqrt_class(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
        
    '''
    #function to apply square root transform
    #takes as arguement pandas dataframe of training and test data (mdf_train), (mdf_test)\
    #and the name of the column string ('column') and parent category (category)
    #applies a square root transform
    #replaces zeros, negative, and missing or improperly formatted data with post-log mean as default infill
    #returns same dataframes with new column of name column + '_log0'
    '''
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    #retrieve normalizastion parameters from postprocess_dict
    normkey = column + '_sqrt'
    
    meansqrt = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['meansqrt']

    if inplace is not True:
      mdf_test[column + '_sqrt'] = mdf_test[column].copy()
    else:
      mdf_test.rename(columns = {column : column + '_sqrt'}, inplace = True)

    #convert all values to either numeric or NaN
    mdf_test[column + '_sqrt'] = pd.to_numeric(mdf_test[column + '_sqrt'], errors='coerce')
    
    #replace all non-positive with nan for the log operation
    mdf_test.loc[mdf_test[column + '_sqrt'] < 0, (column + '_sqrt')] = np.nan
    
    #log transform column
    #note that this replaces negative values with nan which we will infill with meanlog
    mdf_test[column + '_sqrt'] = np.sqrt(mdf_test[column + '_sqrt'])

    #get mean of training data
    meansqrt = meansqrt  

    #replace missing data with training set mean
    mdf_test[column + '_sqrt'] = mdf_test[column + '_sqrt'].fillna(meansqrt)

#     #replace missing data with 0
#     mdf_test[column + '_log0'] = mdf_test[column + '_log0'].fillna(0)

#     #change data type for memory savings
#     mdf_test[column + '_log0'] = mdf_test[column + '_log0'].astype(np.float32)

    return mdf_test

  def postprocess_addd_class(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #postprocess_addd_class(.)
    #function to apply addition transform
    #takes as arguement pandas dataframe of training and test data (mdf_train), (mdf_test)\
    #and the name of the column string ('column') and parent category (category)
    #accepts parameter 'add' for amount of addition, otherwise defaults to adding 1
    #applies an addition transform
    #replaces non-numeric entries with set mean after addition
    #returns same dataframes with new column of name column + '_addd'
    '''
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    #retrieve normalizastion parameters from postprocess_dict
    normkey = column + '_addd'
    
    mean = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['mean']
    
    add = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['add']

    if inplace is not True:
      mdf_test[column + '_addd'] = mdf_test[column].copy()
    else:
      mdf_test.rename(columns = {column : column + '_addd'}, inplace = True)

    #convert all values to either numeric or NaN
    mdf_test[column + '_addd'] = pd.to_numeric(mdf_test[column + '_addd'], errors='coerce')
    
    #lperform addition
    mdf_test[column + '_addd'] = mdf_test[column + '_addd'] + add
    
    #replace missing data with training set mean
    mdf_test[column + '_addd'] = mdf_test[column + '_addd'].fillna(mean)

    return mdf_test
  
  def postprocess_sbtr_class(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #postprocess_sbtr_class(.)
    #function to apply subtraction transform
    #takes as arguement pandas dataframe of training and test data (mdf_train), (mdf_test)\
    #and the name of the column string ('column') and parent category (category)
    #accepts parameter 'subtract' for amount of subtraction, otherwise defaults to subtracting 1
    #applies a subtraction transform
    #replaces non-numeric entries with set mean after subtraction
    #returns same dataframes with new column of name column + '_sbtr'
    '''
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
        
    #retrieve normalizastion parameters from postprocess_dict
    normkey = column + '_sbtr'
    
    mean = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['mean']
    
    subtract = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['subtract']

    if inplace is not True:
      mdf_test[column + '_sbtr'] = mdf_test[column].copy()
    else:
      mdf_test.rename(columns = {column : column + '_sbtr'}, inplace = True)

    #convert all values to either numeric or NaN
    mdf_test[column + '_sbtr'] = pd.to_numeric(mdf_test[column + '_sbtr'], errors='coerce')
    
    #lperform subtraction
    mdf_test[column + '_sbtr'] = mdf_test[column + '_sbtr'] - subtract
    
    #replace missing data with training set mean
    mdf_test[column + '_sbtr'] = mdf_test[column + '_sbtr'].fillna(mean)

    return mdf_test
  
  def postprocess_mltp_class(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #postprocess_mltp_class(.)
    #function to apply multiplication transform
    #takes as arguement pandas dataframe of training and test data (mdf_train), (mdf_test)\
    #and the name of the column string ('column') and parent category (category)
    #accepts parameter 'multiply' for amount of addition, otherwise defaults to multiplying 2
    #applies an multiplication transform
    #replaces non-numeric entries with set mean after addition
    #returns same dataframes with new column of name column + '_mltp'
    '''
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    #retrieve normalizastion parameters from postprocess_dict
    normkey = column + '_mltp'
    
    mean = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['mean']
    
    multiply = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['multiply']

    if inplace is not True:
      mdf_test[column + '_mltp'] = mdf_test[column].copy()
    else:
      mdf_test.rename(columns = {column : column + '_mltp'}, inplace = True)

    #convert all values to either numeric or NaN
    mdf_test[column + '_mltp'] = pd.to_numeric(mdf_test[column + '_mltp'], errors='coerce')
    
    #lperform addition
    mdf_test[column + '_mltp'] = mdf_test[column + '_mltp'] * multiply
    
    #replace missing data with training set mean
    mdf_test[column + '_mltp'] = mdf_test[column + '_mltp'].fillna(mean)

    return mdf_test
    
  def postprocess_divd_class(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #postprocess_divd_class(.)
    #function to apply division transform
    #accepts parameter 'divide' for amount of division, otherwise defaults to dividing by 2
    #applies an division transform
    #replaces non-numeric entries with set mean after division
    #returns same dataframes with new column of name column + '_divd'
    '''
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    #retrieve normalizastion parameters from postprocess_dict
    normkey = column + '_divd'
    
    mean = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['mean']
    
    divide = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['divide']

    if inplace is not True:
      mdf_test[column + '_divd'] = mdf_test[column].copy()
    else:
      mdf_test.rename(columns = {column : column + '_divd'}, inplace = True)

    #convert all values to either numeric or NaN
    mdf_test[column + '_divd'] = pd.to_numeric(mdf_test[column + '_divd'], errors='coerce')
    
    #lperform addition
    mdf_test[column + '_divd'] = mdf_test[column + '_divd'] / divide
    
    #replace missing data with training set mean
    mdf_test[column + '_divd'] = mdf_test[column + '_divd'].fillna(mean)

    return mdf_test
  
  def postprocess_rais_class(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #postprocess_rais_class(.)
    #function to apply raise to a power transform
    #accepts parameter 'raiser' for amount of power, otherwise defaults to square (raise by 2)
    #applies an raise transform
    #replaces non-numeric entries with set mean after raise
    #returns same dataframes with new column of name column + '_rais'
    '''
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
        
    #retrieve normalizastion parameters from postprocess_dict
    normkey = column + '_rais'
    
    mean = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['mean']
    
    raiser = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['raiser']

    if inplace is not True:
      mdf_test[column + '_rais'] = mdf_test[column].copy()
    else:
      mdf_test.rename(columns = {column : column + '_rais'}, inplace = True)

    #convert all values to either numeric or NaN
    mdf_test[column + '_rais'] = pd.to_numeric(mdf_test[column + '_rais'], errors='coerce')
    
    #lperform addition
    mdf_test[column + '_rais'] = mdf_test[column + '_rais'] ** raiser
    
    #replace missing data with training set mean
    mdf_test[column + '_rais'] = mdf_test[column + '_rais'].fillna(mean)

    return mdf_test
    
  def postprocess_absl_class(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #postprocess_absl_class(.)
    #function to apply absolute transform
    #does not accept paraemters
    #applies an absolute transform
    #replaces non-numeric entries with set mean after transform
    #returns same dataframes with new column of name column + '_absl'
    '''
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    #retrieve normalizastion parameters from postprocess_dict
    normkey = column + '_absl'
    
    mean = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['mean']

    if inplace is not True:
      mdf_test[column + '_absl'] = mdf_test[column].copy()
    else:
      mdf_test.rename(columns = {column : column + '_absl'}, inplace = True)

    #convert all values to either numeric or NaN
    mdf_test[column + '_absl'] = pd.to_numeric(mdf_test[column + '_absl'], errors='coerce')
    
    #lperform addition
    mdf_test[column + '_absl'] = mdf_test[column + '_absl'].abs()
    
    #replace missing data with training set mean
    mdf_test[column + '_absl'] = mdf_test[column + '_absl'].fillna(mean)

    return mdf_test

  def postprocess_pwrs_class(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #processes a numerical set by creating bins corresponding to powers
    #of ten in one hot encoded columns
    
    #pwrs will be intended for a raw set that is not yet normalized
    
    #postiive values encoded under column 'column' + '_10^#' where # is power of 10
    #0 and negative values considered infill with no activations
    
    #if all values are infill no columns returned
    
    #accepts boolean 'negvalues' parameter, defaults False, True activates encoding for values <0
    '''
    
    #retrieve normalization parameters from postprocess_dict
    normkey = False
    
    if column in postprocess_dict['origcolumn']:
      
      columnkeylist = postprocess_dict['origcolumn'][column]['columnkeylist']
      
    else:
      
      origcolumn = postprocess_dict['column_dict'][column]['origcolumn']
      
      columnkeylist = postprocess_dict['origcolumn'][origcolumn]['columnkeylist']
    
    for columnkey in columnkeylist:
      
      if column == postprocess_dict['column_dict'][columnkey]['inputcolumn']:

        if 'powerlabelsdict_pwrs' in postprocess_dict['column_dict'][columnkey]['normalization_dict'][columnkey]:

          normkey = columnkey
        
    if normkey is not False:

      #normkey = columnkey

      powerlabelsdict = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['powerlabelsdict_pwrs']
      labels_train = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['labels_train']
      negvalues = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['negvalues']

      textcolumns = postprocess_dict['column_dict'][normkey]['categorylist']

      tempcolumn = column + '_-10^'
      
      #store original column for later reversion
      mdf_test[tempcolumn] = mdf_test[column].copy()

      #convert all values to either numeric or NaN
      mdf_test[tempcolumn] = pd.to_numeric(mdf_test[tempcolumn], errors='coerce')


      #create copy with negative values
      negtempcolumn = column + '_negtemp'
      mdf_test[negtempcolumn] = mdf_test[tempcolumn].copy()

      #convert all values in negtempcolumn >= 0 to Nan
      mdf_test[negtempcolumn] = \
      np.where(mdf_test[negtempcolumn] >= 0, np.nan, mdf_test[negtempcolumn].values)

      #convert all values <= 0 to Nan
      mdf_test[tempcolumn] = \
      np.where(mdf_test[tempcolumn] <= 0, np.nan, mdf_test[tempcolumn].values)

      #log transform column

      #take abs value of negtempcolumn
      mdf_test[negtempcolumn] = mdf_test[negtempcolumn].abs()

      mdf_test[negtempcolumn] = \
      np.where(mdf_test[negtempcolumn] != np.nan, np.floor(np.log10(mdf_test[negtempcolumn])), mdf_test[negtempcolumn].values)

      test_neg_dict = {}
      negunique = mdf_test[negtempcolumn].unique()
      for unique in negunique:
        if unique != unique:
          newunique = np.nan
        else:
          #this is update for difference between pwr2 and pwrs
          if negvalues:
            newunique = column + '_-10^' + str(int(unique))
          else:
            newunique = np.nan
        if newunique in labels_train and newunique == newunique:
          test_neg_dict.update({unique : newunique})
        else:
          test_neg_dict.update({unique : np.nan})

      mdf_test[negtempcolumn] = mdf_test[negtempcolumn].replace(test_neg_dict)

      #now log trasnform positive values in column column 
      mdf_test[tempcolumn] = \
      np.where(mdf_test[tempcolumn] != np.nan, np.floor(np.log10(mdf_test[tempcolumn])), mdf_test[tempcolumn].values)

      test_pos_dict = {}
      posunique = mdf_test[tempcolumn].unique()
      for unique in posunique:
        if unique != unique:
          newunique = np.nan
        else:
          newunique = column + '_10^' + str(int(unique))
        if newunique in labels_train and newunique == newunique:
          test_pos_dict.update({unique : newunique})
        else:
          test_pos_dict.update({unique : np.nan})

      mdf_test[tempcolumn] = mdf_test[tempcolumn].replace(test_pos_dict)    

      #combine the two columns
      mdf_test[tempcolumn] = mdf_test[negtempcolumn].where(mdf_test[negtempcolumn] == mdf_test[negtempcolumn], mdf_test[tempcolumn])

      #apply onehotencoding
      df_test_cat = pd.get_dummies(mdf_test[tempcolumn])

      #Get missing columns in test set that are present in training set
      missing_cols = set( textcolumns ) - set( df_test_cat.columns )

      #Add a missing column in test set with default value equal to 0
      for c in missing_cols:
          df_test_cat[c] = 0

      #Ensure the order of column in the test set is in the same order than in train set
      #Note this also removes categories in test set that aren't present in training set
      df_test_cat = df_test_cat[textcolumns]

      #concatinate the sparse set with the rest of our training data
      mdf_test = pd.concat([mdf_test, df_test_cat], axis=1)

      #replace original column
      del mdf_test[negtempcolumn]

      del mdf_test[tempcolumn]

      #change data types to 8-bit (1 byte) integers for memory savings
      for textcolumn in textcolumns:

        mdf_test[textcolumn] = mdf_test[textcolumn].astype(np.int8)

    return mdf_test
  
  def postprocess_pwor_class(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #processes a numerical set by creating bins coresponding to powers
    #of ten in ordinal encoded columns
    
    #pwrs will be intended for a raw set that is not yet normalized
    
    #infill has 0, other designations are based on the data
    
    #negative values allows, comparable to pwr2
    '''
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    #get normkey
    normkey = column + '_pwor'
    
    #retrieve stuff from normalization dictionary
    train_replace_dict = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['train_replace_dict']
    negvalues = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['negvalues']
    
    pworcolumn = column + '_pwor'

    if inplace is not True:
      mdf_test[pworcolumn] = mdf_test[column].copy()
    else:
      mdf_test.rename(columns = {column : pworcolumn}, inplace = True)

    #convert all values to either numeric or NaN
    mdf_test[pworcolumn] = pd.to_numeric(mdf_test[pworcolumn], errors='coerce')
    
    #copy set for negative values
    negtempcolumn = column + '_negtempcolumn'
    
    mdf_test[negtempcolumn] = mdf_test[pworcolumn].copy()
    
    #convert all values >= 0 to Nan
    mdf_test[negtempcolumn] = \
    np.where(mdf_test[negtempcolumn] >= 0, np.nan, mdf_test[negtempcolumn].values)
    
    #take abs value of negtempcolumn
    mdf_test[negtempcolumn] = mdf_test[negtempcolumn].abs()
    
    #convert all values <= 0 in column to Nan
    mdf_test[pworcolumn] = \
    np.where(mdf_test[pworcolumn] <= 0, np.nan, mdf_test[pworcolumn].values)

    mdf_test[pworcolumn] = \
    np.where(mdf_test[pworcolumn] != np.nan, np.floor(np.log10(mdf_test[pworcolumn])), mdf_test[pworcolumn].values)
    
    #do same for negtempcolumn
    mdf_test[negtempcolumn] = \
    np.where(mdf_test[negtempcolumn] != np.nan, np.floor(np.log10(mdf_test[negtempcolumn])), mdf_test[negtempcolumn].values)

    newunique_list = list(train_replace_dict)
      
    test_neg_dict = {}
    negunique = mdf_test[negtempcolumn].unique()
    for unique in negunique:
      if unique != unique:
        newunique = np.nan
      else:
        #this is update for difference between pwr2 and pwrs
        if negvalues:
          newunique = column + '_-10^' + str(int(unique))
        else:
          newunique = np.nan
      if newunique in newunique_list and newunique == newunique:
        test_neg_dict.update({unique : newunique})
      else:
        test_neg_dict.update({unique : np.nan})
        
    mdf_test[negtempcolumn] = mdf_test[negtempcolumn].replace(test_neg_dict)
    
    #now do same for column
 
    test_pos_dict = {}
    posunique = mdf_test[pworcolumn].unique()
    for unique in posunique:
      if unique != unique:
        newunique = np.nan
      else:
        newunique = column + '_10^' + str(int(unique))
      if newunique in newunique_list and newunique == newunique:
        test_pos_dict.update({unique : newunique})
      else:
        test_pos_dict.update({unique : np.nan})
    
    mdf_test[pworcolumn] = mdf_test[pworcolumn].replace(test_pos_dict)
    
    #combine the two columns
    mdf_test[pworcolumn] = mdf_test[negtempcolumn].where(mdf_test[negtempcolumn] == mdf_test[negtempcolumn], mdf_test[pworcolumn])
    
    test_unique = mdf_test[pworcolumn].unique()
  
    #Get missing entries in test set that are present in training set
    missing_cols = set( list(newunique_list) ) - set( list(test_unique) )
    
    extra_cols = set( list(test_unique) ) - set( list(newunique_list) )
      
    test_replace_dict = {}
    for testunique in test_unique:
      if testunique in newunique_list:
        test_replace_dict.update({testunique : train_replace_dict[testunique]})
      else:
        test_replace_dict.update({testunique : 0})
    
#     pworcolumn = column + '_por2'
#     mdf_test[pworcolumn] = mdf_test[column].copy()
    
    mdf_test[pworcolumn] = mdf_test[pworcolumn].replace(test_replace_dict)

    #replace original column from training data
    del mdf_test[negtempcolumn]    
    
    return mdf_test

  def postprocess_bins_class(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #bins processes a numerical set by creating bins coresponding to post z score
    #normalization of <-2, -2-1, -10, 01, 12, >2 in one hot encoded columns
    
    #bins will be intended for a raw set that is not normalized
    #bint will be intended for a previously normalized set
    
    #bins accepts a parameter bincount 
    #as integer for number of bins
    #where if bincount is an odd number the center bin straddles the mean
    #and if bincount is even the center two bins straddle the mean
    #defaults to 5 bins
    #suffix appender is '_bins_#'' where # is integer for bin id
    '''
    
    #retrieve normalization parameters from postprocess_dict
    normkey = column +'_bins_0'
    
    if normkey in postprocess_dict['column_dict']:

      mean = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['binsmean']
      std = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['binsstd']
      bincuts = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bincuts']
      binlabels = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['binlabels']
      textcolumns = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['binscolumns']
      normalizedinput = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['normalizedinput']

      binscolumn = column + '_bins'

      mdf_test[binscolumn] = mdf_test[column].copy()

      #convert all values to either numeric or NaN
      mdf_test[binscolumn] = pd.to_numeric(mdf_test[binscolumn], errors='coerce')

      #replace missing data with training set mean
      mdf_test[binscolumn] = mdf_test[binscolumn].fillna(mean)
      
      if normalizedinput is False:

        #z-score normalize
        mdf_test[binscolumn] = (mdf_test[binscolumn] - mean) / std

      #create bins based on standard deviation increments
  #     binscolumn = column + '_bins'
      mdf_test[binscolumn] = \
      pd.cut( mdf_test[binscolumn], bins = bincuts,  \
             labels = binlabels, precision=4)

      #process bins as a categorical set
      mdf_test = \
      self.postprocess_textsupport_class(mdf_test, binscolumn, {}, 'tempkey', {'textcolumns':textcolumns})

      #change data type for memory savings
      for textcolumn in textcolumns:
        mdf_test[textcolumn] = mdf_test[textcolumn].astype(np.int8)

      #delete the support column
      del mdf_test[binscolumn]

    return mdf_test

  def postprocess_bsor_class(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #bins processes a numerical set by creating bins coresponding to post z score
    #normalization of <-2, -2-1, -10, 01, 12, >2 in one hot encoded columns
    
    #bins accepts a parameter bincount 
    #as integer for number of bins
    #where if bincount is an odd number the center bin straddles the mean
    #and if bincount is even the center two bins straddle the mean
    #defaults to 5 bins
    #suffix appender is '_bins_#'' where # is integer for bin id
    
    #bsor is comparable to bins but returns ordinal encoded column
    '''
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    if 'bincount' in params:
      bincount = params['bincount']
    else:
      bincount = 6
      
    if bincount > 0:

      #retrieve normalization parameters from postprocess_dict
      normkey = column +'_bsor'
      #normkey = columnkey

      mean = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['binsmean']
      std = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['binsstd']
      ordinal_dict = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['ordinal_dict']
      bincuts = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bincuts']
      binlabels = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['binlabels']
      normalizedinput = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['normalizedinput']
      
      binscolumn = column + '_bsor'

      if inplace is not True:
        mdf_test[binscolumn] = mdf_test[column].copy()
      else:
        mdf_test.rename(columns = {column : binscolumn}, inplace = True)
      
      #convert all values to either numeric or NaN
      mdf_test[binscolumn] = pd.to_numeric(mdf_test[binscolumn], errors='coerce')

      #replace missing data with training set mean
      mdf_test[binscolumn] = mdf_test[binscolumn].fillna(mean)

      if normalizedinput is False:
      
        #z-score normalize
        mdf_test[binscolumn] = (mdf_test[binscolumn] - mean) / std

      #create bins based on standard deviation increments
      mdf_test[binscolumn] = \
      pd.cut( mdf_test[binscolumn], bins = bincuts,  \
             labels = binlabels, precision=4)

      mdf_test[binscolumn] = mdf_test[binscolumn].astype(np.int8)

    return mdf_test
  
  def postprocess_bnwd_class(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #processes a numerical set by creating equal width bins coresponding to 
    #parameter 'width' which defaults to 1
    #and returning in one-hot encoded set
    
    #deletes columns without activations
    
    #can be applied top either a raw set not yet normalized or after normalization
    #such as after z-score normalization)
    '''
    
    if 'width' in params:
      bn_width = params['width']
    else:
      bn_width = 1
    
    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = '_bnwd'
    
    #to retrieve the normalization dictionary we're going to use new method since we don't yet 
    #know what the returned columns titles are yet
    
    normkey = False
    
    if column in postprocess_dict['origcolumn']:
      
      columnkeylist = postprocess_dict['origcolumn'][column]['columnkeylist']
      
    else:
      
      origcolumn = postprocess_dict['column_dict'][column]['origcolumn']
      
      columnkeylist = postprocess_dict['origcolumn'][origcolumn]['columnkeylist']
    
    for columnkey in columnkeylist:
      
      if column == postprocess_dict['column_dict'][columnkey]['inputcolumn']:

        if 'bn_width_bnwd' in postprocess_dict['column_dict'][columnkey]['normalization_dict'][columnkey]:

          if postprocess_dict['column_dict'][columnkey]['normalization_dict'][columnkey]['bn_width_bnwd'] == bn_width \
          and postprocess_dict['column_dict'][columnkey]['normalization_dict'][columnkey]['suffix'] == suffix:

            normkey = columnkey
        
    if normkey is not False:
      
      suffix = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
      mean = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['binsmean']
      bn_min = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_min']
      bn_max = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_max']
      bn_delta = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_delta']
      bn_count = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_count']
      bins_id = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_id']
      bins_cuts = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_cuts']
      bn_width = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_width_bnwd']
      textcolumns = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['textcolumns']
      
      binscolumn = column + suffix
      
      #store original column for later reversion
      mdf_test[binscolumn] = mdf_test[column].copy()

      #convert all values to either numeric or NaN
      mdf_test[binscolumn] = pd.to_numeric(mdf_test[binscolumn], errors='coerce')

      #replace missing data with training set mean
      mdf_test[binscolumn] = mdf_test[binscolumn].fillna(mean)
      
      #create bins based on standard deviation increments
#       binscolumn = column + '_bnwd'

      mdf_test[binscolumn] = \
      pd.cut(mdf_test[binscolumn], bins = bins_cuts,  \
             labels = bins_id, precision=len(str(bn_count)))

      mdf_test = \
      self.postprocess_textsupport_class(mdf_test, binscolumn, {}, 'tempkey', {'textcolumns':textcolumns})

      #change data type for memory savings
      for textcolumn in textcolumns:
        mdf_test[textcolumn] = mdf_test[textcolumn].astype(np.int8)

      #delete the support column
      del mdf_test[binscolumn]

    return mdf_test

  def postprocess_bnwo_class(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #processes a numerical set by creating equal width bins coresponding to 
    #parameter 'width' which defaults to 1
    #and returning in ordinal encoded set
    
    #segments without activations are included
    
    #can be applied top either a raw set not yet normalized or after normalization
    #such as after z-score normalization)
    '''
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = '_bnwo'
    
    #retrieve normalization parameters from postprocess_dict
    normkey = column + suffix
    
    mean = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['binsmean']
    bn_min = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_min']
    bn_max = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_max']
    bn_delta = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_delta']
    bn_count = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_count']
    bins_id = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_id']
    bins_cuts = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_cuts']
    bn_width = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_width']

    binscolumn = column + suffix
    
    if inplace is not True:
      mdf_test[binscolumn] = mdf_test[column].copy()
    else:
      mdf_test.rename(columns = {column : binscolumn}, inplace = True)

    #convert all values to either numeric or NaN
    mdf_test[binscolumn] = pd.to_numeric(mdf_test[binscolumn], errors='coerce')

    #replace missing data with training set mean
    mdf_test[binscolumn] = mdf_test[binscolumn].fillna(mean)

    #create bins based on standard deviation increments
#     binscolumn = column + '_bnwo'
    mdf_test[binscolumn] = \
    pd.cut(mdf_test[binscolumn], bins = bins_cuts,  \
           labels = bins_id, precision=len(str(bn_count)))
    
    #change column dtype
    mdf_test[binscolumn] = mdf_test[binscolumn].astype(int)
    
    return mdf_test

  def postprocess_bnep_class(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #processes a numerical set by creating equal population bins coresponding to 
    #parameter 'bincount' which defaults to 5
    #and returning in one-hot encoded set
    
    #default infill is to have no activations in a row
    
    #can be applied top either a raw set not yet normalized or after normalization
    #such as after z-score normalization)
    '''
    
    if 'bincount' in params:
      bincount = params['bincount']
    else:
      bincount = 5
      
    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = '_bnep'
    
    #to retrieve the normalization dictionary we're going to use new method since we don't yet 
    #know what the returned columns titles are yet
    
    normkey = False
    
    if column in postprocess_dict['origcolumn']:
      
      columnkeylist = postprocess_dict['origcolumn'][column]['columnkeylist']
      
    else:
      
      origcolumn = postprocess_dict['column_dict'][column]['origcolumn']
      
      columnkeylist = postprocess_dict['origcolumn'][origcolumn]['columnkeylist']
    
    for columnkey in columnkeylist:
      
      if column == postprocess_dict['column_dict'][columnkey]['inputcolumn']:

        if 'bincount_bnep' in postprocess_dict['column_dict'][columnkey]['normalization_dict'][columnkey]:

          if postprocess_dict['column_dict'][columnkey]['normalization_dict'][columnkey]['bincount_bnep'] == bincount \
          and postprocess_dict['column_dict'][columnkey]['normalization_dict'][columnkey]['suffix'] == suffix:

            normkey = columnkey
        
    if normkey is not False:
      
      mean = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['binsmean']
      bn_min = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_min']
      bn_max = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_max']
      bn_delta = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_delta']
      bn_count = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_count']
      bins_id = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_id']
      bins_cuts = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_cuts']
      bincount = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bincount_bnep']
      textcolumns = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['textcolumns']
      
      binscolumn = column + suffix
      
      #copy original column
      mdf_test[binscolumn] = mdf_test[column].copy()

      #convert all values to either numeric or NaN
      mdf_test[binscolumn] = pd.to_numeric(mdf_test[binscolumn], errors='coerce')

#       #replace missing data with training set mean
#       mdf_test[binscolumn] = mdf_test[binscolumn].fillna(mean)
      
      #create bins based on standard deviation increments
#       binscolumn = column + '_bnwd'

      if bn_delta > 0 and bn_min == bn_min:

        mdf_test[binscolumn] = \
        pd.cut(mdf_test[binscolumn], bins = bins_cuts,  \
               labels = bins_id, precision=len(str(bn_count)), duplicates='drop')

        mdf_test = \
        self.postprocess_textsupport_class(mdf_test, binscolumn, {}, 'tempkey', {'textcolumns':textcolumns})

        #change data type for memory savings
        for textcolumn in textcolumns:
          mdf_test[textcolumn] = mdf_test[textcolumn].astype(np.int8)

        #delete the support column
        del mdf_test[binscolumn]
        
      else:
        
        mdf_test[binscolumn] = 0
    
    return mdf_test

  def postprocess_bneo_class(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #processes a numerical set by creating equal population bins coresponding to 
    #parameter 'bincount' which defaults to 5
    #and returning in ordinal encoded set
    
    #default infill is adjacent cell infill
    
    #can be applied top either a raw set not yet normalized or after normalization
    #such as after z-score normalization)
    '''
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    if 'suffix' in params:
      suffix = params['suffix']
    else:
      suffix = '_bneo'
    
    #retrieve normalization parameters from postprocess_dict
    normkey = column + suffix
    
    mean = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['binsmean']
    bn_min = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_min']
    bn_max = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_max']
    bn_delta = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_delta']
    bn_count = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_count']
    bins_id = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_id']
    bins_cuts = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_cuts']
    bincount = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bincount']

    binscolumn = column + suffix
    
    if inplace is not True:
      mdf_test[binscolumn] = mdf_test[column].copy()
    else:
      mdf_test.rename(columns = {column : binscolumn}, inplace = True)

    #convert all values to either numeric or NaN
    mdf_test[binscolumn] = pd.to_numeric(mdf_test[binscolumn], errors='coerce')

#     #replace missing data with training set mean
#     mdf_test[binscolumn] = mdf_test[binscolumn].fillna(mean)

    if bn_delta > 0 and bn_min == bn_min:

      #create bins based on prepared increments
  #     binscolumn = column + '_bnwo'
      mdf_test[binscolumn] = \
      pd.cut(mdf_test[binscolumn], bins = bins_cuts,  \
             labels = bins_id, precision=len(str(bn_count)), duplicates='drop')

      #apply ffill to replace NArows with value from adjacent cell in pre4ceding row
      mdf_test[binscolumn] = mdf_test[binscolumn].fillna(method='ffill')

      #we'll follow with a bfill just in case first row had a nan
      mdf_test[binscolumn] = mdf_test[binscolumn].fillna(method='bfill')

      #and if the entire set was nan we'll infill with a 0 plug
      mdf_test[binscolumn] = mdf_test[binscolumn].fillna(0)

      #change column dtype
      mdf_test[binscolumn] = mdf_test[binscolumn].astype(int)
      
    else:
      
      mdf_test[binscolumn] = 0
    
    return mdf_test
  
  def postprocess_tlbn_class(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #processes a numerical set by creating equal population bins coresponding to 
    #parameter 'bincount' which defaults to 9
    #and returning in one-hot encoded set
    
    #default infill is to have no activations in a row
    
    #can be applied top either a raw set not yet normalized or after normalization
    #such as after z-score normalization)
    '''
    
    if 'bincount' in params:
        
      bincount = params['bincount']
    
    else:
      
      bincount = 9
    
    #to retrieve the normalization dictionary we're going to use new method since we don't yet 
    #know what the returned columns titles are yet
    
    normkey = False
    
    if column in postprocess_dict['origcolumn']:
      
      columnkeylist = postprocess_dict['origcolumn'][column]['columnkeylist']
      
    else:
      
      origcolumn = postprocess_dict['column_dict'][column]['origcolumn']
      
      columnkeylist = postprocess_dict['origcolumn'][origcolumn]['columnkeylist']
    
    for columnkey in columnkeylist:
      
      if column == postprocess_dict['column_dict'][columnkey]['inputcolumn']:

        if 'bincount_tlbn' in postprocess_dict['column_dict'][columnkey]['normalization_dict'][columnkey]:

          if postprocess_dict['column_dict'][columnkey]['normalization_dict'][columnkey]['bincount_tlbn'] == bincount:

            normkey = columnkey
        
    if normkey is not False:
      
      mean = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['binsmean']
      bn_min = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_min']
      bn_max = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_max']
      bn_delta = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_delta']
      bn_count = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_count']
      bins_id = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_id']
      bins_cuts = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_cuts']
      bincount = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bincount_tlbn']
      textcolumns = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['textcolumns']
      
      binscolumn = column + '_tlbn'
      
      #copy original column
      mdf_test[binscolumn] = mdf_test[column].copy()

      #convert all values to either numeric or NaN
      mdf_test[binscolumn] = pd.to_numeric(mdf_test[binscolumn], errors='coerce')

#       #replace missing data with training set mean
#       mdf_test[binscolumn] = mdf_test[binscolumn].fillna(mean)
      
      #create bins based on standard deviation increments
#       binscolumn = column + '_bnwd'

      if bn_delta > 0 and bn_min == bn_min:

        mdf_test[binscolumn] = \
        pd.cut(mdf_test[binscolumn], bins = bins_cuts,  \
               labels = bins_id, precision=len(str(bn_count)), duplicates='drop')

        mdf_test = \
        self.postprocess_textsupport_class(mdf_test, binscolumn, {}, 'tempkey', {'textcolumns':textcolumns})
                
        #initialize binscolumn once more
        mdf_test[binscolumn] = mdf_test[column].copy()        
        mdf_test[binscolumn] = pd.to_numeric(mdf_test[binscolumn], errors='coerce')
        
        
        if len(textcolumns) > 1:

          #for i in range(bincount):
          for i in range(len(textcolumns)):

            tlbn_column = binscolumn + '_' + str(i)

            if i == 0:

              mdf_test[tlbn_column] = \
              np.where(mdf_test[tlbn_column] == 1, \
                      (bins_cuts[i+1] - mdf_test[binscolumn]) / (bins_cuts[i+1] - bn_min), -1)

            elif i == bincount - 1:

              mdf_test[tlbn_column] = \
              np.where(mdf_test[tlbn_column] == 1, \
                      (mdf_test[binscolumn] - bins_cuts[i]) / (bn_max - bins_cuts[i]), -1)

            else:

              mdf_test[tlbn_column] = \
              np.where(mdf_test[tlbn_column] == 1, \
                      (mdf_test[binscolumn] - bins_cuts[i]) / (bins_cuts[i+1] - bins_cuts[i]), -1)


#         #change data type for memory savings
#         for textcolumn in textcolumns:
#           mdf_test[textcolumn] = mdf_test[textcolumn].astype(np.int8)

        #delete the support column
        del mdf_test[binscolumn]
        
      else:
        
        mdf_test[binscolumn] = 0

    return mdf_test
  
  def postprocess_bkt1_class(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #processes a numerical set by creating custom bins coresponding to 
    #parameter 'buckets' which defaults to [0,1]
    #first and last buckets unconstrained
    #and returning in one-hot encoded set
    
    #removes buckets without activations in train set
    '''
    
    if 'buckets' in params:
        
      buckets = params['buckets']
    
    else:
      
      buckets = [0,1,2]
    
    #to retrieve the normalization dictionary we're going to use new method since we don't yet 
    #know what the returned columns titles are yet
    
    normkey = False
    
    if column in postprocess_dict['origcolumn']:
      
      columnkeylist = postprocess_dict['origcolumn'][column]['columnkeylist']
      
    else:
      
      origcolumn = postprocess_dict['column_dict'][column]['origcolumn']
      
      columnkeylist = postprocess_dict['origcolumn'][origcolumn]['columnkeylist']
      
    for columnkey in columnkeylist:
      
      if column == postprocess_dict['column_dict'][columnkey]['inputcolumn']:

        if 'buckets_bkt1' in postprocess_dict['column_dict'][columnkey]['normalization_dict'][columnkey]:

          if postprocess_dict['column_dict'][columnkey]['normalization_dict'][columnkey]['buckets_bkt1'] == buckets:

            normkey = columnkey
          
    if normkey is not False:
      
      mean = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['binsmean']
      buckets = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['buckets_bkt1']
      bins_cuts = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_cuts']
      bins_id = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_id']
      textcolumns = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['textcolumns']
      
      binscolumn = column + '_bkt1'
      
      #store original column for later reversion
      mdf_test[binscolumn] = mdf_test[column].copy()

      #convert all values to either numeric or NaN
      mdf_test[binscolumn] = pd.to_numeric(mdf_test[binscolumn], errors='coerce')

      #replace missing data with training set mean
      # mdf_test[binscolumn] = mdf_test[binscolumn].fillna(mean)
      
      #create bins based on standard deviation increments
#       binscolumn = column + '_bnwd'

      mdf_test[binscolumn] = \
      pd.cut(mdf_test[binscolumn], bins = bins_cuts,  \
             labels = bins_id, precision=len(str(len(bins_id))))

      mdf_test = \
      self.postprocess_textsupport_class(mdf_test, binscolumn, {}, 'tempkey', {'textcolumns':textcolumns})

      #change data type for memory savings
      for textcolumn in textcolumns:
        mdf_test[textcolumn] = mdf_test[textcolumn].astype(np.int8)

      #delete the support column
      del mdf_test[binscolumn]

    return mdf_test
  
  def postprocess_bkt2_class(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #processes a numerical set by creating custom bins coresponding to 
    #parameter 'buckets' which defaults to [0,1]
    #first and last buckets unconstrained
    #and returning in one-hot encoded set
    
    #removes buckets without activations in train set
    '''
    
    if 'buckets' in params:
        
      buckets = params['buckets']
    
    else:
      
      buckets = [0,1,2]
    
    #to retrieve the normalization dictionary we're going to use new method since we don't yet 
    #know what the returned columns titles are yet
    
    normkey = False
    
    if column in postprocess_dict['origcolumn']:
      
      columnkeylist = postprocess_dict['origcolumn'][column]['columnkeylist']
      
    else:
      
      origcolumn = postprocess_dict['column_dict'][column]['origcolumn']
      
      columnkeylist = postprocess_dict['origcolumn'][origcolumn]['columnkeylist']
    
    for columnkey in columnkeylist:
      
      if column == postprocess_dict['column_dict'][columnkey]['inputcolumn']:

        if 'buckets_bkt2' in postprocess_dict['column_dict'][columnkey]['normalization_dict'][columnkey]:

          if postprocess_dict['column_dict'][columnkey]['normalization_dict'][columnkey]['buckets_bkt2'] == buckets:

            normkey = columnkey
        
    if normkey is not False:
      
      mean = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['binsmean']
      buckets = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['buckets_bkt2']
      bins_cuts = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_cuts']
      bins_id = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_id']
      textcolumns = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['textcolumns']
      
      binscolumn = column + '_bkt2'
      
      #store original column for later reversion
      mdf_test[binscolumn] = mdf_test[column].copy()

      #convert all values to either numeric or NaN
      mdf_test[binscolumn] = pd.to_numeric(mdf_test[binscolumn], errors='coerce')

      #replace missing data with training set mean
      # mdf_test[binscolumn] = mdf_test[binscolumn].fillna(mean)
      
      #create bins based on standard deviation increments
#       binscolumn = column + '_bnwd'

      mdf_test[binscolumn] = \
      pd.cut(mdf_test[binscolumn], bins = bins_cuts,  \
             labels = bins_id, precision=len(str(len(bins_id))))

      mdf_test = \
      self.postprocess_textsupport_class(mdf_test, binscolumn, {}, 'tempkey', {'textcolumns':textcolumns})

      #change data type for memory savings
      for textcolumn in textcolumns:
        mdf_test[textcolumn] = mdf_test[textcolumn].astype(np.int8)

      #delete the support column
      del mdf_test[binscolumn]
    
    return mdf_test
  
  def postprocess_bkt3_class(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #processes a numerical set by creating custom bins coresponding to 
    #parameter 'buckets' which defaults to [0,1]
    #first and last buckets unconstrained
    #and returning in ordinal encoded set
    
    #segments without activations are included
    '''
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    #retrieve normalization parameters from postprocess_dict
    normkey = column +'_bkt3'
    
    mean = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['binsmean']
    buckets = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['buckets']
    bins_cuts = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_cuts']
    bins_id = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_id']
    ordl_activations_dict = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['ordl_activations_dict']
    infill_activation = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['infill_activation']

    binscolumn = column + '_bkt3'
    
    if inplace is not True:
      mdf_test[binscolumn] = mdf_test[column].copy()
    else:
      mdf_test.rename(columns = {column : binscolumn}, inplace = True)

    #convert all values to either numeric or NaN
    mdf_test[binscolumn] = pd.to_numeric(mdf_test[binscolumn], errors='coerce')

    # #replace missing data with training set mean
    # mdf_test[binscolumn] = mdf_test[binscolumn].fillna(mean)

    #create bins based on standard deviation increments
#     binscolumn = column + '_bnwo'
    mdf_test[binscolumn] = \
    pd.cut(mdf_test[binscolumn], bins = bins_cuts,  \
           labels = bins_id, precision=len(str(len(bins_id))))

    mdf_test[binscolumn] = mdf_test[binscolumn].astype(float)
    
    #replace missing data with infill_activation
    mdf_test[binscolumn] = mdf_test[binscolumn].fillna(infill_activation)
    
    #change column dtype
    mdf_test[binscolumn] = mdf_test[binscolumn].astype(int)
    
    return mdf_test
  
  def postprocess_bkt4_class(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #processes a numerical set by creating custom bins coresponding to 
    #parameter 'buckets' which defaults to [0,1]
    #first and last buckets bounded
    #and returning in ordinal encoded set
    
    #segments without activations are included
    '''
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    #retrieve normalization parameters from postprocess_dict
    normkey = column +'_bkt4'
    
    mean = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['binsmean']
    buckets = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['buckets']
    bins_cuts = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_cuts']
    bins_id = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_id']
    ordl_activations_dict = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['ordl_activations_dict']
    infill_activation = postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['infill_activation']

    binscolumn = column + '_bkt4'
    
    if inplace is not True:
      mdf_test[binscolumn] = mdf_test[column].copy()
    else:
      mdf_test.rename(columns = {column : binscolumn}, inplace = True)

    #convert all values to either numeric or NaN
    mdf_test[binscolumn] = pd.to_numeric(mdf_test[binscolumn], errors='coerce')
    
    #set all values that fall outside of bounded buckets to nan for replacement with mean
    mdf_test.loc[mdf_test[binscolumn] <= buckets[0], (binscolumn)] = np.nan
    
    mdf_test.loc[mdf_test[binscolumn] > buckets[-1], (binscolumn)] = np.nan

    # #replace missing data with training set mean
    # mdf_test[binscolumn] = mdf_test[binscolumn].fillna(mean)

    #create bins based on standard deviation increments
#     binscolumn = column + '_bnwo'
    mdf_test[binscolumn] = \
    pd.cut(mdf_test[binscolumn], bins = bins_cuts,  \
           labels = bins_id, precision=len(str(len(bins_id))))
  
    mdf_test[binscolumn] = mdf_test[binscolumn].astype(float)
    
    #replace missing data with infill_activation
    mdf_test[binscolumn] = mdf_test[binscolumn].fillna(infill_activation)
    
    #change column dtype
    mdf_test[binscolumn] = mdf_test[binscolumn].astype(int)
    
    return mdf_test

  def postprocess_DPnb_class(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #process_DPnb_class(mdf_train, mdf_test, column, category, postprocess_dict, params = {})
    #function to inject noise to training data, such as for differential privacy purposes
    #assumes input is numeric data with z-score normalization to mean 0 and sigma 1
    #adds data sampled from normal distribution with mean 0 and sigma 0.06 by default
    #where noise only injected to a subset of data based on flip_prob defaulting to 0.03
    #the noise properties may be customized with parameters 'mu', 'sigma', 'flip_prob'
    #note that the noise is only injected into the designated training data of df_train
    #for test data this is a pass-through operation
    #note this assumes clean data as input since this will be intended for downstream applicaiton
    #in family trees, so no infill is performed
    #note that for postprocess function in postmunge, determination of whether to treat
    #df_test as train or test data is based on the traindata entry in postprocess_dict
    #in automunge df_test is treated as test data by default
    '''
    
    #retrieve normalizastion parameters from postprocess_dict
    normkey = column + '_DPnb'
    
    mu = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['mu']
    sigma = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['sigma']
    flip_prob = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['flip_prob']
    noisedistribution = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['noisedistribution']

    DPnm_column = column + '_DPnb'
    
    #check if df_test is to be treated as train or test data
    traindata = postprocess_dict['traindata']
    
    if traindata is True:
      
      #first we'll derive our sampled noise for injection
      if noisedistribution == 'normal':
        normal_samples = np.random.normal(loc=mu, scale=sigma, size=(mdf_test.shape[0]))
      elif noisedistribution == 'laplace':
        normal_samples = np.random.laplace(loc=mu, scale=sigma, size=(mdf_test.shape[0]))
        
      binomial_samples = np.random.binomial(n=1, p=flip_prob, size=(mdf_test.shape[0]))
      
      mdf_test[DPnm_column] = pd.DataFrame(normal_samples) * pd.DataFrame(binomial_samples)
    
      #now inject noise
      mdf_test[DPnm_column] = mdf_test[DPnm_column] + mdf_test[column]
      
    elif traindata is False:
      
      #for test data is just pass-through
      mdf_test[DPnm_column] = mdf_test[column]

    return mdf_test

  def postprocess_DPmm_class(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #process_DPmm_class(mdf_train, mdf_test, column, category, postprocess_dict, params = {})
    #function to inject noise to training data, such as for differential privacy purposes
    #assumes input is numeric data min-max scaled within range 0-1
    #adds data sampled from normal distribution with mean 0 and sigma 0.03 by default
    #the noise properties may be customized with parameters 'mu', 'sigma'
    #also accepts parameter 'flip_prob' for ratio of data that will be adjusted (defaults to 1.)
    #noise is scaled based on the recieved points to keep within range 0-1
    #(e.g. for recieved data point 0.1, noise is scaled so as not to fall below -0.1)
    #gaussian noise source is also capped to maintain the range -0.5 to 0.5 (rare outlier points)
    #note that the noise is only injected into the designated training data of df_train
    #for test data this is a pass-through operation
    #note this assumes clean data as input since this will be intended for downstream applicaiton
    #in family trees, so no infill is performed
    #note that for postprocess function in postmunge, determination of whether to treat
    #df_test as train or test data is based on the traindata entry in postprocess_dict
    #in automunge df_test is treated as test data by default
    '''
    
    #retrieve normalizastion parameters from postprocess_dict
    normkey = column + '_DPmm'
    
    mu = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['mu']
    sigma = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['sigma']
    flip_prob = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['flip_prob']
    noisedistribution = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['noisedistribution']

    DPmm_column = column + '_DPmm'
    DPmm_column_temp1 = column + '_DPmm' + '_tmp1'
    
    #check if df_test is to be treated as train or test data
    traindata = postprocess_dict['traindata']
    
    if traindata is True:
      
      if noisedistribution == 'normal':
        normal_samples = np.random.normal(loc=mu, scale=sigma, size=(mdf_test.shape[0]))
      elif noisedistribution == 'laplace':
        normal_samples = np.random.laplace(loc=mu, scale=sigma, size=(mdf_test.shape[0]))
      
      binomial_samples = np.random.binomial(n=1, p=flip_prob, size=(mdf_test.shape[0]))

      mdf_test[DPmm_column] = pd.DataFrame(normal_samples) * pd.DataFrame(binomial_samples)

      #cap outliers
      mdf_test[DPmm_column] = np.where(mdf_test[DPmm_column] < -0.5, np.nan, mdf_test[DPmm_column])
      mdf_test[DPmm_column] = np.where(mdf_test[DPmm_column] > 0.5, np.nan, mdf_test[DPmm_column])

      #adjacent cell infill
      mdf_test[DPmm_column] = mdf_test[DPmm_column].fillna(method='ffill')
      mdf_test[DPmm_column] = mdf_test[DPmm_column].fillna(method='bfill')

      #support column to signal sign of noise, 0 is neg, 1 is pos
      mdf_test[DPmm_column_temp1] = 0
      mdf_test[DPmm_column_temp1] = np.where(mdf_test[DPmm_column] >= 0., 1, mdf_test[DPmm_column_temp1])
      
      #now inject noise, with scaled noise to maintain range 0-1
      #(so if mnmx value <0.5, and neg noise, we scale noise to maintain ratio as if minmax was 0.5, similarly for >0.5 mnmx)
      mdf_test[DPmm_column] = np.where(mdf_test[column] < 0.5, \
                                        mdf_test[column] + \
                                        (1 - mdf_test[DPmm_column_temp1]) * (mdf_test[DPmm_column] * mdf_test[column] / 0.5) + \
                                        (mdf_test[DPmm_column_temp1]) * (mdf_test[DPmm_column]), \
                                        mdf_test[DPmm_column])

      mdf_test[DPmm_column] = np.where(mdf_test[column] >= 0.5, \
                                        mdf_test[column] + \
                                        (1 - mdf_test[DPmm_column_temp1]) * (mdf_test[DPmm_column]) + \
                                        (mdf_test[DPmm_column_temp1]) * (mdf_test[DPmm_column] * (1 - mdf_test[column]) / 0.5), \
                                        mdf_test[DPmm_column])

      #remove support column
      del mdf_test[DPmm_column_temp1]

    elif traindata is False:
      
      #for test data is just pass-through
      mdf_test[DPmm_column] = mdf_test[column]

    return mdf_test

  def postprocess_DPrt_class(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    """
    #process_DPrt_class
    #function to scale data as follows:
    
    # if max >= 0 and min <= 0:
    #   #scaling based on 
    #   x = x / (max - min)

    # elif max >= 0 and min >= 0:
    #   #traditional min/max
    #   x = (x - min) / (max - min)
    
    # elif max <= 0 and min <= 0:
    #   #max/min (retains negative values)
    #   x = (x - max) / (max - min)
    
    #followed by a noise injection similar to DPrt based based on this set's retn range
    
    #replaces missing or improperly formatted data with mean of remaining values
    #(prior to noise injection)
    
    #returns same dataframes with new column of name column + '_DPrt'
    #note this is a "dualprocess" function since is applied to both dataframes
    
    #note with parameters divisor can also be set as standard deviation
    #also aprameters accepted for cap/floor/mulitplier/offset
    #where cap/floor based on pretransform values
    #multiplier/offset based on posttransform values, muoltiplier applied betfore offset
    """
    
    #retrieve normalizastion parameters from postprocess_dict
    normkey = column + '_DPrt'
    
    mean = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['mean']
    minimum = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['minimum']
    maximum = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['maximum']
    scalingapproach = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['scalingapproach']
    offset = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['offset']
    multiplier = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['multiplier']
    cap = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['cap']
    floor = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['floor']
    divisor = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['divisor']
    adjinfill = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['adjinfill']
    
    mu = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['mu']
    sigma = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['sigma']
    flip_prob = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['flip_prob']
    noisedistribution = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['noisedistribution']
    
    #initialize returned column and support columns
    DPrt_column = column + '_DPrt'
    DPrt_column_temp1 = column + '_DPrt' + '_tmp1'
    DPrt_column_temp2 = column + '_DPrt' + '_tmp2'
    
    #copy original column for implementation
    mdf_test[DPrt_column] = mdf_test[column].copy()

    #convert all values to either numeric or NaN
    mdf_test[DPrt_column] = pd.to_numeric(mdf_test[DPrt_column], errors='coerce')
    
    if cap is not False:
      #replace values in test > cap with cap
      mdf_test.loc[mdf_test[DPrt_column] > cap, (DPrt_column)] \
      = cap
    
    if floor is not False:
      #replace values in test < floor with floor
      mdf_test.loc[mdf_test[DPrt_column] < floor, (DPrt_column)] \
      = floor
    
    if adjinfill is True:
      mdf_test[DPrt_column] = mdf_test[DPrt_column].fillna(method='ffill')
      mdf_test[DPrt_column] = mdf_test[DPrt_column].fillna(method='bfill')

    #replace missing data with training set mean
    mdf_test[DPrt_column] = mdf_test[DPrt_column].fillna(mean)
    
    #avoid outlier div by zero when max = min
    maxminusmin = maximum - minimum
    if maxminusmin == 0:
      maxminusmin = 1
    
    if scalingapproach == 'retn':
      
      mdf_test[DPrt_column] = (mdf_test[DPrt_column]) / \
                                    (divisor) * multiplier + offset
      
    elif scalingapproach == 'mnmx':
    
      #perform min-max scaling to test set using values from train
      mdf_test[DPrt_column] = (mdf_test[DPrt_column] - minimum) / \
                                   (divisor) * multiplier + offset
      
    elif scalingapproach == 'mxmn':
    
      #perform min-max scaling to test set using values from train
      mdf_test[DPrt_column] = (mdf_test[DPrt_column] - maximum) / \
                                   (divisor) * multiplier + offset
      
    #check if df_test is to be treated as train or test data
    traindata = postprocess_dict['traindata']
    
    #if this is train data we'll inject noise
    if traindata is True:
      
      #first we'll derive our sampled noise for injection
      
      if noisedistribution == 'normal':
        normal_samples = np.random.normal(loc=mu, scale=sigma, size=(mdf_test.shape[0]))
      elif noisedistribution == 'laplace':
        normal_samples = np.random.laplace(loc=mu, scale=sigma, size=(mdf_test.shape[0]))
      
      binomial_samples = np.random.binomial(n=1, p=flip_prob, size=(mdf_test.shape[0]))

      mdf_test[DPrt_column_temp2] = pd.DataFrame(normal_samples) * pd.DataFrame(binomial_samples)
      
      #cap outliers
      mdf_test[DPrt_column_temp2] = np.where(mdf_test[DPrt_column] < -0.5, np.nan, mdf_test[DPrt_column_temp2])
      mdf_test[DPrt_column_temp2] = np.where(mdf_test[DPrt_column] > 0.5, np.nan, mdf_test[DPrt_column_temp2])

      #adjacent cell infill
      mdf_test[DPrt_column_temp2] = mdf_test[DPrt_column_temp2].fillna(method='ffill')
      mdf_test[DPrt_column_temp2] = mdf_test[DPrt_column_temp2].fillna(method='bfill')
      
      #support column to signal sign of noise, 0 is neg, 1 is pos
      mdf_test[DPrt_column_temp1] = 0
      mdf_test[DPrt_column_temp1] = np.where(mdf_test[DPrt_column_temp2] >= 0., 1, mdf_test[DPrt_column_temp1])
      
      #for noise injection we'll first move data into range 0-1 and then revert after injection
      if scalingapproach == 'retn':
        mdf_test[DPrt_column] = (mdf_test[DPrt_column] - (minimum / divisor) ) / multiplier - offset
      elif scalingapproach == 'mnmx':
        mdf_test[DPrt_column] = (mdf_test[DPrt_column]) / multiplier - offset
      elif scalingapproach == 'mxmn':
        mdf_test[DPrt_column] = (mdf_test[DPrt_column] + (maximum - minimum) / divisor) / multiplier - offset
      
      #now inject noise, with scaled noise to maintain range 0-1
      #(so if mnmx value <0.5, and neg noise, we scale noise to maintain ratio as if minmax was 0.5, similarly for >0.5 mnmx)
      mdf_test[DPrt_column] = np.where(mdf_test[DPrt_column] < 0.5, \
                                        mdf_test[DPrt_column] + \
                                        (1 - mdf_test[DPrt_column_temp1]) * (mdf_test[DPrt_column_temp2] * mdf_test[DPrt_column] / 0.5) + \
                                        (mdf_test[DPrt_column_temp1]) * (mdf_test[DPrt_column_temp2]), \
                                        mdf_test[DPrt_column])

      mdf_test[DPrt_column] = np.where(mdf_test[DPrt_column] >= 0.5, \
                                        mdf_test[DPrt_column] + \
                                        (1 - mdf_test[DPrt_column_temp1]) * (mdf_test[DPrt_column_temp2]) + \
                                        (mdf_test[DPrt_column_temp1]) * (mdf_test[DPrt_column_temp2] * (1 - mdf_test[DPrt_column]) / 0.5), \
                                        mdf_test[DPrt_column])
      
      #remove support columns
      del mdf_test[DPrt_column_temp1]
      del mdf_test[DPrt_column_temp2]
      
      #for noise injection we'll first move data into range 0-1 and then revert after injection
      if scalingapproach == 'retn':
        mdf_test[DPrt_column] = (mdf_test[DPrt_column] + (minimum / divisor) ) * multiplier + offset
      elif scalingapproach == 'mnmx':
        mdf_test[DPrt_column] = (mdf_test[DPrt_column]) * multiplier + offset
      elif scalingapproach == 'mxmn':
        mdf_test[DPrt_column] = (mdf_test[DPrt_column] - (maximum - minimum) / divisor) * multiplier + offset

    return mdf_test

  def postprocess_DPbn_class(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #process_DPbn_class(mdf_train, mdf_test, column, category, postprocess_dict, params = {})
    #function to inject noise to training data, such as for differential privacy purposes
    #assumes input is bnry encoded data (i.e. boolean integers in single column)
    #adds data sampled from Bernoulli distribution with flip_prob 0.03 by default
    #the noise properties may be customized with parameter 'flip_prob'
    #note that the noise is only injected into the designated training data of df_train
    #for test data this is a pass-through operation
    #note this assumes clean data as input since this will be intended for downstream applicaiton
    #in family trees, so no infill is performed
    #note that for postprocess function in postmunge, determination of whether to treat
    #df_test as train or test data is based on the traindata entry in postprocess_dict
    #in automunge df_test is treated as test data by default
    '''
    
    #retrieve normalizastion parameters from postprocess_dict
    normkey = column + '_DPbn'
    
    flip_prob = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['flip_prob']

    DPbn_column = column + '_DPbn'
    
    #check if df_test is to be treated as train or test data
    traindata = postprocess_dict['traindata']
    
    if traindata is True:
      
      #first we'll derive our sampled noise for injection
      mdf_test[DPbn_column] = pd.DataFrame(np.random.binomial(n=1, p=flip_prob, size=(mdf_test.shape[0])))
    
      #now inject noise
      mdf_test[DPbn_column] = abs(mdf_test[column] - mdf_test[DPbn_column])
      
    elif traindata is False:
      
      #for test data is just pass-through
      mdf_test[DPbn_column] = mdf_test[column]

    return mdf_test

  def postprocess_DPod_class(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #process_DPod_class(mdf_train, mdf_test, column, category, postprocess_dict, params = {})
    #function to inject noise to training data, such as for differential privacy purposes
    #assumes input is ordinal encoded data (i.e. categoric by integer in single column)
    #adds data sampled from Bernoulli distribution with flip_prob 0.03 by default
    #the noise properties may be customized with parameter 'flip_prob'
    #when flip activated selects from the set of encodings per level random draw
    #(including potenitally the current encoding for no flip)
    #note that the noise is only injected into the designated training data of df_train
    #for test data this is a pass-through operation
    #note this assumes clean data as input since this will be intended for downstream applicaiton
    #in family trees, so no infill is performed
    #note that for postprocess function in postmunge, determination of whether to treat
    #df_test as train or test data is based on the traindata entry in postprocess_dict
    #in automunge df_test is treated as test data by default
    '''
    
    #retrieve normalizastion parameters from postprocess_dict
    normkey = column + '_DPod'
    
    flip_prob = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['flip_prob']

    DPod_column = column + '_DPod'
    DPod_tempcolumn1 = column + '_DPod_tmp1'
    DPod_tempcolumn2 = column + '_DPod_tmp2'
    
    #check if df_test is to be treated as train or test data
    traindata = postprocess_dict['traindata']
    
    if traindata is True:
      
      ord_encodings = False
      
      #we'll suplement a dif approach to grab ordinal encodings in postmunge
      #since all activations may not be present in test set
      #so we'll rely on convention that ordinal sets have a list of activations in norm_dict
      #saved as 'activations_list'
      if column in postprocess_dict['column_dict']:
        if 'activations_list' in postprocess_dict['column_dict'][column]['normalization_dict'][column]:
          ord_encodings = np.array(postprocess_dict['column_dict'][column]['normalization_dict'][column]['activations_list'])
      
      #else just grab set of unique entries
      if ord_encodings is False:
        ord_encodings = mdf_test[column].unique()
      
      #now we'll derive our sampled noise for injection
      mdf_test[DPod_tempcolumn1] = pd.DataFrame(np.random.binomial(n=1, p=flip_prob, size=(mdf_test.shape[0])))
      mdf_test[DPod_tempcolumn2] = pd.DataFrame(np.random.choice(ord_encodings, size=(mdf_test.shape[0])))
    
      #now inject noise
    
      #initialize return column
      mdf_test[DPod_column] = 0
    
      for entry in list(ord_encodings):
        
        #this returns column value when DPod_tempcolumn1 is 0 or DPod_tempcolumn2 when DPod_tempcolumn1 is 1
        mdf_test[DPod_column] = \
        np.where(mdf_test[column] == entry, \
                 mdf_test[column] * (1 - mdf_test[DPod_tempcolumn1]) + mdf_test[DPod_tempcolumn1] * mdf_test[DPod_tempcolumn2], \
                 mdf_test[DPod_column])
      
      del mdf_test[DPod_tempcolumn1]
      del mdf_test[DPod_tempcolumn2]
      
    elif traindata is False:
      
      #for test data is just pass-through
      mdf_test[DPod_column] = mdf_test[column]

    return mdf_test

  def postprocess_exc2_class(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #here we'll address any columns that returned a 'excl' category
    #note this is a. singleprocess transform
    #we'll simply maintain the same column but with a suffix to the header
    '''
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    #retrieve normalizastion parameters from postprocess_dict
    normkey = column + '_exc2'
    
    fillvalue = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['fillvalue']
    
    exclcolumn = column + '_exc2'
    
    if inplace is not True:
      mdf_test[exclcolumn] = mdf_test[column].copy()
    else:
      mdf_test.rename(columns = {column : exclcolumn}, inplace = True)
    
    #del df[column]
    
    mdf_test[exclcolumn] = pd.to_numeric(mdf_test[exclcolumn], errors='coerce')
    
    #fillvalue = mdf_train[exclcolumn].mode()[0]
    
    #replace missing data with fill value
    mdf_test[exclcolumn] = mdf_test[exclcolumn].fillna(fillvalue)

    return mdf_test
  
  def postprocess_exc5_class(self, mdf_test, column, postprocess_dict, columnkey, params = {}):
    '''
    #here we'll address any columns that returned a 'excl' category
    #note this is a. singleprocess transform
    #we'll simply maintain the same column but with a suffix to the header
    '''
    
    if 'inplace' in params:
      inplace = params['inplace']
    else:
      inplace = False
    
    #retrieve normalizastion parameters from postprocess_dict
    normkey = column + '_exc5'
    
    fillvalue = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['fillvalue']
    
    exclcolumn = column + '_exc5'
    
    if inplace is not True:
      mdf_test[exclcolumn] = mdf_test[column].copy()
    else:
      mdf_test.rename(columns = {column : exclcolumn}, inplace = True)
    
    #del df[column]
    
    mdf_test[exclcolumn] = pd.to_numeric(mdf_test[exclcolumn], errors='coerce')
    
    #non integers are subject to infill
    mdf_test[exclcolumn] = np.where(mdf_test[exclcolumn] == mdf_test[exclcolumn].round(), mdf_test[exclcolumn], np.nan)
    
    #fillvalue = mdf_train[exclcolumn].mode()[0]
    
    #replace missing data with fill value
    mdf_test[exclcolumn] = mdf_test[exclcolumn].fillna(fillvalue)

    return mdf_test

  def createpostMLinfillsets(self, df_test, column, testNArows, category, \
                             postprocess_dict, columnslist = [], categorylist = []):
    '''
    #createpostMLinfillsets(df_test, column, testNArows, category, \
    #columnslist = []) function that when fed dataframe of
    #test set, column id, df of True/False corresponding to rows from original \
    #sets with missing values, a string category of 'text', 'date', 'nmbr', or \
    #'bnry', and a list of column id's for the text category if applicable. The \
    #function returns a series of dataframes which can be applied to apply a \
    #machine learning model previously trained on our train set as part of the 
    #original automunge application to predict apppropriate infill values for those\
    #points that had missing values from the original sets, returning the dataframe\
    #df_test_fillfeatures
    '''
    
    MLinfilltype = postprocess_dict['process_dict'][category]['MLinfilltype']

    #if category in ['nmbr', 'nbr2', 'bxcx', 'bnry', 'text', 'bins', 'bint']:
    if MLinfilltype in ['numeric', 'singlct', 'binary', \
                        'multirt', '1010', \
                        'concurrent_act', 'concurrent_nmbr']:

      #if this is a single column set or concurrent_act
      if len(categorylist) == 1 or \
      postprocess_dict['process_dict'][category]['MLinfilltype'] in ['concurrent_act', 'concurrent_nmbr']:

        #first concatinate the NArows True/False designations to df_train & df_test
  #       df_train = pd.concat([df_train, trainNArows], axis=1)
        df_test = pd.concat([df_test, testNArows], axis=1)

  #       #create copy of df_train to serve as training set for fill
  #       df_train_filltrain = df_train.copy()
  #       #now delete rows coresponding to True
  #       df_train_filltrain = df_train_filltrain[df_train_filltrain[trainNArows.columns[0]] == False]

  #       #now delete columns = columnslist and the NA labels (orig column+'_NArows') from this df
  #       df_train_filltrain = df_train_filltrain.drop(columnslist, axis=1)
  #       df_train_filltrain = df_train_filltrain.drop([trainNArows.columns[0]], axis=1)

  #       #create a copy of df_train[column] for fill train labels
  #       df_train_filllabel = pd.DataFrame(df_train[column].copy())
  #       #concatinate with the NArows
  #       df_train_filllabel = pd.concat([df_train_filllabel, trainNArows], axis=1)
  #       #drop rows corresponding to True
  #       df_train_filllabel = df_train_filllabel[df_train_filllabel[trainNArows.columns[0]] == False]

  #       #delete the NArows column
  #       df_train_filllabel = df_train_filllabel.drop([trainNArows.columns[0]], axis=1)

  #       #create features df_train for rows needing infill
  #       #create copy of df_train (note it already has NArows included)
  #       df_train_fillfeatures = df_train.copy()
  #       #delete rows coresponding to False
  #       df_train_fillfeatures = df_train_fillfeatures[(df_train_fillfeatures[trainNArows.columns[0]])]
  #       #delete columnslist and column+'_NArows'
  #       df_train_fillfeatures = df_train_fillfeatures.drop(columnslist, axis=1)
  #       df_train_fillfeatures = df_train_fillfeatures.drop([trainNArows.columns[0]], axis=1)

        #create features df_test for rows needing infill
        #create copy of df_test (note it already has NArows included)
        df_test_fillfeatures = df_test.copy()
        #delete rows coresponding to False
        df_test_fillfeatures = df_test_fillfeatures[(df_test_fillfeatures[testNArows.columns[0]])]
        #delete column and column+'_NArows'
        df_test_fillfeatures = df_test_fillfeatures.drop(columnslist, axis=1)
        df_test_fillfeatures = df_test_fillfeatures.drop([testNArows.columns[0]], axis=1)

        #delete NArows from df_train, df_test
  #       df_train = df_train.drop([trainNArows.columns[0]], axis=1)
        df_test = df_test.drop([testNArows.columns[0]], axis=1)

      #else if categorylist wasn't empty
      else:

        #create a list of columns representing columnslist exlucding elements from
        #categorylist
        noncategorylist = columnslist[:]
        #this removes categorylist elements from noncategorylist
        noncategorylist = list(set(noncategorylist).difference(set(categorylist)))

        #first concatinate the NArows True/False designations to df_train & df_test
  #       df_train = pd.concat([df_train, trainNArows], axis=1)
        df_test = pd.concat([df_test, testNArows], axis=1)

  #       #create copy of df_train to serve as training set for fill
  #       df_train_filltrain = df_train.copy()
  #       #now delete rows coresponding to True
  #       df_train_filltrain = df_train_filltrain[df_train_filltrain[trainNArows.columns[0]] == False]

  #       #now delete columns = columnslist and the NA labels (orig column+'_NArows') from this df
  #       df_train_filltrain = df_train_filltrain.drop(columnslist, axis=1)
  #       df_train_filltrain = df_train_filltrain.drop([trainNArows.columns[0]], axis=1)

  #       #create a copy of df_train[columnslist] for fill train labels
  #       df_train_filllabel = df_train[columnslist].copy()
  #       #concatinate with the NArows
  #       df_train_filllabel = pd.concat([df_train_filllabel, trainNArows], axis=1)
  #       #drop rows corresponding to True
  #       df_train_filllabel = df_train_filllabel[df_train_filllabel[trainNArows.columns[0]] == False]

  #       #now delete columns = noncategorylist from this df
  #       df_train_filltrain = df_train_filltrain.drop(noncategorylist, axis=1)

  #       #delete the NArows column
  #       df_train_filllabel = df_train_filllabel.drop([trainNArows.columns[0]], axis=1)

  #       #create features df_train for rows needing infill
  #       #create copy of df_train (note it already has NArows included)
  #       df_train_fillfeatures = df_train.copy()
  #       #delete rows coresponding to False
  #       df_train_fillfeatures = df_train_fillfeatures[(df_train_fillfeatures[trainNArows.columns[0]])]
  #       #delete columnslist and column+'_NArows'
  #       df_train_fillfeatures = df_train_fillfeatures.drop(columnslist, axis=1)
  #       df_train_fillfeatures = df_train_fillfeatures.drop([trainNArows.columns[0]], axis=1)

        #create features df_test for rows needing infill
        #create copy of df_test (note it already has NArows included)
        df_test_fillfeatures = df_test.copy()
        #delete rows coresponding to False
        df_test_fillfeatures = df_test_fillfeatures[(df_test_fillfeatures[testNArows.columns[0]])]
        #delete column and column+'_NArows'
        df_test_fillfeatures = df_test_fillfeatures.drop(columnslist, axis=1)
        df_test_fillfeatures = df_test_fillfeatures.drop([testNArows.columns[0]], axis=1)

        #delete NArows from df_train, df_test
  #       df_train = df_train.drop([trainNArows.columns[0]], axis=1)
        
        df_test = df_test.drop([testNArows.columns[0]], axis=1)

    #if category == 'date':
    #if MLinfilltype in ['exclude']:
    else:

      #create empty sets for now
      #an extension of this method would be to implement a comparable method \
      #for the time category, based on the columns output from the preprocessing
  #     df_train_filltrain = pd.DataFrame({'foo' : []}) 
  #     df_train_filllabel = pd.DataFrame({'foo' : []})
  #     df_train_fillfeatures = pd.DataFrame({'foo' : []})
      df_test_fillfeatures = pd.DataFrame({'foo' : []})
    
    return df_test_fillfeatures

  def predictpostinfill(self, category, model, df_test_fillfeatures, \
                        postprocess_dict, ML_cmnd, autoMLer, printstatus, categorylist = []):
    '''
    #predictpostinfill(category, model, df_test_fillfeatures, \
    #categorylist = []), function that takes as input \
    #a category string, a model trained as part of automunge on the coresponding \
    #column from the train set, the output of createpostMLinfillsets(.), a seed \
    #for randomness, and a list of columns \
    #produced by a text class preprocessor when applicable and returns \
    #predicted infills for the test feature sets as df_testinfill based on \
    #derivations using scikit-learn, with the lenth of \
    #infill consistent with the number of True values from NArows
    #a reasonable extension of this funciton would be to allow ML inference with \
    #other ML architectures such a SVM or something SGD based for instance
    '''
    
    #MLinfilltype distinguishes between classifier/regressor, single/multi column, ordinal/onehot/binary, etc
    #see potential values documented in assembleprocessdict function
    MLinfilltype = postprocess_dict['process_dict'][category]['MLinfilltype']
    
    #grab autoML_type from ML_cmnd, this will be one of our keys for autoMLer dictionary
    autoML_type = ML_cmnd['autoML_type']
    
    #ony run the following if we successfuly trained a model in automunge
    if model is not False:
      
      #if target category is numeric:
      if MLinfilltype in ['numeric', 'concurrent_nmbr']:
        
        ML_application = 'regression'
    
        #only run following if we have any test rows needing infill
        if df_test_fillfeatures.shape[0] > 0:
          df_testinfill = autoMLer[autoML_type][ML_application]['predict'](ML_cmnd, model, df_test_fillfeatures, printstatus, categorylist)
        else:
          df_testinfill = np.array([0])

        #convert infill values to dataframe
        df_testinfill = pd.DataFrame(df_testinfill, columns = ['infill'])
        
      #if target category is single column categoric (eg ordinal or boolean integer)
      if MLinfilltype in ['singlct', 'binary', 'concurrent_act']:
        
        if MLinfilltype == 'singlct':
          ML_application = 'ordinalclassification'
        else:
          ML_application = 'booleanclassification'
        
        #only run following if we have any test rows needing infill
        if df_test_fillfeatures.shape[0] > 0:
          df_testinfill = autoMLer[autoML_type][ML_application]['predict'](ML_cmnd, model, df_test_fillfeatures, printstatus, categorylist)
        else:
          df_testinfill = np.array([0])

        #convert infill values to dataframe
        df_testinfill = pd.DataFrame(df_testinfill, columns = ['infill'])
        
      #if target category is multi-column categoric (one hot encoding) / (binary encoded sets handled sepreately)
      if MLinfilltype in ['multirt']:
        
        ML_application = 'onehotclassification'
        
        #only run following if we have any test rows needing infill
        if df_test_fillfeatures.shape[0] > 0:
          df_testinfill = autoMLer[autoML_type][ML_application]['predict'](ML_cmnd, model, df_test_fillfeatures, printstatus, categorylist)
        else:
          #this needs to have same number of columns
          df_testinfill = np.zeros(shape=(1,len(categorylist)))
          
        #convert infill values to dataframe
        df_testinfill = pd.DataFrame(df_testinfill, columns = categorylist)
      
      #if target is binary encoded
      if MLinfilltype in ['1010']:
        
        ML_application = 'onehotclassification'

        _1010_categorylist_proxy_for_postmunge_MLinfill = \
        postprocess_dict['column_dict'][categorylist[0]]['_1010_categorylist_proxy_for_postmunge_MLinfill']
        
        #only run following if we have any test rows needing infill
        if df_test_fillfeatures.shape[0] > 0:
          df_testinfill = autoMLer[autoML_type][ML_application]['predict'](ML_cmnd, model, df_test_fillfeatures, printstatus, _1010_categorylist_proxy_for_postmunge_MLinfill)
          
          df_testinfill = \
          self.convert_onehot_to_1010(df_testinfill)
          
        else:
          #this needs to have same number of columns
          df_testinfill = np.zeros(shape=(1,len(categorylist)))
        
        #convert infill values to dataframe
        df_testinfill = pd.DataFrame(df_testinfill, columns = categorylist)         
        
      #if target is exlcuded from ML infill
      if MLinfilltype in ['exclude', 'boolexclude', 'totalexclude']:

        #create empty sets
        df_testinfill = pd.DataFrame({'infill' : [0]}) 

    #else if we didn't have a trained model let's create some plug values
    else:

      df_testinfill = np.zeros(shape=(1,len(categorylist)))
      df_testinfill = pd.DataFrame(df_testinfill, columns = categorylist) 
    
    return df_testinfill

  def postMLinfillfunction(self, df_test, column, postprocess_dict, \
                            masterNArows_test, printstatus):

    '''
    #new function ML infill, generalizes the MLinfill application
    #def MLinfill (df_train, df_test, column, postprocess_dict, \
    #masterNArows_train, masterNArows_test, randomseed)
    #function that applies series of functions of createMLinfillsets, 
    #predictinfill, and insertinfill to a categorical encoded set.
    #for the record I'm sure that the conversion of the single column
    #series to a dataframe is counter to the intent of pandas
    #it's probably less memory efficient but it's the current basis of
    #the functions so we're going to maintain that approach for now
    #the revision of these functions to accept pandas series is a
    #possible future extension
    '''
    
    if postprocess_dict['column_dict'][column]['infillcomplete'] is False:

      columnslist = postprocess_dict['column_dict'][column]['columnslist']
      categorylist = postprocess_dict['column_dict'][column]['categorylist']
      origcolumn = postprocess_dict['column_dict'][column]['origcolumn']
      category = postprocess_dict['column_dict'][column]['category']
      model = postprocess_dict['column_dict'][column]['infillmodel']
      
      if len(categorylist) == 1 or \
      postprocess_dict['process_dict'][postprocess_dict['column_dict'][column]['category']]['MLinfilltype'] \
      in ['concurrent_act', 'concurrent_nmbr']:
        #copy the datatype to ensure returned set is consistent
        df_temp_dtype = pd.DataFrame(df_test[column][:0]).copy()

      elif len(categorylist) > 1:
        #copy the datatype to ensure returned set is consistent
        df_temp_dtype = pd.DataFrame(df_test[categorylist][:0]).copy()
      
      #createMLinfillsets
      df_test_fillfeatures = \
      self.createpostMLinfillsets(df_test, column, \
                         pd.DataFrame(masterNArows_test[origcolumn+'_NArows']), \
                         category, postprocess_dict, \
                         columnslist = columnslist, \
                         categorylist = categorylist)

      #predict infill values using defined function predictinfill(.)
      df_testinfill = \
      self.predictpostinfill(category, model, df_test_fillfeatures, \
                            postprocess_dict, postprocess_dict['ML_cmnd'], postprocess_dict['autoMLer'], \
                            printstatus, categorylist = categorylist)

      #if model is not False:
      if postprocess_dict['column_dict'][column]['infillmodel'] is not False:

        df_test = self.insertinfill(df_test, column, df_testinfill, category, \
                               pd.DataFrame(masterNArows_test[origcolumn+'_NArows']), \
                               postprocess_dict, columnslist = columnslist, \
                               categorylist = categorylist)
          
      #now change the infillcomplete marker in the text_dict for each \
      #associated text column unless in concurrent_activations MLinfilltype
      if postprocess_dict['process_dict'][postprocess_dict['column_dict'][column]['category']]['MLinfilltype'] \
      in ['concurrent_act', 'concurrent_nmbr']:
        postprocess_dict['column_dict'][column]['infillcomplete'] = True
        
      else:
        for columnname in categorylist:
          postprocess_dict['column_dict'][columnname]['infillcomplete'] = True
        
      #reset data type to ensure returned data is consistent with what was passed
      if len(categorylist) == 1 or \
      postprocess_dict['process_dict'][postprocess_dict['column_dict'][column]['category']]['MLinfilltype'] \
      in ['concurrent_act', 'concurrent_nmbr']:
        df_test[column] = \
        df_test[column].astype({column:df_temp_dtype[column].dtypes})

      elif len(categorylist) > 1:
        for dtype_column in categorylist:
          df_test[dtype_column] = \
          df_test[dtype_column].astype({dtype_column:df_temp_dtype[dtype_column].dtypes})

    return df_test, postprocess_dict

  def postcreatePCAsets(self, df_test, postprocess_dict):
    '''
    Function that takes as input the dataframes df_train and df_test 
    Removes those columns associated with the PCAexcl (which are the original 
    columns passed to automunge which are to be exlcuded from PCA), and returns 
    those sets as PCAset_trian, PCAset_test, and the list of columns extracted as
    PCAexcl_posttransform.
    '''

    PCAexcl = postprocess_dict['PCAexcl']

    #initiate list PCAexcl_postransform
    PCAexcl_posttransform = []

    #derive the excluded columns post-transform using postprocess_dict
    for exclcolumn in PCAexcl:
      
      #if this is one of the original columns (pre-transform)
      if exclcolumn in postprocess_dict['origcolumn']:
      
        #get a column key for this column (used to access stuff in postprofcess_dict)
        exclcolumnkey = postprocess_dict['origcolumn'][exclcolumn]['columnkey']

        #get the columnslist from this columnkey
        exclcolumnslist = postprocess_dict['column_dict'][exclcolumnkey]['columnslist']

        #add these items to PCAexcl_posttransform
        PCAexcl_posttransform.extend(exclcolumnslist)
        
      #if this is a post-transformation column
      elif exclcolumn in postprocess_dict['column_dict']:
        
        #if we hadn't already done another column from the same source
        if exclcolumn not in PCAexcl_posttransform:
          
          #add these items to PCAexcl_posttransform
          PCAexcl_posttransform.extend([exclcolumn])

    #assemble the sets by dropping the columns excluded
    PCAset_test = df_test.drop(PCAexcl_posttransform, axis=1)

    return PCAset_test, PCAexcl_posttransform

  def postPCAfunction(self, PCAset_test, postprocess_dict):
    '''
    Function that takes as input the train and test sets intended for PCA
    dimensionality reduction. Returns a trained PCA model saved in postprocess_dict
    and trasnformed sets.
    '''

    PCAmodel = postprocess_dict['PCAmodel']

    #convert PCAsets to numpy arrays
    PCAset_test = PCAset_test.values

    #apply the transform
    PCAset_test = PCAmodel.transform(PCAset_test)

    #get new number of columns
    newcolumncount = np.size(PCAset_test,1)

    #generate a list of column names for the conversion to pandas
    columnnames = ['PCAcol'+str(y) for y in range(newcolumncount)]

    #convert output to pandas
    PCAset_test = pd.DataFrame(PCAset_test, columns = columnnames)

    return PCAset_test, postprocess_dict

  def postfeatureselect(self, df_test, labelscolumn, testID_column, \
                        postprocess_dict, printstatus):
    '''
    featureselect is a function called within automunge() that applies methods
    to evaluate predictive power of derived features towards a downstream model
    such as to trim the branches of the transform tree.
    
    The function returns a list of column names that "made the cut" so that
    automunge() can then remove extraneous branches.
    '''
    
    #now we'll use automunge() to prepare the subset for feature evaluation
    #note the passed arguments, these are all intentional (no MLinfill applied,
    #primary goal here is to produce a processed dataframe for df_subset
    #with corresponding labels)
    
        
    #printout display progress
    if printstatus is True:
      print("_______________")
      print("Begin Feature Importance evaluation")
      print("")


    #this converts any numeric columns labels, such as from a passed numpy array, to strings
    testlabels=[]
    for column in df_test.columns:
      testlabels.append(str(column))
    df_test.columns = testlabels

    if labelscolumn is False or labelscolumn is True:
      if postprocess_dict['labels_column'] in list(df_test):
        labelscolumn = postprocess_dict['labels_column']
      
    if labelscolumn is False:
      
      FSmodel = False

      baseaccuracy = False
      
      #printout display progress
      if printstatus is True:
        print("_______________")
        print("No labels_column passed, Feature Importance halted")
        print("")
        
    elif labelscolumn is not False:
    
      #copy postprocess_dict to customize for feature importance evaluation
      FSpostprocess_dict = deepcopy(postprocess_dict)
      testID_column = testID_column
      labelscolumn = labelscolumn
      pandasoutput = True
      printstatus = printstatus
      TrainLabelFreqLevel = False
      featureeval = False
      FSpostprocess_dict['shuffletrain'] = True
      FSpostprocess_dict['TrainLabelFreqLevel'] = False
      FSpostprocess_dict['MLinfill'] = False
      FSpostprocess_dict['featureselection'] = False
      FSpostprocess_dict['PCAn_components'] = None
      FSpostprocess_dict['Binary'] = False
      FSpostprocess_dict['excl_suffix'] = True
      FSpostprocess_dict['ML_cmnd']['PCA_type'] = 'off'
      FSpostprocess_dict['assigninfill'] = {'stdrdinfill':[], 'MLinfill':[], 'zeroinfill':[], 'oneinfill':[], \
                                             'adjinfill':[], 'meaninfill':[], 'medianinfill':[]}
      randomseed = FSpostprocess_dict['randomseed']
      process_dict = FSpostprocess_dict['process_dict']
      ML_cmnd = FSpostprocess_dict['ML_cmnd']

      FS_LabelSmoothing = False

  #     #FUTURE EXTENSION
  #     FS_LabelSmoothing = FSpostprocess_dict['ML_cmnd']['MLinfill_cmnd']['RandomForestClassifier']['LabelSmoothing']

  #     #but first real quick we'll just deal with PCA default functionality for FS
  #     FSML_cmnd = deepcopy(ML_cmnd)
  #     FSML_cmnd['PCA_type'] = 'off'

      #totalvalidation = valpercent1 + valpercent2

      #if totalvalidation == 0:
      totalvalidation = 0.2

      #prepare sets for FS with postmunge
      am_train, _1, am_labels, labelsencoding_dict, finalcolumns_train = \
      self.postmunge(FSpostprocess_dict, df_test, testID_column = testID_column, \
                     labelscolumn = labelscolumn, pandasoutput = pandasoutput, printstatus = printstatus, \
                     TrainLabelFreqLevel = TrainLabelFreqLevel, featureeval = featureeval, \
                     LabelSmoothing = FS_LabelSmoothing, shuffletrain = True)

      #prepare validaiton sets for FS
      am_train, am_validation1 = \
      self.df_split(am_train, totalvalidation, False, randomseed)

      am_labels, am_validationlabels1 = \
      self.df_split(am_labels, totalvalidation, False, randomseed)


      #this is the returned process_dict
      #(remember "processdict" is what we pass to automunge() call, "process_dict" is what is 
      #assembled inside automunge, there is a difference)
      FSprocess_dict = FSpostprocess_dict['process_dict']

      if am_labels.empty is True:
        FSmodel = False

        baseaccuracy = False
        
        #printout display progress
        if printstatus is True:
          print("_______________")
          print("No labels returned from postmunge(.), Feature Importance halted")
          print("")
    
      #if am_labels is not an empty set
      if am_labels.empty is False:

        #find origcateogry of am_labels from FSpostprocess_dict
        labelcolumnkey = list(am_labels)[0]
        origcolumn = FSpostprocess_dict['column_dict'][labelcolumnkey]['origcolumn']
        origcategory = FSpostprocess_dict['column_dict'][labelcolumnkey]['origcategory']

        #find labelctgy from process_dict based on this origcategory
        labelctgy = process_dict[origcategory]['labelctgy']

        am_categorylist = []

        for am_label_column in am_labels.columns:

          if FSpostprocess_dict['column_dict'][am_label_column]['category'] == labelctgy:

            am_categorylist = FSpostprocess_dict['column_dict'][am_label_column]['categorylist']
            
            #we'll follow convention that if target label category MLinfilltype is concurrent
            #we'll arbitrarily take the first column and use that as target
            if FSpostprocess_dict['process_dict'][labelctgy]['MLinfilltype'] \
            in ['concurrent_act', 'concurrent_nmbr']:
              
              am_categorylist = [am_categorylist[0]]
              
            break

        if len(am_categorylist) == 0:
          if printstatus is True:
            #this is a remote edge case, printout added for troubleshooting support
            print("Label category processdict entry contained a labelctgy entry not found in transformdict entry")
            print("Feature Seclection model training will not run without valid labelgctgy processdict entry")
            print()

        elif len(am_categorylist) == 1:
          am_labels = pd.DataFrame(am_labels[am_categorylist[0]])
          am_validationlabels1 = pd.DataFrame(am_validationlabels1[am_categorylist[0]])

        else:
          am_labels = am_labels[am_categorylist]
          am_validationlabels1 = am_validationlabels1[am_categorylist]

        #if there's a bug occuring after this point it might mean the labelctgy wasn't
        #properly populated in the process_dict for the root category assigned to the labels
        #again the labelctgy entry to process_dict represents for labels returned in 
        #multiple configurations the trasnofrmation category whose returned set will be
        #used to train the feature selection model

        #printout display progress
        if printstatus is True:
          print("_______________")
          print("Training feature importance evaluation model")
          print("")

        #apply function trainFSmodel
        #FSmodel, baseaccuracy = \
        FSmodel = \
        self.trainFSmodel(am_train, am_labels, randomseed, \
                          FSprocess_dict, FSpostprocess_dict, labelctgy, ML_cmnd, \
                          printstatus)
        
        if FSmodel is False:
          
          #initialize dictionary FScolumn_dict = {}
          FScolumn_dict = {}
          
          FS_origcolumns = list(FSpostprocess_dict['origcolumn'])

          baseaccuracy = False
          
          #printout display progress
          if printstatus is True:
            print("_______________")
            print("No model returned from training, Feature Importance halted")
            print("")
          
        elif FSmodel is not False:

          #update v2.11 baseaccuracy should be based on validation set
          baseaccuracy = self.shuffleaccuracy(am_validation1, am_validationlabels1, \
                                              FSmodel, randomseed, am_categorylist, \
                                              FSprocess_dict, labelctgy, FSpostprocess_dict)

          if printstatus is True:
            print("Base Accuracy of feature importance model:")
            print(baseaccuracy)
            print()

          #get list of columns
          am_train_columns = list(am_train)

          #initialize dictionary FScolumn_dict = {}
          FScolumn_dict = {}
          
          FS_origcolumns = list(FSpostprocess_dict['origcolumn'])

          #assemble FScolumn_dict to support the feature evaluation
          for column in am_train_columns:

            #pull categorylist, category, columnslist
            categorylist = FSpostprocess_dict['column_dict'][column]['categorylist']
            category = FSpostprocess_dict['column_dict'][column]['category']
            columnslist = FSpostprocess_dict['column_dict'][column]['columnslist']
            origcolumn = FSpostprocess_dict['column_dict'][column]['origcolumn']

            #create entry to FScolumn_dict
            FScolumn_dict.update({column : {'categorylist' : categorylist, \
                                            'category' : category, \
                                            'columnslist' : columnslist, \
                                            'origcolumn' : origcolumn, \
                                            'FScomplete' : False, \
                                            'shuffleaccuracy' : None, \
                                            'shuffleaccuracy2' : None, \
                                            'baseaccuracy' : baseaccuracy, \
                                            'metric' : None, \
                                            'metric2' : None}})

          #printout display progress
          if printstatus is True:
            print("_______________")
            print("Evaluating feature importances")
            print("")

          #perform feature evaluation on each column
          for column in am_train_columns:

#             if FSpostprocess_dict['column_dict'][column]['category'] != 'NArw' \
#             and FScolumn_dict[column]['FScomplete'] is False:
            if FScolumn_dict[column]['FScomplete'] is False:

              #categorylist = FScolumn_dict[column]['categorylist']
              #update version 1.80, let's perform FS on columnslist instead of categorylist
              columnslist = FScolumn_dict[column]['columnslist']

              #create set with columns shuffle from columnslist
              #shuffleset = self.createFSsets(am_train, column, categorylist, randomseed)
              #shuffleset = self.createFSsets(am_train, column, columnslist, randomseed)
              shuffleset = self.createFSsets(am_validation1, column, columnslist, randomseed)

              #determine resulting accuracy after shuffle
              columnaccuracy = self.shuffleaccuracy(shuffleset, am_validationlabels1, \
                                                    FSmodel, randomseed, am_categorylist, \
                                                    FSprocess_dict, labelctgy, FSpostprocess_dict)

              #I think this will clear some memory
              del shuffleset

              #category accuracy penalty metric
              metric = baseaccuracy - columnaccuracy
              #metric2 = baseaccuracy - columnaccuracy2
              
              #save accuracy to FScolumn_dict and set FScomplete to True
              #(for each column in the categorylist)
              #for categorycolumn in FSpostprocess_dict['column_dict'][column]['categorylist']:
              for categorycolumn in FSpostprocess_dict['column_dict'][column]['columnslist']:

                FScolumn_dict[categorycolumn]['FScomplete'] = True
                FScolumn_dict[categorycolumn]['shuffleaccuracy'] = columnaccuracy
                FScolumn_dict[categorycolumn]['metric'] = metric
                #FScolumn_dict[categorycolumn]['shuffleaccuracy2'] = columnaccuracy2
                #FScolumn_dict[categorycolumn]['metric2'] = metric2

            columnslist = FScolumn_dict[column]['columnslist']

            #create second set with all but one columns shuffled from columnslist
            #this will allow us to compare the relative importance between columns
            #derived from the same parent
            #shuffleset2 = self.createFSsets2(am_train, column, columnslist, randomseed)
            shuffleset2 = self.createFSsets2(am_validation1, column, columnslist, randomseed)

            #determine resulting accuracy after shuffle
    #           columnaccuracy2 = self.shuffleaccuracy(shuffleset2, am_labels, FSmodel, \
    #                                                 randomseed, \
    #                                                 process_dict)
            columnaccuracy2 = self.shuffleaccuracy(shuffleset2, am_validationlabels1, \
                                                  FSmodel, randomseed, am_categorylist, \
                                                  FSprocess_dict, labelctgy, FSpostprocess_dict)

            metric2 = baseaccuracy - columnaccuracy2

            FScolumn_dict[column]['shuffleaccuracy2'] = columnaccuracy2
            FScolumn_dict[column]['metric2'] = metric2

    #     madethecut = self.assemblemadethecut(FScolumn_dict, featurepct, featuremetric, \
    #                                          featuremethod, am_train_columns)


        #if the only column left in madethecut from origin column is a NArw, delete from the set
        #(this is going to lean on the column ID string naming conventions)
        #couldn't get this to work, this functionality a future extension
    #     trimfrommtc = []
    #     for traincolumn in list(df_train):
    #       if (traincolumn + '_') not in [checkmtc[:(len(traincolumn)+1)] for checkmtc in madethecut]:
    #         for mtc in madethecut:
    #           #if mtc originated from traincolumn
    #           if mtc[:(len(traincolumn)+1)] == traincolumn + '_':
    #             #count the number of same instance in madethecut set
    #             madethecut_trim = [mdc_trim[:(len(traincolumn)+1)] for mdc_trim in madethecut]
    #             if madethecut_trim.count(mtc[:(len(traincolumn)+1)]) == 1 \
    #             and mtc[-5:] == '_NArw':
    #               trimfrommtc = trimfrommtc + [mtc]
    #     madethecut = list(set(madethecut).difference(set(trimfrommtc)))


        #apply function madethecut(FScolumn_dict, featurepct)
        #return madethecut
        #where featurepct is the percent of features that we intend to keep
        #(might want to make this a passed argument from automunge)

        #I think this will clear some memory
    #     del am_train, _1, am_labels, am_validation1, _3, \
    #     am_validationlabels1, _5, _6, _7, \
    #     _8, _9, labelsencoding_dict, finalcolumns_train, _10,  \
    #     FSpostprocess_dict

          del am_train, _1, am_labels, labelsencoding_dict, finalcolumns_train, am_validation1, am_validationlabels1

          if printstatus is True:
            print("_______________")
            print("Feature Importance results:")
            print("")

          #to inspect values returned in featureimportance object one could run
          if printstatus is True:
            for keys,values in FScolumn_dict.items():
              print(keys)
              print('metric = ', values['metric'])
              print('metric2 = ', values['metric2'])
              print()
              
    FS_sorted = {'baseaccuracy':baseaccuracy, \
                 'metric_key':{}, \
                 'column_key':{}, \
                 'metric2_key':{}, \
                 'metric2_column_key':{}}
    
    #first we'll handle first metric based on source column
    for FS_origcolumn in FS_origcolumns:
      for key in FScolumn_dict:
        if FScolumn_dict[key]['origcolumn'] == FS_origcolumn:
          if FScolumn_dict[key]['metric'] in FS_sorted['metric_key']:
            if isinstance(FS_sorted['metric_key'][FScolumn_dict[key]['metric']], list):
              FS_sorted['metric_key'][FScolumn_dict[key]['metric']].append(FS_origcolumn)
            else:
              FS_sorted['metric_key'][FScolumn_dict[key]['metric']] = \
              [FS_sorted['metric_key'][FScolumn_dict[key]['metric']]]
              FS_sorted['metric_key'][FScolumn_dict[key]['metric']].append(FS_origcolumn)
          else:
            FS_sorted['metric_key'].update({FScolumn_dict[key]['metric'] : [FS_origcolumn]})
          break

    FS_sorted['metric_key'] = dict(sorted(FS_sorted['metric_key'].items(), reverse=True))
    
    for key in FS_sorted['metric_key']:
      for entry in FS_sorted['metric_key'][key]:
        entry_index = FS_sorted['metric_key'][key].index(entry)
        FS_sorted['column_key'].update({FS_sorted['metric_key'][key][entry_index] : key})
      
    #now for metric2 based on derived columns relative importance, note sorted in other order
    for FS_origcolumn in FS_origcolumns:
      FS_sorted['metric2_key'].update({FS_origcolumn : {}})
      for key in FScolumn_dict:
        if FScolumn_dict[key]['origcolumn'] == FS_origcolumn:
          if FScolumn_dict[key]['metric2'] in FS_sorted['metric2_key'][FS_origcolumn]:
            if isinstance(FS_sorted['metric2_key'][FS_origcolumn][FScolumn_dict[key]['metric2']], list):
              FS_sorted['metric2_key'][FS_origcolumn][FScolumn_dict[key]['metric2']].append(key)
            else:
              FS_sorted['metric2_key'][FS_origcolumn][FScolumn_dict[key]['metric2']] = \
              [FS_sorted['metric2_key'][FS_origcolumn][FScolumn_dict[key]['metric2']]]
              FS_sorted['metric2_key'][FS_origcolumn][FScolumn_dict[key]['metric2']].append(key)
          else:
            FS_sorted['metric2_key'][FS_origcolumn].update({FScolumn_dict[key]['metric2'] : [key]})
    
    for key in FS_sorted['metric2_key']:
      FS_sorted['metric2_key'][key] = dict(sorted(FS_sorted['metric2_key'][key].items(), reverse=False))
    
    for key1 in FS_sorted['metric2_key']:
      FS_sorted['metric2_column_key'].update({key1 : {}})
      for key2 in FS_sorted['metric2_key'][key1]:
        for entry in FS_sorted['metric2_key'][key1][key2]:
          entry_index = FS_sorted['metric2_key'][key1][key2].index(entry)
          FS_sorted['metric2_column_key'][key1].update({FS_sorted['metric2_key'][key1][key2][entry_index] : key2})
        
    if printstatus is True:
      print()
      print("______________________")
      print("sorted metric results:")
      print()
      for keys,values in FS_sorted['metric_key'].items():
        for entry in values:
          print(entry)
          print(keys)
          print()
      print("______________________")
      print("sorted metric2 results:")
      print()
      for key in FS_sorted['metric2_key']:
        print("for source column: ", key)
        for keys,values in FS_sorted['metric2_key'][key].items():
          for entry in values:
            print(entry)
            print(keys)
            print()
        print()
        
    if FSmodel is False:
      
      FScolumn_dict = {}
    
    #printout display progress
    if printstatus is True:
      
      print("")
      print("_______________")
      print("Feature Importance evaluation complete")
      print("")

    return FSmodel, FScolumn_dict, FS_sorted
  
  def prepare_driftreport(self, df_test, postprocess_dict, printstatus):
    """
    #driftreport uses the processfamily functions as originally implemented
    #in automunge to recalculate normalization parameters based on the test
    #set passed to postmunge and print a comparison with those original 
    #normalization parameters saved in the postprocess_dict, such as may 
    #prove useful to track drift from original training data.
    #returns a store of the temporary postprocess_dict containing the newly 
    #calculated normalziation parameters and a report of the results
    """
    
    if printstatus is True:
      print("_______________")
      print("Preparing Drift Report:")
      print("")
      
    #initialize empty dictionary to store results
    drift_report = {}
    
    #temporary store for updated normalization parameters
    #we'll copy all the support stuff from original pp_d but delete the 'column_dict'
    #entries for our new derivations below
    drift_ppd = deepcopy(postprocess_dict)
    drift_ppd['column_dict'] = {}
    
    #for each column in df_test
    for drift_column in df_test:
      
      returnedcolumns = postprocess_dict['origcolumn'][drift_column]['columnkeylist']
      returnedcolumns.sort()
      
      if printstatus is True:
        print("______")
        print("Preparing drift report for columns derived from: ", drift_column)
        print("")
        print("original returned columns:")
        print(returnedcolumns)
        print("")
        
      if len(returnedcolumns) > 0:

        drift_category = \
        postprocess_dict['column_dict'][postprocess_dict['origcolumn'][drift_column]['columnkey']]['origcategory']
        
      else:
        
        drift_category = 'null'
      
      #update driftreport with this column
      drift_report.update({drift_column : {'origreturnedcolumns_list':returnedcolumns, \
                                           'newreturnedcolumns_list':[], \
                                           'drift_category' : drift_category, \
                                           'orignotinnew' : {}, \
                                           'newnotinorig' : {}, \
                                           'newreturnedcolumn':{}}})
      
      drift_process_dict = \
      postprocess_dict['process_dict']
      
      drift_transform_dict = \
      postprocess_dict['transform_dict']
      
      drift_assign_param = \
      postprocess_dict['assign_param']
      
      #we're only going to copy one source column at a time, as should be 
      #more memory efficient than copying the entire set
      df_test2_temp = pd.DataFrame(df_test[drift_column].copy())
      
      #then a second copy set, here of just a few rows, to follow convention of 
      #automunge processfamily calls
#       df_test3_temp = df_test2_temp[0:10].copy()
      df_test3_temp = df_test2_temp[0:1].copy()
      
      #here's a templist to support the columnkey entry below
      templist1 = list(df_test2_temp)
    
      #now process family
      df_test2_temp, df_test3_temp, drift_ppd = \
      self.processfamily(df_test2_temp, df_test3_temp, drift_column, drift_category, \
                         drift_category, drift_process_dict, drift_transform_dict, \
                         drift_ppd, drift_assign_param)

      #here's a second templist to support the columnkey entry below
      templist2 = list(df_test2_temp)
      
      #ok now we're going to pick one of the new entries of. returned columns to serve 
      #as a "columnkey" for pulling datas from the postprocess_dict 
      #columnkeylist = list(set(templist2) - set(templist1))[0]
      columnkeylist = list(set(templist2) - set(templist1))

      #so last line I believe returns string if only one entry, so let's run a test
      if isinstance(columnkeylist, str):
        columnkey = columnkeylist
      else:
        #if list is empty
        if len(columnkeylist) == 0:
          columnkey = drift_column
        else:
          columnkey = columnkeylist[0]
      
      #if drift_ppd['origcolumn'][drift_column]['columnkey'] not in drift_ppd['column_dict']:
      if len(columnkeylist) == 0:
        
        if printstatus is True:
          print("no new returned columns:")
          print("")
        
        newreturnedcolumns = []
        
      else:
        
#         newreturnedcolumns = \
#         drift_ppd['column_dict'][drift_ppd['origcolumn'][drift_column]['columnkey']]['columnslist']
        newreturnedcolumns = \
        drift_ppd['column_dict'][columnkey]['columnslist']

        newreturnedcolumns.sort()

        if printstatus is True:
          print("new returned columns:")
          print(newreturnedcolumns)
          print("")
          
        #add to driftreport
        drift_report[drift_column]['newreturnedcolumns_list'] = newreturnedcolumns
      
      for origreturnedcolumn in returnedcolumns:
        if origreturnedcolumn not in newreturnedcolumns:
          if printstatus is True:
            print("___")
            print("original derived column not in new returned column: ", origreturnedcolumn)
            print("")
            print("original automunge normalization parameters:")
            print(postprocess_dict['column_dict'][origreturnedcolumn]['normalization_dict'][origreturnedcolumn])
            print("")
          
          drift_report[drift_column]['orignotinnew'].update({origreturnedcolumn:{'orignormparam':\
          postprocess_dict['column_dict'][origreturnedcolumn]['normalization_dict'][origreturnedcolumn]}})
      
      for returnedcolumn in newreturnedcolumns:
        
        drift_report[drift_column]['newreturnedcolumn'].update(\
        {returnedcolumn:{'orignormparam':{}, 'newnormparam':{}}})
        
        if printstatus is True:
          print("___")
          print("derived column: ", returnedcolumn)
          print("")
          
        if returnedcolumn in returnedcolumns:
          if printstatus is True:
            print("original automunge normalization parameters:")
            
            print(postprocess_dict['column_dict'][returnedcolumn]['normalization_dict'][returnedcolumn])
            print("")
            
          #add to driftreport
          drift_report[drift_column]['newreturnedcolumn'][returnedcolumn]['orignormparam'] \
          = postprocess_dict['column_dict'][returnedcolumn]['normalization_dict'][returnedcolumn]
          
        else:
          if printstatus is True:
            print("new derived column not in original returned columns: ", returnedcolumn)
            print("")
            
          drift_report[drift_column]['newnotinorig'].update({returnedcolumn:{'newnormparam':\
          drift_ppd['column_dict'][returnedcolumn]['normalization_dict'][returnedcolumn]}})
          
        if printstatus is True:
          print("new postmunge normalization parameters:")
          print(drift_ppd['column_dict'][returnedcolumn]['normalization_dict'][returnedcolumn])
          print("")
          
        #add to driftreport
        drift_report[drift_column]['newreturnedcolumn'][returnedcolumn]['newnormparam'] \
        = drift_ppd['column_dict'][returnedcolumn]['normalization_dict'][returnedcolumn]
      
      #free up some memory
      del df_test2_temp, df_test3_temp, returnedcolumns
      
    if printstatus is True:
      print("")
      print("_______________")
      print("Drift Report Complete")
      print("")
      
    return drift_ppd, drift_report

  def postmunge(self, postprocess_dict, df_test, \
                testID_column = False, labelscolumn = False, \
                pandasoutput = False, printstatus = True, \
                TrainLabelFreqLevel = False, featureeval = False, driftreport = False, \
                LabelSmoothing = False, LSfit = False, inversion = False, traindata = False, \
                returnedsets = True, shuffletrain = False):
    """
    #This function documented in READ ME, available online at:
    # (anonymized)
    """

    # #copy postprocess_dict into internal state so don't edit external object
    # #(going to leave this out for now in case has large memory overhead impact
    # #as in some scenarios postprocess_dict can be a large file)
    # postprocess_dict = deepcopy(postprocess_dict)
    #I believe the only edits made to postproces_dict in postmunge are to track infill status
    #which are reset after use

    #traindata only matters when transforms apply different methods for train vs test
    #such as for noise injection to train data for differential privacy
    if traindata is True:
      postprocess_dict['traindata'] = True
    else:
      postprocess_dict['traindata'] = False

    indexcolumn = postprocess_dict['indexcolumn']
    testID_column_orig = testID_column

    #quick conversion of any passed column idenitfiers to str
    labelscolumn = self.parameter_str_convert(labelscolumn)
    testID_column = self.parameter_str_convert(testID_column)
    
    #check the range of parameters 
    #(generally speaking other than passed dictionaries, dataframes, or column identifiers)
    pm_miscparameters_results = \
    self.check_pm_miscparameters(pandasoutput, printstatus, TrainLabelFreqLevel, \
                                featureeval, driftreport, LabelSmoothing, LSfit, \
                                returnedsets, shuffletrain, inversion, traindata)
    
    #printout display progress
    if printstatus is True:
      print("_______________")
      print("Begin Postmunge processing")
      print("")
    
    #feature selection analysis performed here if elected
    if featureeval is True:

      if inversion is not False:
        print("featureselection not available when performing inversion")
        print()
        
        madethecut = postprocess_dict['madethecut']
        FSmodel = False
        FScolumn_dict = {}
        FS_sorted = {}
      
      elif postprocess_dict['labels_column'] is False:
        print("featureselection not available without labels_column in training set")
        print()
        
        madethecut = postprocess_dict['madethecut']
        FSmodel = False
        FScolumn_dict = {}
        FS_sorted = {}

      else:

        FSmodel, FScolumn_dict, FS_sorted = \
        self.postfeatureselect(df_test, labelscolumn, testID_column, \
                               postprocess_dict, printstatus)

        madethecut = postprocess_dict['madethecut']

    else:

      madethecut = postprocess_dict['madethecut']
      FSmodel = None
      FScolumn_dict = {}
      FS_sorted = {}

    check_FSmodel_result = self.check_FSmodel(featureeval, FSmodel)
    pm_miscparameters_results.update({'FSmodel_valresult' : check_FSmodel_result})

    #initialize postreports_dict
    postreports_dict = {'featureimportance':FScolumn_dict, \
                        'FS_sorted' : FS_sorted, \
                        'finalcolumns_test':[], \
                        'driftreport':{}, \
                        'pm_miscparameters_results':pm_miscparameters_results}

    #functionality to support passed numpy arrays
    #if passed object was a numpy array, convert to pandas dataframe
    checknp = np.array([])
    if isinstance(checknp, type(df_test)):
      df_test = pd.DataFrame(df_test)

      #this converts to original headers
      if len(df_test.columns) == len(postprocess_dict['origcolumns_all']):
        df_test.columns = postprocess_dict['origcolumns_all']

      if labelscolumn is False and postprocess_dict['labels_column'] is not False:
        if len(df_test.columns) == len(postprocess_dict['origcolumns_all']) - 1:
          origcolumns_all_excluding_label = postprocess_dict['origcolumns_all'].copy()
          origcolumns_all_excluding_label.remove(postprocess_dict['labels_column'])
          df_test.columns = origcolumns_all_excluding_label

    #this converts any numeric columns labels, such as from a passed numpy array, to strings
    testlabels=[]
    for column in df_test.columns:
      testlabels.append(str(column))
    df_test.columns = testlabels

    if labelscolumn is False or labelscolumn is True:
      if postprocess_dict['labels_column'] in list(df_test):
        labelscolumn = postprocess_dict['labels_column']

    #initialize processing dicitonaries

    powertransform = postprocess_dict['powertransform']
    binstransform = postprocess_dict['binstransform']
    NArw_marker = postprocess_dict['NArw_marker']
    floatprecision = postprocess_dict['floatprecision']

#     transform_dict = self.assembletransformdict(binstransform, NArw_marker)

#     if bool(postprocess_dict['transformdict']) is not False:
#       transform_dict.update(postprocess_dict['transformdict'])

#     process_dict = self.assembleprocessdict()

    transform_dict = postprocess_dict['transform_dict']
    process_dict = postprocess_dict['process_dict']

#     assign_param = self.assembleassignparam()

#     if bool(postprocess_dict['assignparam']) is not False:
#       assign_param.update(postprocess_dict['assignparam'])
      
    assign_param = postprocess_dict['assign_param']

    #copy input dataframes to internal state so as not to edit exterior objects
#     if inplace is False:
    df_test = df_test.copy()
#     elif inplace is True:
#       pass

    #_______
    #here is where inversion is performed if selected
    if inversion is not False:

      df_test = self.inversion_numpy_support(df_test, postprocess_dict, inversion)

      df_test = self.inversion_privacy_support(df_test, postprocess_dict, inversion)

      df_test, recovered_list, inversion_info_dict = \
      self.inversion_parent(inversion, df_test, postprocess_dict, printstatus, \
                            pandasoutput, LabelSmoothing)
      
      return df_test, recovered_list, inversion_info_dict
    #_______

    #we'll have convention that if testID_column=False, if trainID_column in df_test etc
    trainID_columns_in_df_test = False
    if testID_column is False or testID_column is True:
      if postprocess_dict['trainID_column_orig'] is not False:
        trainID_columns_in_df_test = True
        if isinstance(postprocess_dict['trainID_column_orig'], list):
          for trainIDcolumn in postprocess_dict['trainID_column_orig']:
            if trainIDcolumn not in df_test.columns:
              trainID_columns_in_df_test = False
              break
        elif isinstance(postprocess_dict['trainID_column_orig'], str):
          if postprocess_dict['trainID_column_orig'] not in df_test.columns:
            trainID_columns_in_df_test = False
    if trainID_columns_in_df_test is True:
      testID_column = postprocess_dict['trainID_column_orig']

    if testID_column is False:
      testID_column = []
    elif isinstance(testID_column, str):
      testID_column = [testID_column]
    elif not isinstance(testID_column, list):
      if testID_column is not True:
        print("error, testID_column allowable values are boolean, string, or list")
    if testID_column is True:
      if isinstance(postprocess_dict['trainID_column_orig'], str):
        testID_column = [postprocess_dict['trainID_column_orig']]
      elif isinstance(postprocess_dict['trainID_column_orig'], list):
        testID_column = postprocess_dict['trainID_column_orig']
      elif postprocess_dict['trainID_column_orig'] is False:
        testID_column = []

    if type(df_test.index) != pd.RangeIndex:
      #if df_train.index.names == [None]:
      if None in df_test.index.names:
        df_test = df_test.rename_axis('Orig_index_' +  str(postprocess_dict['application_number']))
      testID_column = testID_column + list(df_test.index.names)
      df_test = df_test.reset_index(drop=False)

    # if labelscolumn is False or labelscolumn is True:
    #   if postprocess_dict['labels_column'] in list(df_test):
    #     labelscolumn = postprocess_dict['labels_column']

    if labelscolumn is not False:
      labels_column = postprocess_dict['labels_column']
    
#         df_test = df_test.dropna(subset=[labels_column])

      if labelscolumn is not True:
        if labelscolumn != labels_column:
          print("error, labelscolumn in test set passed to postmunge must have same column")
          print("labeling convention, labels column from automunge was: ", labels_column)

      df_testlabels = pd.DataFrame(df_test[labels_column])
      del df_test[labels_column]
      
      #if we only had one (label) column to begin with we'll create a dummy test set
      if df_test.shape[1] == 0:
#         df_test = df_testlabels[0:10].copy()
        df_test = df_testlabels[0:1].copy()
  
    else:
      df_testlabels = pd.DataFrame()

    #here we derive a range integer index for inclusion in the test ID sets
    tempIDlist = []
    df_test_tempID = pd.DataFrame({indexcolumn:range(0,df_test.shape[0])})

    if testID_column is not False:

      df_testID = pd.DataFrame(df_test[testID_column])

      # if isinstance(testID_column, str):
      #   testID_column = [testID_column]
      # elif isinstance(testID_column, list):
      #   testID_column = testID_column
      # else:
      #   print("error, testID_column value must be False, str, or list")
      
      df_test_tempID.index = df_testID.index
      
      df_testID = pd.concat([df_testID, df_test_tempID], axis=1)

      for IDcolumn in testID_column:
        del df_test[IDcolumn]

      #then append the indexcolumn to testID_column list for use in later methods
      testID_column.append(indexcolumn)
      
    else:
      df_test_tempID.index = df_test.index
      df_testID = df_test_tempID.copy()
      testID_column = [indexcolumn]

    del df_test_tempID

    #confirm consistency of train an test sets

    #check columns passed to postmunge(.) are consistent with train set passed to automunge(.)
    if len(set(postprocess_dict['origtraincolumns']) - set(df_test)) > 0 \
    or len(set(df_test) - set(postprocess_dict['origtraincolumns'])) > 0:
      print("Error, inconsistent columns between train set passed to automunge(.)")
      print("and test set passed to postmunge(.)")
      print()
      print("__________")
      print("original columns passed to automunge(.) (exluding any labels_column and/or trainID_column):")
      print()
      print(postprocess_dict['origtraincolumns'])
      print()
      print("__________")
      print("current columns passed to postmunge(.) (exluding any labelscolumn and/or testID_column):")
      print()
      print(list(df_test))
      print()
      if len(set(postprocess_dict['origtraincolumns']) - set(df_test)) > 0:
        print("__________")
        print("missing following columns in df_test passed to postmunge(.):")
        print()
        print(list(set(postprocess_dict['origtraincolumns']) - set(df_test)))
        print()
        print("If this is a label column requires designation in automunge(.)")
        print("via the labels_column parameter.")
        print()
      if len(set(df_test) - set(postprocess_dict['origtraincolumns'])) > 0:
        print("__________")
        print("extra columns passed in df_test to postmunge(.) are:")
        print()
        print(list(set(df_test) - set(postprocess_dict['origtraincolumns'])))
        print()
        print("Note that extra columns can be carved out in postmunge(.)")
        print("with testID_column parameter.")
        print()
      
      return

    #check order of column headers are consistent
    columns_train = postprocess_dict['origtraincolumns']
    columns_test = list(df_test)
    if columns_train != columns_test:
      print("error, different order of column labels in the train and test set")
      print()
      print("__________")
      print("original columns passed to automunge(.) (exluding any labels_column and/or trainID_column):")
      print()
      print(postprocess_dict['origtraincolumns'])
      print()
      print("__________")
      print("current columns passed to postmunge(.) (exluding any labelscolumn and/or testID_column):")
      print()
      print(list(df_test))
      print()
      return

    #__________
    #here we'll perform drift report if elected
    #if driftreport is True:
    if driftreport in [True, 'report_full']:

      #returns a new partially populated postprocess_dict containing
      #column_dict entries populated with newly calculated normalization parameters
      #for now we'll just print the results in the function, a future expansion may
      #return these to the user somehow, need to put some thought into that
      drift_ppd, drift_report = self.prepare_driftreport(df_test, postprocess_dict, printstatus)

      postreports_dict['driftreport'] = drift_report
      
    if driftreport in ['report_full', 'report_effic']:
      
      postdrift_dict = {}

      if printstatus is True:
        print("_______________")
        print("Preparing Source Column Drift Report:")
        print("")
      
      for column in df_test:

        if column in postprocess_dict['drift_dict']:

          if printstatus is True:
            print("______")
            print("Preparing source column drift report for column: ", column)
            print("")
            print("original drift stats:")
            print(postprocess_dict['drift_dict'][column])
            print("")

          category = postprocess_dict['origcolumn'][column]['category']

          _1, postdrift_dict = \
          self.getNArows(df_test, column, category, postprocess_dict, postdrift_dict, True)

          if printstatus is True:
            print("new drift stats:")
            print(postdrift_dict[column])
            print("")
          
      postreports_dict.update({'sourcecolumn_drift' : {'orig_driftstats' : postprocess_dict['drift_dict'], \
                                                       'new_driftstats' : postdrift_dict}})

      if printstatus is True:
        print("_______________")
        print("Source Column Drift Report Complete")
        print("")
      
      return [], [], [], [], postreports_dict
    #end drift report section
    #__________

    #create an empty dataframe to serve as a store for each column's NArows
    #the column id's for this df will follow convention from NArows of 
    #column+'_NArows' for each column in columns_train
    #these are used in the ML infill methods
    #masterNArows_train = pd.DataFrame()
    masterNArows_test = pd.DataFrame()
    
    #initialize postdrift_dict
    postdrift_dict = {}

    #For each column, determine appropriate processing function
    #processing function will be based on evaluation of train set
    for column in columns_train:

      columnkey = postprocess_dict['origcolumn'][column]['columnkey']        
      #traincategory = postprocess_dict['column_dict'][columnkey]['origcategory']
      traincategory = postprocess_dict['origcolumn'][column]['category']

      #originally I seperately used evalcategory to check the actual category of
      #the test set, but now that we are allowing assigned categories that could
      #get too complex, this type of functionality could be a future extension
      #for now let's just make explicit assumption that test set has same 
      #properties as train set
      #(this approach greatly benefits latency)

      category = traincategory

      #printout display progress
      if printstatus is True:
        print("______")
        print("")
        print("processing column: ", column)
        print("    root category: ", category)
        print("")

      #here we'll delete any columns that returned a 'null' category
      if category == 'null':
        df_test = df_test.drop([column], axis=1)

      #so if we didn't delete the column let's proceed
      else:

        #assignnan application
        df_test = self.assignnan_convert(df_test, column, category, postprocess_dict['assignnan'], postprocess_dict)

        #we also have convention that infinity values are by default subjected to infill
        df_test = self.convert_inf_to_nan(df_test, column, category, postprocess_dict)

        #create NArows (column of True/False where True coresponds to missing data)
        if driftreport in ['efficient', True]:
          testNArows, postdrift_dict = \
          self.getNArows(df_test, column, category, postprocess_dict, postdrift_dict, True)

          if printstatus is True:
            print("original source column drift stats:")
            print(postprocess_dict['drift_dict'][column])
            print("")
            print("new source column drift stats:")
            print(postdrift_dict[column])
            print("")

        else:
          testNArows = self.getNArows(df_test, column, category, postprocess_dict)

        #now append that NArows onto a master NA rows df
        masterNArows_test = pd.concat([masterNArows_test, testNArows], axis=1)

        #process family
        df_test = \
        self.postprocessfamily(df_test, column, category, category, process_dict, \
                              transform_dict, postprocess_dict, columnkey, assign_param)

        #delete columns subject to replacement
        df_test = \
        self.postcircleoflife(df_test, column, category, category, process_dict, \
                              transform_dict, postprocess_dict, columnkey)

#         #now we'll apply the floatprecision transformation
#         columnkeylist = postprocess_dict['origcolumn'][column]['columnkeylist']
#         df_test = self.floatprecision_transform(df_test, columnkeylist, floatprecision)

        #printout display progress
        if printstatus is True:
          print(" returned columns:")
          print(postprocess_dict['origcolumn'][column]['columnkeylist'])
          print("")

    #process labels consistent to the train set if any are included in the postmunge test set

    #first let's get the name of the labels column from postprocess_dict
    labels_column = postprocess_dict['labels_column']

    #ok now let's check if that labels column is present in the test set
    
    if labelscolumn is not False:
      if labelscolumn is not True:
        if labelscolumn != labels_column:
          print("error, labelscolumn in test set passed to postmunge must have same column")
          print("labeling convention, labels column from automunge was: ", labels_column)

      #initialize processing dicitonaries (we'll use same as for train set)
      #a future extension may allow custom address for labels
      labelstransform_dict = transform_dict
      labelsprocess_dict = process_dict

      columnkey = postprocess_dict['origcolumn'][labels_column]['columnkey']        
      #traincategory = postprocess_dict['column_dict'][columnkey]['origcategory']
      labelscategory = postprocess_dict['origcolumn'][labels_column]['category']

      #apply assignnan_convert
      df_testlabels = self.assignnan_convert(df_testlabels, labels_column, labelscategory, postprocess_dict['assignnan'], postprocess_dict)
      
      #apply convert_inf_to_nan
      df_testlabels = self.convert_inf_to_nan(df_testlabels, labels_column, labelscategory, postprocess_dict)

      if printstatus is True:
        #printout display progress
        print("______")
        print("")
        print("processing label column: ", labels_column)
        print("    root label category: ", labelscategory)
        print("")

      #process family
      df_testlabels = \
      self.postprocessfamily(df_testlabels, labels_column, labelscategory, labelscategory, process_dict, \
                             transform_dict, postprocess_dict, columnkey, assign_param)

      #delete columns subject to replacement
      df_testlabels = \
      self.postcircleoflife(df_testlabels, labels_column, labelscategory, labelscategory, process_dict, \
                            transform_dict, postprocess_dict, columnkey)
      
      #now we'll apply the floatprecision transformation
#       columnkeylist = postprocess_dict['origcolumn'][labels_column]['columnkeylist']
#       df_testlabels = self.floatprecision_transform(df_testlabels, columnkeylist, floatprecision)

      #marker for printouts
      pmsmoothing = False
    
      #marker
      match_testLS_to_train = False
      
      #special case for postmunge label smoothing, 
      #if user passes True grab LabelSmoothing_train from postprocess_dict to apply consistently as automunge
      if str(LabelSmoothing) == 'True':
        LabelSmoothing = postprocess_dict['LabelSmoothing_train']
#         LSfit = postprocess_dict['LSfit']
        LSfitparams_dict = postprocess_dict['LSfitparams_dict']
        match_testLS_to_train = True
      
      #apply label smoothing to test set if elected
      if LabelSmoothing > 0.0 and LabelSmoothing < 1.0 and str(LabelSmoothing) != 'False':
        
        pmsmoothing = True
        
        testsmoothing = True
        
        #this will be our marker to indicate if labelsmoothing is already conducted
        categorycomplete_dict = dict(zip(postprocess_dict['origcolumn'][labels_column]['columnkeylist'], [False]*len(postprocess_dict['origcolumn'][labels_column]['columnkeylist'])))

        
        #for column in label columns
        for labelsmoothingcolumn in postprocess_dict['origcolumn'][labels_column]['columnkeylist']:
          
          if categorycomplete_dict[labelsmoothingcolumn] is False:
            
            if match_testLS_to_train is False:
            
              label_categorylist = postprocess_dict['column_dict'][labelsmoothingcolumn]['categorylist']
              label_category = postprocess_dict['column_dict'][labelsmoothingcolumn]['category']

              df_testlabels, categorycomplete_dict, _1 = \
              self.apply_LabelSmoothing(df_testlabels, labelsmoothingcolumn, LabelSmoothing, label_categorylist, label_category, categorycomplete_dict, LSfit, {})
              
              del _1
              
            else:
              
              df_testlabels, categorycomplete_dict = \
              self.postapply_LabelSmoothing(df_testlabels, labelsmoothingcolumn, categorycomplete_dict, LSfitparams_dict)

      #printout display progress
      if printstatus is True:
        print(" returned columns:")
        print(postprocess_dict['origcolumn'][labels_column]['columnkeylist'])
        print("")
        
        if pmsmoothing:
          print("Label Smoothing applied to labels")
          print("")

    labelsencoding_dict = postprocess_dict['labelsencoding_dict']

    #now that we've pre-processed all of the columns, let's run through them again\
    #using infill to derive plug values for the previously missing cells
    
    #printout display progress
    if printstatus is True:
      print("______")
      print("")
    
    infillcolumns_list = list(df_test)

    #access infill assignments derived in automunge(.) call
    postprocess_assigninfill_dict = \
    postprocess_dict['postprocess_assigninfill_dict']
    
    df_test = \
    self.apply_pm_infill(df_test, postprocess_assigninfill_dict, \
                        postprocess_dict, printstatus, infillcolumns_list, \
                        masterNArows_test, process_dict)

    #trim branches associated with feature selection
    if postprocess_dict['featureselection'] is True:

      #get list of columns currently included
      currentcolumns = list(df_test)

      #get list of columns to trim
      trimcolumns = [b for b in currentcolumns if b not in madethecut]

      if len(trimcolumns) > 0:
        #printout display progress
        if printstatus is True:
          print("_______________")
          print("Begin feature importance dimensionality reduction")
          print("")
          print("   method: ", postprocess_dict['featuremethod'])
          if postprocess_dict['featuremethod'] == 'pct':
            print("threshold: ", postprocess_dict['featurepct'])
          if postprocess_dict['featuremethod'] == 'metric':
            print("threshold: ", postprocess_dict['featuremetric'])
          print("")
          print("trimmed columns: ")
          print(trimcolumns)
          print("")

        #trim columns manually
        for trimmee in trimcolumns:
          del df_test[trimmee]
        
      if len(trimcolumns) > 0:
        if printstatus is True:
          print("returned columns: ")
          print(list(df_test))
          print("")

    #first this check allows for backward compatibility with published demonstrations
    if 'PCAn_components' in postprocess_dict:
      #grab parameters from postprocess_dict
      PCAn_components = postprocess_dict['PCAn_components']
      #prePCAcolumns = postprocess_dict['prePCAcolumns']

      if PCAn_components != None:

        PCAset_test, PCAexcl_posttransform = \
        self.postcreatePCAsets(df_test, postprocess_dict)

        #printout display progress
        if printstatus is True:
          print("_______________")
          print("Applying PCA dimensionality reduction")
          print("")
          if len(postprocess_dict['PCAexcl']) > 0:
            print("columns excluded from PCA: ")
            print(postprocess_dict['PCAexcl'])
            print("")

        PCAset_test, postprocess_dict = \
        self.postPCAfunction(PCAset_test, postprocess_dict)

        #reattach the excluded columns to PCA set
        df_test = pd.concat([PCAset_test.set_index(df_test.index), df_test[PCAexcl_posttransform]], axis=1)

        #printout display progress
        if printstatus is True:
          print("returned PCA columns: ")
          print(list(PCAset_test))
          print("")

        del PCAset_test

    #Binary dimensionality reduction goes here
    #we'll only apply to test data not labels
    #making an executive decvision for now that ordinal encoded columns will be excluded
    if postprocess_dict['Binary'] in [True, 'retain']:
      
      #printout display progress
      if printstatus is True:
        print("_______________")
        print("Begin Binary dimensionality reduction")
        print("")
        print("Before transform test set column count = ")
        print(df_test.shape[1])
        print("")
      
      Binary_dict = postprocess_dict['Binary_dict']
      Binary = postprocess_dict['Binary']
      
      if printstatus is True:
        print("Consolidating boolean columns:")
        print(Binary_dict['bool_column_list'])
        print()
          
      df_test = self.postBinary_convert(df_test, Binary_dict, Binary)
      
      #printout display progress
      if printstatus is True:
        print("Boolean column count = ")
        print(len(Binary_dict['bool_column_list']))
        print("")
        print("After transform test set column count = ")
        print(df_test.shape[1])
        print("")

    #here is the process to levelize the frequency of label rows in train data
    #currently only label categories of 'bnry' or 'text' are considered
    #a future extension will include numerical labels by adding supplemental 
    #label columns to designate inclusion in some fractional bucket of the distribution
    #e.g. such as quintiles for instance
    if TrainLabelFreqLevel is True \
    and labelscolumn is not False:

      #printout display progress
      if printstatus is True:
        print("_______________")
        print("Begin label rebalancing")
        print("")
        print("Before rebalancing row count = ")
        print(df_testlabels.shape[0])
        print("")

  #       train_df = pd.DataFrame(np_train, columns = finalcolumns_train)
  #       labels_df = pd.DataFrame(np_labels, columns = finalcolumns_labels)
      if testID_column is not False:
  #         trainID_df = pd.DataFrame(np_trainID, columns = [trainID_column])
        #add trainID set to train set for consistent processing
  #         train_df = pd.concat([train_df, trainID_df], axis=1)                        
        df_test = pd.concat([df_test, df_testID], axis=1)                        

      #apply LabelFrequencyLevelizer defined function
      df_test, df_testlabels = \
      self.LabelFrequencyLevelizer(df_test, df_testlabels, \
                                   postprocess_dict, process_dict, LabelSmoothing)

      #extract trainID
      if testID_column is not False:

        df_testID = pd.DataFrame(df_test[testID_column])

        if isinstance(testID_column, str):
          tempIDlist = [testID_column]
        elif isinstance(testID_column, list):
          tempIDlist = testID_column
        for IDcolumn in tempIDlist:
          del df_test[IDcolumn]
        #del df_train[trainID_column]

      #printout display progress
      if printstatus is True:
        print("After rebalancing row count = ")
        print(df_testlabels.shape[0])
        print("")

    #if shuffletrain passed to postmunge it takes place here
    #(postmunge does not default to consistent shuffle as train set, relies on parameter)
    if shuffletrain is True:
      #shuffle training set and labels
      df_test = self.df_shuffle(df_test, postprocess_dict['randomseed'])
      df_testlabels = self.df_shuffle(df_testlabels, postprocess_dict['randomseed'])

      if testID_column is not False:
        df_testID = self.df_shuffle(df_testID, postprocess_dict['randomseed'])

    #a special case, those columns that we completely excluded from processing via excl
    #we'll scrub the suffix appender
    #(we won't perform this step to test data if PCA was applied)
    if postprocess_dict['excl_suffix'] is False:
      df_test.columns = [column[:-5] if column in postprocess_dict['column_dict'] and \
                         postprocess_dict['column_dict'][column]['category'] == 'excl' \
                         else column for column in df_test.columns]
      
    if labelscolumn is not False and postprocess_dict['excl_suffix'] is False:
      df_testlabels.columns = [column[:-5] if column in postprocess_dict['column_dict'] and \
                              postprocess_dict['column_dict'][column]['category'] == 'excl' \
                              else column for column in df_testlabels.columns]

    if postprocess_dict['privacy_encode'] is True:

      df_test = df_test.rename(columns = postprocess_dict['privacy_headers_train_dict'])

      if labelscolumn is not False:

        df_testlabels = df_testlabels.rename(columns = postprocess_dict['privacy_headers_labels_dict'])

      df_testID = df_testID.rename(columns = postprocess_dict['privacy_headers_testID_dict'])

    #here's a list of final column names saving here since the translation to \
    #numpy arrays scrubs the column names
    finalcolumns_test = list(df_test)

    postreports_dict['finalcolumns_test'] = finalcolumns_test
    
    postreports_dict.update({'sourcecolumn_drift' : {'orig_driftstats' : postprocess_dict['drift_dict'], \
                                                     'new_driftstats' : postdrift_dict}})

    #printout display progress
    if printstatus is True:

      print("_______________")
      if df_testID.empty is False:
        print("Postmunge returned ID column set: ")
        print(list(df_testID))
        print("")

      print("Postmunge returned test column set: ")
      print(list(df_test))
      print("")

      if labelscolumn is not False:
        print("Postmunge returned label column set: ")
        print(list(df_testlabels))
        print("")

    #now we'll apply the floatprecision transformation
    floatcolumns_test = finalcolumns_test
    floatcolumns_testlabels = list(df_testlabels)
    if postprocess_dict['privacy_encode'] is True:
      floatcolumns_test = postprocess_dict['privacy_headers_train']
      if labelscolumn is not False:
        floatcolumns_testlabels = postprocess_dict['privacy_headers_labels']

    #now we'll apply the floatprecision transformation
    df_test = self.floatprecision_transform(df_test, floatcolumns_test, floatprecision)
    if labelscolumn is not False:
      finalcolumns_labels = list(df_testlabels)
      df_testlabels = self.floatprecision_transform(df_testlabels, floatcolumns_testlabels, floatprecision)

    if testID_column is not False:
      #testID = df_testID
      #pass
      if returnedsets in ['test_ID', 'test_ID_labels']:
        df_test = pd.concat([df_test, df_testID], axis=1)
      
    else:
      df_testID = pd.DataFrame()

    if labelscolumn is not False:
      #testlabels = df_testlabels
      #pass
      if returnedsets in ['test_labels', 'test_ID_labels']:
        df_test = pd.concat([df_test, df_testlabels], axis=1)
      
    else:
      df_testlabels = pd.DataFrame()

    #else output numpy arrays
    #else:
    if pandasoutput is False:
      #global processing to test set including conversion to numpy array
      df_test = df_test.values

      if testID_column is not False \
      and returnedsets not in [False, 'test_ID', 'test_labels', 'test_ID_labels']:
        df_testID = df_testID.values
      else:
        df_testID = []

      if labelscolumn is not False \
      and returnedsets not in [False, 'test_ID', 'test_labels', 'test_ID_labels']:
        df_testlabels = df_testlabels.values

        #apply ravel to labels if appropriate - converts from eg [[1,2,3]] to [1,2,3]
        if df_testlabels.ndim == 2 and df_testlabels.shape[1] == 1:
          df_testlabels = np.ravel(df_testlabels)

      else:
        df_testlabels = []

    #printout display progress
    if printstatus is True:

      print("_______________")
      print("Postmunge Complete")
      print("")
    
    if returnedsets is True:
    
      return df_test, df_testID, df_testlabels, labelsencoding_dict, postreports_dict
    
    else:
      
      return df_test
    
  def populate_categorytree(self, postprocess_dict):
    """
    #Populates mirror tree of transformations
    #to facilitate translation between transformation category space 
    #and suffix appender space
    
    #As an example of a populated tree for bxcx root cateogry transform 
    #which returns columns ['column_bxcx_nmbr', 'column_NArw']
    
    categorytree = \
    {'__root__' : \
    ['__root__', '', ['__root__'], {'NArw': ['sup', inputcol, categorylist, {}], \
                                    'bxcx': ['rep', inputcol, categorylist, \
                                            {'nmbr': ['rep', inputcol, categorylist, {}]}]}]}
    
    #here 'sup'/'rep' refers to distinction between supplement and replace primitives
    #and 'root' is for the source column entry
    
    #inputcol and categorylist access from column_dict from key of one of entries in categorylist
    #so we'll search column_dict for key with entries that match category 
    #and inputcolumn that matches categorylist entry of preceding entry
    #(convention is we only have downstream transforms applied to single entry categorylists)
    
    #Once we have this populated, we'll translate it to an inverse
    #with returned columns in the root and root in each final
    #such as to facilitate any translation from returned sets to source sets
    #(such as for converting predictions back to original label formatting)
    
    #I think this will work let's give it a try
    """
    
    #these are the columns passed to automunge(.) including labels
    source_columns = list(postprocess_dict['origcolumn'])
    
    #these are derived columns including replaced columns and labels
    produced_columns = list(postprocess_dict['column_dict'])
    
    #initialize categorytree
    categorytree = {'__root__' : {}}
    
    for origcolumn in source_columns:
      
      categorytree['__root__'].update({origcolumn : ['__root__', '', ['__root__'], {}]})
      
      root_category = postprocess_dict['origcolumn'][origcolumn]['category']
          
      parents     = postprocess_dict['transform_dict'][root_category]['parents']
      siblings    = postprocess_dict['transform_dict'][root_category]['siblings']
      auntsuncles = postprocess_dict['transform_dict'][root_category]['auntsuncles']
      cousins     = postprocess_dict['transform_dict'][root_category]['cousins']
      
      categorytree_entry = self.populate_family(postprocess_dict, categorytree['__root__'][origcolumn][3], origcolumn, '__root__', \
                                           parents, siblings, auntsuncles, cousins)
      
      categorytree['__root__'][origcolumn][3].update(categorytree_entry)
      
    return categorytree

  def populate_family(self, postprocess_dict, categorytree, inputcolumn, inputcategory, \
                      parents, siblings, auntsuncles, cousins):
    """
    #populates categorytree entries from seeding of a source-column and root category
    
    #we will run in order of
    #parents, auntsuncles, siblings, cousins
    
    #see also notes for populate_categorytree function
    """
    
    for entry in parents:
      
      if entry != None:
      
        categorylist = self.get_categorylist(postprocess_dict, inputcolumn, entry)
        
        if entry not in categorytree:
          categorytree.update({entry : {}})

        #parents have downstream offspring
        children      = postprocess_dict['transform_dict'][entry]['children']
        niecesnephews = postprocess_dict['transform_dict'][entry]['niecesnephews']
        coworkers     = postprocess_dict['transform_dict'][entry]['coworkers']
        friends       = postprocess_dict['transform_dict'][entry]['friends']

        #convention is downstream offspring only allowed on sets returned with single column categorylist
        if len(categorylist) == 1:

          parentcolumn = categorylist[0]
          
          #parents is replace primitive
          categorytree[entry].update({parentcolumn : ['rep', inputcolumn, categorylist, {}]})

          categorytree_entry = self.populate_family(postprocess_dict, categorytree[entry][parentcolumn][3], parentcolumn, entry, \
                                               children, niecesnephews, coworkers, friends)
          
          categorytree[entry][parentcolumn][3].update(categorytree_entry)
          
        else:
          
          for category_column in categorylist:
            
            categorytree[entry].update({category_column : ['rep', inputcolumn, categorylist, {}]})
          
    for entry in auntsuncles:
      
      if entry != None:
        
        if entry not in categorytree:
          categorytree.update({entry : {}})
      
        categorylist = self.get_categorylist(postprocess_dict, inputcolumn, entry)
#         categorylist = get_categorylist(postprocess_dict, inputcolumn, entry)
        
        for category_column in categorylist:

          #auntsuncles is replace primitive
          categorytree[entry].update({category_column : ['rep', inputcolumn, categorylist, {}]})

          #auntsuncles have no downstream offspring
      
    for entry in siblings:
      
      if entry != None:
      
        categorylist = self.get_categorylist(postprocess_dict, inputcolumn, entry)

        if entry not in categorytree:
          categorytree.update({entry : {}})

        #siblings have downstream offspring
        children      = postprocess_dict['transform_dict'][entry]['children']
        niecesnephews = postprocess_dict['transform_dict'][entry]['niecesnephews']
        coworkers     = postprocess_dict['transform_dict'][entry]['coworkers']
        friends       = postprocess_dict['transform_dict'][entry]['friends']

        #convention is downstream offspring only allowed on sets returned with single column categorylist
        if len(categorylist) == 1:

          parentcolumn = categorylist[0]
          
          #siblings is supplement primitive
          categorytree[entry].update({parentcolumn : ['sup', inputcolumn, categorylist, {}]})


          categorytree_entry = self.populate_family(postprocess_dict, categorytree[entry][parentcolumn][3], parentcolumn, entry, \
                                               children, niecesnephews, coworkers, friends)
          
          categorytree[entry][parentcolumn][3].update(categorytree_entry)
          
        else:
          for category_column in categorylist:
            
            categorytree[entry].update({category_column : ['sup', inputcolumn, categorylist, {}]})
        
    for entry in cousins:
      
      if entry != None:
      
        categorylist = self.get_categorylist(postprocess_dict, inputcolumn, entry)

        if entry not in categorytree:
          categorytree.update({entry : {}})

        #cousins have no downstream offspring
        
        for category_column in categorylist:

          #cousins is supplement primitive
          categorytree[entry].update({category_column : ['sup', inputcolumn, categorylist, {}]})

          #auntsuncles have no downstream offspring
      
    return categorytree
    
  def get_categorylist(self, postprocess_dict, inputcolumn, category):
    """
    #access a returned categorylist
    #corresponding to the category of transformation applied to an inputcolumn
    #by searching entries to postprocess_dict['column_dict']
    """
    
    categorylist = []
    
    for entry in postprocess_dict['column_dict']:
      
      if postprocess_dict['column_dict'][entry]['inputcolumn'] == inputcolumn \
      and postprocess_dict['column_dict'][entry]['category'] == category:
        
        categorylist = postprocess_dict['column_dict'][entry]['categorylist']
          
        break
        
    #this is for edge case when a transform does not return columns
    if categorylist == None:
      categorylist = []
        
    return categorylist
      
  def populate_inverse_categorytree(self, postprocess_dict):
    """
    #So this is similar to the categorytree in that we are mirror the transformations
    #in a populated data structure
    #but in this inverse version the bottom tier are the conclusion branches
    #which progress back to the root
    #note that we'll allow redundant entries in first tiers
    #such as to aggregate each distinct path by common starting point in list
    
    #here we'll want additional data points for:
    #- depth of branch
    #- information retention of transform
    #- availability of inverse transform
    
    #As an example of excerpt from a populated tree for bxcx root category transform 
    #which returns columns ['column_bxcx_nmbr', 'column_NArw']
    
    inverse_categorytree['nmbr'] = \
    {'Age_bxcx_nmbr': ['sup', 'Age_bxcx', ['Age_bxcx_nmbr'], ['Age_bxcx_nmbr', 'Age_NArw'],
                      2, False, False, \
                      {'bxcx': {'Age_bxcx': ['rep', 'Age', ['Age_bxcx'], ['Age_bxcx_nmbr', 'Age_NArw'],
                                             1, False, False, \
                                             {'__root__': {'Age': ['__root__','__root__','__root__',\
                                                                   '__root__', 1 , False , False, \
                                                                    {}]}}]}}]}
    """
    
    #all returned columns including labels
    returned_columns = \
    postprocess_dict['pre_dimred_finalcolumns_train'] + postprocess_dict['finalcolumns_labels']
    
    #these are all derived columns including replaced columns and labels
    produced_columns = list(postprocess_dict['column_dict'])
    
    #these are the columns passed to automunge(.) including labels
    source_columns = list(postprocess_dict['origcolumn'])
    
    #initialize inverse_categorytree
    inverse_categorytree = {}
    
    for returned_column in returned_columns:
      
      category     = postprocess_dict['column_dict'][returned_column]['category']
      inputcolumn  = postprocess_dict['column_dict'][returned_column]['inputcolumn']
      origcolumn   = postprocess_dict['column_dict'][returned_column]['origcolumn']
      columnslist  = postprocess_dict['column_dict'][returned_column]['columnslist']
      categorylist = postprocess_dict['column_dict'][returned_column]['categorylist']
      
      if category not in inverse_categorytree:
      
        inverse_categorytree.update(
        {category : {}}
        )
      
      depth = 1
      
      info_retention = False
      if 'info_retention' in postprocess_dict['process_dict'][category]:
        if postprocess_dict['process_dict'][category]['info_retention'] is True:
          info_retention = True
      
      transforms_avail = False
      if 'inverseprocess' in postprocess_dict['process_dict'][category]:
        if callable(postprocess_dict['process_dict'][category]['inverseprocess']):
          transforms_avail = True
      
      for entry in categorylist:
        
        if entry in returned_columns:
          sup_or_rep = 'sup'
        else:
          sup_or_rep = 'rep'

        #note the index number of entries in this list are used to access
        #so any added points here should be tacked on end (after {})
        #and mirrored in other function
        inverse_categorytree[category].update(
        {entry : [sup_or_rep, 
                  inputcolumn, 
                  categorylist, 
                  columnslist, 
                  depth, 
                  info_retention,
                  transforms_avail,
                  {}]}
        )
        
        if inputcolumn not in source_columns:

          inverse_categorytree_entry, depth_, info_retention_, transforms_avail_ = \
          self.populate_inverse_family(
            postprocess_dict, inverse_categorytree[category][entry][7], inputcolumn,
            returned_columns, source_columns
          )
          
          inverse_categorytree[category][entry][4] += depth_
          
          inverse_categorytree[category][entry][5] = \
          inverse_categorytree[category][entry][5] and info_retention_
          
          inverse_categorytree[category][entry][6] = \
          inverse_categorytree[category][entry][6] and transforms_avail_
          
          inverse_categorytree[category][entry][7].update(inverse_categorytree_entry)
        
        else:
          
          inverse_categorytree[category][entry][7].update(
          {'__root__' : {inputcolumn : ['__root__', 
                                        '__root__', 
                                        '__root__', 
                                        '__root__', 
                                        depth, 
                                        info_retention,
                                        transforms_avail,
                                        {}]}}
          )
          
    return inverse_categorytree
  
  def populate_inverse_family(self, postprocess_dict, inverse_categorytree, column, \
                              returned_columns, source_columns):
    """
    #populates inverse_categorytree entries from seeding of an inputcolumn
    
    #see also notes for populate_inverse_categorytree function
    """
      
    category     = postprocess_dict['column_dict'][column]['category']
    inputcolumn  = postprocess_dict['column_dict'][column]['inputcolumn']
    origcolumn   = postprocess_dict['column_dict'][column]['origcolumn']
    columnslist  = postprocess_dict['column_dict'][column]['columnslist']
    categorylist = postprocess_dict['column_dict'][column]['categorylist']
    
    if category not in inverse_categorytree:
    
      inverse_categorytree.update(
      {category : {}}
      )
    
    depth = 1
    
    info_retention = False
    if 'info_retention' in postprocess_dict['process_dict'][category]:
      if postprocess_dict['process_dict'][category]['info_retention'] is True:
        info_retention = True

    transforms_avail = False
    if 'inverseprocess' in postprocess_dict['process_dict'][category]:
      if callable(postprocess_dict['process_dict'][category]['inverseprocess']):
        transforms_avail = True
        
    for entry in categorylist:

      if entry in returned_columns:
        sup_or_rep = 'sup'
      else:
        sup_or_rep = 'rep'
      
      #note the index number of entries in this list are used to access
      #so any added points here should be tacked on end (after {})
      #and mirrored in other function
      inverse_categorytree[category].update(
      {entry : [sup_or_rep, 
                inputcolumn, 
                categorylist, 
                columnslist, 
                depth, 
                info_retention,
                transforms_avail,
                {}]}
      )
      
      if inputcolumn not in source_columns:
        
        inverse_categorytree_entry, depth_, info_retention_, transforms_avail_ = \
        self.populate_inverse_family(
          postprocess_dict, inverse_categorytree[category][entry][7], inputcolumn,
          returned_columns, source_columns
        )
  
        inverse_categorytree[category][entry][4] += depth_

        inverse_categorytree[category][entry][5] = \
        inverse_categorytree[category][entry][5] and info_retention_

        inverse_categorytree[category][entry][6] = \
        inverse_categorytree[category][entry][6] and transforms_avail_

        inverse_categorytree[category][entry][7].update(inverse_categorytree_entry)

      else:

        inverse_categorytree[category][entry][7].update(
        {'__root__' : {inputcolumn : ['__root__', 
                                      '__root__', 
                                      '__root__', 
                                      '__root__', 
                                      depth, 
                                      info_retention,
                                      transforms_avail,
                                      {}]}}
        )
        
        
    depth            = inverse_categorytree[category][entry][4]
    info_retention   = inverse_categorytree[category][entry][5]
    transforms_avail = inverse_categorytree[category][entry][6]
    
    return inverse_categorytree, depth, info_retention, transforms_avail
  
  def populate_inputcolumn_dict(self, postprocess_dict):
    """
    #we'll create another structure, this one flatted, similar to origcolumn or column_dict
    #this one as inputcolumn_dict
    
    #this will populate a strucutre with example entry
    
    #inputcolumn_dict = {inputcolumn : {category : {column : column_dict[column]}}}
    
    #where inputcolumn is a column serving as input to a specific generation (set) of trasnformtions
    #such as either entries to parents/siblings/auntsuncles/cousins
    #or for downstream generations entries to children/niecesnephews/coworkers/friends
    """
    
    inputcolumn_dict = {}
    
    for column in postprocess_dict['column_dict']:
      
      inputcolumn = postprocess_dict['column_dict'][column]['inputcolumn']
      
      if inputcolumn not in inputcolumn_dict:
        
        inputcolumn_dict.update({inputcolumn : {}})
        
      category = postprocess_dict['column_dict'][column]['category']
      
      if category not in inputcolumn_dict[inputcolumn]:
        
        inputcolumn_dict[inputcolumn].update({category:{}})
        
      inputcolumn_dict[inputcolumn][category].update({column : postprocess_dict['column_dict'][column]})
      
    return inputcolumn_dict
  
  def LS_invert(self, LabelSmoothing, df, categorylist, postprocess_dict):
    """
    #Converts smoothed labels back to one-hot encoding
    #for a particular categorylist
    """
    
    if LabelSmoothing is True:
      
      LabelSmoothing = postprocess_dict['LabelSmoothing_train']
    
    if LabelSmoothing > 0 and LabelSmoothing < 1:
      
      for categorylist_entry in categorylist:
        
        df[categorylist_entry] = np.where(df[categorylist_entry] == LabelSmoothing, 1, 0)
        
        df[categorylist_entry] = df[categorylist_entry].astype(np.int8)
        
    return df
  
  def meta_LS_invert(self, LabelSmoothing, df, postprocess_dict):
    """
    #Performs labels smoothing inversion
    #Note that label smoothing may have been applied to multiple distinct sets
    #originating from same source column
    """
    
    if LabelSmoothing is True or (LabelSmoothing > 0 and LabelSmoothing < 1):

      labels_column = postprocess_dict['labels_column']

      labels_root_category = postprocess_dict['origcolumn'][labels_column]['category']

      label_returnedcolumns = postprocess_dict['origcolumn'][labels_column]['columnkeylist']

      inverse_categorytree = postprocess_dict['inverse_categorytree']

      #we'll strike columns from this list if they are in another column's categorylist
      returnedcolumns_copy = label_returnedcolumns.copy()

      for returnedcolumn in label_returnedcolumns:

        #only continue if not previiously stricken from antoher column's cateogrylist
        if returnedcolumn in returnedcolumns_copy:

          returned_categorylist = postprocess_dict['column_dict'][returnedcolumn]['categorylist']

          for returned_categorylist_entry in returned_categorylist:

            returnedcolumns_copy.remove(returned_categorylist_entry)

          returned_category = postprocess_dict['column_dict'][returnedcolumn]['category']

          #if column is from a multicolumn boolean MLinfilltype then we'll apply LSinvert
          if postprocess_dict['process_dict'][returned_category]['MLinfilltype'] in \
          ['multirt']:

            df = self.LS_invert(LabelSmoothing, df, returned_categorylist, postprocess_dict)
            
    return df

  def inversion_numpy_support(self, df, postprocess_dict, inversion):
    """
    #Checks if a data set passed to inversion is numpy
    #and if so checks the column count
    #and if it matches column count of test or testlabels set
    #then it converts to dataframe and applies the correct headers
    #such as to enable inversion_parent function
    #relies on the inversion parameter for whether to 
    #apply label column headers or test set column headers
    #note that currently numpy is only supported for full set inverison
    #eg by passing a full test set or a full label set
    #if user passes a list of column headers to inversion
    #this method assumes that list is for a test inversion
    """
      
    if inversion == 'labels':

      if len(df.columns) == len(postprocess_dict['finalcolumns_labels']):

        if set(df.columns) != set(postprocess_dict['finalcolumns_labels']):

          df.columns = postprocess_dict['finalcolumns_labels']

    else:

      if len(df.columns) == len(postprocess_dict['finalcolumns_train']):

        if set(df.columns) != set(postprocess_dict['finalcolumns_train']):

          df.columns = postprocess_dict['finalcolumns_train']
      
    return df

  def inversion_privacy_support(self, df, postprocess_dict, inversion):
    """
    #Checks if privacy_encoding option is active
    #and if so converts df from privacy version of encodings to suffix version
    #note that the rename function can be run even when dictionary doesn't have matches
    #so it is ok to run this on a set that had already been converted from numpy 
    #such as in inversion_numpy_support function
    #note that privacy headers are integers so won't overlap with strings
    #relies on the inversion parameter for whether to 
    #apply label column headers or test set column headers
    """
    
    if postprocess_dict['privacy_encode'] is True:
      
      if inversion == 'labels':
        
        df = df.rename(columns = postprocess_dict['inverse_privacy_headers_labels_dict'])

      else:

        df = df.rename(columns = postprocess_dict['inverse_privacy_headers_train_dict'])
      
    return df

  def inverseprocess_nmbr(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_numerical_class
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    """
    
    normkey = categorylist[0]
    
    mean = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['mean']
    std = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['std']
    offset = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['offset']
    multiplier = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['multiplier']
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    df[inputcolumn] = ((df[normkey] - offset) * std / multiplier ) + mean
    
    return df, inputcolumn

  def inverseprocess_year(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_year_class
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    """
    
    normkey = categorylist[0]
    
    mean = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['meanyear']
    std = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['stdyear']
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    df[inputcolumn] = ((df[normkey]) * std + mean )
    
    return df, inputcolumn
  
  def inverseprocess_mean(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_mean_class
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    """
    
    normkey = categorylist[0]
    
    mean = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['mean']
    maxminusmin = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['maxminusmin']
    offset = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['offset']
    multiplier = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['multiplier']
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    df[inputcolumn] = ((df[normkey] - offset) * maxminusmin / multiplier ) + mean
    
    return df, inputcolumn
  
  def inverseprocess_MADn(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_MADn_class
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    """
    
    normkey = categorylist[0]
    
    mean = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['mean']
    MAD = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['MAD']
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    df[inputcolumn] = df[normkey] * MAD + mean
    
    return df, inputcolumn
  
  def inverseprocess_MAD3(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_MAD3_class
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    """
    
    normkey = categorylist[0]
    
    mean = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['mean']
    MAD = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['MAD']
    datamax = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['datamax']
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    df[inputcolumn] = df[normkey] * MAD + datamax
    
    return df, inputcolumn
  
  def inverseprocess_mnmx(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_mnmx_class
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    """
    
    normkey = categorylist[0]
    
    minimum = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['minimum']
    maximum = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['maximum']
    maxminusmin = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['maxminusmin']
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    df[inputcolumn] = df[normkey] * maxminusmin + minimum
    
    return df, inputcolumn

  def inverseprocess_mnm3(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_mnmx_class
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    """
    
    normkey = categorylist[0]
    
    minimum = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['quantilemin']
    maximum = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['quantilemax']
    maxminusmin = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['maxminusmin']
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    df[inputcolumn] = df[normkey] * maxminusmin + minimum
    
    return df, inputcolumn
  
  def inverseprocess_retn(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_retn_class
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    """
    
    normkey = categorylist[0]
    
    minimum = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['minimum']
    maximum = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['maximum']
    divisor = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['divisor']
    multiplier = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['multiplier']
    offset = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['offset']

    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    if maximum >= 0 and minimum <= 0:
      
      df[inputcolumn] = (df[normkey] - offset) * divisor / multiplier
      
    elif maximum >= 0 and minimum >= 0:
      
      df[inputcolumn] = (df[normkey] - offset) * divisor / multiplier + minimum
      
    elif maximum <= 0 and minimum <= 0:
      
      df[inputcolumn] = (df[normkey] - offset) * divisor / multiplier + maximum
    
    return df, inputcolumn

  def inverseprocess_shft(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_shft_class
    #assumes any relevant parameters were saved in normalization_dict
    #applies zzzinfill infill
    """
    
    normkey = categorylist[0]
    
    periods = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['periods']
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    df[inputcolumn] = df[normkey].shift(periods = -periods)
    
    df[inputcolumn] = df[inputcolumn].fillna('zzzinfill')
        
    return df, inputcolumn
  
  def inverseprocess_log0(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_log0_class
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    """
    
    normkey = categorylist[0]
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    df[inputcolumn] = 10 ** df[normkey]
    
    return df, inputcolumn
  
  def inverseprocess_logn(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_logn_class
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    """
    
    normkey = categorylist[0]
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    df[inputcolumn] = np.e ** df[normkey]
    
    return df, inputcolumn
  
  def inverseprocess_addd(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_addd_class
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    """
    
    normkey = categorylist[0]
    
    add = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['add']
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    df[inputcolumn] = df[normkey] - add
    
    return df, inputcolumn
  
  def inverseprocess_sbtr(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_sbtr_class
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    """
    
    normkey = categorylist[0]
    
    subtract = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['subtract']
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    df[inputcolumn] = df[normkey] + subtract
    
    return df, inputcolumn
  
  def inverseprocess_mltp(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_mltp_class
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    """
    
    normkey = categorylist[0]
    
    multiply = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['multiply']
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    df[inputcolumn] = df[normkey] / multiply
    
    return df, inputcolumn
  
  def inverseprocess_divd(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_divd_class
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    """
    
    normkey = categorylist[0]
    
    divide = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['divide']
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    df[inputcolumn] = df[normkey] * divide
    
    return df, inputcolumn
  
  def inverseprocess_rais(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_rais_class
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    """
    
    normkey = categorylist[0]
    
    raiser = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['raiser']
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    df[inputcolumn] = df[normkey] ** (1 / raiser)
    
    return df, inputcolumn
  
  def inverseprocess_absl(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_absl_class
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    
    #incomplete info retention, no transformation performed
    #this funciton only populated to support partial info recovery
    #in case a full info_retention path is not available
    """
    
    normkey = categorylist[0]
    
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    df[inputcolumn] = df[normkey]
    
    return df, inputcolumn
  
  def inverseprocess_sqrt(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_sqrt_class
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    """
    
    normkey = categorylist[0]
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    df[inputcolumn] = df[normkey] ** 2
    
    return df, inputcolumn
  
  def inverseprocess_UPCS(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_UPCS_class
    #is simply a pass-through function, original character cases not retained
    #does not perform infill, assumes clean data
    """
    
    normkey = categorylist[0]
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    df[inputcolumn] = df[normkey]
    
    return df, inputcolumn
  
  def inverseprocess_excl(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_excl_class
    #is simply a pass-through function, original character cases not retained
    #does not perform infill, assumes clean data
    """
    
    normkey = categorylist[0]
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    #this is a hack for special treatment associated wiuth excl suffix edge case
    if normkey not in df.columns:
      
      if normkey[-5:] == '_excl':
        
        if normkey[:-5] in df.columns:
          
          df.rename(columns={normkey[:-5]:normkey}, inplace=True)
    
    df[inputcolumn] = df[normkey]
    
    return df, inputcolumn
  
  def inverseprocess_pwr2(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_pwr2_class
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    
    #note that pwr2 differs from pwrs in that negative numbers are allowed
    
    #this only achieves partial information recovery as power level returned to single column
    
    #in interest of expediency, building this method to extract power
    #level from column suffix appender
    #a potential improvement would be to add an additional entry to pwrs normalization_dict
    #matching column header to power, eg {'column_10^-1' : -1}
    #saving that for a future update
    #(same functionality, but would better match convention of library for use of column headers)
    """
    
    normkey = categorylist[0]
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    df[inputcolumn] = 0
    
    #for pwr2 trasnform suffix is either '_10^#' for positive values or '_-10^#' for negative
    for column in categorylist:
      
      if column[len(inputcolumn)+1] == '1':
      
        power = int(column.replace(inputcolumn + '_10^', ''))
      
        df[inputcolumn] = np.where(df[column] == 1, 10 ** power, df[inputcolumn])
        
      if column[len(inputcolumn)+1] == '-':
        
        power = int(column.replace(inputcolumn + '_-10^', ''))
        
        df[inputcolumn] = np.where(df[column] == 1, -(10 ** power), df[inputcolumn])
        
    return df, inputcolumn
  
  def inverseprocess_por2(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_por2_class
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    """
    
    normkey = categorylist[0]
    
    #this has keys of pwr2 equivalent column headers and values of ordinal values found in column
    train_replace_dict = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['train_replace_dict']
    
    inverse_train_replace_dict = {value:key for key,value in train_replace_dict.items()}
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    df[inputcolumn] = 0

    #we'll convert the input to integers
    df[normkey] = df[normkey].astype(int, errors='ignore')
        
    #for pwr2 trasnform suffix is either '_10^#' for positive values or '_-10^#' for negative
    for column in train_replace_dict:
      
      #this dictionary is including a nan key entry for infill points, we'll leave these points as 0
      if column == column:

        if column[len(inputcolumn)+1] == '1':

          power = int(column.replace(inputcolumn + '_10^', ''))

          df[inputcolumn] = np.where(df[normkey] == train_replace_dict[column], 10 ** power, df[inputcolumn])

        if column[len(inputcolumn)+1] == '-':

          power = int(column.replace(inputcolumn + '_-10^', ''))

          df[inputcolumn] = np.where(df[normkey] == train_replace_dict[column], -(10 ** power), df[inputcolumn])
        
    return df, inputcolumn

  def inverseprocess_bins(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_bins_class
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    
    #incomplete information recovery
    """
    
    normkey = categorylist[0]
    
    binsmean = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['binsmean']
    binsstd = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['binsstd']
    bincuts = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bincuts']
    binlabels = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['binlabels']
    textcolumns = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['binscolumns']
    bincount = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bincount']
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    #we'll follow convention that values placed in mid point of buckets
    #eg for bucket with boundaries mean through mean+stsndard devation
    #we'll return value mean + 1/2 standard deviation
    #and for tails we'll arbitraricly follow consistent deltas between buckets
    #(so this loses a lot of tail distribution information in the inversion)
    returned_values_dict = {}
    
    i = - (bincount - 2) / 2 - 0.5
    
    for bucket in binlabels:
      
      #relies on suffix appender conventions
      column = inputcolumn + '_bins_' + bucket
      value = (i * binsstd + binsmean)
      
      returned_values_dict.update({column : value})
      
      i += 1
      
    df[inputcolumn] = 'zzzinfill'
    
    for column in categorylist:
        
      df[inputcolumn] = np.where(df[column] == 1, returned_values_dict[column], df[inputcolumn])
      
    return df, inputcolumn

  def inverseprocess_bsor(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_bsor_class
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    
    #incomplete information recovery
    """
    
    normkey = categorylist[0]
    
    #this has keys of pwr2 equivalent column headers and values of ordinal values found in column
    binsmean = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['binsmean']
    binsstd = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['binsstd']
    ordinal_dict = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['ordinal_dict']
    bincount = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bincount']
    binlabels = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['binlabels']
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    #we'll follow convention that values placed in mid point of buckets
    #eg for bucket with boundaries mean through mean+stsndard devation
    #we'll return value mean + 1/2 standard deviation
    #and for tails we'll arbitraricly follow consistent deltas between buckets
    #(so this loses a lot of tail distribution information in the inversion)
    returned_values_dict = {}

    #we'll convert the input to integers
    df[normkey] = df[normkey].astype(int, errors='ignore')
    
    i = - (bincount - 2) / 2 - 0.5
    
    df[inputcolumn] = 'zzzinfill'
    
    for bucket in binlabels:
      
      df[inputcolumn] = np.where(df[normkey] == bucket, (i * binsstd + binsmean), df[inputcolumn])

      i += 1

    return df, inputcolumn
  
  def inverseprocess_bnwd(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_bnwd_class
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    
    #this only achieves partial information recovery as bins sets aggregated to single value per bin
    """
    
    normkey = categorylist[0]
    
    bins_cuts = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_cuts']
    bn_min = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_min']
    bn_delta = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_delta']
    bins_id = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_id']
    bn_delta = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_delta']
    bn_width_bnwd = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_width_bnwd']
    suffix = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
    

    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    df[inputcolumn] = 0
    
    for i in range(len(bins_id)):
      
      _id =  bins_id[i]
      
      column = inputcolumn + suffix + '_' + _id
      
      if column in df.columns:
      
        df[inputcolumn] = np.where(df[column] == 1, i * bn_width_bnwd + bn_min, df[inputcolumn])
    
    return df, inputcolumn
  
  def inverseprocess_bnwo(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_bnwo_class
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    
    #this only achieves partial information recovery as bins sets aggregated to single value per bin
    """
    
    normkey = categorylist[0]
    
    bins_cuts = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_cuts']
    bn_min = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_min']
    bn_delta = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_delta']
    bins_id = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_id']
    bn_width = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_width']
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    df[inputcolumn] = df[normkey].copy()

    df[inputcolumn] = df[inputcolumn].astype(int, errors='ignore')
    
    df[inputcolumn] = df[inputcolumn] * bn_width + bn_min
    
    return df, inputcolumn
  
  def inverseprocess_bnep(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_bnep_class
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    
    #this only achieves partial information recovery as bins sets aggregated to single value per bin
    """
    
    normkey = categorylist[0]
    
    textcolumns = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['textcolumns']
    bins_id = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_id']
    bn_min = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_min']
    bn_delta = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_delta']
    bn_count = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_count']
    bins_cuts = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_cuts']
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    df[inputcolumn] = 0
    
    for i in range(len(textcolumns)):
      
      column = textcolumns[i]
      _id = int(bins_id[i])
      
      if i == 0:
        
        value = bins_cuts[i+1]
        
        df[inputcolumn] = np.where(df[column] == 1, value, df[inputcolumn])
        
      elif i == len(textcolumns)-1:
        
        value = bins_cuts[i]
        
        df[inputcolumn] = np.where(df[column] == 1, value, df[inputcolumn])
        
      else:
        
        value = (bins_cuts[i] + bins_cuts[i+1]) / 2
        
        df[inputcolumn] = np.where(df[column] == 1, value, df[inputcolumn])
    
    return df, inputcolumn
  
  def inverseprocess_bneo(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_bneo_class
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    
    #this only achieves partial information recovery as bins sets aggregated to single value per bin
    """
    
    normkey = categorylist[0]
    
    bins_id = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_id']
    bn_min = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_min']
    bn_delta = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_delta']
    bn_count = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bn_count']
    bins_cuts = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_cuts']
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    df[inputcolumn] = 0

    #we'll convert the input to integers
    df[normkey] = df[normkey].astype(int, errors='ignore')
    
    for i in range(len(bins_id)):
      
      _id = int(bins_id[i])
      
      if i == 0:
        
        value = bins_cuts[i+1]
        
        df[inputcolumn] = np.where(df[normkey] == _id, value, df[inputcolumn])
        
      elif i == len(bins_id)-1:
        
        value = bins_cuts[i]
        
        df[inputcolumn] = np.where(df[normkey] == _id, value, df[inputcolumn])
        
      else:
        
        value = (bins_cuts[i] + bins_cuts[i+1]) / 2
        
        df[inputcolumn] = np.where(df[normkey] == _id, value, df[inputcolumn])
    
    return df, inputcolumn
  
  def inverseprocess_bkt1(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_bkt1_class
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    
    #this only achieves partial information recovery as bins sets aggregated to single value per bin
    """
    
    normkey = categorylist[0]
    
    textcolumns = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['textcolumns']
    bins_cuts = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_cuts']
    bins_id = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_id']
    buckets_bkt1 = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['buckets_bkt1']
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    #we'll use convention that bucket's replaced with the midpoint value
    #except for bottom bucket replaced with ceiling and top bucket replaced with floor
    #so don't have to deal with inf
    
    df[inputcolumn] = 0
    
    bucket_ids = []
    for textcolumn in textcolumns:
      bucket_id = textcolumn.replace(inputcolumn + '_bkt1_', '')
      bucket_ids.append(int(bucket_id))

    for i in bucket_ids:
      
      textcolumn = inputcolumn + '_bkt1_' + str(i)
      
      if i == 0:
        
        value = buckets_bkt1[i]
        
      elif i == bins_id[-1]:
        
        value = buckets_bkt1[-1]
        
      else:
        
        value = (buckets_bkt1[i-1] + buckets_bkt1[i]) / 2
        
      df[inputcolumn] = np.where(df[textcolumn] == 1, value, df[inputcolumn])
    
    return df, inputcolumn
  
  def inverseprocess_bkt2(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_bkt2_class
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    
    #this only achieves partial information recovery as bins sets aggregated to single value per bin
    """
    
    normkey = categorylist[0]
    
    buckets_bkt2 = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['buckets_bkt2']
    bins_cuts = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_cuts']
    bins_id = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_cuts']
    textcolumns = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['textcolumns']
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    #we'll use convention that bucket's replaced with the midpoint value
    #except for bottom bucket replaced with ceiling and top bucket replaced with floor
    #so don't have to deal with inf
    
    df[inputcolumn] = 0
    
    bucket_ids = []
    for textcolumn in textcolumns:
      bucket_id = textcolumn.replace(inputcolumn + '_bkt2_', '')
      bucket_ids.append(int(bucket_id))
      
    for i in bucket_ids:
      
      value = (bins_cuts[i] + bins_cuts[i+1]) / 2
      
      df[inputcolumn] = np.where(df[inputcolumn + '_bkt2_' + str(i)]==1, value, df[inputcolumn])
    
    return df, inputcolumn
  
  def inverseprocess_bkt3(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_bkt3_class
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    
    #this only achieves partial information recovery as bins sets aggregated to single value per bin
    """
    
    normkey = categorylist[0]
    
    buckets = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['buckets']
    bins_cuts = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_cuts']
    bins_id = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_id']
    ordl_activations_dict = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['ordl_activations_dict']
    infill_activation = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['infill_activation']
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    #we'll use convention that bucket's replaced with the midpoint value
    #except for bottom bucket replaced with ceiling and top bucket replaced with floor
    #so don't have to deal with inf

    #we'll convert the input to integers
    df[normkey] = df[normkey].astype(int, errors='ignore')
    
    df[inputcolumn] = 0

    #infill recovery
    df[inputcolumn] = np.where(df[normkey] == infill_activation, 'zzzinfill', df[inputcolumn])
    
    for i in bins_id:
      
      if i == 0:
        
        value = bins_cuts[i+1]
        
      elif i == len(bins_id)-1:
        
        value = bins_cuts[i]
        
      else:
        
        value = (bins_cuts[i] + bins_cuts[i+1]) / 2
        
      df[inputcolumn] = np.where(df[normkey] == i, value, df[inputcolumn])
    
    return df, inputcolumn
  
  def inverseprocess_bkt4(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_bkt4_class
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    
    #this only achieves partial information recovery as bins sets aggregated to single value per bin
    """
    
    normkey = categorylist[0]
    
    buckets = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['buckets']
    bins_cuts = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_cuts']
    bins_id = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['bins_id']
    infill_activation = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['infill_activation']
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    #we'll use convention that bucket's replaced with the midpoint value
    #except for bottom bucket replaced with ceiling and top bucket replaced with floor
    #so don't have to deal with inf
    
    df[inputcolumn] = 0

    #we'll convert the input to integers
    df[normkey] = df[normkey].astype(int, errors='ignore')

    #infill recovery
    df[inputcolumn] = np.where(df[normkey] == infill_activation, 'zzzinfill', df[inputcolumn])
    
    for i in bins_id:
      
      value = (bins_cuts[i] + bins_cuts[i+1]) / 2
        
      df[inputcolumn] = np.where(df[normkey] == i, value, df[inputcolumn])
    
    return df, inputcolumn
  
  def inverseprocess_onht(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_text_class
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    #note that this will return numeric entries as str
    """
    
    normkey = categorylist[0]
    
    inverse_labels_dict = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['inverse_labels_dict']
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    df[inputcolumn] = 'zzzinfill'
    
    for categorylist_entry in categorylist:
      
      df[inputcolumn] = \
      np.where(df[categorylist_entry], inverse_labels_dict[categorylist_entry], df[inputcolumn])
      
    return df, inputcolumn
    
  def inverseprocess_text(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_text_class
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    #note that this will return numeric entries as str
    """
    
    normkey = categorylist[0]
    
    textlabelsdict_text = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['textlabelsdict_text']
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    df[inputcolumn] = 'zzzinfill'
    
    for categorylist_entry in categorylist:
      
      df[inputcolumn] = \
      np.where(df[categorylist_entry], textlabelsdict_text[categorylist_entry], df[inputcolumn])
      
    return df, inputcolumn
  
  def inverseprocess_ordl(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_ordl_class
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    #note that this will return numeric entries as str
    """
    
    normkey = categorylist[0]
    
    ordinal_dict = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['ordinal_dict']
    overlap_replace = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['ordinal_overlap_replace']
    
    inverse_ordinal_dict = {value:key for key,value in ordinal_dict.items()}
    inverse_overlap_replace = {value:key for key,value in overlap_replace.items()}
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    df[inputcolumn] = \
    df[normkey].replace(inverse_ordinal_dict)

    if df[inputcolumn].dtype.name != 'object':
      df[inputcolumn] = df[inputcolumn].astype('object')
    
    df[inputcolumn] = \
    df[inputcolumn].replace(inverse_overlap_replace)
    
    return df, inputcolumn
  
  def inverseprocess_ord3(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_ord3_class
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    #note that this will return numeric entries as str
    """
    
    normkey = categorylist[0]
    
    ordinal_dict = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['ordinal_dict']
    overlap_replace = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['ordinal_overlap_replace']
    
    inverse_ordinal_dict = {value:key for key,value in ordinal_dict.items()}
    inverse_overlap_replace = {value:key for key,value in overlap_replace.items()}
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']

    #we'll convert the input to integers
    df[normkey] = df[normkey].astype(int, errors='ignore')
    
    df[inputcolumn] = \
    df[normkey].replace(inverse_ordinal_dict)

    if df[inputcolumn].dtype.name != 'object':
      df[inputcolumn] = df[inputcolumn].astype('object')
    
    df[inputcolumn] = \
    df[inputcolumn].replace(inverse_overlap_replace)
    
    return df, inputcolumn

  def inverseprocess_strg(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_strg_class
    #converts strings back to integers
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    #note that this will return numeric entries as str
    """
    
    normkey = categorylist[0]
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    df[inputcolumn] = \
    df[normkey].astype(int, errors='ignore')
    
    return df, inputcolumn
  
  def inverseprocess_bnry(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_bnry_class
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    """
    
    normkey = categorylist[0]
    
    onevalue = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey][1]
    zerovalue = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey][0]
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    df[inputcolumn] = \
    np.where(df[normkey] == 1, onevalue, 0)
    
    df[inputcolumn] = \
    np.where(df[normkey] == 0, zerovalue, df[inputcolumn])
      
    return df, inputcolumn

  def inverseprocess_1010(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_1010_class
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    #note that this will return numeric entries as str
    """
    
    normkey = categorylist[0]
    
    _1010_binary_encoding_dict = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['_1010_binary_encoding_dict']
    _1010_overlap_replace = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['_1010_overlap_replace']
    
    inverse_binary_encoding_dict = {value:key for key,value in _1010_binary_encoding_dict.items()}
    inverse_overlap_replace = {value:key for key,value in _1010_overlap_replace.items()}
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    for categorylist_entry in categorylist:
      
      if categorylist_entry == categorylist[0]:
        
        df[inputcolumn] = df[categorylist_entry].astype(int).astype(str)
        
      else:
        
        df[inputcolumn] = df[inputcolumn] + df[categorylist_entry].astype(int).astype(str)
        
    df[inputcolumn] = df[inputcolumn].replace(inverse_binary_encoding_dict)

    if df[inputcolumn].dtype.name != 'object':
      df[inputcolumn] = df[inputcolumn].astype('object')

    df[inputcolumn] = df[inputcolumn].replace(inverse_overlap_replace)
      
    return df, inputcolumn

  def inverseprocess_splt(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_sp1t_class
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    
    #this only achieves partial information recovery, missing entries without overlap
    
    #returns the overlaps, not the full entries
    #since it doesn't know which of the full entries to return
    """
    
    normkey = categorylist[0]
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    newcolumns = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['splt_newcolumns_splt']
    preint_newcolumns = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['preint_newcolumns']
    overlap_dict = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['overlap_dict']
    int_headers = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['int_headers']
    suffix = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']
    
    df[inputcolumn] = 'zzzinfill'
    
    column_conversion_dict = dict(zip(preint_newcolumns, newcolumns))
    
    preint_newcolumns = sorted(preint_newcolumns, reverse = False, key=len)
    
    if int_headers is False:

      for column in preint_newcolumns:

        overlap = column.replace(inputcolumn + suffix + '_', '')

        df[inputcolumn] = np.where(df[column] == 1, overlap, df[inputcolumn])
    
    else:

      for column in preint_newcolumns:
        
        newcolumn = column_conversion_dict[column]
        
        overlap = column.replace(inputcolumn + suffix + '_', '')
        
        df[inputcolumn] = np.where(df[newcolumn] == 1, overlap, df[inputcolumn])

    return df, inputcolumn
  
  def inverseprocess_spl2(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_spl2_class
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    
    #this only achieves partial information recovery, missing entries without overlap
    #also missing entries that didn't have any overlaps identified (returned as 0)
    
    #returns the overlaps, not the full entries
    #since it doesn't know which of the full entries to return
    
    #returning zeros from inversion is counter to the convention used in other transforms
    #So we'll replace zeros with 'zzzinfill'
    """
    
    normkey = categorylist[0]
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    consolidate_nonoverlaps = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['consolidate_nonoverlaps']
    
    df[inputcolumn] = df[normkey]
    
    if consolidate_nonoverlaps is True:

      df[inputcolumn] = np.where(df[inputcolumn] == '0', 'zzzinfill', df[inputcolumn])
  
    return df, inputcolumn

  def inverseprocess_sp19(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_sp19_class
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    
    #this only achieves partial information recovery, missing entries without overlap
    
    #returns the overlaps, not the full entries
    #since it doesn't know which of the full entries to return
    """
    
    normkey = categorylist[0]
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    newcolumns = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['splt_newcolumns_sp19']
    preint_newcolumns = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['preint_newcolumns']
    overlap_dict = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['overlap_dict']
    int_headers = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['int_headers']
    _1010_binary_encoding_dict = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['_1010_binary_encoding_dict']
    _1010_binary_column_count = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['_1010_binary_column_count']
    _1010_activations_dict = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['_1010_activations_dict']
    categorylist = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['categorylist']
    
    inverse_binary_encoding_dict = {value:key for key,value in _1010_binary_encoding_dict.items()}
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    for categorylist_entry in categorylist:
      
      if categorylist_entry == categorylist[0]:
        
        df[inputcolumn] = df[categorylist_entry].astype(int).astype(str)
        
      else:
        
        df[inputcolumn] = df[inputcolumn] + df[categorylist_entry].astype(int).astype(str)
        
    df[inputcolumn] = df[inputcolumn].replace(inverse_binary_encoding_dict)
    
    df[inputcolumn] = df[inputcolumn].str.replace('activations_', '')
    
    #now let's extract the encoding
    i=0
    for newcolumn in newcolumns:
      
      df[newcolumn] = df[inputcolumn].str.slice(i,i+1).astype(np.int8)
      
      i+=1
    
    df[inputcolumn] = 'zzzinfill'
    
    column_conversion_dict = dict(zip(preint_newcolumns, newcolumns))
    
    preint_newcolumns = sorted(preint_newcolumns, reverse = False, key=len)
    
    if int_headers is False:

      for column in preint_newcolumns:

        overlap = column.replace(inputcolumn + '_sp15_', '')

        df[inputcolumn] = np.where(df[column] == 1, overlap, df[inputcolumn])
    
    else:

      for column in preint_newcolumns:
        
        newcolumn = column_conversion_dict[column]
        
        overlap = column.replace(inputcolumn + '_sp15_', '')
        
        df[inputcolumn] = np.where(df[newcolumn] == 1, overlap, df[inputcolumn])

    for newcolumn in newcolumns:
      
      del df[newcolumn]

    return df, inputcolumn

  def inverseprocess_sbst(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_sbst_class
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    
    #this only achieves partial information recovery, missing entries without overlap
    
    #returns the overlaps, not the full entries
    #since it doesn't know which of the full entries to return
    """
    
    normkey = categorylist[0]
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    newcolumns = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['splt_newcolumns_sbst']
    preint_newcolumns = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['preint_newcolumns']
    overlap_dict = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['overlap_dict']
    int_headers = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['int_headers']
    suffix = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['suffix']

    
    df[inputcolumn] = 'zzzinfill'
    
    if int_headers is False:

      for column in preint_newcolumns:

        overlap = column.replace(inputcolumn + suffix + '_', '')

        df[inputcolumn] = np.where(df[column] == 1, overlap, df[inputcolumn])
    
    else:
      
      i = 0
      for column in preint_newcolumns:
        
        newcolumn = newcolumns[i]
        
        overlap = column.replace(inputcolumn + suffix + '_', '')
        
        df[inputcolumn] = np.where(df[newcolumn] == 1, overlap, df[inputcolumn])
        
        i += 1
    
    return df, inputcolumn

  def inverseprocess_sbs3(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_sbs3_class
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    
    #this only achieves partial information recovery, missing entries without overlap
    
    #returns the overlaps, not the full entries
    #since it doesn't know which of the full entries to return
    """
    
    normkey = categorylist[0]
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    newcolumns = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['splt_newcolumns_sbs3']
    preint_newcolumns = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['preint_newcolumns']
    overlap_dict = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['overlap_dict']
    int_headers = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['int_headers']
    _1010_binary_encoding_dict = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['_1010_binary_encoding_dict']
    _1010_binary_column_count = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['_1010_binary_column_count']
    _1010_activations_dict = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['_1010_activations_dict']
    categorylist = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['categorylist']
    
    inverse_binary_encoding_dict = {value:key for key,value in _1010_binary_encoding_dict.items()}
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    for categorylist_entry in categorylist:
      
      if categorylist_entry == categorylist[0]:
        
        df[inputcolumn] = df[categorylist_entry].astype(int).astype(str)
        
      else:
        
        df[inputcolumn] = df[inputcolumn] + df[categorylist_entry].astype(int).astype(str)
        
    df[inputcolumn] = df[inputcolumn].replace(inverse_binary_encoding_dict)
    
    df[inputcolumn] = df[inputcolumn].str.replace('activations_', '')
    
    #now let's extract the encoding
    i=0
    for newcolumn in newcolumns:
      
      df[newcolumn] = df[inputcolumn].str.slice(i,i+1).astype(np.int8)
      
      i+=1
    
    df[inputcolumn] = 'zzzinfill'
    
    column_conversion_dict = dict(zip(preint_newcolumns, newcolumns))
    
    preint_newcolumns = sorted(preint_newcolumns, reverse = False, key=len)
    
    if int_headers is False:

      for column in preint_newcolumns:

        overlap = column.replace(inputcolumn + '_sbst_', '')

        df[inputcolumn] = np.where(df[column] == 1, overlap, df[inputcolumn])
    
    else:

      for column in preint_newcolumns:
        
        newcolumn = column_conversion_dict[column]
        
        overlap = column.replace(inputcolumn + '_sbst_', '')
        
        df[inputcolumn] = np.where(df[newcolumn] == 1, overlap, df[inputcolumn])

    for newcolumn in newcolumns:
      
      del df[newcolumn]

    return df, inputcolumn
  
  def inverseprocess_srch(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_srch_class
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    
    #this only achieves partial information recovery, missing entries without srch term
    
    #returns the search term, not the full entries
    #since it doesn't know which of the full entries to return
    """
    
    normkey = categorylist[0]
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    search_dict = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['search_dict']
    
    df[inputcolumn] = 'zzzinfill'
    
    #to match convention of prioritizing search parameter entries at end of list
    inverse_search = list(search_dict)
    inverse_search.reverse()
    
    for column in inverse_search:
      
      search = search_dict[column]
      
      if column in df.columns:

        df[inputcolumn] = np.where((df[column] == 1) & (df[inputcolumn] == 'zzzinfill'), search, df[inputcolumn])
    
    return df, inputcolumn
  
  def inverseprocess_src2(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_src2_class
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    
    #this only achieves partial information recovery, missing entries without search term
    
    #returns the search term, not the full entries
    #since it doesn't know which of the full entries to return
    """
    
    normkey = categorylist[0]
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    newcolumns = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['src2_newcolumns_src2']
#     overlap_dict = \
#     postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['overlap_dict']
    
    df[inputcolumn] = 'zzzinfill'
    
    #to match convention of prioritizing search parameter entries at end of list
    newcolumns.reverse()
    
    for column in newcolumns:
      
      searchterm = column.replace(inputcolumn + '_src2_', '')
      
      if column in df.columns:
      
        df[inputcolumn] = np.where((df[column] == 1) & (df[inputcolumn] == 'zzzinfill'), searchterm, df[inputcolumn])
    
    return df, inputcolumn
  
  def inverseprocess_src3(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_src3_class
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    
    #this only achieves partial information recovery, missing entries without search term
    
    #returns the search term, not the full entries
    #since it doesn't know which of the full entries to return
    """
    
    normkey = categorylist[0]
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    newcolumns = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['srch_newcolumns_src3']
#     overlap_dict = \
#     postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['overlap_dict']
    
    df[inputcolumn] = 'zzzinfill'
    
    #to match convention of prioritizing search parameter entries at end of list
    newcolumns.reverse()
    
    for column in newcolumns:
      
      searchterm = column.replace(inputcolumn + '_src3_', '')
      
      if column in df.columns:
      
        df[inputcolumn] = np.where((df[column] == 1) & (df[inputcolumn] == 'zzzinfill'), searchterm, df[inputcolumn])
    
    return df, inputcolumn
  
  def inverseprocess_src4(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_src4_class
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    
    #this only achieves partial information recovery, missing entries without search term
    
    #returns the search term, not the full entries
    #since it doesn't know which of the full entries to return
    """
    
    normkey = categorylist[0]
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    ordl_dict1 = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['ordl_dict1']
    
    df[inputcolumn] = 'zzzinfill'
    
    #to match convention of prioritizing search parameter entries at end of list
    keys = list(ordl_dict1)
    keys.reverse()

    #we'll convert the input to integers
    df[normkey] = df[normkey].astype(int, errors='ignore')
    
    for key in keys:
      
      searchterm = ordl_dict1[key].replace(inputcolumn + '_src4_', '')
      
      df[inputcolumn] = np.where((df[normkey] == key) & (df[inputcolumn] == 'zzzinfill'), searchterm, df[inputcolumn])
    
    return df, inputcolumn
  
  def inverseprocess_nmrc(self, df, categorylist, postprocess_dict):
    """
    #inverse transform corresponding to process_nmrc_class
    #assumes any relevant parameters were saved in normalization_dict
    #does not perform infill, assumes clean data
    
    #this only achieves partial information recovery, missing entries without search term
    
    #returns the search term, not the full entries
    #since it doesn't know which of the full entries to return
    """
    
    normkey = categorylist[0]
    
    inputcolumn = postprocess_dict['column_dict'][normkey]['inputcolumn']
    
    overlap_dict = \
    postprocess_dict['column_dict'][normkey]['normalization_dict'][normkey]['overlap_dict']
    
    df[inputcolumn] = 'zzzinfill'
    
    for key in overlap_dict:
      
      extract = overlap_dict[key]
      
      df[inputcolumn] = np.where((df[normkey] == extract) & (df[inputcolumn] == 'zzzinfill'), key, df[inputcolumn])
    
    return df, inputcolumn
  
  def df_inversion(self, categorylist_entry, df_test, postprocess_dict, inverse_categorytree, printstatus):
    """
    #support function for df_inversion_meta
    #this is where the inverseprocess functions are applied
    """
    
    origcolumn = postprocess_dict['column_dict'][categorylist_entry]['origcolumn']
    category = postprocess_dict['column_dict'][categorylist_entry]['category']
    categorylist = postprocess_dict['column_dict'][categorylist_entry]['categorylist']
    
    columns_before_inversion = set(df_test)
    
    if 'inverseprocess' in postprocess_dict['process_dict'][category]:
      
      if callable(postprocess_dict['process_dict'][category]['inverseprocess']):
        
        df_test, inputcolumn = \
        postprocess_dict['process_dict'][category]['inverseprocess'](df_test, categorylist, postprocess_dict)
        #df, categorylist, postprocess_dict
    
    columns_after_inversion = set(df_test)
    
    #convention is that inversion always returns a single column
    #(multi-column transformations only performed on final leaf of a forward pass branch)
    if len(list(columns_after_inversion - columns_before_inversion)) > 0:
    
      inputcolumn = list(columns_after_inversion - columns_before_inversion)[0]
      
    #this only happens for excl suffix edge case
    else:
      inputcolumn = origcolumn
      
    
    if inputcolumn != origcolumn:
      
      df_test, inputcolumn = \
      self.df_inversion(inputcolumn, df_test, postprocess_dict, inverse_categorytree, printstatus)
    
    return df_test, inputcolumn
    
  def df_inversion_meta(self, df_test, source_columns, postprocess_dict, printstatus):
    """
    #Performs inversion of transformation sets
    #Relies on optional processdict entries of info_retention and inverseprocess
    #Uses the inverse_categorytree populated during automunge
    #For all entries associated with a single source column, 
    #Selects the returned categorylist with the lowest depth with info retention through the branch
    #If info retention not available, instead takes the shortest depth with inverse transformations available
    #If full set of inverse transfomations not available, does not perfgorm inversion for that source column
    #For each inversion path selected performs the sets of transfomrations in inverse order to the original path's population steps
    #All returned sets are added to the dataframe, and at conclusion of the function
    #any columns not matching a source column are removed form the set
    #also returns a list of the recovered columns
    #relies of column headers of received df_test matching the column headers of original columns returned from automunge
    #note that this function may be applied consistently to test or label sets
    """
    
    inverse_categorytree = postprocess_dict['inverse_categorytree']
    
    #we'll store the inversion paths info in this dictionary
    inversion_info_dict = {}
    
    #this will be a list of columns successfully recovered
    recovered_list = []
    
    for source_column in source_columns:
      
      if printstatus is True:
        print("Evaluating inversion paths for columns derived from: ", source_column)
      
      returned_columns = postprocess_dict['origcolumn'][source_column]['columnkeylist']
      
      #we'll just take one of the columns here if it is part of a multicolumn set
      returned_columns_clean = returned_columns.copy()

      for returned_column in returned_columns:
        
        if returned_column in returned_columns_clean:
        
          categorylist = postprocess_dict['column_dict'][returned_column]['categorylist']

          for categorylist_entry in categorylist:

            if categorylist_entry != returned_column:
              
              #we'll just take one of the columns here if it is part of a multicolumn set
              returned_columns_clean.remove(categorylist_entry)
      
      #initialize for ranking paths of transformation inversions
      path_depth_eval         = {}
      inforetention_eval      = {}
      transformavailable_eval = {}
      
      for returned_column in returned_columns_clean:
        
        category = postprocess_dict['column_dict'][returned_column]['category']
        
        path_depth_eval.update({returned_column : inverse_categorytree[category][returned_column][4]})
        inforetention_eval.update({returned_column : inverse_categorytree[category][returned_column][5]})
        transformavailable_eval.update({returned_column : inverse_categorytree[category][returned_column][6]})
        
      #now invert the path_depth_eval dictionary for sorting
      #inverse_path_depth_eval = {value:key for key,value in path_depth_eval.items()}
      inverse_path_depth_eval = {}
      for key, value in path_depth_eval.items():
        if value not in inverse_path_depth_eval:
          inverse_path_depth_eval.update({value : [key]})
        else:
          inverse_path_depth_eval[value].append(key)
          
      #and sort it to find shortest depth
      inverse_path_depth_eval = dict(sorted(inverse_path_depth_eval.items()))
      
      #note that this sorted by depth method is based on a heuristic
      #that the fewest number of transforms will be the most efficient
      
      #now let's select our returned column for the path
      best_path = False
      info_retention_marker = False
      
      for depth in inverse_path_depth_eval:
        
        for path in inverse_path_depth_eval[depth]:
        
          inforetention = inforetention_eval[path]
          transformavailable = transformavailable_eval[path]

          if inforetention and transformavailable:

            best_path = path
            info_retention_marker = True

            break

          elif transformavailable and best_path is False:

            best_path = path
            
        if info_retention_marker is True:
          
          break
          
      if best_path is not False:
        #check that best path has all categorylist entries present
        if not set(postprocess_dict['column_dict'][best_path]['categorylist']).issubset(set(df_test)):
          if printstatus is True:
            print("Inversion path selected based on returned column ", best_path)
            print("Inversion not available due to incomplete set of categorylist entries.")
          best_path = False
          
      if printstatus is True:
        
        if best_path is not False:
          
          if info_retention_marker is True:
          
            print("Inversion path selected based on returned column ", best_path)
            print("With full recovery.")
            
          else:
            
            print("Inversion path selected based on returned column ", best_path)
            print("With partial recovery.")
          
        else:
          
          print("No inversion path available for source column: ", source_column)
          print()
          
      #great we've selected our path for this source column's inversion
      inversion_info_dict.update({source_column : {'best_path' : best_path, \
                                                   'info_retention' : info_retention_marker}})
      
      #now let's apply our inversion transforms
      #if best_path is not False and callable(postprocess_dict['process_dict'][postprocess_dict['column_dict'][best_path]['category']]['inverseprocess']):
      if best_path is not False:
        
        columns_before_inversion = set(df_test)
        
        df_test, _1 = self.df_inversion(best_path, df_test, postprocess_dict, inverse_categorytree, printstatus)
        
        columns_after_inversion = set(df_test)
        
        #this gives, for the target source columns, returned columns, source columns, and intermediate columns
        full_returned_columns = columns_after_inversion - (columns_before_inversion - set(returned_columns))
        
      else:
        
        full_returned_columns = returned_columns
    
      for column in full_returned_columns:

        if column in source_columns:
          
          recovered_list += [column]

        else:

          #we're only retaining successfully recovered source columns in the returned df
          #this deletion is performed sequentially for columns returned from given source column for memory management
          if column in df_test.columns:
            del df_test[column]
          
      if printstatus is True:
        
        if best_path is not False:
          
          print("Recovered source column: ", source_column)
          print()
        
    #special case for excl suffix
    for column in source_columns:
      if column not in recovered_list \
      and column in postprocess_dict['excl_columns_without_suffix'] \
      and column in postprocess_dict['finalcolumns_train']:
        recovered_list.append(column)
    
    return df_test, recovered_list, inversion_info_dict
  
  def inversion_parent(self, inversion, df_test, postprocess_dict, printstatus, \
                       pandasoutput, LabelSmoothing):
    
    if inversion == 'test' and postprocess_dict['PCAmodel'] is not None:
      print("error: full test set inversion not currently supported with PCA.")
      print("user can pass partial list of columns to inversion parameter instead")
      print()
      inversion = False

    if isinstance(inversion, list):
      #convert list entries to string
      inversion = [str(entry) for entry in inversion]

    #initialize objects that may be adjusted in case of Binary
    Binary_finalcolumns_train = postprocess_dict['finalcolumns_train']
    Binary_inversion_marker = False

    #if Binary was performed, treatment depends on whether it was a replace or retain
    if len(list(postprocess_dict['Binary_dict']['column_dict'])) > 0:
      #if Binary was a 'retain' call then we don't need these columns for inversion
      if postprocess_dict['Binary'] == 'retain' and inversion == 'test':
        inversion = list(df_test)
        for entry in postprocess_dict['Binary_dict']['column_dict']:
          if entry in inversion:
            inversion.remove(entry)
      if postprocess_dict['Binary'] == 'retain' and isinstance(inversion, list):
        #we'll have convention that if Binary didn't replace columns 
        #partial inversion only available for Binary source columns
        for entry in postprocess_dict['Binary_dict']['column_dict']:
          if entry in inversion:
            print("please note partial inversion lists only supported for columns not returned from Binary")
            print("when Binary was not performed with replacement")
            inversion.remove(entry)
      if postprocess_dict['Binary'] == True and inversion == 'test':
        Binary_inversion_marker = True
      if postprocess_dict['Binary'] == True and isinstance(inversion, list):
        if set(list(postprocess_dict['Binary_dict']['column_dict'])).issubset(set(inversion)):
          Binary_inversion_marker = True
          for entry in postprocess_dict['Binary_dict']['column_dict']:
            inversion.remove(entry)
          inversion += postprocess_dict['Binary_dict']['bool_column_list']
        elif bool(set(postprocess_dict['Binary_dict']['column_dict']) & set(inversion)):
          print("error: partial inversion lists only supported for columns returned from Binary")
          print("when entire set of Binary columns are included in the inversion list")

    if Binary_inversion_marker is True:

      print("Recovering columns from Binary dimensionality reduction.")

      df_test = self.meta_inverseprocess_Binary(df_test, postprocess_dict)

      for entry in postprocess_dict['Binary_dict']['column_dict']:

        if entry != 'Binary':

          Binary_finalcolumns_train.remove(entry)

      Binary_finalcolumns_train += postprocess_dict['Binary_dict']['bool_column_list']

      print("Recovered columns:")
      print(postprocess_dict['Binary_dict']['bool_column_list'])
      print()
      
    #this is relevant for when feature importance dimensionality reduction was performed
    if inversion == 'test':
      if set(Binary_finalcolumns_train).issubset(set(postprocess_dict['pre_dimred_finalcolumns_train'])):
        inversion = Binary_finalcolumns_train
        
    if inversion == 'test':

      #this is to handle edge case of excl transforms
      #which after processing have their suffix removed from header
      finalcolumns_labels = Binary_finalcolumns_train
      source_columns = postprocess_dict['origtraincolumns']

      finalcolumns_labels = [str(c)+'_excl' if c in source_columns else c for c in finalcolumns_labels]

      #confirm consistency of train an test sets

      #check number of columns is consistent
      if len(finalcolumns_labels)!= df_test.shape[1]:
        print("error, different number of returned columns in train and test sets")
        return

      #check order of column headers are consistent
      columns_test = list(df_test)
      if set(finalcolumns_labels) == set(columns_test):
        if finalcolumns_labels != columns_test:
          print("error, different order of column labels in the train and test set")
          return
      #this is for excl edge case again in case we had any updates to finalcolumns_labels above
      elif set(postprocess_dict['finalcolumns_train']) == set(columns_test):
        if postprocess_dict['finalcolumns_train'] != columns_test:
          print("error, different order of column labels in the train and test set")
          return

      #assign labels to column headers if they weren't passed
      if finalcolumns_labels != columns_test:
        df_test.columns = finalcolumns_labels


      if printstatus is True:
        print("Performing inversion recovery of original columns for test set.")
        print()

      df_test, recovered_list, inversion_info_dict = \
      self.df_inversion_meta(df_test, postprocess_dict['origtraincolumns'], postprocess_dict, printstatus)

      if printstatus is True:
        print("Inversion succeeded in recovering original form for columns:")
        print(recovered_list)
        print()

      if pandasoutput is False:

        df_test = df_test.values

      return df_test, recovered_list, inversion_info_dict

    if inversion == 'labels':

      #this is to handle edge case of excl transforms
      #which after processing have their suffix removed from header
      finalcolumns_labels = postprocess_dict['finalcolumns_labels']
      source_columns = postprocess_dict['labels_column']

      finalcolumns_labels = [str(c)+'_excl' if c in source_columns else c for c in finalcolumns_labels]

      #confirm consistency of label sets

      #check number of columns is consistent
      if len(finalcolumns_labels)!= df_test.shape[1]:
        print("error, different number of returned label columns in train and test sets")
        return

      #check order of column headers are consistent
      columns_test = list(df_test)
      if set(finalcolumns_labels) == set(columns_test):
        if finalcolumns_labels != columns_test:
          print("error, different order of column labels in the train and test set")
          return
      #this is for excl edge case again in case we had any updates to finalcolumns_labels above
      elif set(postprocess_dict['finalcolumns_labels']) == set(columns_test):
        if postprocess_dict['finalcolumns_labels'] != columns_test:
          print("error, different order of column labels in the train and test set")
          return

      #assign labels to column headers if they weren't passed
      if finalcolumns_labels != columns_test:
        df_test.columns = finalcolumns_labels

      if printstatus is True:
        print("Performing inversion recovery of original columns for label set.")
        print()

      #first revert any label smoothing to one-hot encoding, LabelSmoothing can be passed as True
      #for basis of LabelSmoothing_train passed to automunge, or float 0-1 matching activation setting 
      #assumes if labels encoded in multiple smoothed configurations they have consistent activations
      df_test = self.meta_LS_invert(LabelSmoothing, df_test, postprocess_dict)

      df_test, recovered_list, inversion_info_dict = \
      self.df_inversion_meta(df_test, [postprocess_dict['labels_column']], postprocess_dict, printstatus)

      if printstatus is True:
        print("Inversion succeeded in recovering original form for columns:")
        print(recovered_list)
        print()

      if pandasoutput is False:

        df_test = df_test.values

      return df_test, recovered_list, inversion_info_dict

    if isinstance(inversion, list):

      #this is to handle edge case of excl transforms
      #which after processing have their suffix removed from header
      finalcolumns_train = postprocess_dict['finalcolumns_train']
      source_columns = postprocess_dict['origtraincolumns']

      inversion = [str(c)+'_excl' if c in source_columns and c in finalcolumns_train else c for c in inversion]

      finalcolumns_train = [str(c)+'_excl' if c in source_columns else c for c in finalcolumns_train]

#         #for inversion need source columns
#         inversion = [postprocess_dict['column_dict'][entry]['origcolumn'] if entry in finalcolumns_train else entry for entry in inversion]

      #for inversion need source columns
      inversion = [postprocess_dict['column_dict'][entry]['origcolumn'] if entry in finalcolumns_train or entry in postprocess_dict['excl_columns_with_suffix'] else entry for entry in inversion]

      #consolidate redundancies
      inversion_copy = inversion.copy()
      inversion = []
      for entry in inversion_copy:
        if entry not in inversion:
          inversion.append(entry)

      #check these are all valid source columns
      for entry in inversion:

        if entry not in source_columns:

          print("error: entry passed to inversion parameter list not matching a source or derived column")
          print("for entry: ", entry)

      df_test, recovered_list, inversion_info_dict = \
      self.df_inversion_meta(df_test, inversion, postprocess_dict, printstatus)

      if printstatus is True:
        print("Inversion succeeded in recovering original form for columns:")
        print(recovered_list)
        print()

      if pandasoutput is False:

        df_test = df_test.values

      return df_test, recovered_list, inversion_info_dict