##########################################
# To parse log files generated by abelian.
# Author: Gurbinder Gill
# Email: gurbinder533@gmail.com
#########################################

import re
import os
import sys, getopt
import csv
import numpy

######## NOTES:
# All time values are in sec by default.


def sd_iterations(inputFile, outputFile, outputFile_mainfile, benchmark, runs, time_unit, hostNum, iterationNum, variant, input_graph, deviceKind, devices, partition):

  mean_time = 0.0;
  recvNum_total = 0
  recvBytes_total = 0
  sendNum_total = 0
  sendBytes_total = 0
  sync_pull_avg_time_total = 0.0;
  extract_avg_time_total = 0.0;
  set_avg_time_total = 0.0;
  sync_push_avg_time_total = 0.0;
  graph_init_time = 0
  hg_init_time = 0
  total_time = 0

  if(benchmark == "cc"):
    benchmark = "ConnectedComp"

  if (time_unit == 'seconds'):
    divisor = 1000
  else:
    divisor = 1

  log_data = open(inputFile).read()

  data = [variant, input_graph, hostNum, benchmark, partition, deviceKind, devices]
  fd_outputFile = open(outputFile, 'a')
  fd_outputFile_main = open(outputFile_mainfile, 'a')

  rep_regex = re.compile(r'.*,\(NULL\),0\s,\sREPLICATION_FACTOR_0_0,(\d*),\d*,(.*)')

  Total_mean_compute = 0.0
  Total_rsd_compute = 0.0
  rep_search = rep_regex.search(log_data)
  if rep_search is not None:
    rep_factor = rep_search.group(2)
    rep_factor = round(float(rep_factor), 3)
    print ("FOUND  : ", rep_factor)

  iterNum_start = 0
  #do_all_regex = re.compile(r'.*,\(NULL\),0\s,\sDO_ALL_IMPL_FirstItr_(?i)' + re.escape(benchmark) + r'_0_' + r',.*' + r',\d*,(\d*)')
  #do_all_all_hosts = re.findall(do_all_regex, log_data)
  #num_arr = numpy.array(map(int,do_all_all_hosts))

  #if(num_arr.size > 0):
    #sd = numpy.std(num_arr, axis=0)
    #mean = numpy.mean(num_arr, axis=0)
    #var = numpy.var(num_arr, axis=0)

    #complete_data = data + [rep_factor,iterNum, mean, var, sd, sd/mean]
    #wr = csv.writer(fd_outputFile, quoting=csv.QUOTE_NONE, lineterminator='\n')
    #wr.writerow(complete_data)
    #iterNum_start += 1

    #Total_mean_compute += mean
    #Total_rsd_compute += sd/mean



  for iterNum in range(iterNum_start, int(iterationNum)):
    do_all_regex = re.compile(r'.*,\(NULL\),0\s,\sDO_ALL_IMPL_(?i)' + re.escape(benchmark) + r'_0_' + re.escape(str(iterNum))  +r',.*' + r',\d*,(\d*)')
    do_all_all_hosts = re.findall(do_all_regex, log_data)
    num_arr_tmp = numpy.array(map(int,do_all_all_hosts))
    if(num_arr_tmp.size < int(hostNum) and iterNum == 0):
      num_arr = numpy.zeros(int(hostNum));
      for i in range(0, num_arr_tmp.size):
          num_arr[i] = num_arr_tmp[i]
    else:
      num_arr = num_arr_tmp
    print num_arr

    if(num_arr.size < int(hostNum)):
      print "SOME DATA IS MISSING\n"
      #sys.exit("aa! errors! SOME DATA MISSING IN THE LOG FILES!!")

    sd=0.0
    mean=0.0
    var=0.0
    try:
      if(num_arr.size > 0):
        sd = numpy.std(num_arr, axis=0)
        mean = numpy.mean(num_arr, axis=0)
        var = numpy.var(num_arr, axis=0)
    except ValueError:
      pass

    rsd = 0.0;
    if(mean > 0):
      rsd = sd/mean
    complete_data = data + [rep_factor,iterNum, mean, var, sd, rsd]
    wr = csv.writer(fd_outputFile, quoting=csv.QUOTE_NONE, lineterminator='\n')
    wr.writerow(complete_data)

    Total_mean_compute += mean
    Total_rsd_compute += rsd
    print ("MEAN : ", Total_mean_compute)
    print ("RSD : ", Total_rsd_compute)


  Total_mean_compute = round(Total_mean_compute,3)
  Total_rsd_compute = round(Total_rsd_compute/int(iterationNum),3)

  print ("Total_mean_compute : ", Total_mean_compute)
  print ("Total_rsd_compute : ", Total_rsd_compute)

  complete_data = data + [rep_factor,iterNum, Total_mean_compute, Total_rsd_compute]
  wr = csv.writer(fd_outputFile_main, quoting=csv.QUOTE_NONE, lineterminator='\n')
  wr.writerow(complete_data)
  fd_outputFile_main.close();
  fd_outputFile.close()

def get_basicInfo(fileName):

  hostNum_regex = re.compile(r'.*,\(NULL\),0\s,\sHosts,0,0,(\d*)')
  cmdLine_regex = re.compile(r'.*,\(NULL\),0\s,\sCommandLine,0,0,(.*)')
  threads_regex = re.compile(r'.*,\(NULL\),0\s,\sThreads,0,0,(\d*)')
  runs_regex = re.compile(r'.*,\(NULL\),0\s,\sRuns,0,0,(\d*)')
  num_itr_regex = re.compile(r'.*,\(NULL\),0\s,\sNUM_ITERATIONS_\d*,0,0,(\d*)')

  log_data = open(fileName).read()

  hostNum    = ''
  cmdLine    = ''
  threads    = ''
  runs       = ''
  benchmark  = ''
  variant  = ''
  cut_type   = ''
  input_graph = ''
  iterationNum = '' 

  hostNum_search = hostNum_regex.search(log_data)
  if hostNum_search is not None:
    hostNum = hostNum_search.group(1)

  cmdLine_search = cmdLine_regex.search(log_data)
  if cmdLine_search is not None:
    cmdLine = cmdLine_search.group(1)

  threads_search = threads_regex.search(log_data)
  if threads_search is not None:
    threads = threads_search.group(1)

  runs_search    = runs_regex.search(log_data)
  if runs_search is not None:
    runs = runs_search.group(1)
 
  num_itr_search = num_itr_regex.search(log_data)
  if num_itr_search is not None:
    iterationNum = num_itr_search.group(1)

  split_cmdLine_algo = cmdLine.split()[0].split("/")[-1].split("_")
  benchmark, variant =  split_cmdLine_algo

  split_cmdLine_input = cmdLine.split()[1].split("/")
  input_graph_name = split_cmdLine_input[-1]
  input_graph = input_graph_name.split(".")[0]

  split_cmdLine = cmdLine.split()
  cut_type = "edge-cut"
  for index in range(0, len(split_cmdLine)):
    if split_cmdLine[index] == "-enableVertexCut=1":
      cut_type = "vertex-cut"
      break
    elif split_cmdLine[index] == "-enableVertexCut":
         cut_type = "vertex-cut"
         break
    elif split_cmdLine[index] == "-enableVertexCut=0":
         cut_type = "edge-cut"
         break

  #cut_type = "edge-cut"
  #for index in range(0, len(split_cmdLine_input)):
    #if split_cmdLine_input[index] == "-enableVertexCut":
      #cut_type = "vertex-cut"
      #break

  devices = str(hostNum) + " CPU"
  deviceKind = "CPU"
  for index in range(2, len(cmdLine.split())):
    split_cmdLine_devices = cmdLine.split()[index].split("=")
    if split_cmdLine_devices[0] == '-pset':
      devices_str = split_cmdLine_devices[-1]
      cpus = devices_str.count('c')
      gpus = devices_str.count('g')
      if str(cpus + gpus) == hostNum and gpus > 0:
        if cpus == 0:
          devices = str(gpus) + " GPU"
          deviceKind = "GPU"
        else:
          devices = str(cpus) + " CPU + " + str(gpus) + " GPU"
          deviceKind = "CPU+GPU"
          hostNum = str(int(hostNum) - cpus)
      break

  return hostNum, cmdLine, threads, runs, benchmark, variant, cut_type, input_graph, devices, deviceKind, iterationNum




def main(argv):
  inputFile = ''
  forHost = '0'
  outputFile = 'LOG_output.csv'
  time_unit = 'seconds'
  try:
    opts, args = getopt.getopt(argv,"hi:n:o:md",["ifile=","node=","ofile=","milliseconds"])
  except getopt.GetoptError:
    print 'abelian_log_parser.py -i <inputFile> [-o <outputFile> -n <hostNumber 0 to hosts-1> --milliseconds]'
    sys.exit(2)
  for opt, arg in opts:
    if opt == '-h':
      print 'abelian_log_parser.py -i <inputFile> [-o <outputFile> -n <hostNumber 0 to hosts-1> --milliseconds]'
      sys.exit()
    elif opt in ("-i", "--ifile"):
      inputFile = arg
    elif opt in ("-n", "--node"):
      forHost = arg
    elif opt in ("-o", "--ofile"):
      outputFile = arg
    elif opt in ("-m", "--milliseconds"):
      time_unit = 'milliseconds'

  if inputFile == '':
    print 'abelian_log_parser_analysis.py -i <inputFile> [-o <outputFile> -n <hostNumber 0 to hosts-1> --milliseconds]'
    sys.exit(2)

  print 'Input file is : ', inputFile
  print 'Output file is : ', outputFile
  print 'Data for host : ', forHost

  hostNum, cmdLine, threads, runs, benchmark, variant, cut_type, input_graph, devices, deviceKind, iterationNum = get_basicInfo(inputFile)

  #shorten the graph names:
  if input_graph == "twitter-ICWSM10-component_withRandomWeights" or input_graph == "twitter-ICWSM10-component-transpose" or input_graph == "twitter-ICWSM10-component":
    input_graph = "twitter-50"
  elif input_graph == "twitter-WWW10-component_withRandomWeights" or input_graph == "twitter-WWW10-component-transpose" or input_graph == "twitter-WWW10-component":
    input_graph = "twitter-40"

  print 'Hosts : ', hostNum , ' CmdLine : ', cmdLine, ' Threads : ', threads , ' Runs : ', runs, ' benchmark :' , benchmark , ' variant :', variant, ' cut_type : ', cut_type, ' input_graph : ', input_graph, 'iterationNum :', iterationNum
  print 'Devices : ', devices


  header_csv_str = "variant,input,hosts,benchmark,partition,"
  header_csv_str += "deviceKind,devices,replication,iteration,mean,variance,sd,sdByMean"

  header_csv_str_mainfile = "variant,input,hosts,benchmark,partition,"
  header_csv_str_mainfile += "deviceKind,devices,replication,total_mean_compute,rsd_total"

  output_str = variant + ',' + input_graph + ',' + hostNum + ',' + benchmark + ','
  output_str += deviceKind  + ',' + devices  + ','


  header_csv_list = header_csv_str.split(',')
  header_csv_list_mainfile = header_csv_str_mainfile.split(',')

  outputFile_mainfile = outputFile
  outputFile = outputFile + ".csv"
  #if outputFile is empty add the header to the file
  try:
    if os.path.isfile(outputFile) is False:
      fd_outputFile = open(outputFile, 'wb')
      wr = csv.writer(fd_outputFile, quoting=csv.QUOTE_NONE, lineterminator='\n')
      wr.writerow(header_csv_list)
      fd_outputFile.close()
      print "Adding header to the empty file."
    else:
      print "outputFile : ", outputFile, " exists, results will be appended to it."
  except OSError:
    print "Error in outfile opening\n"

  outputFile_mainfile = outputFile_mainfile + "_main.csv"
  try:
    if os.path.isfile(outputFile_mainfile) is False:
      fd_outputFile = open(outputFile_mainfile, 'wb')
      wr = csv.writer(fd_outputFile, quoting=csv.QUOTE_NONE, lineterminator='\n')
      wr.writerow(header_csv_list_mainfile)
      fd_outputFile.close()
      print "Adding header to the empty file."
    else:
      print "outputFile_mainfile : ", outputFile_mainfile, " exists, results will be appended to it."
  except OSError:
    print "Error in outfile opening\n"




  sd_iterations(inputFile, outputFile, outputFile_mainfile, benchmark, runs, time_unit, hostNum, iterationNum, variant, input_graph, deviceKind, devices, cut_type)
'''
  data_list = list(data) #[data] #list(data)
  #data_list.extend((total_SendBytes, total_SendBytes_pull_sync, total_SendBytes_pull_reply, total_SendBytes_push_sync))
  complete_data = output_str.split(",") + data_list + [rep_factor]#+ list(sendBytes_list)
  fd_outputFile = open(outputFile, 'a')
  wr = csv.writer(fd_outputFile, quoting=csv.QUOTE_NONE, lineterminator='\n')
  wr.writerow(complete_data)
  fd_outputFile.close()
'''
'''
  ## Write ghost and slave nodes to a file.
  ghost_array = build_master_ghost_matrix(inputFile, benchmark, cut_type, hostNum, runs, threads)
  ghostNodes_file = outputFile + "_" + cut_type
  fd_ghostNodes_file = open(ghostNodes_file, 'ab')
  fd_ghostNodes_file.write("\n--------------------------------------------------------------\n")
  fd_ghostNodes_file.write("\nHosts : " + hostNum + "\nInputFile : "+ inputFile + "\nBenchmark: " + benchmark + "\nPartition: " + cut_type + "\n\n")
  numpy.savetxt(fd_ghostNodes_file, ghost_array, delimiter=',', fmt='%d')
  fd_ghostNodes_file.write("\n--------------------------------------------------------------\n")
  fd_ghostNodes_file.close()
'''

if __name__ == "__main__":
  main(sys.argv[1:])

