##########################################
# To parse log files generated by abelian.
# Author: Gurbinder Gill
# Email: gurbinder533@gmail.com
#########################################

import re
import os
import sys, getopt
import csv
import numpy

######## NOTES:
# All time values are in sec by default.


def match_timers(fileName, benchmark, forHost, numRuns, numThreads, time_unit, total_hosts, partition):

  mean_time = 0.0;
  recvNum_total = 0
  recvBytes_total = 0
  sendNum_total = 0
  sendBytes_total = 0
  sync_pull_avg_time_total = 0.0;
  extract_avg_time_total = 0.0;
  set_avg_time_total = 0.0;
  sync_push_avg_time_total = 0.0;
  graph_init_time = 0
  hg_init_time = 0
  total_time = 0

  if(benchmark == "cc"):
    benchmark = "ConnectedComp"

  if (time_unit == 'seconds'):
    divisor = 1000
  else:
    divisor = 1
  #e2901bc2-f648-4ff4-9976-ac3b4c794a6a,(NULL),0 , TIMER_2,7,0,79907
  timer_regex = re.compile(r'.*,\(NULL\),0\s,\sTIMER_(\d*),(\d*),0,(\d*)')
  #timer_regex = re.compile(r'.*,\(NULL\),0\s,\sTIMER_(\d*),7,\d*,(\d*)')

  log_data = open(fileName).read()

  timers = re.findall(timer_regex, log_data)
  print timers

  time = []
  for i in range(int(numRuns)):
    time.append(0)
  for timer in timers:
    run_num = int(timer[0])
    host = int(timer[1])
    host_time = float(timer[2])
    if time[run_num] < host_time:
      time[run_num] = host_time
  for i in range(int(numRuns)):
    mean_time = mean_time + time[i]
  if(len(time) > 0):
    mean_time /= int(numRuns) 
  mean_time /= divisor
  mean_time = round(mean_time, 3)
  print "Mean time: ", mean_time

  #total_cpu_do_all_impl = 0.0
  #max_cpu_do_all_impl = 0.0;
  #min_cpu_do_all_impl = sys.maxint;
  #for host in range(int(total_hosts)):
  #  cpu_do_all_impl_regex = re.compile(r'.*,\(NULL\),0\s,\sDO_ALL_IMPL_(?i)' + re.escape(benchmark) + r'_..*,'+ re.escape(str(host)) + r',\d*,(\d*)')
  #  cpu_do_all_impl_per_host = re.findall(cpu_do_all_impl_regex, log_data)
  #  time_per_host = 0.0
  #  for cpu_do_all_time in cpu_do_all_impl_per_host:
  #    time_per_host += float(cpu_do_all_time)
  #    #print time_per_host
  #  time_per_host /= int(numRuns)
  #  total_cpu_do_all_impl += time_per_host
  #  if(max_cpu_do_all_impl < time_per_host):
  #    max_cpu_do_all_impl = time_per_host
  #  if(min_cpu_do_all_impl > time_per_host):
  #    min_cpu_do_all_impl = time_per_host
  #total_cpu_do_all_impl /= divisor
  #total_cpu_do_all_impl = round(total_cpu_do_all_impl, 3)
  #mean_cpu_do_all_impl = total_cpu_do_all_impl/int(total_hosts)
  #mean_cpu_do_all_impl = round(mean_cpu_do_all_impl, 3)
  #max_cpu_do_all_impl /= divisor
  #max_cpu_do_all_impl = round(max_cpu_do_all_impl, 3)
  #min_cpu_do_all_impl /= divisor
  #min_cpu_do_all_impl = round(min_cpu_do_all_impl, 3)
  #print "total_cpu_do_all : ", total_cpu_do_all_impl
  #print "mean_cpu_do_all : ", mean_cpu_do_all_impl

  #total_cuda_do_all_impl = 0.0
  #max_cuda_do_all_impl = 0.0;
  #min_cuda_do_all_impl = sys.maxint;
  #for host in range(int(total_hosts)):
  #  cuda_do_all_impl_regex = re.compile(r'.*,\(NULL\),0\s,\sCUDA_DO_ALL_IMPL_(?i)' + re.escape(benchmark) + r'_..*,'+ re.escape(str(host)) + r',\d*,(\d*)')
  #  cuda_do_all_impl_per_host = re.findall(cuda_do_all_impl_regex, log_data)
  #  time_per_host = 0.0
  #  for cuda_do_all_time in cuda_do_all_impl_per_host:
  #    time_per_host += float(cuda_do_all_time)
  #    #print time_per_host
  #  time_per_host /= int(numRuns)
  #  total_cuda_do_all_impl += time_per_host
  #  if(max_cuda_do_all_impl < time_per_host):
  #    max_cuda_do_all_impl = time_per_host
  #  if(min_cuda_do_all_impl > time_per_host):
  #    min_cuda_do_all_impl = time_per_host
  #total_cuda_do_all_impl /= divisor
  #total_cuda_do_all_impl = round(total_cuda_do_all_impl, 3)
  #mean_cuda_do_all_impl = total_cuda_do_all_impl/int(total_hosts)
  #mean_cuda_do_all_impl = round(mean_cuda_do_all_impl, 3)
  #max_cuda_do_all_impl /= divisor
  #max_cuda_do_all_impl = round(max_cuda_do_all_impl, 3)
  #min_cuda_do_all_impl /= divisor
  #min_cuda_do_all_impl = round(min_cuda_do_all_impl, 3)
  #print "total_cuda_do_all : ", total_cuda_do_all_impl
  #print "mean_cuda_do_all : ", mean_cuda_do_all_impl

  #TOTAL_DO_ALL_IMPL all hosts
  #414c1fb5-0df1-4741-a0ee-cee82f2fc83b,(NULL),0 , DO_ALL_IMPL_bfs,0,0,389
  total_do_all_impl = 0.0
  max_do_all_impl = 0.0;
  min_do_all_impl = sys.maxint;
  #6d3d9407-ed61-4fc9-a4ee-dd9af891b47a,(NULL),0 , DO_ALL_IMPL_BFS_0_1,0,0,5145
  #35537c51-6ba3-47aa-afa0-72edec803b75,(NULL),0 , DO_ALL_IMPL_bfs,3,0,41104
  #0d0cd9b5-f61d-4bd7-963d-f5d129cd711e,(NULL),0 , DO_ALL_IMPL_BFS_0_1,0,0,8007
  for host in range(int(total_hosts)):
    do_all_impl_regex = re.compile(r'.*,\(NULL\),0\s,\s.*DO_ALL_IMPL_(?i)' + re.escape(benchmark) + r'_..*,'+ re.escape(str(host)) + r',\d*,(\d*)')
    do_all_impl_per_host = re.findall(do_all_impl_regex, log_data)
    #print do_all_impl_all
    time_per_host = 0.0
    #print "----> ", do_all_impl_per_host
    for do_all_time in do_all_impl_per_host:
      if (do_all_time != ""):
        time_per_host += float(do_all_time)
    if time_per_host == 0.0:
      continue
    time_per_host /= len(do_all_impl_per_host) 
    total_do_all_impl += time_per_host
    if(max_do_all_impl < time_per_host):
      max_do_all_impl = time_per_host
    if(min_do_all_impl > time_per_host):
      min_do_all_impl = time_per_host
  total_do_all_impl /= divisor
  total_do_all_impl = round(total_do_all_impl, 3)
  mean_do_all_impl = total_do_all_impl/int(total_hosts)
  mean_do_all_impl = round(mean_do_all_impl, 3)
  max_do_all_impl /= divisor
  max_do_all_impl = round(max_do_all_impl, 3)
  min_do_all_impl /= divisor
  min_do_all_impl = round(min_do_all_impl, 3)
  #print "total_do_all : ", total_do_all_impl
  #print "mean_do_all : ", mean_do_all_impl
  #print "max_do_all : ", max_do_all_impl
  #print "min_do_all : ", min_do_all_impl

  total_comm_time = 0.0
  max_comm_time = 0.0;
  min_comm_time = sys.maxint;
  for host in range(int(total_hosts)):
    comm_time_regex = re.compile(r'.*,\(NULL\),0\s,\sSYNC_PU.._(?i)' + re.escape(benchmark) + r'_..*,'+ re.escape(str(host)) + r',\d*,(\d*)')
    #comm_time_regex = re.compile(r'.*,\(NULL\),0\s,\s.*DO_ALL_IMPL_(?i)' + re.escape(benchmark) + r'_..*,'+ re.escape(str(host)) + r',\d*,(\d*)')
    comm_time_per_host = re.findall(comm_time_regex, log_data)
    #print comm_time_per_host
    time_per_host = 0.0
    #print "----> ", comm_time_per_host
    for comm_time in comm_time_per_host:
      if (comm_time != ""):
        time_per_host += float(comm_time)
    if time_per_host == 0.0:
      continue
    time_per_host /= len(comm_time_per_host) 
    total_comm_time += time_per_host
    if(max_comm_time < time_per_host):
      max_comm_time = time_per_host
    if(min_comm_time > time_per_host):
      min_comm_time = time_per_host
  total_comm_time /= divisor
  total_comm_time = round(total_comm_time, 3)
  mean_comm_time = total_comm_time/int(total_hosts)
  mean_comm_time = round(mean_comm_time, 3)
  max_comm_time /= divisor
  max_comm_time = round(max_comm_time, 3)
  min_comm_time /= divisor
  min_comm_time = round(min_comm_time, 3)
  #print "total_comm_time : ", total_comm_time
  #print "mean_comm_time : ", mean_comm_time
  #print "max_comm_time : ", max_comm_time
  #print "min_comm_time : ", min_comm_time

  total_send_bytes = 0;
  #6d3d9407-ed61-4fc9-a4ee-dd9af891b47a,BFS,0 , SEND_BYTES_SYNC_PULL,0,0,4209914580
  if(partition == "edge-cut"):
    send_bytes_regex = re.compile(r'.*,\(NULL\),0\s,\sSEND_BYTES_SYNC_(PUSH|PULL)_(?i)' + re.escape(benchmark) + r'.*,\d*,\d*,(\d*)')
    send_bytes_firstItr_regex = re.compile(r'.*,\(NULL\),0\s,\sSEND_BYTES_SYNC_(PUSH|PULL)_FirstItr_(?i)' + re.escape(benchmark) + r'.*,\d*,\d*,(\d*)')

    send_bytes_host = re.findall(send_bytes_regex, log_data)
    send_bytes_firstItr_host = re.findall(send_bytes_firstItr_regex, log_data)

    for byte in send_bytes_host:
      if (byte[1] != ""):
        total_send_bytes += (int(byte[1]))
      #print("->", byte[0], " , " , byte[1])
      #print("->", byte_firstItr[0], " , " , byte_firstItr[1])

    for byte_firstItr in send_bytes_firstItr_host:
      if (byte_firstItr[1] != ""):
        total_send_bytes += int(byte_firstItr[1])

    total_send_bytes /= int(numRuns)

  elif(partition == "vertex-cut"):
    #35537c51-6ba3-47aa-afa0-72edec803b75,BFS,0 , SEND_BYTES_SYNC_PULL,0,0,4210027620
    send_bytes_regex = re.compile(r'.*,(?i)' + re.escape(benchmark) + r',0\s,\sSEND_BYTES_SYNC_(PUSH|PULL),\d*,\d*,(\d*)')
    send_bytes_firstItr_regex = re.compile(r'.*,FirstItr_(?i)' + re.escape(benchmark) + r',0\s,\sSEND_BYTES_SYNC_(PUSH|PULL),\d*,\d*,(\d*)')

    send_bytes_host = re.findall(send_bytes_regex, log_data)
    send_bytes_firstItr_host = re.findall(send_bytes_firstItr_regex, log_data)

    for byte in send_bytes_host:
      if (byte[1] != ""):
        total_send_bytes += (int(byte[1]))
      #print("->", byte[0], " , " , byte[1])
      #print("->", byte_firstItr[0], " , " , byte_firstItr[1])

    for byte_firstItr in send_bytes_firstItr_host:
      if (byte_firstItr[1] != ""):
        total_send_bytes += int(byte_firstItr[1])

    total_send_bytes /= int(numRuns)

  ## SYNC_PULL and SYNC_PUSH total average over runs.
  #num_iterations = 0
  #for i in range(0, int(numRuns)):
    # find extract
    #extract_regex = re.compile(r'\[' + re.escape(forHost) + r'\]STAT,\(NULL\),SYNC_EXTRACT_(?i)' + re.escape(benchmark) + r'\w*_' + re.escape(str(i)) + r'_(\d*),\d*,(\d*),(\d*).*')
    #extract_lines = re.findall(extract_regex, log_data)
    #for j in range (0, len(extract_lines)):
      #extract_avg_time_total += float(extract_lines[j][2])

    # find set
    #set_regex = re.compile(r'\[' + re.escape(forHost) + r'\]STAT,\(NULL\),SYNC_SET_(?i)' + re.escape(benchmark) + r'\w*_' + re.escape(str(i)) + r'_(\d*),\d*,(\d*),(\d*).*')
    #set_lines = re.findall(set_regex, log_data)
    #for j in range (0, len(set_lines)):
      #set_avg_time_total += float(set_lines[j][2])

    # find sync_pull
    #sync_pull_regex = re.compile(r'\[' + re.escape(forHost) + r'\]STAT,\(NULL\),SYNC_PULL_(?i)' + re.escape(benchmark) + r'\w*_' + re.escape(str(i)) + r'_(\d*),\d*,(\d*),(\d*).*')
    #sync_pull_lines = re.findall(sync_pull_regex, log_data)
    #num_iterations = len(sync_pull_lines);
    #for j in range (0, len(sync_pull_lines)):
      #sync_pull_avg_time_total += float(sync_pull_lines[j][2])

    # find sync_push
    #sync_push_regex = re.compile(r'\[' + re.escape(forHost) + r'\]STAT,\(NULL\),SYNC_PUSH_(?i)' + re.escape(benchmark) + r'\w*_'+ re.escape(str(i)) + r'_(\d*),\d*,(\d*),(\d*).*')
    #sync_push_lines = re.findall(sync_push_regex, log_data)

    #if(num_iterations == 0):
    #  num_iterations = len(sync_push_lines)

    #for j in range (0, len(sync_push_lines)):
    #  sync_push_avg_time_total += float(sync_push_lines[j][2])

  #extract_avg_time_total /= int(numRuns)
  #extract_avg_time_total /= divisor
  #extract_avg_time_total = round(extract_avg_time_total, 0)

  #set_avg_time_total /= int(numRuns)
  #set_avg_time_total /= divisor
  #set_avg_time_total = round(set_avg_time_total, 0)

  #sync_pull_avg_time_total /= int(numRuns)
  #sync_pull_avg_time_total /= divisor
  #sync_pull_avg_time_total = round(sync_pull_avg_time_total, 0)

  #sync_push_avg_time_total /= int(numRuns)
  #sync_push_avg_time_total /= divisor
  #sync_push_avg_time_total = round(sync_push_avg_time_total, 0)

  ## sendBytes and recvBytes.
  #recvBytes_regex = re.compile(r'\[' + re.escape(forHost) + r'\]STAT,\(NULL\),RecvBytes,\d*,(\d*),(\d*),.*')
  #recvBytes_search = recvBytes_regex.search(log_data)
  #if recvBytes_search is not None:
     #recvBytes_total = float(recvBytes_search.group(1))/int(numRuns)

  #sendBytes_regex = re.compile(r'\[' + re.escape(forHost) + r'\]STAT,\(NULL\),SendBytes,\d*,(\d*),(\d*),.*')
  #sendBytes_search = sendBytes_regex.search(log_data)
  #if sendBytes_search is not None:
    #sendBytes_total = float(sendBytes_search.group(1))/int(numRuns)

  ## sendNum and recvNum.
  #recvNum_regex = re.compile(r'\[' + re.escape(forHost) + r'\]STAT,\(NULL\),RecvNum,\d*,(\d*),(\d*),.*')
  #recvNum_search = recvNum_regex.search(log_data)
  #if recvNum_search is not None:
    #recvNum_total = float(recvNum_search.group(1))/int(numRuns)

  #sendNum_regex = re.compile(r'\[' + re.escape(forHost) + r'\]STAT,\(NULL\),SendNum,\d*,(\d*),(\d*),.*')
  #sendNum_search = sendNum_regex.search(log_data)
  #if sendNum_search is not None:
    #sendNum_total = float(sendNum_search.group(1))/int(numRuns)

  ## Get Graph_init, HG_init, total
  timer_graph_init_regex = re.compile(r'\[' + re.escape(forHost) + r'\]STAT,\(NULL\),TIMER_GRAPH_INIT,' + re.escape(numThreads) + r',(\d*),(\d*).*')
  timer_hg_init_regex = re.compile(r'\[' + re.escape(forHost) + r'\]STAT,\(NULL\),TIMER_HG_INIT,' + re.escape(numThreads) + r',(\d*),(\d*).*')
  timer_total_regex = re.compile(r'\[' + re.escape(forHost) + r'\]STAT,\(NULL\),TIMER_TOTAL,' + re.escape(numThreads) + r',(\d*),(\d*).*')


  timer_graph_init = timer_graph_init_regex.search(log_data)
  timer_hg_init = timer_hg_init_regex.search(log_data)
  timer_total = timer_total_regex.search(log_data)

  if timer_graph_init is not None:
    graph_init_time = float(timer_graph_init.group(1))
    graph_init_time /= divisor
    graph_init_time = round(graph_init_time, 0)

  if timer_hg_init is not None:
    hg_init_time = float(timer_hg_init.group(1))
    hg_init_time /= divisor
    hg_init_time = round(hg_init_time, 0)

  if timer_total is not None:
    total_time = float(timer_total.group(1))
    total_time /= divisor
    total_time = round(total_time, 0)

  ## Get Commits, Conflicts, Iterations, Pushes for worklist versions:
  commits_search = re.compile(r'\[' + re.escape(forHost) + r'\]STAT,(?i)' + re.escape(benchmark) + '\w*,Commits,' + re.escape(numThreads) + r',(\d*),(\d*).*').search(log_data)
  conflicts_search = re.compile(r'\[' + re.escape(forHost) + r'\]STAT,(?i)' + re.escape(benchmark) + r'\w*,Conflicts,' + re.escape(numThreads) + r',(\d*),(\d*).*').search(log_data)
  iterations_search = re.compile(r'\[' + re.escape(forHost) + r'\]STAT,(?i)' + re.escape(benchmark) + r'\w*,Iterations,' + re.escape(numThreads) + r',(\d*),(\d*).*').search(log_data)
  pushes_search = re.compile(r'\[' + re.escape(forHost) + r'\]STAT,(?i)' + re.escape(benchmark) + r'\w*,Pushes,' + re.escape(numThreads) + r',(\d*),(\d*).*').search(log_data)

  commits    = 0
  conflicts  = 0
  iterations = 0
  pushes     = 0
  if commits_search is not None:
    commits = int(commits_search.group(1))
    commits /= int(numRuns)
  if conflicts_search is not None:
    conflicts = int(conflicts_search.group(1))
    conflicts /= int(numRuns)
  if iterations_search is not None:
    iterations = int(iterations_search.group(1))
    iterations /= int(numRuns)
  if pushes_search is not None:
    pushes = int(pushes_search.group(1))
    pushes /= int(numRuns)

  #return mean_time,graph_init_time,hg_init_time,total_time,sync_pull_avg_time_total,sync_push_avg_time_total,recvNum_total,recvBytes_total,sendNum_total,sendBytes_total,commits,conflicts,iterations, pushes
  #return mean_time,graph_init_time,hg_init_time,total_time,extract_avg_time_total,set_avg_time_total,sync_pull_avg_time_total,sync_push_avg_time_total,num_iterations,commits,conflicts,iterations, pushes
  return mean_time,total_do_all_impl,mean_do_all_impl,max_do_all_impl,min_do_all_impl,total_comm_time,mean_comm_time,max_comm_time,min_comm_time,total_send_bytes


def sendRecv_bytes_all(fileName, benchmark, total_hosts, numRuns, numThreads):
  sendBytes_list = [0]*256 #Max host number is 256
  recvBytes_list = [0]*256 #Max host number is 256

  log_data = open(fileName).read()

  if(benchmark == "cc"):
    benchmark = "ConnectedComp"

  ## sendBytes and recvBytes.
  total_SendBytes = 0;
  for host in range(0,int(total_hosts)):
    sendBytes_regex = re.compile(r'\[' + re.escape(str(host)) + r'\]STAT,\(NULL\),SendBytes,\d*,(\d*),(\d*),.*')
    sendBytes_search = sendBytes_regex.search(log_data)
    if sendBytes_search is not None:
      sendBytes_list[host] = float(sendBytes_search.group(1))/int(numRuns)

  total_SendBytes = sum(sendBytes_list)

  total_RecvBytes = 0;
  for host in range(0,int(total_hosts)):
    recvBytes_regex = re.compile(r'\[' + re.escape(str(host)) + r'\]STAT,\(NULL\),RecvBytes,\d*,(\d*),(\d*),.*')
    recvBytes_search = recvBytes_regex.search(log_data)
    if recvBytes_search is not None:
       recvBytes_list[host] = float(recvBytes_search.group(1))/int(numRuns)

  total_RecvBytes = sum(recvBytes_list)
  return total_SendBytes, sendBytes_list



def sendBytes_syncOnly(fileName, benchmark, total_hosts, numRuns, numThreads):
  sendBytes_total_list = [0]*256 #Max host number is 256
  sendBytes_pull_sync_list = [0]*256 #Max host number is 256
  sendBytes_push_sync_list = [0]*256 #Max host number is 256
  sendBytes_pull_sync_reply_list = [0]*256 #Max host number is 256

  log_data = open(fileName).read()

  if(benchmark == "cc"):
    benchmark = "ConnectedComp"

  ## sendBytes from sync_pull.
  total_SendBytes_pull_sync = 0;
  for host in range(0,int(total_hosts)):
    sendBytes_sync_pull_regex = re.compile(r'\[' + re.escape(str(host)) + r'\]STAT,\(NULL\),SEND_BYTES_SYNC_PULL_(?i)'+ re.escape(benchmark) + r'_0_\d*,\d*,(\d*),(\d*),.*')
    sendBytes_sync_pull_lines = re.findall(sendBytes_sync_pull_regex, log_data)
    print sendBytes_sync_pull_lines

    if len(sendBytes_sync_pull_lines) > 0:
      sendBytes_pull_sync_list[host] = float(sendBytes_sync_pull_lines[0][0]) * len(sendBytes_sync_pull_lines)
      sendBytes_total_list[host] += sendBytes_pull_sync_list[host]
      print "-------> : ", host , " val : " , sendBytes_pull_sync_list[host]

  total_SendBytes_pull_sync = sum(sendBytes_pull_sync_list)

  ## sendBytes from sync_pull_reply.
  total_SendBytes_pull_reply = 0;
  for host in range(0,int(total_hosts)):
    sendBytes_sync_pull_reply_regex = re.compile(r'\[' + re.escape(str(host)) + r'\]STAT,\(NULL\),SEND_BYTES_SYNC_PULL_REPLY_(?i)'+ re.escape(benchmark) + r'_0_\d*,\d*,(\d*),(\d*),.*')
    sendBytes_sync_pull_reply_lines = re.findall(sendBytes_sync_pull_reply_regex, log_data)
    print sendBytes_sync_pull_reply_lines

    if len(sendBytes_sync_pull_reply_lines) > 0:
      sendBytes_pull_sync_reply_list[host] = float(sendBytes_sync_pull_reply_lines[0][0]) * len(sendBytes_sync_pull_reply_lines)
      sendBytes_total_list[host] += sendBytes_pull_sync_reply_list[host]
      #print "-------> : ", host , " val : " , sendBytes_pull_sync_reply_list[host]

  total_SendBytes_pull_reply = sum(sendBytes_pull_sync_reply_list)

  #[2]STAT,(NULL),SEND_BYTES_SYNC_PUSH_BFS_0_0,15,33738828,33738828,0,0,0,0,0,0,0,0,0,0,0,0,0,0
   ## sendBytes from sync_push.
  total_SendBytes_push_sync = 0;
  for host in range(0,int(total_hosts)):
    sendBytes_sync_push_regex = re.compile(r'\[' + re.escape(str(host)) + r'\]STAT,\(NULL\),SEND_BYTES_SYNC_PUSH_(?i)'+ re.escape(benchmark) + r'_0_\d*,\d*,(\d*),(\d*),.*')
    sendBytes_sync_push_lines = re.findall(sendBytes_sync_push_regex, log_data)
    print sendBytes_sync_push_lines

    if len(sendBytes_sync_push_lines) > 0:
      sendBytes_push_sync_list[host] = float(sendBytes_sync_push_lines[0][0]) * len(sendBytes_sync_push_lines)
      sendBytes_total_list[host] += sendBytes_push_sync_list[host]
      #print "-------> : ", host , " val : " , sendBytes_push_sync_list[host]

  total_SendBytes_push_sync = sum(sendBytes_push_sync_list)

  total_SendBytes = total_SendBytes_pull_sync + total_SendBytes_pull_reply + total_SendBytes_push_sync

  return total_SendBytes, total_SendBytes_pull_sync, total_SendBytes_pull_reply, total_SendBytes_push_sync, sendBytes_total_list



def replication_factor(fileName, benchmark, partition, total_hosts, numRuns, numThreads, input_graph):
  log_data = open(fileName).read()
  total_nodes = 0
  if(input_graph == "rmat28"):
    total_nodes = 268435456
  elif(input_graph == "twitter-50"):
    total_nodes = 51161011
  elif(input_graph == "rmat25"):
    total_nodes = 33554432
  elif(input_graph == "twitter-40"):
    total_nodes = 41652230
  else:
    return 0

  print "total_nodes : ", total_nodes
  rep_regex = re.compile(r'.*,\(NULL\),0\s,\sREPLICATION_FACTOR_0_0,(\d*),\d*,(.*)')

  rep_search = rep_regex.search(log_data)
  if rep_search is not None:
    rep_factor = rep_search.group(2)
    rep_factor = round(float(rep_factor), 3)
    print ("FOUND  : ", rep_factor)
    return rep_factor

  if partition == "edge-cut":
    total_ghost = 0
    #7fee06cb-4c74-458f-a761-ddf6997a1edd,(NULL),0 , TotalGhostNodes,3,0,28215509
    ghost_from_re = re.compile(r'.*,\(NULL\),0\s,\sTotalGhostNodes,(\d*),\d*,(\d*)')
    ghost_from_lines = re.findall(ghost_from_re, log_data)
    for line in ghost_from_lines:
      #print int(line[1])
      total_ghost += int(line[1])

    rep_factor = float(total_nodes + total_ghost)/float(total_nodes)
    rep_factor = round(rep_factor, 3)
    return rep_factor
  elif partition == "vertex-cut" or partition == "vertex-cut-balanced":
    total_slave = 0
    #8190584f-391e-45ca-9d3b-bf1d0d682fad,(NULL),0 , SLAVE_NODES_FROM_0,0,0,83207225
    slave_from_re = re.compile(r'.*,\(NULL\),0\s,\sSLAVE_NODES_FROM_(\d*),(\d*),\d*,(\d*)')
    slave_from_lines = re.findall(slave_from_re, log_data)
    for line in slave_from_lines:
      #print "v", int(line[2])
      total_slave += int(line[2])

    rep_factor = float(float(total_slave)/float(total_nodes))
    rep_factor = round(rep_factor, 3)
    return rep_factor



def build_master_ghost_matrix(fileName, benchmark, partition, total_hosts, numRuns, numThreads):
  #[1]STAT,(NULL),GhostNodes_from_1,15,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
  log_data = open(fileName).read()
  if partition == "edge-cut":
    GhostNodes_array = numpy.zeros((int(total_hosts), int(total_hosts)))
    for host in range(0, int(total_hosts)):
      #(NULL),0 , GhostNodes_from_1,3,0,45865
      #ghost_from_re = re.compile(r'\[' + re.escape(str(host)) + r'\]STAT,\(NULL\),GhostNodes_from_(\d*),\d*,(\d*),.*')
      ghost_from_re = re.compile(r'\(NULL\),\d* , GhostNodes_from_(\d*),' + re.escape(str(host)) + r',\d*,(\d*)')
      ghost_from_lines = re.findall(ghost_from_re, log_data)
      if(len(ghost_from_lines) > 0):
        for line in ghost_from_lines:
          GhostNodes_array[host][int(line[0])] = int(line[1])
    return GhostNodes_array
  #[1]STAT,(NULL),SLAVE_NODES_FROM_0,15,21693895,21693895,0,0,0,0,0,0,0,0,0,0,0,0,0,0
  elif partition == "vertex-cut" or partition == "vertex-cut-balanced":
    SlaveNodes_array = numpy.zeros((int(total_hosts), int(total_hosts)))
    for host in range(0, int(total_hosts)):
      slave_from_re = re.compile(r'\[' + re.escape(str(host)) + r'\]STAT,\(NULL\),SLAVE_NODES_FROM_(\d*),\d*,(\d*),.*')
      slave_from_lines = re.findall(slave_from_re, log_data)
      if(len(slave_from_lines) > 0):
        for line in slave_from_lines:
          SlaveNodes_array[host][int(line[0])] = int(line[1])
    return SlaveNodes_array


#[0]STAT,(NULL),SYNC_PULL_BARRIER_BFS_0_1,15,992,992,0,0,0,0,0,0,0,0,0,0,0,0,0,0
#[0]STAT,(NULL),SYNC_PULL_BARRIER_BFS_0_2,15,538,538,0,0,0,0,0,0,0,0,0,0,0,0,0,0
#[0]STAT,(NULL),SYNC_PULL_BARRIER_BFS_0_3,15,1408,1408,0,0,0,0,0,0,0,0,0,0,0,0,0,0
#[0]STAT,(NULL),SYNC_PULL_BARRIER_BFS_1_1,15,1458,1458,0,0,0,0,0,0,0,0,0,0,0,0,0,0
#[0]STAT,(NULL),SYNC_PULL_BARRIER_BFS_1_2,15,1568,1568,0,0,0,0,0,0,0,0,0,0,0,0,0,0
#[0]STAT,(NULL),SYNC_PULL_BARRIER_BFS_1_3,15,2766,2766,0,0,0,0,0,0,0,0,0,0,0,0,0,0
def time_at_barrier(fileName, benchmark, total_hosts, numRuns, numThreads):
  log_data = open(fileName).read()
  thousand = 1000.0
  sync_pull_barrier_avg_time_total = [0.0]*256
  sync_pull_avg_time_total = [0.0]*256

  if(benchmark == "cc"):
    benchmark = "ConnectedComp"

  for host in range(0, int(total_hosts)):
      for i in range(0, int(numRuns)):
        # find sync_pull
        sync_pull_barrier_regex = re.compile(r'\[' + re.escape(str(host)) + r'\]STAT,\(NULL\),SYNC_PULL_BARRIER_(?i)' + re.escape(benchmark) + r'\w*_' + re.escape(str(i)) + r'_(\d*),\d*,(\d*),(\d*).*')
        sync_pull_barrier_lines = re.findall(sync_pull_barrier_regex, log_data)
        num_iterations = len(sync_pull_barrier_lines);
        for j in range (0, len(sync_pull_barrier_lines)):
          sync_pull_barrier_avg_time_total[host] += float(sync_pull_barrier_lines[j][2])

      sync_pull_barrier_avg_time_total[host] /= int(numRuns)
      sync_pull_barrier_avg_time_total[host] /= thousand

  for host in range(0, int(total_hosts)):
      for i in range(0, int(numRuns)):
        # find sync_pull
        sync_pull_regex = re.compile(r'\[' + re.escape(str(host)) + r'\]STAT,\(NULL\),SYNC_PULL_(?i)' + re.escape(benchmark) + r'\w*_' + re.escape(str(i)) + r'_(\d*),\d*,(\d*),(\d*).*')
        sync_pull_lines = re.findall(sync_pull_regex, log_data)
        num_iterations = len(sync_pull_lines);
        for j in range (0, len(sync_pull_lines)):
          sync_pull_avg_time_total[host] += float(sync_pull_lines[j][2])

      sync_pull_avg_time_total[host] /= int(numRuns)
      sync_pull_avg_time_total[host] /= thousand

  print sync_pull_barrier_avg_time_total
  print sync_pull_avg_time_total


#63719d90-126e-4bdb-87d2-b7d878a23abc,(NULL),0 , CommandLine,0,0,/work/02982/ggill0/Distributed_latest/build_dist_hetero/release_new_gcc/exp/apps/compiler_outputs/bfs_push-topological_edge-cut /scratch/01131/rashid/inputs/rmat16-2e28-a=0.57-b=0.19-c=0.19-d=0.05.rgr -startNode=155526494 -maxIterations=10000 -verify=0 -t=15
#63719d90-126e-4bdb-87d2-b7d878a23abc,(NULL),0 , Threads,0,0,15
#63719d90-126e-4bdb-87d2-b7d878a23abc,(NULL),0 , Hosts,0,0,4
#63719d90-126e-4bdb-87d2-b7d878a23abc,(NULL),0 , Runs,0,0,3

def get_basicInfo(fileName):

  hostNum_regex = re.compile(r'.*,\(NULL\),0\s,\sHosts,0,0,(\d*)')
  cmdLine_regex = re.compile(r'.*,\(NULL\),0\s,\sCommandLine,0,0,(.*)')
  threads_regex = re.compile(r'.*,\(NULL\),0\s,\sThreads,0,0,(\d*)')
  runs_regex = re.compile(r'.*,\(NULL\),0\s,\sRuns,0,0,(\d*)')

  log_data = open(fileName).read()

  hostNum    = ''
  cmdLine    = ''
  threads    = ''
  runs       = ''
  benchmark  = ''
  algo_type  = ''
  cut_type   = ''
  input_graph = ''

  hostNum_search = hostNum_regex.search(log_data)
  if hostNum_search is not None:
    hostNum = hostNum_search.group(1)

  cmdLine_search = cmdLine_regex.search(log_data)
  if cmdLine_search is not None:
    cmdLine = cmdLine_search.group(1)

  threads_search = threads_regex.search(log_data)
  if threads_search is not None:
    threads = threads_search.group(1)

  runs_search    = runs_regex.search(log_data)
  if runs_search is not None:
    runs = runs_search.group(1)
  if runs == "":
    runs = "3"
 
  split_cmdLine_algo = cmdLine.split()[0].split("/")[-1].split("_")
  benchmark, algo_type = split_cmdLine_algo

  split_cmdLine_input = cmdLine.split()[1].split("/")
  input_graph_name = split_cmdLine_input[-1]
  input_graph = input_graph_name.split(".")[0]
  cut_type = "edge-cut"
  for index in range(0, len(split_cmdLine_input)):
    if split_cmdLine_input[index] == "-enableVertexCut":
      cut_type = "vertex-cut"
      break

  devices = str(hostNum) + " CPU"
  deviceKind = "CPU"
  for index in range(2, len(cmdLine.split())):
    split_cmdLine_devices = cmdLine.split()[index].split("=")
    if split_cmdLine_devices[0] == '-pset':
      devices_str = split_cmdLine_devices[-1]
      cpus = devices_str.count('c')
      gpus = devices_str.count('g')
      if str(cpus + gpus) == hostNum and gpus > 0:
        if cpus == 0:
          devices = str(gpus) + " GPU"
          deviceKind = "GPU"
        else:
          devices = str(cpus) + " CPU + " + str(gpus) + " GPU"
          deviceKind = "CPU+GPU"
          hostNum = str(int(hostNum) - cpus)
      break

  return hostNum, cmdLine, threads, runs, benchmark, algo_type, cut_type, input_graph, devices, deviceKind

def format_str(col):
  max_len = 0
  for c in col:
    if max_len < len(str(c)):
      max_len = len(str(c))
  return max_len

def main(argv):
  inputFile = ''
  forHost = '0'
  outputFile = 'LOG_output.csv'
  time_unit = 'seconds'
  try:
    opts, args = getopt.getopt(argv,"hi:n:o:md",["ifile=","node=","ofile=","milliseconds"])
  except getopt.GetoptError:
    print 'abelian_log_parser.py -i <inputFile> [-o <outputFile> -n <hostNumber 0 to hosts-1> --milliseconds]'
    sys.exit(2)
  for opt, arg in opts:
    if opt == '-h':
      print 'abelian_log_parser.py -i <inputFile> [-o <outputFile> -n <hostNumber 0 to hosts-1> --milliseconds]'
      sys.exit()
    elif opt in ("-i", "--ifile"):
      inputFile = arg
    elif opt in ("-n", "--node"):
      forHost = arg
    elif opt in ("-o", "--ofile"):
      outputFile = arg
    elif opt in ("-m", "--milliseconds"):
      time_unit = 'milliseconds'

  if inputFile == '':
    print 'abelian_log_parser.py -i <inputFile> [-o <outputFile> -n <hostNumber 0 to hosts-1> --milliseconds]'
    sys.exit(2)

  print 'Input file is : ', inputFile
  print 'Output file is : ', outputFile
  print 'Data for host : ', forHost

  hostNum, cmdLine, threads, runs, benchmark, algo_type, cut_type, input_graph, devices, deviceKind = get_basicInfo(inputFile)

  #shorten the graph names:
  if input_graph == "twitter-ICWSM10-component_withRandomWeights" or input_graph == "twitter-ICWSM10-component-transpose" or input_graph == "twitter-ICWSM10-component":
    input_graph = "twitter-50"
  elif input_graph == "twitter-WWW10-component_withRandomWeights" or input_graph == "twitter-WWW10-component-transpose" or input_graph == "twitter-WWW10-component":
    input_graph = "twitter-40"

  print 'Hosts : ', hostNum , ' CmdLine : ', cmdLine, ' Threads : ', threads , ' Runs : ', runs, ' benchmark :' , benchmark , ' algo_type :', algo_type, ' cut_type : ', cut_type, ' input_graph : ', input_graph
  print 'Devices : ', devices
  data = match_timers(inputFile, benchmark, forHost, runs, threads, time_unit, hostNum, cut_type)
  rep_factor = replication_factor(inputFile, benchmark, cut_type, hostNum, runs, threads, input_graph)
  print "rep factor : " , rep_factor
  #total_SendBytes, sendBytes_list = sendRecv_bytes_all(inputFile, benchmark, hostNum, runs, threads)
  #total_SendBytes, total_SendBytes_pull_sync, total_SendBytes_pull_reply, total_SendBytes_push_sync, sendBytes_list = sendBytes_syncOnly(inputFile, benchmark, hostNum, runs, threads)
  print data

  output_str = benchmark + ',' + 'abelian' + ',' + hostNum  + ',' + threads  + ','
  output_str += deviceKind  + ',' + devices  + ','
  output_str += input_graph  + ',' + algo_type  + ',' + cut_type
  #time_at_barrier(inputFile, benchmark, forHost, runs, threads)

  #output_str = benchmark + ',' + 'abelian'  + ',' + hostNum  + ',' + threads  + ',' + input_graph  + ',' + algo_type  + ',' + cut_type

  #for d in data:
    #output_str += ','
    #output_str += str(d)
  print output_str


  header_csv_str = "benchmark,platform,host,threads,"
  header_csv_str += "deviceKind,devices,"
  header_csv_str += "input,variant,partition,mean_time,total_comp_time,mean_comp_time,max_comp_time,min_comp_time,total_comm_time,mean_comm_time,max_comm_time,min_comm_time,total_bytes_sent,rep_factor" #,graph_init_time,hg_init_time,total_time,extract_avg_time,set_avg_time,sync_pull_avg_time,sync_push_avg_time,converge_iterations,commits,conflicts,iterations,pushes,total_sendBytes, total_sendBytes_pull_sync, total_sendBytes_pull_reply, total_sendBytes_push_sync"

  #for i in range(0,256):
    #header_csv_str += ","
    #header_csv_str += ("SB_" + str(i))

  header_csv_list = header_csv_str.split(',')
  #if outputFile is empty add the header to the file
  try:
    if os.path.isfile(outputFile) is False:
      fd_outputFile = open(outputFile, 'wb')
      wr = csv.writer(fd_outputFile, quoting=csv.QUOTE_NONE, lineterminator='\n')
      wr.writerow(header_csv_list)
      fd_outputFile.close()
      print "Adding header to the empty file."
    else:
      print "outputFile : ", outputFile, " exists, results will be appended to it."
  except OSError:
    print "Error in outfile opening\n"

  data_list = list(data) #[data] #list(data)
  #data_list.extend((total_SendBytes, total_SendBytes_pull_sync, total_SendBytes_pull_reply, total_SendBytes_push_sync))
  complete_data = output_str.split(",") + data_list + [rep_factor]#+ list(sendBytes_list)
  fd_outputFile = open(outputFile, 'a')
  wr = csv.writer(fd_outputFile, quoting=csv.QUOTE_NONE, lineterminator='\n')
  wr.writerow(complete_data)
  fd_outputFile.close()

'''
  ## Write ghost and slave nodes to a file.
  ghost_array = build_master_ghost_matrix(inputFile, benchmark, cut_type, hostNum, runs, threads)
  ghostNodes_file = outputFile + "_" + cut_type
  fd_ghostNodes_file = open(ghostNodes_file, 'ab')
  fd_ghostNodes_file.write("\n--------------------------------------------------------------\n")
  fd_ghostNodes_file.write("\nHosts : " + hostNum + "\nInputFile : "+ inputFile + "\nBenchmark: " + benchmark + "\nPartition: " + cut_type + "\n\n")
  numpy.savetxt(fd_ghostNodes_file, ghost_array, delimiter=',', fmt='%d')
  fd_ghostNodes_file.write("\n--------------------------------------------------------------\n")
  fd_ghostNodes_file.close()
'''

if __name__ == "__main__":
  main(sys.argv[1:])

