#!/usr/bin/perl -w
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
#
# Authors : David Verbeiren from Intel Corporation - July 2007
#           Xavier Simonart from Intel Corporation - July 2007
#           Philippe Lecluse from Intel Corporation - July 2007
#           Patrice Buriez from Intel Corporation - December 2008
#
use Config;
use XML::Simple;
use IO::File;
use strict;

my $InputFile = "report.xml";

my $OsWin = 0;
# Defaults (Linux assumed)
if ($Config{'osname'} eq "MSWin32") {
  $OsWin = 1;
}

my $DO_COPY = 1;
my $DO_MERGE = 2;
my $DO_CLEAN = 4;

my $t_totmerge = 0;


sub GetMilli
{
  use Time::HiRes qw(gettimeofday);
  my ($seconds, $milliseconds) = gettimeofday;
  return ($seconds * 1000) + $milliseconds/1000;
}

sub get_number_cpu
{
  if ($OsWin == 1) {
    return 1;
  } else {
    my $n = 0;
    open(DATA, "cat /proc/cpuinfo |") || die ("Can't open /proc/cpuinfo");
    while(<DATA>)
    {
      chomp;
      if (/processor\s*:\s*(\d*)/) {
        $n++;
      }
    }
    return $n;
  }
}

sub merge_sipp_data
{
  my ($ts_ids, $out_fn, $ext) = @_;

  my $t0 = GetMilli();
  my @forked_pids = ();
  my ($i, $j, $k, $l, $pid, $fn, $max_fork);
  my $n_CPU = get_number_cpu();
  $| = 1;

  my $num_ts = @$ts_ids;
  print " Merging SIPp scenario stats for $num_ts SIPp instances (", join(",", @$ts_ids), ")\n";
  print " using $n_CPU CPUs...\n";

  if ($num_ts < $n_CPU) {
    $max_fork = $num_ts;
  } else {
    $max_fork = $n_CPU;
  }
  print(" $max_fork processes forked\n");

  # Split the files in smaller files
  for ($j=0;$j<$max_fork;$j++) {
    $pid = 0;
    if ($j < $max_fork-1) {
      $pid = fork;
      push(@forked_pids, $pid);
    }
    if ($pid == 0) {
      $i = 0; $k = 0; $l = 0;
      while($j+$max_fork*$l < $num_ts) {
        my $ts_id = $ts_ids->[$j+$max_fork*$l];
        $fn = "sipp_${ts_id}_$ext.csv";
        print "  Splitting $fn...\n";
        open(CSV, $fn) || die "Could not open $fn";
        $fn = "sipp_".($j)."_".($k)."_".($ext).".tmp";
        $k++;
        open(OUT, ">$fn") || die "Could not open $fn";
        while (<CSV>) {
          if ($i++ < 100000) {
            print(OUT "$_");
          } else {
            #print("In $fn...");
            if (close(OUT) <= 0) {
              print("Error closing file: $!\n");
              exit(0);
            }
            $fn = "sipp_".($j)."_".($k)."_".($ext).".tmp";
            $k++;
            open(OUT, ">$fn") || die "Could not open $fn";
            print(OUT "$_");
            $i = 0;
          }
        }
        if (close(OUT) <= 0) {
          print("Error closing file : $!\n");
          exit(0);
        }
        close(CSV);
        $l++;
      }
      if ($j < $max_fork-1) {
        exit(0);
      }
    }
  }
  #print(" Waiting for pid @forked_pids\n");
  foreach $pid (@forked_pids) {
    waitpid($pid, 0);
  }
  
  print(" Splitting done!\n");
  @forked_pids = ();

  # Build a table with file names
  opendir(DIR, ".");
  my @allFiles = ();
  my $file;
  while(defined ($file = readdir(DIR))) {
    if ($file =~ /(.+).tmp/) {
      push(@allFiles, $1);
    }
  }
  close(DIR);
  
  my $n = @allFiles;
  print(" $n files to sort\n");
  for ($i=0;$i<$n_CPU;$i++) {
    $pid = fork;
    if ($pid == 0) {
      $j = 0;
      while($j*$n_CPU+$i < @allFiles) {
        my @data = ();
        open(CSV, "$allFiles[$j*$n_CPU+$i]".".tmp") ||die "Could not open $allFiles[$j*$n_CPU+$i].tmp";        
        while (<CSV>) {
          my ($time, $data) = /^([0-9.]*);(.*)/g;
          my $el = {t => $time, d => $data};
          push(@data, $el);
        }
        close(CSV);
        my @sorted = sort { $a->{t} <=> $b->{t} } @data;
        $fn = "$allFiles[$j*$n_CPU+$i]";
        open(CSV, ">$fn") || die "Could not open $fn";
        foreach my $el (@sorted) {
          printf(CSV "%.3f;%s\n", $el->{t}, $el->{d});
        }
        close(CSV);
        unlink("$allFiles[$j*$n_CPU+$i]".".tmp");
        $j++;
       }
       exit(0);
    }
    push(@forked_pids, $pid);
  }
  #print(" Waiting for pid @forked_pids\n");
  foreach $pid (@forked_pids) {
    waitpid($pid, 0);
  }
  @forked_pids = ();

  my $nfiles = @allFiles;

  print(" $nfiles Files individually sorted; now merging\n");
  while($nfiles != 1) {
    print("  $nfiles remaining\n");
    for ($k=0;$k<$n_CPU;$k++) {
      $j = 0;
      # Execute last loop in main process - no need to launch an extra fork.
      $pid = 0;
      if ($k != $n_CPU-1) { 
        $pid = fork;
        push(@forked_pids, $pid);
      }
      if ($pid == 0) {
        my @file = ();
        my @candi = ();
        my @candi_data = ();
        my $i = 0;
        while(1) {
          if ($j*2*$n_CPU+2*$k+1 >= @allFiles) {
            if ($j*2*$n_CPU+2*$k < @allFiles) {
              rename($allFiles[$j*2*$n_CPU+2*$k], "$allFiles[$j*2*$n_CPU+2*$k].tmp".$nfiles);
            }
            if ($k != $n_CPU-1) {
              exit(0);
            } else {
              last;
            }
          }
          #print(" merging $allFiles[$j*2*$n_CPU+2*$k] and $allFiles[$j*2*$n_CPU+2*$k+1]\n");
          for ($i=0;$i<2;$i++) {
            # Open file            
            $file[$i] = IO::File->new();
            $fn = "$allFiles[$j*2*$n_CPU+2*$k+$i]";        
            $file[$i]->open("<$fn") || die "Could not reopen $fn";
            # Read first line of data
            if ($_ = readline($file[$i])) {
              my ($time, $data) =  /^([0-9.]*);(.*)/g;
              $candi[$i] = $time;
              $candi_data[$i] = $data;
            } else {
              $candi[$i] = -1;
            }
            if(!defined $candi[$i]) {
              printf("*****undefined!!!!!\n");
            }
          }
          my $done = 0;
          my $line;
          my $out = IO::File->new();
          $out->open(">$allFiles[$j*2*$n_CPU+2*$k].tmp".$nfiles);
          while (! $done) {
            # Compute min of all candidates
            my $where = 0;
            if(!defined $candi[1]) {
              printf("*****undefined!!!!!\n");
            }
            $where = 1 if (($candi[1] != -1) && (($candi[1] < $candi[0]) || ($candi[0] == -1)));
            if ($candi[$where] == -1) {
              $done = 1;
            } else {
              # Dump lowest
              print $out "$candi[$where];$candi_data[$where]\n";;
              # Load new value from the file from which the dumped value came
              if ($_ = readline($file[$where])) {
                my ($time, $data) =  /^([0-9.]*);(.*)/g;
                $candi[$where] = $time;
                $candi_data[$where] = $data;
              }
              else {
                $candi[$where] = -1;
                if ($file[$where]->close() <= 0) {
                  print("Error closing file: $!\n");
                  exit(0);
                }
              }
            }
          }
          if ($out->close() <= 0) {
            print("Error closing file: $!\n");
            exit(0);
          }
          $j++;
        }
      }
    } #
    #print(" Waiting for pid @forked_pids\n");
    foreach $pid (@forked_pids) {
      waitpid($pid, 0);
    }
    @forked_pids = ();
    foreach $file (@allFiles) {
      unlink($file);
    }
    # Build a table with file names
    opendir(DIR, ".");
    @allFiles = ();
    while(defined ($file = readdir(DIR))) {
      if ($file =~ /(.+).tmp$nfiles/) {
        push(@allFiles, $file);
      }
    }
    close(DIR);
    $nfiles = @allFiles;
  }
  if ($ext eq "scen") {
    rename($allFiles[0], "sipp.csv");
  } else {
    rename($allFiles[0], "sipp_retrans.csv");
  }
  my $t_merge = GetMilli();
  print " Merge time: ", $t_merge-$t0, "ms\n";
  $t_totmerge += $t_merge -$t0;
}

sub merge_files
{
  my $xmlData = shift;

  print("\nSorting and merging files...\n");

  my $config = $xmlData->{configuration};
  my ($i, $ip, @ts_ids);

  for ($i=0; $i < 100; $i++) {
    my $name = "TS$i";
    if ($config->{system}->{$name}) {
      my $system = $config->{system}->{$name};
      my $ip = $system->{ip};
      push(@ts_ids, $system->{id});
    }
  }

  merge_sipp_data(\@ts_ids, "sipp.csv", "scen");
  merge_sipp_data(\@ts_ids, "sipp_retrans.csv", "retrans");

  print ("Sorting and merging took $t_totmerge ms\n");
}

sub copy_files
{
  my $xmlData = shift;

  print("Copying files...\n");

  # Note: CPU & MEM data is collected by the manager communicating with cpumem tool on the SUT

  my $config = $xmlData->{configuration};
  my @all_machines = keys %{$config->{system}};
  foreach my $name (@all_machines) {
    if ($config->{system}->{$name}) {
      my $system = $config->{system}->{$name};
      my $ipv4 = $system->{ip};
      my $ip = $ipv4; $ip =~ s/IP4:\s*//;
      my $path = $system->{path};
      my $ts_id = $system->{id};
      my $cmd = "scp $ip:$path/sipp_".$ts_id."_*.csv .";
      print(" $cmd\n");
      if (system($cmd) < 0) {
        print("Error issuing $cmd \n");
        exit(0);
      }
    }
  }
}


sub clean_files
{
  my $xmlData = shift;

  print("Clean files...\n");

  my $config = $xmlData->{configuration};
  my @all_machines = keys %{$config->{system}};
  foreach my $name (@all_machines) {
    if ($config->{system}->{$name}) {
      my $system = $config->{system}->{$name};
      my $ts_id = $system->{id};
      my $cmd = "unlink sipp_".$ts_id."_scen.csv";
      print(" $cmd\n");
      system($cmd);
      $cmd = "unlink sipp_".$ts_id."_retrans.csv";
      print(" $cmd\n");
      system($cmd);
    }
  }
}


my $Mode = $DO_COPY | $DO_MERGE;  # By default, copy and merge
my $n_args = @ARGV;
for (my $i=0;$i<$n_args;$i++) {
  if ($ARGV[$i] eq "-r") {
    $i++;
    $InputFile = $ARGV[$i];
  } elsif ($ARGV[$i] eq "-copy") {
    $Mode = $DO_COPY;
  } elsif ($ARGV[$i] eq "-merge") {
    $Mode = $DO_MERGE;
  } elsif ($ARGV[$i] eq "-clean") {
    $Mode = $DO_CLEAN;
  } elsif ($ARGV[$i] =~ /-\?/) {
    #######01234567890123456789012345678901234567890123456789012345678901234567890123456789
    print("Copies - using scp - the SIPp result files from the Test Systems onto the local\n");
    print("system and merges the individual result files into a single file (sipp.csv)\n");
    print("that the doReport.pl script takes as input.\n");
    print("\n");
    print("Syntax: getResults.pl [-r <report_file>] [-copy|-merge|-clean]\n");
    print("\n");
    print(" -r specifies the raw benchmark report file (default: report.xml) generated by\n");
    print("    the manager during a benchmark run.\n");
    print(" -copy tells the script to only perform the copy of the SIPp result files and\n");
    print("    not the merge operation.\n");
    print("    (sipp.csv)\n");
    print(" -merge is the opposite of -copy and tells the script to assume the SIPp results\n");
    print("    have already been copied onto the local system and to only merge them.\n");
    print(" -clean remove intermediate files that are not longer required after a successful\n");
    print("    merge\n");
    print(" -? to get this help.\n");
    exit(0);
  } else {
    print("unknown parameter $ARGV[$i]\n");
    exit(1);
  }
}

print("Loading data from $InputFile\n");

my $xmlData = XMLin($InputFile, forcearray => [ qw(run step system scenario) ]);

if ($Mode & $DO_COPY) {
  copy_files($xmlData);
}
if ($Mode & $DO_MERGE) {
  merge_files($xmlData);
}
if ($Mode & $DO_CLEAN) {
  clean_files($xmlData);
}