~kiithsacmp/miniini/trunk

#!/usr/bin/python3

# Copyright (C) 2009-2010 Ferdinand Majerech
# This file is part of MiniINI
# For conditions of distribution and use, see copyright notice in LICENSE.txt

import subprocess
import getopt
import sys
import re
import time
import os.path
import configparser

output_dir = "testout"
#base name of output files
name = "testout"
#output a summary file?
summary_name = name + ".sum.ini"
#keep files generated by tests (e.g. cachegrind)?
keep_files = False
verbosity = 0
#print human readable summary to standard output?
human_readable = False

#run given command and return its return value
def run_cmd(cmd):
    if verbosity > 0:
        print (cmd)
    return subprocess.call(cmd, shell=True)

#does the given command (e.g a progam) exist?
def command_exists(command):
    if run_cmd("type " + command + " > /dev/null") == 0:
        return True
    else:
        return False

#return lines from file which contain given regexp
def grep(exp, infile):
    infile.seek(0)
    grepre = re.compile(exp)
    out = []
    for line in infile:
        if(grepre.search(line)):
            out.append(line)
    return out

#extract number strings from lines containing given regexp from given file
def extract_numbers(exp, infile):
    numre = re.compile("\d+\.?\d*")
    line = grep(exp, infile)[0]
    line = line.replace(",", "")
    return numre.findall(line)

#handles all test runs
class TestRunner:
    def __init__(self):
        print("Initializing TestRunner")
        self.__tests = []

    #runs all registered tests
    def run_tests(self):
        print("Running tests")
        for test in self.__tests:
            try:
                print("Running test:", test.name())
                test.run()
            except Exception as error:
                print("WARNING: Test failed: ", str(error))
                raise error
        if human_readable:
            print()
            print("----------")
            print("SUMMARY:")
            print("----------")
        with open(os.path.join(output_dir, summary_name), 'w') as sumfile:
            sumfile.write(";testutil.py run summary:\n")
        for test in self.__tests:
            test.write()

    #called by tests to register themselves
    def register(self, test):
        print("Registering test:", test.name())
        self.__tests.append(test)

test_runner = TestRunner()

#test run base class
class TestRun:
    def __init__(self):
        self._summary = configparser.ConfigParser()
        test_runner.register(self)

    def concurrent(self):
        raise NotImplementedError

    def name(self):
        raise NotImplementedError

    def run(self):
        raise NotImplementedError

    #write summary to file and, if wanted, to standard output
    def write(self):
        if human_readable:
            print()
            for section in self._summary.sections():
                print(section, ":")
                print()
                for tag in self._summary.options(section):
                    print("   ", tag," = ", self._summary.get(section, tag, raw=True))
        with open(os.path.join(output_dir, summary_name), "a+") as outfile:
            self._summary.write(outfile)

#base class for test runs based on commands that can be run once and processed
class TestRunCommand(TestRun):
    def __init__(self, log_file_name, command):
        #output of command will be redirected here
        self.__log_file_name = log_file_name
        self.__command = command
        TestRun.__init__(self)

    def run(self):
        run_cmd(self.__command)
        self._summary.add_section(self.name())
        with open(self.__log_file_name, "r") as self.infile:
            self.summarize(self.infile, self._summary)
        if not keep_files:
            run_cmd("rm " + self.__log_file_name)

    def extract_data(self, tag, grep, index):
        try:
            self._summary.set(self.name(), tag,
                              extract_numbers(grep, self.infile)[index])
        except NameError:
            print("ERROR: calling extract_data when self.inifile is nonexistent")

    def summarize(self, infile, summary):
        raise NotImplementedError

class MemCheck(TestRunCommand):
    def __init__(self, tested_command):
        if not command_exists("valgrind"):
            print("WARNING: valgrind not installed, cannot use memcheck")
        else:
            log_file_name = os.path.join(output_dir, name + "-memcheck.txt")
            command = "valgrind " + tested_command + " > " + log_file_name + " 2>&1"
            TestRunCommand.__init__(self, log_file_name, command)

    def concurrent(self):
        return True

    def name(self):
        return "memcheck"
    def summarize(self, infile, summary):
        self.extract_data("bytes_at_exit", "in use at exit", 1)
        self.extract_data("blocks_at_exit", "in use at exit", 2)
        self.extract_data("total_allocs", "total heap usage", 1)
        self.extract_data("total_frees", "total heap usage", 2)
        self.extract_data("total_bytes_allocated", "total heap usage", 3)
        self.extract_data("errors", "ERROR SUMMARY", 1)

class Callgrind(TestRunCommand):
    def __init__(self, tested_command):
        if not command_exists("valgrind"):
            print("WARNING: valgrind not installed, cannot use callgrind")
        else:
            log_file_name = os.path.join(output_dir, name + "-callgrind.txt")
            self.callgrind_file_name = os.path.join(output_dir,
                                                    name + "-callgrind.out")
            command = ("valgrind --tool=callgrind --callgrind-out-file=" +
                       self.callgrind_file_name + " " + tested_command + " > " +
                       log_file_name + " 2>&1")
            TestRunCommand.__init__(self, log_file_name, command)

    def concurrent(self):
        return True

    def name(self):
        return "callgrind"

    def summarize(self, infile, summary):
        self.extract_data("instruction_refs", "I   refs", 1)
        if not keep_files:
            run_cmd("rm " + self.callgrind_file_name)

class Cachegrind(TestRunCommand):
    def __init__(self, tested_command):
        if not command_exists("valgrind"):
            print("WARNING: valgrind not installed, cannot use cachegrind")
        else:
            log_file_name = os.path.join(output_dir, name + "-cachegrind.txt")
            self.cachegrind_file_name = os.path.join(output_dir,
                                                     name + "-cachegrind.out")
            command = ("valgrind --tool=cachegrind --cachegrind-out-file=" +
                       self.cachegrind_file_name + " " + tested_command + " > " +
                       log_file_name + " 2>&1")
            TestRunCommand.__init__(self, log_file_name, command)

    def concurrent(self):
        return True

    def name(self):
        return "cachegrind"

    def summarize(self, infile, summary):
        self.extract_data("instruction_refs", "I   refs", 1)
        self.extract_data("data_refs", "D   refs", 1)
        instruction_refs = summary.getint(self.name(), "instruction_refs")
        data_refs = summary.getint(self.name(), "data_refs")

        #L1 cache
        self.extract_data("l1_instruction_misses", "I1  misses", 2)
        l1_instruction_misses = summary.getint(self.name(), "l1_instruction_misses")
        summary.set(self.name(), "l1_instruction_miss_rate",
                    l1_instruction_misses / instruction_refs)
        self.extract_data("l1_data_misses", "D1  misses", 2)
        l1_data_misses = summary.getint(self.name(), "l1_data_misses")
        summary.set(self.name(), "l1_data_miss_rate", l1_data_misses / data_refs)
        l1_total_misses = l1_instruction_misses + l1_data_misses
        summary.set(self.name(), "l1_total_misses", l1_total_misses)
        summary.set(self.name(), "l1_total_miss_rate", l1_total_misses /
                    (instruction_refs + data_refs))

        #L2 cache
        self.extract_data("l2_references", "L2 refs", 2)
        self.extract_data("l2_instruction_misses", "L2i misses", 2)
        l2_instruction_misses = summary.getint(self.name(), "l2_instruction_misses")
        summary.set(self.name(), "l2_instruction_miss_rate",
                    l2_instruction_misses / instruction_refs)
        self.extract_data("l2_data_misses", "L2d misses", 2)
        l2_data_misses = summary.getint(self.name(), "l2_data_misses")
        summary.set(self.name(), "l2_data_miss_rate", l2_data_misses / data_refs)

        l2_total_misses = l2_instruction_misses + l2_data_misses
        summary.set(self.name(), "l2_total_misses", l2_total_misses)
        summary.set(self.name(), "l2_total_miss_rate", l2_total_misses /
                    (instruction_refs + data_refs))

        #Branch prediction
        self.extract_data("branches", "Branches", 1)
        self.extract_data("mispredicts", "Mispredicts", 1)
        branches = summary.getint(self.name(), "branches")
        mispredicts = summary.getint(self.name(), "mispredicts")
        summary.set(self.name(), "mispredict_rate", mispredicts / branches)

        if not keep_files:
            run_cmd("rm " + self.cachegrind_file_name)

class Massif(TestRunCommand):
    def __init__(self, tested_command):
        if not command_exists("valgrind"):
            print("WARNING: valgrind not installed, cannot use massif")
        else:
            base_file_name = os.path.join(output_dir, name + "-massif")
            self.log_file_name = base_file_name + ".txt"
            self.massif_file_name = base_file_name + ".out"
            command = ("valgrind --tool=massif --massif-out-file=" +
                       self.massif_file_name + " " + tested_command +
                       " > /dev/null 2>&1 ; ms_print --x=120 --y=30 " +
                       self.massif_file_name + " > " + self.log_file_name + " 2>&1")
            TestRunCommand.__init__(self, self.log_file_name, command)

    def concurrent(self):
        return False

    def name(self):
        return "massif"

    def summarize(self, infile, summary):
        unitline = grep("^\s+\S+", infile)[0]
        peak = float(extract_numbers("^\d+\.?\d*", infile)[0])
        #convert peak to bytes
        if("KB" in unitline):
            peak = int(peak * 1024)
        elif("MB" in unitline):
            peak = int(peak * 1024 * 1024)
        elif("GB" in unitline):
            peak = int(peak * 1024 * 1024 * 1024)
        elif("B" in unitline):
            peak = int(peak)
        else:
            #no support for other units
            raise NotImplementedError
        summary.set(self.name(), "peak_memory_usage", peak)
        if not keep_files:
            run_cmd("rm " + self.massif_file_name)

class RunTime(TestRun):
    def __init__(self, tested_command, runs):
        self.__runs = runs
        self.__command = tested_command + " > /dev/null"
        TestRun.__init__(self)

    def concurrent(self):
        return False

    def name(self):
        return "runtime"

    def run(self):
        start = time.time()
        for run in range(0, self.__runs):
            run_cmd(self.__command)
        end = time.time()

        self._summary.add_section(self.name())
        self._summary.set(self.name(), "average_run_time", (end - start) / self.__runs)
        self._summary.set(self.name(), "run_count", self.__runs)

class ExtraTest(TestRun):
    def __init__(self, tested_command, runs, number):
        self.__runs = runs
        self.__name =  "EXTRA" + str(number)
        self.__log_file_name = os.path.join(output_dir, name + "-" +
                                            self.__name+ ".txt")
        self.__command = tested_command + " > " + self.__log_file_name + " 2>&1"
        self.__number = number
        TestRun.__init__(self)

    def concurrent(self):
        return False

    def name(self):
        return self.__name

    def run(self):
        #list of 2-lists
        values = []

        #run the command first time, extract values from output
        run_cmd(self.__command)
        self.__process_log_file()
        with open(self.__log_file_name) as log_file:
            log_ini = configparser.ConfigParser()
            log_ini.readfp(log_file)
            for item in log_ini.items("SECTION"):
                values.append([item[0], float(item[1])])

        run_cmd("rm " + self.__log_file_name)

        #run the command multiple times, compute average values
        for run in range(1, self.__runs):
            run_cmd(self.__command)
            self.__process_log_file()
            with open(self.__log_file_name) as log_file:
                log_ini = configparser.ConfigParser()
                log_ini.readfp(log_file)
                for item in log_ini.items("SECTION"):
                    for val in values:
                        if val[0] == item[0]:
                            val[1] = (run * val[1] + float(item[1])) / (run + 1)
            run_cmd("rm " + self.__log_file_name)

        #write averages to summary
        self._summary.add_section(self.name())
        self._summary.set(self.name(), "run_count", self.__runs)
        for val in values:
            self._summary.set(self.name(), val[0], val[1])

    def __process_log_file(self):
        lines = []
        with open(self.__log_file_name, "r") as log_file_in:
            lines = log_file_in.readlines()

        #remove lines not starting with TESTUTIL
        for line in lines:
            if not line.startswith("TESTUTIL"):
                lines.remove(line)

        #remove TESTUTIL and leading space from remaining lines
        for line in range(0, len(lines)):
            lines[line] = lines[line][len("TESTUTIL "):]

        #write lines back as an INI file
        with open(self.__log_file_name, "w") as log_file_out:
            log_file_out.write("[SECTION]\n")
            log_file_out.writelines(lines)

#write a custom summary section with tags and values given from command line
def custom(args):
    if human_readable:
        print("\ncustom :\n")
        for arg in args:
            print("    " + arg)
        print()
    with open(os.path.join(output_dir, summary_name), 'a+') as outfile:
        outfile.write("[custom]\n")
        for arg in args:
            outfile.write(arg + "\n")
        outfile.write("\n")

def help():
    print("MiniINI benchmark/command run analyzer")
    print("Copyright (C) 2009-2010 Ferdinand Majerech")
    print("Usage: testutil.py [OPTION...] COMMAND TO TEST [CUSTOM]")
    print("Runs memcheck, cachegrind, callgrind and so on on given command")
    print("and summarizes their results.")
    print("Optionally, CUSTOM must be a list of custom tag=value pairs to write")
    print("to the [custom] section in the summary output file.")
    print(" -h --help           display this help and exit")
    print(" -v --verbose        more detailed output - can be repeated to")
    print("                     increase verbosity further.")
    print(" -t --tests      val tests to perform - every character specifies a")
    print("                     test to run: ")
    print("                     m : memcheck")
    print("                     c : callgrind")
    print("                     C : cachegrind")
    print("                     M : massif")
    print("                     t : run time test")
    print("                     example: -t mcC will run memcheck, callgrind and")
    print("                     cachegrind on given command")
    print(" -f --base-fname val base file name of output files")
    print(" -o --out-dir    val directory to save output files to")
    print(" -k --keep           keep all files produced by tests performed.")
    print(" -H --human          print human readable summary to standard output")
    print(" -T --time-runs  val if run time test, or any extra tests are used,")
    print("                     this option sets the number of times the command ")
    print("                     is run to get average statistics")
    print(" -e --extra-test val extra test command to run. Output of the command")
    print("                     should contain lines in format:")
    print("                     TESTUTIL NAME = VALUE")
    print("                     these will be processed and added to summary")
    print("                     this option can be repeaded to give multiple")
    print("                     extra commands")
    print("                     example: bin/benchmark has a command line option")
    print("                     to provide extra profiling information: --extra")
    print("                     so we could add: -e 'bin/benchmark --extra'")
    print("                     to testutil.py options (note the quotes)")

def main():
    global name
    global output_dir
    global summary_name
    global keep_files
    global verbosity
    global human_readable

    test = "mc"
    timeruns = 128
    extra_tests = []

    try:
        opts, args = getopt.getopt(sys.argv[1:], "hvkt:f:o:T:e:H",
                     ["help", "verbose", "keep", "tests=", "base-fname=", "out-dir=",
                      "time-runs", "extra-test=", "human"])
    except getopt.GetoptError:
        print('Input error')
        help()
        sys.exit(1)
    for opt, arg in opts:
        if opt in ("-h", "--help"):
            help()
            sys.exit(0)
        elif opt in ("-t", "--tests"):
            tests = arg
        elif opt in ("-f", "--base-fname"):
            name = arg
            summary_name = name + ".sum.ini"
        elif opt in ("-o", "--out-dir"):
            output_dir = arg
        elif opt in ("-k", "--keep"):
            keep_files = True
        elif opt in ("-v", "--verbose"):
            verbosity += 1
        elif opt in ("-T", "--time-runs"):
            timeruns = int(arg)
        elif opt in ("-N", "--no-summary"):
            do_summarize = False
        elif opt in ("-e", "--extra-test"):
            extra_tests.append(arg)
        elif opt in ("-H", "--human"):
            human_readable = True
    if len(args):
        run_cmd("mkdir -p " + output_dir)
        if "t" in tests:
            RunTime(args[0], timeruns)
        if "c" in tests:
            Callgrind(args[0])
        if "C" in tests:
            Cachegrind(args[0])
        if "M" in tests:
            Massif(args[0])
        if "m" in tests:
            MemCheck(args[0])
    else:
        help()
        sys.exit(1)
    for test_index in range(0, len(extra_tests)):
        ExtraTest(extra_tests[test_index], timeruns, test_index)
    test_runner.run_tests()
    if(len(args) > 1):
        custom(args[1:])

if __name__ == '__main__':
    main()