benchmark.h

Go to the documentation of this file.

// Evo C++ Library
/* Copyright 2019 Justin Crowell
Distributed under the BSD 2-Clause License -- see included file LICENSE.txt for details.
*/

#pragma once
#ifndef INCL_evo_benchmark_h
#define INCL_evo_benchmark_h

#include "fmt.h"
#include "io.h"
#include "thread.h"
#include "timer.h"


#define EVO_BENCH_SETUP(F, WARMUP) Benchmark bench(0, WARMUP); bench.scale(F)

#define EVO_BENCH_RUN(F) bench.run(#F, F)

#define EVO_BENCH_RUN2(F) { String evo_bench_name_(#F); F.get_name(evo_bench_name_); bench.run(evo_bench_name_, F); }


namespace evo {


class Benchmark {
public:
    Benchmark(ulongl default_count=0, ulongl default_warmup_count=0) : default_count_(default_count), default_warmup_count_(default_warmup_count) {
    }

    ~Benchmark() {
        if (report_.size() > 0)
            report();
    }

    template<class T>
    ulongl scale(const T& func, uint factor=1) {
        const ulong THRESHOLD_MSEC  = 100;
        const ulong THRESHOLD_COUNT = 1000000000;
        const ulongl MAX_COUNT = ULongL::MAX;
        assert( factor > 0 );

        Timer timer;
        timer.start();

        ulongl count = 1;
        func();
        while (timer.msec() < THRESHOLD_MSEC && count < MAX_COUNT) {
            const ulongl target = count * (count > THRESHOLD_COUNT ? 2 : 10);
            while (count < target) {
                EVO_ATOMIC_FENCE(EVO_ATOMIC_SYNC);
                func();
                ++count;
            }
        }

        if (count > MAX_COUNT / factor)
            count = MAX_COUNT;
        else
            count *= factor;

        default_count_ = count;
        return count;
    }

    template<class T>
    Benchmark& run(const SubString& name, const T& func, ulongl count, ulongl warmup_count=0) {
        const ulongl WARMUP_COUNT = 100;
        if (count < 1)
            count = 1;
        const ulongl start_count = count;

        if (warmup_count == 0)
            warmup_count = WARMUP_COUNT;
        for (; warmup_count > 0; --warmup_count)
            func(); // warmup

        // Start a monitor thread to keep run from taking too long
        AtomicInt monitor_flag;
        monitor_flag.store(0);
        Thread monitor(monitor_thread, &monitor_flag);
        monitor.thread_start();

        Timer walltimer;
        TimerCpu cputimer;
        walltimer.start();
        cputimer.start();
        for (; count > 0 && monitor_flag.load() == 0; --count)
            func();
        cputimer.stop();
        walltimer.stop();

        // Stop thread, if needed
        monitor_flag.store(1);
        monitor.thread_join();

        report_.add(ReportItem(name, walltimer.nsec(), cputimer.nsec(), start_count - count));
        return *this;
    }

    template<class T>
    Benchmark& run(const SubString& name, const T& func) {
        return run(name, func, default_count_, default_warmup_count_);
    }

    Benchmark& clear() {
        report_.clear();
        return *this;
    }

    template<class T>
    Benchmark& report_out(T& out, FmtTable::Type type=FmtTable::tTEXT) {
        if (!report_.empty()) {
            const SubString COLUMN_NAMES[] = {
                "Name",
                "Time(nsec)",
                "CPU(nsec)",
                "Count",
                "AvgTime(nsec)",
                "AvgCPU(nsec)",
                "DiffBest(nsec)",
                ""
            };

            // Find lowest cpu time
            const ReportItem* fastest = NULL;
            for (ReportList::Iter iter(report_); iter; ++iter) {
                const ReportItem& item = *iter;
                if (fastest == NULL || item.cputime_nsec < fastest->cputime_nsec)
                    fastest = &item;
            }

            // Tests
            FmtTable table(COLUMN_NAMES, 0);
            FmtTableOut<T> table_out(out, table, type);
            for (ReportList::Iter iter(report_); iter; ++iter) {
                const ReportItem& item = *iter;
                const double cpu_avg_diff = (fastest == NULL ? 0.0 : ((double)item.cputime_nsec / item.count) - ((double)fastest->cputime_nsec / fastest->count));
                table_out
                    << item.name << item.walltime_nsec << item.cputime_nsec << item.count
                    << ((double)item.walltime_nsec / item.count) << ((double)item.cputime_nsec / item.count)
                    << cpu_avg_diff << NL;
            }
            table_out << fFLUSH;
            out << NL;

            report_.clear();
        }
        return *this;
    }

    Benchmark& report(FmtTable::Type type=FmtTable::tTEXT) {
        return report_out(con().out, type);
    }

private:
    // Disable copying
    Benchmark(const Benchmark&) EVO_ONCPP11(= delete);
    Benchmark& operator=(const Benchmark&) EVO_ONCPP11(= delete);

    struct ReportItem {
        String name;
        ulongl walltime_nsec;
        ulongl cputime_nsec;
        ulongl count;

        ReportItem() : walltime_nsec(0), cputime_nsec(0), count(0) {
        }

        ReportItem(const SubString& name, ulongl walltime_nsec, ulongl cputime_nsec, ulongl count) : name(name), walltime_nsec(walltime_nsec), cputime_nsec(cputime_nsec), count(count) {
        }

        ReportItem(const ReportItem& src) : name(src.name), walltime_nsec(src.walltime_nsec), cputime_nsec(src.cputime_nsec), count(src.count) {
        }
    };

    typedef List<ReportItem> ReportList;

    ulongl default_count_;
    ulongl default_warmup_count_;
    ReportList report_;

    static void monitor_thread(void* arg) {
        const uint WAIT_TIME = 5000;
        const uint WAIT_INC  = 200;
        AtomicInt& flag = *(AtomicInt*)arg;
        for (uint msec = 0; msec < WAIT_TIME && flag.load() == 0; msec += WAIT_INC)
            sleepms(WAIT_INC);
        flag.store(1);
    }
};


}
#endif