olm/lib/doctest/scripts/bench/bench.py

238 lines
8.8 KiB
Python
Raw Normal View History

#!/usr/bin/python3
import os
import sys
if sys.version_info[0] < 3: raise Exception("Python 3 or a more recent version is required.")
import pprint
import argparse
import urllib.request
from datetime import datetime
import shutil
from time import sleep
# ==============================================================================
# == ARGUMENTS =================================================================
# ==============================================================================
def addCommonFlags(parser):
parser.add_argument("compiler", choices=['msvc', 'gcc', 'clang'], default='msvc', help = "compiler to use")
parser.add_argument("--debug", action = "store_true", help = "build in debug")
parser.add_argument("--catch", action = "store_true", help = "use Catch instead of doctest")
parser.add_argument("--disabled", action = "store_true", help = "DOCTEST_CONFIG_DISABLE / CATCH_CONFIG_DISABLE")
parser.add_argument("--fast", action = "store_true", help = "define the doctest/Catch fast config identifier")
parser.add_argument("--files", type=int, default=1, help = "number of source files (besides the implementation)")
parser.add_argument("--tests", type=int, default=1, help = "number of test cases per source file")
parser.add_argument("--checks", type=int, default=1, help = "number of asserts per test case")
parser.add_argument("--asserts", choices=['normal', 'binary'], default="normal",
help = "<doctest> type of assert used - Catch: only normal")
parser = argparse.ArgumentParser()
subparsers = parser.add_subparsers()
parser_c = subparsers.add_parser('compile', help='benchmark compile times')
addCommonFlags(parser_c)
parser_c.add_argument("--implement", action = "store_true", help = "implement the framework test runner")
parser_c.add_argument("--header", action = "store_true", help = "include the framework header everywhere")
parser_r = subparsers.add_parser('runtime', help='benchmark runtime')
addCommonFlags(parser_r)
parser_r.add_argument("--loop-iters", type=int, default=1000, help = "loop N times all asserts in each test case")
parser_r.add_argument("--info", action = "store_true", help = "log the loop variable with INFO()")
def compile(args): args.compile = True; args.runtime = False
def runtime(args): args.compile = False; args.runtime = True
parser_c.set_defaults(func=compile)
parser_r.set_defaults(func=runtime)
args = parser.parse_args()
args.func(args)
print("== PASSED OPTIONS TO BENCHMARK SCRIPT:")
pprint.pprint(vars(args), width = 1)
# ==============================================================================
# == SETUP ENVIRONMENT =========================================================
# ==============================================================================
# catch version
catch_ver = "2.3.0"
catch_header = "catch." + catch_ver + ".hpp"
# get the catch header
if not os.path.exists("catch." + catch_ver + ".hpp"):
urllib.request.urlretrieve("https://github.com/catchorg/Catch2/releases/download/v" + catch_ver + "/catch.hpp", catch_header)
# folder with generated code
the_folder = 'project'
# delete the folder
if os.path.exists(the_folder):
shutil.rmtree(the_folder)
# wait a bit or the script might fail...
sleep(2)
# create the folder
if not os.path.exists(the_folder):
os.makedirs(the_folder)
# enter folder
os.chdir(the_folder);
# ==============================================================================
# == DO STUFF ==================================================================
# ==============================================================================
# setup defines used
defines = ""
if args.catch and args.disabled:
defines += "#define CATCH_CONFIG_DISABLE\n"
if not args.catch and args.disabled:
defines += "#define DOCTEST_CONFIG_DISABLE\n"
if args.catch and args.fast:
defines += "#define CATCH_CONFIG_FAST_COMPILE\n"
if not args.catch and args.fast:
defines += "#define DOCTEST_CONFIG_SUPER_FAST_ASSERTS\n"
define_implement = "#define DOCTEST_CONFIG_IMPLEMENT\n"
if args.catch:
define_implement = "#define CATCH_CONFIG_RUNNER\n"
# setup the macros used
macro = " CHECK(a == b);\n"
if args.runtime:
macro = " CHECK(i == i);\n"
if not args.catch and args.asserts == "binary":
macro = " CHECK_EQ(a, b);\n"
# setup the header used
include = '#include "doctest.h"\n'
if args.catch:
include = '#include "' + catch_header + '"\n'
# ==============================================================================
# == GENERATE SOURCE CODE ======================================================
# ==============================================================================
# make the source files
for i in range(0, args.files):
f = open(str(i) + '.cpp', 'w')
if args.runtime or args.header:
f.write(defines)
f.write(include)
for t in range(0, args.tests):
f.write('TEST_CASE("") {\n')
f.write(' int a = 5;\n')
f.write(' int b = 5;\n')
if args.runtime and args.loop_iters > 0:
f.write(' for(int i = 0; i < ' + str(args.loop_iters) + '; ++i) {\n')
if args.runtime and args.info:
f.write(' INFO(i);\n')
for a in range(0, args.checks):
if args.runtime and args.loop_iters > 0:
f.write(' ')
f.write(macro)
if args.runtime and args.loop_iters > 0:
f.write(' }\n')
f.write('}\n\n')
f.write('int f' + str(i) + '() { return ' + str(i) + '; }\n\n')
f.close()
# the main file
f = open('main.cpp', 'w')
if args.runtime or args.implement or args.header:
f.write(defines)
f.write(define_implement)
f.write(include)
f.write('int main(int argc, char** argv) {\n')
if args.runtime or args.implement or args.header:
if not args.catch: f.write(' int res = doctest::Context(argc, argv).run();\n')
else: f.write(' int res = Catch::Session().run(argc, argv);\n')
else:
f.write(' int res = 0;\n')
for i in range(0, args.files):
f.write(' int f' + str(i) + '(); res += f' + str(i) + '();\n')
f.write(' return res;\n}\n')
f.close()
# the cmake file
f = open('CMakeLists.txt', 'w')
f.write('cmake_minimum_required(VERSION 2.8)\n\n')
f.write('project(bench)\n\n')
f.write('if(NOT MSVC)\n')
f.write('set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")\n')
f.write('endif()\n\n')
if not args.catch: f.write('include_directories("../../../doctest/")\n\n')
else: f.write('include_directories("../")\n\n')
f.write('add_executable(bench main.cpp\n')
for i in range(0, args.files):
f.write(' ' + str(i) + '.cpp\n')
f.write(')\n')
f.close()
# ==============================================================================
# == INVOKE CMAKE ==============================================================
# ==============================================================================
compiler = ""
if args.compiler == 'clang':
compiler = " -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_CXX_FLAGS=-w"
if args.compiler == 'gcc':
compiler = " -DCMAKE_CXX_COMPILER=g++ -DCMAKE_CXX_FLAGS=-w"
# setup cmake command
cmake_command = 'cmake . -G "Visual Studio 15 Win64"' # MSVC 2017
if args.compiler != 'msvc':
cmake_command = 'cmake . -G "MinGW Makefiles" -DCMAKE_BUILD_TYPE=' + ('Debug' if args.debug else 'Release')
if os.name != "nt":
cmake_command = 'cmake . -DCMAKE_BUILD_TYPE=' + ('Debug' if args.debug else 'Release')
os.system(cmake_command + compiler)
# ==============================================================================
# == BUILD PROJECT =============================================================
# ==============================================================================
the_config = ''
if args.compiler == 'msvc':
if args.debug: the_config = ' --config Debug'
else: the_config = ' --config Release'
# build it
start = datetime.now()
os.system('cmake --build .' + the_config)
end = datetime.now()
if not args.runtime:
print("Time running compiler (+ linker) in seconds: " + str((end - start).total_seconds()))
# ==============================================================================
# == RUN PROJECT ===============================================================
# ==============================================================================
if args.runtime:
start = datetime.now()
if args.compiler == 'msvc':
os.system(('Debug' if args.debug else 'Release') + '\\bench.exe')
elif os.name == "nt":
os.system('bench.exe')
else:
os.system('./bench')
end = datetime.now()
print("Time running the tests in seconds: " + str((end - start).total_seconds()))
# leave folder
os.chdir("../");