#!/usr/bin/python3 import os import sys if sys.version_info[0] < 3: raise Exception("Python 3 or a more recent version is required.") import pprint import argparse import urllib.request from datetime import datetime import shutil from time import sleep # ============================================================================== # == ARGUMENTS ================================================================= # ============================================================================== def addCommonFlags(parser): parser.add_argument("compiler", choices=['msvc', 'gcc', 'clang'], default='msvc', help = "compiler to use") parser.add_argument("--debug", action = "store_true", help = "build in debug") parser.add_argument("--catch", action = "store_true", help = "use Catch instead of doctest") parser.add_argument("--disabled", action = "store_true", help = "DOCTEST_CONFIG_DISABLE / CATCH_CONFIG_DISABLE") parser.add_argument("--fast", action = "store_true", help = "define the doctest/Catch fast config identifier") parser.add_argument("--files", type=int, default=1, help = "number of source files (besides the implementation)") parser.add_argument("--tests", type=int, default=1, help = "number of test cases per source file") parser.add_argument("--checks", type=int, default=1, help = "number of asserts per test case") parser.add_argument("--asserts", choices=['normal', 'binary'], default="normal", help = " type of assert used - Catch: only normal") parser = argparse.ArgumentParser() subparsers = parser.add_subparsers() parser_c = subparsers.add_parser('compile', help='benchmark compile times') addCommonFlags(parser_c) parser_c.add_argument("--implement", action = "store_true", help = "implement the framework test runner") parser_c.add_argument("--header", action = "store_true", help = "include the framework header everywhere") parser_r = subparsers.add_parser('runtime', help='benchmark runtime') addCommonFlags(parser_r) parser_r.add_argument("--loop-iters", type=int, default=1000, help = "loop N times all asserts in each test case") parser_r.add_argument("--info", action = "store_true", help = "log the loop variable with INFO()") def compile(args): args.compile = True; args.runtime = False def runtime(args): args.compile = False; args.runtime = True parser_c.set_defaults(func=compile) parser_r.set_defaults(func=runtime) args = parser.parse_args() args.func(args) print("== PASSED OPTIONS TO BENCHMARK SCRIPT:") pprint.pprint(vars(args), width = 1) # ============================================================================== # == SETUP ENVIRONMENT ========================================================= # ============================================================================== # catch version catch_ver = "2.3.0" catch_header = "catch." + catch_ver + ".hpp" # get the catch header if not os.path.exists("catch." + catch_ver + ".hpp"): urllib.request.urlretrieve("https://github.com/catchorg/Catch2/releases/download/v" + catch_ver + "/catch.hpp", catch_header) # folder with generated code the_folder = 'project' # delete the folder if os.path.exists(the_folder): shutil.rmtree(the_folder) # wait a bit or the script might fail... sleep(2) # create the folder if not os.path.exists(the_folder): os.makedirs(the_folder) # enter folder os.chdir(the_folder); # ============================================================================== # == DO STUFF ================================================================== # ============================================================================== # setup defines used defines = "" if args.catch and args.disabled: defines += "#define CATCH_CONFIG_DISABLE\n" if not args.catch and args.disabled: defines += "#define DOCTEST_CONFIG_DISABLE\n" if args.catch and args.fast: defines += "#define CATCH_CONFIG_FAST_COMPILE\n" if not args.catch and args.fast: defines += "#define DOCTEST_CONFIG_SUPER_FAST_ASSERTS\n" define_implement = "#define DOCTEST_CONFIG_IMPLEMENT\n" if args.catch: define_implement = "#define CATCH_CONFIG_RUNNER\n" # setup the macros used macro = " CHECK(a == b);\n" if args.runtime: macro = " CHECK(i == i);\n" if not args.catch and args.asserts == "binary": macro = " CHECK_EQ(a, b);\n" # setup the header used include = '#include "doctest.h"\n' if args.catch: include = '#include "' + catch_header + '"\n' # ============================================================================== # == GENERATE SOURCE CODE ====================================================== # ============================================================================== # make the source files for i in range(0, args.files): f = open(str(i) + '.cpp', 'w') if args.runtime or args.header: f.write(defines) f.write(include) for t in range(0, args.tests): f.write('TEST_CASE("") {\n') f.write(' int a = 5;\n') f.write(' int b = 5;\n') if args.runtime and args.loop_iters > 0: f.write(' for(int i = 0; i < ' + str(args.loop_iters) + '; ++i) {\n') if args.runtime and args.info: f.write(' INFO(i);\n') for a in range(0, args.checks): if args.runtime and args.loop_iters > 0: f.write(' ') f.write(macro) if args.runtime and args.loop_iters > 0: f.write(' }\n') f.write('}\n\n') f.write('int f' + str(i) + '() { return ' + str(i) + '; }\n\n') f.close() # the main file f = open('main.cpp', 'w') if args.runtime or args.implement or args.header: f.write(defines) f.write(define_implement) f.write(include) f.write('int main(int argc, char** argv) {\n') if args.runtime or args.implement or args.header: if not args.catch: f.write(' int res = doctest::Context(argc, argv).run();\n') else: f.write(' int res = Catch::Session().run(argc, argv);\n') else: f.write(' int res = 0;\n') for i in range(0, args.files): f.write(' int f' + str(i) + '(); res += f' + str(i) + '();\n') f.write(' return res;\n}\n') f.close() # the cmake file f = open('CMakeLists.txt', 'w') f.write('cmake_minimum_required(VERSION 2.8)\n\n') f.write('project(bench)\n\n') f.write('if(NOT MSVC)\n') f.write('set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")\n') f.write('endif()\n\n') if not args.catch: f.write('include_directories("../../../doctest/")\n\n') else: f.write('include_directories("../")\n\n') f.write('add_executable(bench main.cpp\n') for i in range(0, args.files): f.write(' ' + str(i) + '.cpp\n') f.write(')\n') f.close() # ============================================================================== # == INVOKE CMAKE ============================================================== # ============================================================================== compiler = "" if args.compiler == 'clang': compiler = " -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_CXX_FLAGS=-w" if args.compiler == 'gcc': compiler = " -DCMAKE_CXX_COMPILER=g++ -DCMAKE_CXX_FLAGS=-w" # setup cmake command cmake_command = 'cmake . -G "Visual Studio 15 Win64"' # MSVC 2017 if args.compiler != 'msvc': cmake_command = 'cmake . -G "MinGW Makefiles" -DCMAKE_BUILD_TYPE=' + ('Debug' if args.debug else 'Release') if os.name != "nt": cmake_command = 'cmake . -DCMAKE_BUILD_TYPE=' + ('Debug' if args.debug else 'Release') os.system(cmake_command + compiler) # ============================================================================== # == BUILD PROJECT ============================================================= # ============================================================================== the_config = '' if args.compiler == 'msvc': if args.debug: the_config = ' --config Debug' else: the_config = ' --config Release' # build it start = datetime.now() os.system('cmake --build .' + the_config) end = datetime.now() if not args.runtime: print("Time running compiler (+ linker) in seconds: " + str((end - start).total_seconds())) # ============================================================================== # == RUN PROJECT =============================================================== # ============================================================================== if args.runtime: start = datetime.now() if args.compiler == 'msvc': os.system(('Debug' if args.debug else 'Release') + '\\bench.exe') elif os.name == "nt": os.system('bench.exe') else: os.system('./bench') end = datetime.now() print("Time running the tests in seconds: " + str((end - start).total_seconds())) # leave folder os.chdir("../");