1 /** 2 * Unit test helper 3 * 4 * Copyright: 5 * (C) 2014-2015 Jack Lloyd 6 * (C) 2014-2015 Etienne Cimon 7 * 8 * License: 9 * Botan is released under the Simplified BSD License (see LICENSE.md) 10 */ 11 module botan.test; 12 13 import botan.constants; 14 //static if (BOTAN_TEST): 15 16 public import std.stdio : File, writeln; 17 public import std.algorithm : sort, canFind; 18 public import std.range : walkLength; 19 public import std..string : indexOf, lastIndexOf; 20 public import botan.utils.types; 21 private import botan.libstate.libstate; 22 import memutils.hashmap; 23 import std.file; 24 import std.array; 25 import std.exception; 26 import std.datetime.stopwatch : StopWatch; 27 import std.datetime.date; 28 import std.datetime.interval; 29 30 @property bool ok(File f) { return f.isOpen && !f.eof() && !f.error(); } 31 32 string CHECK_MESSAGE (string expr, string print) { 33 return " 34 { 35 import core.atomic : atomicOp; 36 static if (is(typeof(total_tests) == shared)) atomicOp!`+=`(total_tests, cast(size_t) 1); 37 else total_tests++; 38 try { 39 if (!(" ~ expr ~ ")) { 40 ++fails; 41 logError( `" ~ print ~ "` ); 42 } 43 } 44 catch(Exception e) 45 { 46 logError(__FUNCTION__, ` : ` ~ e.msg); 47 } 48 }"; 49 } 50 51 string CHECK (string expr) { 52 return ` 53 { 54 import core.atomic : atomicOp; 55 static if (is(typeof(total_tests) == shared)) atomicOp!"+="(total_tests, cast(size_t) 1); 56 else total_tests++; 57 mixin( q{ 58 bool success = ` ~ expr ~ `; 59 } ); 60 try { 61 if (!success) 62 { ++fails; logError( q{ ` ~ expr ~ ` } ); } 63 } 64 catch(Exception e) 65 { 66 logError(__FUNCTION__ ~ " : " ~ e.msg); 67 } 68 }`; 69 } 70 71 72 string[] listDir(string dir_path) 73 { 74 auto dirfiles = dirEntries(dir_path, "*.vec", SpanMode.depth); 75 string[] files; 76 foreach(file; dirfiles) { 77 files ~= file.name; 78 } 79 files.sort(); 80 return files; 81 } 82 83 StopWatch g_sw; 84 85 size_t runTestsInDir(string dir, size_t delegate(string) fn) 86 { 87 88 assert(exists(cast(char[])dir), "Directory `" ~ dir ~ "` does not exist"); 89 logTrace("Running tests for directory: " ~ dir); 90 import std.parallelism; 91 import core.atomic; 92 shared(size_t) shared_fails; 93 auto dirs = listDir(dir); 94 foreach (vec; dirs) { 95 size_t local_fails = fn(vec); 96 if (local_fails > 0) { 97 assert(false); 98 } 99 atomicOp!"+="(shared_fails, local_fails); 100 } 101 return cast(size_t)atomicLoad(shared_fails); 102 } 103 104 void testReport(string name, size_t ran, size_t failed) 105 { 106 static long last_msecs; 107 if (failed) 108 logError(name, " ... ", failed, " / ", ran, " ************** FAILED ****************"); 109 else 110 logDebug(name, " ... PASSED (all of ", ran, " tests in ", g_sw.peek().total!"msecs" - last_msecs, " msecs)"); 111 last_msecs = g_sw.peek().total!"msecs"; 112 } 113 114 size_t runTestsBb(ref File src, 115 string name_key, 116 string output_key, 117 bool clear_between_cb, 118 size_t delegate(ref HashMap!(string, string)) cb) 119 { 120 121 if (src.eof || src.error) 122 { 123 logError("Could not open input file for " ~ name_key); 124 return 1; 125 } 126 if (!g_sw.running) 127 g_sw.start(); 128 HashMap!(string, string) vars; 129 vars[name_key] = name_key; 130 size_t test_fails = 0, algo_fail = 0; 131 size_t test_count = 0, algo_count = 0; 132 133 string fixed_name = name_key; 134 135 string line; 136 137 while(!src.eof && !src.error) 138 { 139 140 line = src.readln(); 141 if (line.length > 0) 142 line = line[0 .. $-1]; 143 144 if (line.length == 0) 145 continue; 146 147 if (line[0] == '#') 148 continue; 149 150 if (line[0] == '[' && line[$-1] == ']') 151 { 152 if (fixed_name != "" && algo_count > 0) 153 testReport(fixed_name, algo_count, algo_fail); 154 155 test_count += algo_count; 156 test_fails += algo_fail; 157 algo_count = 0; 158 algo_fail = 0; 159 fixed_name = line[1 .. $ - 1]; 160 vars[name_key] = fixed_name; 161 continue; 162 } 163 import std..string : strip; 164 if (line.indexOf('=') == -1) continue; 165 assert(line[line.indexOf('=') - 1] == ' ' && line[line.indexOf('=') + 1] == ' ', "= must be wrapped with spaces"); 166 const string key = line[0 .. line.indexOf('=') - 1].strip; 167 const string val = line[line.indexOf('=') + 2 .. $].strip; 168 169 vars[key] = val; 170 171 if (key == name_key) 172 fixed_name.length = 0; 173 174 if (key == output_key) 175 { 176 ++algo_count; 177 try 178 { 179 const size_t fails = cb(vars); 180 if (fails) 181 { 182 logError(vars[name_key] ~ " test ", algo_count, " : ", fails, " failure"); 183 algo_fail += fails; 184 } 185 } 186 catch(Exception e) 187 { 188 logError(vars[name_key] ~ " test ", algo_count, " failed: " ~ e.msg); 189 ++algo_fail; 190 assert(false); 191 } 192 193 if (clear_between_cb) 194 { 195 vars.clear(); 196 vars[name_key] = fixed_name; 197 } 198 } 199 } 200 test_count += algo_count; 201 test_fails += algo_fail; 202 203 if (fixed_name != "" && (algo_count > 0 || algo_fail > 0)) 204 testReport(fixed_name, algo_count, algo_fail); 205 else 206 testReport(name_key, test_count, test_fails); 207 208 return test_fails; 209 } 210 211 size_t runTests(string filename, 212 string name_key, 213 string output_key, 214 bool clear_between_cb, 215 string delegate(ref HashMap!(string, string)) cb) 216 { 217 File vec = File(filename, "r"); 218 219 if (vec.error || vec.eof) 220 { 221 logError("Failure opening " ~ filename); 222 return 1; 223 } 224 225 return runTests(vec, name_key, output_key, clear_between_cb, cb); 226 } 227 228 size_t runTests(ref File src, 229 string name_key, 230 string output_key, 231 bool clear_between_cb, 232 string delegate(ref HashMap!(string, string)) cb) 233 { 234 return runTestsBb(src, name_key, output_key, clear_between_cb, 235 (ref HashMap!(string, string) vars) 236 { 237 const string got = cb(vars); 238 if (got != vars[output_key]) 239 { 240 logTrace(name_key ~ ' ' ~ vars[name_key] ~ " got " ~ got ~ " expected " ~ vars[output_key]); 241 return 1; 242 } 243 return 0; 244 }); 245 }