1 /**
2 * Unit test helper
3 * 
4 * Copyright:
5 * (C) 2014-2015 Jack Lloyd
6 * (C) 2014-2015 Etienne Cimon
7 *
8 * License:
9 * Botan is released under the Simplified BSD License (see LICENSE.md)
10 */
11 module botan.test;
12 
13 import botan.constants;
14 //static if (BOTAN_TEST):
15 
16 public import std.stdio : File, writeln;
17 public import std.algorithm : sort, canFind;
18 public import std.range : walkLength;
19 public import std..string : indexOf, lastIndexOf;
20 public import botan.utils.types;
21 private import botan.libstate.libstate;
22 import memutils.hashmap;
23 import std.file;
24 import std.array;
25 import std.exception;
26 import std.datetime.stopwatch : StopWatch;
27 import std.datetime.date;
28 import std.datetime.interval;
29 
30 @property bool ok(File f) { return f.isOpen && !f.eof() && !f.error(); }
31 
32 string CHECK_MESSAGE (string expr, string print) {
33     return "
34     {
35         import core.atomic : atomicOp;
36         static if (is(typeof(total_tests) == shared)) atomicOp!`+=`(total_tests, cast(size_t) 1);
37         else total_tests++;
38         try { 
39             if (!(" ~ expr ~ ")) { 
40                 ++fails; 
41                 logError( `" ~ print ~ "` ); 
42             } 
43         } 
44         catch(Exception e) 
45         { 
46             logError(__FUNCTION__, ` : ` ~ e.msg); 
47         }
48     }";
49 }
50 
51 string CHECK (string expr) {
52     return `
53     {
54         import core.atomic : atomicOp;
55         static if (is(typeof(total_tests) == shared)) atomicOp!"+="(total_tests, cast(size_t) 1);
56         else total_tests++;
57         mixin( q{
58             bool success = ` ~ expr ~ `;
59         } );
60         try { 
61             if (!success)
62             { ++fails; logError( q{ ` ~ expr ~ ` } ); } 
63         } 
64         catch(Exception e) 
65         { 
66             logError(__FUNCTION__ ~ " : " ~ e.msg); 
67         }
68     }`;
69 }
70 
71 
72 string[] listDir(string dir_path)
73 {
74     auto dirfiles = dirEntries(dir_path, "*.vec", SpanMode.depth);
75     string[] files;
76     foreach(file; dirfiles) {
77         files ~= file.name;
78     }
79     files.sort();
80     return files;
81 }
82 
83 StopWatch g_sw;
84 
85 size_t runTestsInDir(string dir, size_t delegate(string) fn)
86 {
87 
88     assert(exists(cast(char[])dir), "Directory `" ~ dir ~ "` does not exist");
89     logTrace("Running tests for directory: " ~ dir);
90     import std.parallelism;
91     import core.atomic;
92     shared(size_t) shared_fails;
93     auto dirs = listDir(dir);
94     foreach (vec; dirs) {
95         size_t local_fails = fn(vec);
96         if (local_fails > 0) {
97             assert(false);
98         }
99         atomicOp!"+="(shared_fails, local_fails);
100     }
101     return cast(size_t)atomicLoad(shared_fails);
102 }
103 
104 void testReport(string name, size_t ran, size_t failed)
105 {    
106 	static long last_msecs;
107     if (failed)
108         logError(name, " ... ", failed, " / ", ran, " ************** FAILED ****************");
109     else
110 		logDebug(name, " ... PASSED (all of ", ran, " tests in ", g_sw.peek().total!"msecs" - last_msecs, " msecs)");
111 	last_msecs = g_sw.peek().total!"msecs";
112 }
113 
114 size_t runTestsBb(ref File src,
115                   string name_key,
116                   string output_key,
117                   bool clear_between_cb,
118                   size_t delegate(ref HashMap!(string, string)) cb)
119 {
120 
121     if (src.eof || src.error)
122     {
123         logError("Could not open input file for " ~ name_key);
124         return 1;
125     }
126 	if (!g_sw.running)
127 		g_sw.start();
128     HashMap!(string, string) vars;
129     vars[name_key] = name_key;
130     size_t test_fails = 0, algo_fail = 0;
131     size_t test_count = 0, algo_count = 0;
132     
133     string fixed_name = name_key;
134     
135     string line;
136 
137     while(!src.eof && !src.error)
138     {
139 
140         line = src.readln();
141         if (line.length > 0)
142             line = line[0 .. $-1];
143 
144         if (line.length == 0)
145             continue;
146 
147         if (line[0] == '#')
148             continue;
149         
150         if (line[0] == '[' && line[$-1] == ']')
151         {
152             if (fixed_name != "" && algo_count > 0)
153                 testReport(fixed_name, algo_count, algo_fail);
154             
155             test_count += algo_count;
156             test_fails += algo_fail;
157             algo_count = 0;
158             algo_fail = 0;
159             fixed_name = line[1 .. $ - 1];
160             vars[name_key] = fixed_name;
161             continue;
162         }
163         import std..string : strip;
164         if (line.indexOf('=') == -1) continue;
165         assert(line[line.indexOf('=') - 1] == ' ' && line[line.indexOf('=') + 1] == ' ', "= must be wrapped with spaces");
166         const string key = line[0 .. line.indexOf('=') - 1].strip;
167         const string val = line[line.indexOf('=') + 2 .. $].strip;
168         
169         vars[key] = val;
170         
171         if (key == name_key)
172             fixed_name.length = 0;
173         
174         if (key == output_key)
175         {
176             ++algo_count;
177             try
178             {
179                 const size_t fails = cb(vars);
180                 if (fails)
181                 {
182                     logError(vars[name_key] ~ " test ", algo_count, " : ", fails, " failure");
183                     algo_fail += fails;
184                 }
185             }
186             catch(Exception e)
187             {
188                 logError(vars[name_key] ~ " test ", algo_count, " failed: " ~ e.msg);
189                 ++algo_fail;
190                 assert(false);
191             }
192             
193             if (clear_between_cb)
194             {
195                 vars.clear();
196                 vars[name_key] = fixed_name;
197             }
198         }
199     }
200     test_count += algo_count;
201     test_fails += algo_fail;
202     
203     if (fixed_name != "" && (algo_count > 0 || algo_fail > 0))
204         testReport(fixed_name, algo_count, algo_fail);
205     else
206         testReport(name_key, test_count, test_fails);
207     
208     return test_fails;
209 }
210 
211 size_t runTests(string filename,
212                  string name_key,
213                  string output_key,
214                  bool clear_between_cb,
215                  string delegate(ref HashMap!(string, string)) cb)
216 {
217     File vec = File(filename, "r");
218 
219     if (vec.error || vec.eof)
220     {
221         logError("Failure opening " ~ filename);
222         return 1;
223     }
224     
225     return runTests(vec, name_key, output_key, clear_between_cb, cb);
226 }
227 
228 size_t runTests(ref File src,
229                  string name_key,
230                  string output_key,
231                  bool clear_between_cb,
232                  string delegate(ref HashMap!(string, string)) cb)
233 {
234     return runTestsBb(src, name_key, output_key, clear_between_cb, 
235         (ref HashMap!(string, string) vars)
236         {
237             const string got = cb(vars);
238             if (got != vars[output_key])
239             {
240                 logTrace(name_key ~ ' ' ~ vars[name_key] ~ " got " ~ got ~ " expected " ~ vars[output_key]);
241                 return 1;
242             }
243             return 0;
244         });
245 }