1 /**
2 * Unit test helper
3 * 
4 * Copyright:
5 * (C) 2014-2015 Jack Lloyd
6 * (C) 2014-2015 Etienne Cimon
7 *
8 * License:
9 * Botan is released under the Simplified BSD License (see LICENSE.md)
10 */
11 module botan.test;
12 
13 import botan.constants;
14 //static if (BOTAN_TEST):
15 
16 public import std.stdio : File, writeln;
17 public import std.algorithm : sort, canFind;
18 public import std.range : walkLength;
19 public import std.string : indexOf, lastIndexOf;
20 public import botan.utils.types;
21 private import botan.libstate.libstate;
22 import memutils.hashmap;
23 import std.file;
24 import std.array;
25 import std.exception;
26 
27 @property bool ok(File f) { return f.isOpen && !f.eof() && !f.error(); }
28 
29 string CHECK_MESSAGE (string expr, string print) {
30     return "
31     {
32         import core.atomic : atomicOp;
33         static if (is(typeof(total_tests) == shared)) atomicOp!`+=`(total_tests, cast(size_t) 1);
34         else total_tests++;
35         try { 
36             if (!(" ~ expr ~ ")) { 
37                 ++fails; 
38                 logError( `" ~ print ~ "` ); 
39             } 
40         } 
41         catch(Exception e) 
42         { 
43             logError(__FUNCTION__, ` : ` ~ e.msg); 
44         }
45     }";
46 }
47 
48 string CHECK (string expr) {
49     return `
50     {
51         import core.atomic : atomicOp;
52         static if (is(typeof(total_tests) == shared)) atomicOp!"+="(total_tests, cast(size_t) 1);
53         else total_tests++;
54         mixin( q{
55             bool success = ` ~ expr ~ `;
56         } );
57         try { 
58             if (!success)
59             { ++fails; logError( q{ ` ~ expr ~ ` } ); } 
60         } 
61         catch(Exception e) 
62         { 
63             logError(__FUNCTION__ ~ " : " ~ e.msg); 
64         }
65     }`;
66 }
67 
68 
69 string[] listDir(string dir_path)
70 {
71     auto dirfiles = dirEntries(dir_path, "*.vec", SpanMode.depth);
72     string[] files;
73     foreach(file; dirfiles) {
74         files ~= file.name;
75     }
76     files.sort();
77     return files;
78 }
79 
80 import std.datetime : StopWatch;
81 StopWatch g_sw;
82 
83 size_t runTestsInDir(string dir, size_t delegate(string) fn)
84 {
85 
86     assert(exists(cast(char[])dir), "Directory `" ~ dir ~ "` does not exist");
87     logTrace("Running tests for directory: " ~ dir);
88     import std.parallelism;
89     import core.atomic;
90     shared(size_t) shared_fails;
91     auto dirs = listDir(dir);
92     foreach (vec; dirs) {
93         size_t local_fails = fn(vec);
94         if (local_fails > 0) {
95             assert(false);
96         }
97         atomicOp!"+="(shared_fails, local_fails);
98     }
99     return cast(size_t)atomicLoad(shared_fails);
100 }
101 
102 void testReport(string name, size_t ran, size_t failed)
103 {    
104 	static long last_msecs;
105     if (failed)
106         logError(name, " ... ", failed, " / ", ran, " ************** FAILED ****************");
107     else
108 		logDebug(name, " ... PASSED (all of ", ran, " tests in ", g_sw.peek().msecs() - last_msecs, " msecs)");
109 	last_msecs = g_sw.peek().msecs();
110 }
111 
112 size_t runTestsBb(ref File src,
113                   string name_key,
114                   string output_key,
115                   bool clear_between_cb,
116                   size_t delegate(ref HashMap!(string, string)) cb)
117 {
118 
119     if (src.eof || src.error)
120     {
121         logError("Could not open input file for " ~ name_key);
122         return 1;
123     }
124 	if (!g_sw.running)
125 		g_sw.start();
126     HashMap!(string, string) vars;
127     vars[name_key] = name_key;
128     size_t test_fails = 0, algo_fail = 0;
129     size_t test_count = 0, algo_count = 0;
130     
131     string fixed_name = name_key;
132     
133     string line;
134 
135     while(!src.eof && !src.error)
136     {
137 
138         line = src.readln();
139         if (line.length > 0)
140             line = line[0 .. $-1];
141 
142         if (line.length == 0)
143             continue;
144 
145         if (line[0] == '#')
146             continue;
147         
148         if (line[0] == '[' && line[$-1] == ']')
149         {
150             if (fixed_name != "" && algo_count > 0)
151                 testReport(fixed_name, algo_count, algo_fail);
152             
153             test_count += algo_count;
154             test_fails += algo_fail;
155             algo_count = 0;
156             algo_fail = 0;
157             fixed_name = line[1 .. $ - 1];
158             vars[name_key] = fixed_name;
159             continue;
160         }
161         import std.string : strip;
162         if (line.indexOf('=') == -1) continue;
163         assert(line[line.indexOf('=') - 1] == ' ' && line[line.indexOf('=') + 1] == ' ', "= must be wrapped with spaces");
164         const string key = line[0 .. line.indexOf('=') - 1].strip;
165         const string val = line[line.indexOf('=') + 2 .. $].strip;
166         
167         vars[key] = val;
168         
169         if (key == name_key)
170             fixed_name.length = 0;
171         
172         if (key == output_key)
173         {
174             ++algo_count;
175             try
176             {
177                 const size_t fails = cb(vars);
178                 if (fails)
179                 {
180                     logError(vars[name_key] ~ " test ", algo_count, " : ", fails, " failure");
181                     algo_fail += fails;
182                 }
183             }
184             catch(Exception e)
185             {
186                 logError(vars[name_key] ~ " test ", algo_count, " failed: " ~ e.msg);
187                 ++algo_fail;
188                 assert(false);
189             }
190             
191             if (clear_between_cb)
192             {
193                 vars.clear();
194                 vars[name_key] = fixed_name;
195             }
196         }
197     }
198     test_count += algo_count;
199     test_fails += algo_fail;
200     
201     if (fixed_name != "" && (algo_count > 0 || algo_fail > 0))
202         testReport(fixed_name, algo_count, algo_fail);
203     else
204         testReport(name_key, test_count, test_fails);
205     
206     return test_fails;
207 }
208 
209 size_t runTests(string filename,
210                  string name_key,
211                  string output_key,
212                  bool clear_between_cb,
213                  string delegate(ref HashMap!(string, string)) cb)
214 {
215     File vec = File(filename, "r");
216 
217     if (vec.error || vec.eof)
218     {
219         logError("Failure opening " ~ filename);
220         return 1;
221     }
222     
223     return runTests(vec, name_key, output_key, clear_between_cb, cb);
224 }
225 
226 size_t runTests(ref File src,
227                  string name_key,
228                  string output_key,
229                  bool clear_between_cb,
230                  string delegate(ref HashMap!(string, string)) cb)
231 {
232     return runTestsBb(src, name_key, output_key, clear_between_cb, 
233         (ref HashMap!(string, string) vars)
234         {
235             const string got = cb(vars);
236             if (got != vars[output_key])
237             {
238                 logTrace(name_key ~ ' ' ~ vars[name_key] ~ " got " ~ got ~ " expected " ~ vars[output_key]);
239                 return 1;
240             }
241             return 0;
242         });
243 }