1 /**
2 * Unit test helper
3 * 
4 * Copyright:
5 * (C) 2014-2015 Jack Lloyd
6 * (C) 2014-2015 Etienne Cimon
7 *
8 * License:
9 * Botan is released under the Simplified BSD License (see LICENSE.md)
10 */
11 module botan.test;
12 
13 import botan.constants;
14 //static if (BOTAN_TEST):
15 
16 public import std.stdio : File, writeln;
17 public import std.algorithm : sort, canFind;
18 public import std.range : walkLength;
19 public import std.string : indexOf, lastIndexOf;
20 public import botan.utils.types;
21 public import botan.libstate.libstate;
22 import memutils.hashmap;
23 import std.file;
24 import std.array;
25 import std.exception;
26 
27 @property bool ok(File f) { return f.isOpen && !f.eof() && !f.error(); }
28 
29 string CHECK_MESSAGE (string expr, string print) {
30     return "
31     {
32         import core.atomic : atomicOp;
33         static if (is(typeof(total_tests) == shared)) atomicOp!`+=`(total_tests, cast(size_t) 1);
34         else total_tests++;
35         try { 
36             if (!(" ~ expr ~ ")) { 
37                 ++fails; 
38                 logError( `" ~ print ~ "` ); 
39             } 
40         } 
41         catch(Exception e) 
42         { 
43             logError(__FUNCTION__, ` : ` ~ e.msg); 
44         }
45     }";
46 }
47 
48 string CHECK (string expr) {
49     return `
50     {
51         import core.atomic : atomicOp;
52         static if (is(typeof(total_tests) == shared)) atomicOp!"+="(total_tests, cast(size_t) 1);
53         else total_tests++;
54         mixin( q{
55             bool success = ` ~ expr ~ `;
56         } );
57         try { 
58             if (!success)
59             { ++fails; logError( q{ ` ~ expr ~ ` } ); } 
60         } 
61         catch(Exception e) 
62         { 
63             logError(__FUNCTION__ ~ " : " ~ e.msg); 
64         }
65     }`;
66 }
67 
68 
69 string[] listDir(string dir_path)
70 {
71     auto dirfiles = dirEntries(dir_path, "*.vec", SpanMode.depth);
72     string[] files;
73     foreach(file; dirfiles) {
74         files ~= file.name;
75     }
76     files.sort();
77     return files;
78 }
79 
80 size_t runTestsInDir(string dir, size_t delegate(string) fn)
81 {
82     assert(exists(cast(char[])dir), "Directory `" ~ dir ~ "` does not exist");
83     logTrace("Running tests for directory: " ~ dir);
84     import std.parallelism;
85     import core.atomic;
86     shared(size_t) shared_fails;
87     auto dirs = listDir(dir);
88     foreach (vec; dirs) {
89         size_t local_fails = fn(vec);
90         if (local_fails > 0) {
91             assert(false);
92         }
93         atomicOp!"+="(shared_fails, local_fails);
94     }
95     return cast(size_t)atomicLoad(shared_fails);
96 }
97 
98 void testReport(string name, size_t ran, size_t failed)
99 {    
100     if (failed)
101         logError(name, " ... ", failed, " / ", ran, " ************** FAILED ****************");
102     else
103         logDebug(name, " ... PASSED (all of ", ran, " tests)");
104 }
105 
106 size_t runTestsBb(ref File src,
107                   string name_key,
108                   string output_key,
109                   bool clear_between_cb,
110                   size_t delegate(ref HashMap!(string, string)) cb)
111 {
112     if (src.eof || src.error)
113     {
114         logError("Could not open input file for " ~ name_key);
115         return 1;
116     }
117     
118     HashMap!(string, string) vars;
119     vars[name_key] = name_key;
120     size_t test_fails = 0, algo_fail = 0;
121     size_t test_count = 0, algo_count = 0;
122     
123     string fixed_name = name_key;
124     
125     string line;
126 
127     while(!src.eof && !src.error)
128     {
129 
130         line = src.readln();
131         if (line.length > 0)
132             line = line[0 .. $-1];
133 
134         if (line.length == 0)
135             continue;
136 
137         if (line[0] == '#')
138             continue;
139         
140         if (line[0] == '[' && line[$-1] == ']')
141         {
142             if (fixed_name != "" && algo_count > 0)
143                 testReport(fixed_name, algo_count, algo_fail);
144             
145             test_count += algo_count;
146             test_fails += algo_fail;
147             algo_count = 0;
148             algo_fail = 0;
149             fixed_name = line[1 .. $ - 1];
150             vars[name_key] = fixed_name;
151             continue;
152         }
153         import std.string : strip;
154         if (line.indexOf('=') == -1) continue;
155         assert(line[line.indexOf('=') - 1] == ' ' && line[line.indexOf('=') + 1] == ' ', "= must be wrapped with spaces");
156         const string key = line[0 .. line.indexOf('=') - 1].strip;
157         const string val = line[line.indexOf('=') + 2 .. $].strip;
158         
159         vars[key] = val;
160         
161         if (key == name_key)
162             fixed_name.length = 0;
163         
164         if (key == output_key)
165         {
166             ++algo_count;
167             try
168             {
169                 const size_t fails = cb(vars);
170                 if (fails)
171                 {
172                     logError(vars[name_key] ~ " test ", algo_count, " : ", fails, " failure");
173                     algo_fail += fails;
174                 }
175             }
176             catch(Exception e)
177             {
178                 logError(vars[name_key] ~ " test ", algo_count, " failed: " ~ e.msg);
179                 ++algo_fail;
180                 assert(false);
181             }
182             
183             if (clear_between_cb)
184             {
185                 vars.clear();
186                 vars[name_key] = fixed_name;
187             }
188         }
189     }
190     test_count += algo_count;
191     test_fails += algo_fail;
192     
193     if (fixed_name != "" && (algo_count > 0 || algo_fail > 0))
194         testReport(fixed_name, algo_count, algo_fail);
195     else
196         testReport(name_key, test_count, test_fails);
197     
198     return test_fails;
199 }
200 
201 size_t runTests(string filename,
202                  string name_key,
203                  string output_key,
204                  bool clear_between_cb,
205                  string delegate(ref HashMap!(string, string)) cb)
206 {
207     File vec = File(filename, "r");
208     
209     if (vec.error || vec.eof)
210     {
211         logError("Failure opening " ~ filename);
212         return 1;
213     }
214     
215     return runTests(vec, name_key, output_key, clear_between_cb, cb);
216 }
217 
218 size_t runTests(ref File src,
219                  string name_key,
220                  string output_key,
221                  bool clear_between_cb,
222                  string delegate(ref HashMap!(string, string)) cb)
223 {
224     return runTestsBb(src, name_key, output_key, clear_between_cb, 
225         (ref HashMap!(string, string) vars)
226         {
227             const string got = cb(vars);
228             if (got != vars[output_key])
229             {
230                 logTrace(name_key ~ ' ' ~ vars[name_key] ~ " got " ~ got ~ " expected " ~ vars[output_key]);
231                 return 1;
232             }
233             return 0;
234         });
235 }