botan.cert.x509.test source code

1 /**
2 * Unit test helper
3 * 
4 * Copyright:
5 * (C) 2014-2015 Jack Lloyd
6 * (C) 2014-2015 Etienne Cimon
7 *
8 * License:
9 * Botan is released under the Simplified BSD License (see LICENSE.md)
10 */
11 module botan.cert.x509.test;
12 
13 /*
14   Code to run the X.509v3 processing tests described in "Conformance
15   Testing of Relying Party Client Certificate Path Proccessing Logic",
16   which is available on NIST's web site.
17 
18 Known Failures/Problems
19 
20 TLSPolicy extensions are not implemented, so we skip tests #34-#53.
21 
22 Tests #75 and #76 are skipped as they make use of relatively obscure CRL
23 extensions which are not supported.
24 */
25 import botan.constants;
26 static if (BOTAN_TEST && BOTAN_HAS_X509_CERTIFICATES):
27 
28 import botan.test;
29 import botan.cert.x509.x509path;
30 import botan.utils.types;
31 import std.file;
32 import std.array;
33 
34 string[] dirListing(string dir_path)
35 {
36     auto dirfiles = dirEntries(dir_path, "*", SpanMode.shallow);
37     string[] files;
38     foreach(file; dirfiles) {
39         files ~= file.name;
40     }
41     files.sort();
42     return files;
43 }
44 
45 /*
46   The expected results are essentially the error codes that best coorespond
47   to the problem described in the testing documentation.
48 
49   There are a few cases where the tests say there should or should not be an
50   error, and I disagree. A few of the tests have test results different from
51   what they "should" be: these changes are marked as such, and have comments
52   explaining the problem at hand.
53 */
54 CertificateStatusCode[] getExpected()
55 {
56     CertificateStatusCode[] expected_results;
57     expected_results.length = 75;
58     
59     /* OK, not a super great way of doing this... */
60     expected_results[1] = CertificateStatusCode.VERIFIED;
61     expected_results[2] = CertificateStatusCode.SIGNATURE_ERROR;
62     expected_results[3] = CertificateStatusCode.SIGNATURE_ERROR;
63     expected_results[4] = CertificateStatusCode.VERIFIED;
64     expected_results[5] = CertificateStatusCode.CERT_NOT_YET_VALID;
65     expected_results[6] = CertificateStatusCode.CERT_NOT_YET_VALID;
66     expected_results[7] = CertificateStatusCode.VERIFIED;
67     expected_results[8] = CertificateStatusCode.CERT_NOT_YET_VALID;
68     expected_results[9] = CertificateStatusCode.CERT_HAS_EXPIRED;
69     expected_results[10] = CertificateStatusCode.CERT_HAS_EXPIRED;
70     expected_results[11] = CertificateStatusCode.CERT_HAS_EXPIRED;
71     expected_results[12] = CertificateStatusCode.VERIFIED;
72     expected_results[13] = CertificateStatusCode.CERT_ISSUER_NOT_FOUND;
73     
74     expected_results[14] = CertificateStatusCode.CERT_ISSUER_NOT_FOUND;
75     expected_results[15] = CertificateStatusCode.VERIFIED;
76     expected_results[16] = CertificateStatusCode.VERIFIED;
77     expected_results[17] = CertificateStatusCode.VERIFIED;
78     expected_results[18] = CertificateStatusCode.VERIFIED;
79     
80     expected_results[19] = CertificateStatusCode.NO_REVOCATION_DATA;
81     expected_results[20] = CertificateStatusCode.CERT_IS_REVOKED;
82     expected_results[21] = CertificateStatusCode.CERT_IS_REVOKED;
83     
84     expected_results[22] = CertificateStatusCode.CA_CERT_NOT_FOR_CERT_ISSUER;
85     expected_results[23] = CertificateStatusCode.CA_CERT_NOT_FOR_CERT_ISSUER;
86     expected_results[24] = CertificateStatusCode.VERIFIED;
87     expected_results[25] = CertificateStatusCode.CA_CERT_NOT_FOR_CERT_ISSUER;
88     expected_results[26] = CertificateStatusCode.VERIFIED;
89     expected_results[27] = CertificateStatusCode.VERIFIED;
90     expected_results[28] = CertificateStatusCode.CA_CERT_NOT_FOR_CERT_ISSUER;
91     expected_results[29] = CertificateStatusCode.CA_CERT_NOT_FOR_CERT_ISSUER;
92     expected_results[30] = CertificateStatusCode.VERIFIED;
93     
94     expected_results[31] = CertificateStatusCode.CA_CERT_NOT_FOR_CRL_ISSUER;
95     expected_results[32] = CertificateStatusCode.CA_CERT_NOT_FOR_CRL_ISSUER;
96     expected_results[33] = CertificateStatusCode.VERIFIED;
97     
98     /*
99      TLSPolicy tests: a little trickier because there are other inputs
100      which affect the result.
101 
102      In the case of the tests currently in the suite, the default
103      method (with acceptable policy being "any-policy" and with no
104      explict policy required), will almost always result in a verified
105      status. This is not particularly helpful. So, we should do several
106      different tests for each test set:
107 
108          1) With the user policy as any-policy and no explicit policy
109          2) With the user policy as any-policy and an explicit policy required
110          3) With the user policy as test-policy-1 (2.16.840.1.101.3.1.48.1) and
111              an explict policy required
112          4) With the user policy as either test-policy-1 or test-policy-2 and an
113              explicit policy required
114 
115       This provides reasonably good coverage of the possible outcomes.
116     */
117     
118     expected_results[34] = CertificateStatusCode.VERIFIED;
119     expected_results[35] = CertificateStatusCode.VERIFIED;
120     expected_results[36] = CertificateStatusCode.VERIFIED;
121     expected_results[37] = CertificateStatusCode.VERIFIED;
122     expected_results[38] = CertificateStatusCode.VERIFIED;
123     expected_results[39] = CertificateStatusCode.VERIFIED;
124     expected_results[40] = CertificateStatusCode.VERIFIED;
125     expected_results[41] = CertificateStatusCode.VERIFIED;
126     expected_results[42] = CertificateStatusCode.VERIFIED;
127     expected_results[43] = CertificateStatusCode.VERIFIED;
128     expected_results[44] = CertificateStatusCode.VERIFIED;
129     
130     //expected_results[45] = CertificateStatusCode.EXPLICT_POLICY_REQUIRED;
131     //expected_results[46] = CertificateStatusCode.ACCEPT;
132     //expected_results[47] = CertificateStatusCode.EXPLICT_POLICY_REQUIRED;
133     
134     expected_results[48] = CertificateStatusCode.VERIFIED;
135     expected_results[49] = CertificateStatusCode.VERIFIED;
136     expected_results[50] = CertificateStatusCode.VERIFIED;
137     expected_results[51] = CertificateStatusCode.VERIFIED;
138     expected_results[52] = CertificateStatusCode.VERIFIED;
139     expected_results[53] = CertificateStatusCode.VERIFIED;
140     
141     expected_results[54] = CertificateStatusCode.CERT_CHAIN_TOO_LONG;
142     expected_results[55] = CertificateStatusCode.CERT_CHAIN_TOO_LONG;
143     expected_results[56] = CertificateStatusCode.VERIFIED;
144     expected_results[57] = CertificateStatusCode.VERIFIED;
145     expected_results[58] = CertificateStatusCode.CERT_CHAIN_TOO_LONG;
146     expected_results[59] = CertificateStatusCode.CERT_CHAIN_TOO_LONG;
147     expected_results[60] = CertificateStatusCode.CERT_CHAIN_TOO_LONG;
148     expected_results[61] = CertificateStatusCode.CERT_CHAIN_TOO_LONG;
149     expected_results[62] = CertificateStatusCode.VERIFIED;
150     expected_results[63] = CertificateStatusCode.VERIFIED;
151     
152     expected_results[64] = CertificateStatusCode.CRL_BAD_SIGNATURE;
153     
154     expected_results[65] = CertificateStatusCode.NO_REVOCATION_DATA;
155     expected_results[66] = CertificateStatusCode.NO_REVOCATION_DATA;
156     
157     expected_results[67] = CertificateStatusCode.VERIFIED;
158     
159     expected_results[68] = CertificateStatusCode.CERT_IS_REVOKED;
160     expected_results[69] = CertificateStatusCode.CERT_IS_REVOKED;
161     expected_results[70] = CertificateStatusCode.CERT_IS_REVOKED;
162     expected_results[71] = CertificateStatusCode.CERT_IS_REVOKED;
163     expected_results[72] = CertificateStatusCode.CRL_HAS_EXPIRED;
164     expected_results[73] = CertificateStatusCode.CRL_HAS_EXPIRED;
165     expected_results[74] = CertificateStatusCode.VERIFIED;
166     
167     /* These tests use weird CRL extensions which aren't supported yet */
168     //expected_results[75] = ;
169     //expected_results[76] = ;
170     
171     return expected_results;
172 }
173 
174 static if (BOTAN_HAS_TESTS && !SKIP_X509_TEST) unittest
175 {
176     import botan.libstate.global_state;
177     auto state = globalState(); // ensure initialized
178 
179     logDebug("Testing x509/test.d ...");
180     const string root_test_dir = "../test_data/nist_x509/";
181     
182     size_t unexp_failure = 0;
183     size_t unexp_success = 0;
184     size_t wrong_error = 0;
185     size_t skipped = 0;
186     size_t ran = 0;
187     
188     CertificateStatusCode[] expected_results = getExpected();
189     
190     try {
191         
192         const string[] test_dirs = dirListing(root_test_dir);
193         
194         for(size_t i = 0; i != 74; i++)
195         {
196             const size_t test_no = i+1;
197             logDebug("NIST X.509 test #", test_no);
198             
199             const string test_dir = test_dirs[i];
200             const string[] all_files = dirListing(test_dir);
201             
202             Vector!string certs, crls;
203             string root_cert, to_verify;
204             
205             for(size_t k = 0; k != all_files.length; k++)
206             {
207                 const string current = all_files[k];
208                 
209                 if (current.canFind("int") && current.canFind(".crt"))
210                     certs.pushBack(current);
211                 else if (current.canFind("root.crt"))
212                     root_cert = current;
213                 else if (current.canFind("end.crt"))
214                     to_verify = current;
215                 else if (current.canFind(".crl"))
216                     crls.pushBack(current);
217             }
218             
219             if (expected_results.canFind(i+1) == -1)
220             {
221                 skipped++;
222                 continue;
223             }
224             
225             ++ran;
226             
227             auto store = scoped!CertificateStoreInMemory();
228             
229             //logTrace(root_cert);
230             store.addCertificate(X509Certificate(root_cert));
231             
232             X509Certificate end_user = X509Certificate(to_verify);
233             foreach(cert; certs[])
234                 store.addCertificate(X509Certificate(cert));
235             
236             foreach(crl; crls[])
237             {
238                 DataSourceStream input = DataSourceStream(crl);
239                 X509CRL crl_ = X509CRL(cast(DataSource)input);
240                 store.addCrl(crl_);
241             }
242             
243             auto restrictions = PathValidationRestrictions(true);
244             
245             PathValidationResult validation_result = x509PathValidate(end_user, restrictions, store);
246             auto expected = expected_results[test_no];
247             CertificateStatusCode result = validation_result.result();
248             if (result != expected) {
249                 logError("NIST X.509 test #", test_no, " : ");
250                 const string result_str = PathValidationResult.statusString(result);
251                 const string exp_str = PathValidationResult.statusString(expected);
252                 if (expected == CertificateStatusCode.VERIFIED) {
253                     logError("unexpected failure: " ~ result_str);
254                     unexp_failure++;
255                 }
256                 else if (result == CertificateStatusCode.VERIFIED) {
257                     logError("unexpected success, expected " ~ exp_str);
258                     unexp_success++;
259                 } 
260                 else {
261                     logError("wrong error, got '" ~ result_str ~ "' expected '" ~ exp_str ~ "'");
262                     wrong_error++;
263                     assert(false);
264                 }
265             }
266         }
267     }
268     catch(Throwable e)
269     {
270         logError(e.toString());
271         logTrace(e.msg);
272     }
273     
274     const size_t all_failures = unexp_failure + unexp_success + wrong_error;
275     
276     testReport("NIST X.509 path validation", ran, all_failures);
277 }