1 /*
2 Code to run the X.509v3 processing tests described in "Conformance
3 Testing of Relying Party Client Certificate Path Proccessing Logic",
4 which is available on NIST's web site.
5 */
6
7 #include <botan/x509stor.h>
8 #include <botan/init.h>
9 using namespace Botan;
10
11 #include <algorithm>
12 #include <iostream>
13 #include <iomanip>
14 #include <string>
15 #include <vector>
16 #include <map>
17 #include <cstdlib>
18
19 #include <dirent.h>
20
21 std::vector<std::string> dir_listing(const std::string&);
22
23 void run_one_test(u32bit, X509_Code,
24 std::string, std::string,
25 std::vector<std::string>,
26 std::vector<std::string>);
27
28 std::map<u32bit, X509_Code> expected_results;
29
30 u32bit unexp_failure, unexp_success, wrong_error, skipped;
31
32 void populate_expected_results();
33
main()34 int main()
35 {
36 const std::string root_test_dir = "tests/";
37 unexp_failure = unexp_success = wrong_error = skipped = 0;
38
39 try {
40
41 LibraryInitializer init;
42
43 populate_expected_results();
44
45 std::vector<std::string> test_dirs = dir_listing(root_test_dir);
46 std::sort(test_dirs.begin(), test_dirs.end());
47
48 for(size_t j = 0; j != test_dirs.size(); j++)
49 {
50 const std::string test_dir = root_test_dir + test_dirs[j] + "/";
51 std::vector<std::string> all_files = dir_listing(test_dir);
52
53 std::vector<std::string> certs, crls;
54 std::string root_cert, to_verify;
55
56 for(size_t k = 0; k != all_files.size(); k++)
57 {
58 const std::string current = all_files[k];
59 if(current.find("int") != std::string::npos &&
60 current.find(".crt") != std::string::npos)
61 certs.push_back(test_dir + current);
62 else if(current.find("root.crt") != std::string::npos)
63 root_cert = test_dir + current;
64 else if(current.find("end.crt") != std::string::npos)
65 to_verify = test_dir + current;
66 else if(current.find(".crl") != std::string::npos)
67 crls.push_back(test_dir + current);
68 }
69
70 if(expected_results.find(j+1) == expected_results.end())
71 {
72 #if 0
73 std::cout << "Testing disabled for test #" << j+1
74 << " <skipped>" << std::endl;
75 #endif
76 skipped++;
77 continue;
78 }
79
80 run_one_test(j+1, expected_results[j+1],
81 root_cert, to_verify, certs, crls);
82 }
83
84 }
85 catch(std::exception& e)
86 {
87 std::cout << e.what() << std::endl;
88 return 1;
89 }
90
91 std::cout << "Total unexpected failures: " << unexp_failure << std::endl;
92 std::cout << "Total unexpected successes: " << unexp_success << std::endl;
93 std::cout << "Total incorrect failures: " << wrong_error << std::endl;
94 std::cout << "Tests skipped: " << skipped << std::endl;
95
96 return 0;
97 }
98
run_one_test(u32bit test_no,X509_Code expected,std::string root_cert,std::string to_verify,std::vector<std::string> certs,std::vector<std::string> crls)99 void run_one_test(u32bit test_no, X509_Code expected,
100 std::string root_cert, std::string to_verify,
101 std::vector<std::string> certs,
102 std::vector<std::string> crls)
103 {
104 std::cout << "Processing test #" << test_no << "... ";
105 std::cout.flush();
106
107 X509_Code result = VERIFIED;
108
109 X509_Store store;
110
111 store.add_cert(X509_Certificate(root_cert), true);
112
113 X509_Certificate end_user(to_verify);
114
115 for(size_t j = 0; j != certs.size(); j++)
116 store.add_cert(X509_Certificate(certs[j]));
117
118 for(size_t j = 0; j != crls.size(); j++)
119 {
120 DataSource_Stream in(crls[j]);
121
122 X509_CRL crl(in);
123 /*
124 std::vector<CRL_Entry> crl_entries = crl.get_revoked();
125 for(u32bit k = 0; k != crl_entries.size(); k++)
126 {
127 std::cout << "Revoked: " << std::flush;
128 for(u32bit l = 0; l != crl_entries[k].serial.size(); l++)
129 printf("%02X", crl_entries[k].serial[l]);
130 std::cout << std::endl;
131 }
132 */
133 result = store.add_crl(crl);
134 if(result != VERIFIED)
135 break;
136 }
137
138 /* if everything has gone well up until now */
139
140 if(result == VERIFIED)
141 {
142 result = store.validate_cert(end_user);
143
144 X509_Code result2 = store.validate_cert(end_user);
145
146 if(result != result2)
147 std::cout << "Two runs, two answers: " << result << " "
148 << result2 << std::endl;
149 }
150
151 if(result == expected)
152 {
153 std::cout << "passed" << std::endl;
154 return;
155 }
156
157 if(expected == VERIFIED)
158 {
159 std::cout << "unexpected failure: " << result << std::endl;
160 unexp_failure++;
161 }
162 else if(result == VERIFIED)
163 {
164 std::cout << "unexpected success: " << expected << std::endl;
165 unexp_success++;
166 }
167 else
168 {
169 std::cout << "wrong error: " << result << "/" << expected << std::endl;
170 wrong_error++;
171 }
172 }
173
dir_listing(const std::string & dir_name)174 std::vector<std::string> dir_listing(const std::string& dir_name)
175 {
176 DIR* dir = opendir(dir_name.c_str());
177 if(!dir)
178 {
179 std::cout << "Error, couldn't open dir " << dir_name << std::endl;
180 std::exit(1);
181 }
182
183 std::vector<std::string> listing;
184
185 while(true)
186 {
187 struct dirent* dir_ent = readdir(dir);
188
189 if(dir_ent == 0)
190 break;
191 const std::string entry = dir_ent->d_name;
192 if(entry == "." || entry == "..")
193 continue;
194
195 listing.push_back(entry);
196 }
197 closedir(dir);
198
199 return listing;
200 }
201
202 /*
203 The expected results are essentially the error codes that best coorespond
204 to the problem described in the testing documentation.
205
206 There are a few cases where the tests say there should or should not be an
207 error, and I disagree. A few of the tests have test results different from
208 what they "should" be: these changes are marked as such, and have comments
209 explaining the problem at hand.
210 */
populate_expected_results()211 void populate_expected_results()
212 {
213 /* OK, not a super great way of doing this... */
214 expected_results[1] = VERIFIED;
215 expected_results[2] = SIGNATURE_ERROR;
216 expected_results[3] = SIGNATURE_ERROR;
217 expected_results[4] = VERIFIED;
218 expected_results[5] = CERT_NOT_YET_VALID;
219 expected_results[6] = CERT_NOT_YET_VALID;
220 expected_results[7] = VERIFIED;
221 expected_results[8] = CERT_NOT_YET_VALID;
222 expected_results[9] = CERT_HAS_EXPIRED;
223 expected_results[10] = CERT_HAS_EXPIRED;
224 expected_results[11] = CERT_HAS_EXPIRED;
225 expected_results[12] = VERIFIED;
226 expected_results[13] = CERT_ISSUER_NOT_FOUND;
227
228 // FIXME: we get the answer right for the wrong reason
229 // ummm... I don't know if that is still true. I wish I had thought to
230 // write down exactly what this 'wrong reason' was in the first place.
231 expected_results[14] = CERT_ISSUER_NOT_FOUND;
232 expected_results[15] = VERIFIED;
233 expected_results[16] = VERIFIED;
234 expected_results[17] = VERIFIED;
235 expected_results[18] = VERIFIED;
236
237 /************* CHANGE OF TEST RESULT FOR TEST #19 ************************
238 One of the certificates has no attached CRL. By strict X.509 rules, if
239 there is no good CRL in hand, then the certificate shouldn't be used for
240 CA stuff. But while this is usually a good idea, it interferes with simple
241 uses of certificates which shouldn't (IMO) force the use of CRLs. There is
242 no assigned error code for this scenario because I don't consider it to be
243 an error (probably would be something like NO_REVOCATION_DATA_AVAILABLE)
244 **************************************************************************/
245 expected_results[19] = VERIFIED;
246 expected_results[20] = CERT_IS_REVOKED;
247 expected_results[21] = CERT_IS_REVOKED;
248
249 expected_results[22] = CA_CERT_NOT_FOR_CERT_ISSUER;
250 expected_results[23] = CA_CERT_NOT_FOR_CERT_ISSUER;
251 expected_results[24] = VERIFIED;
252 expected_results[25] = CA_CERT_NOT_FOR_CERT_ISSUER;
253 expected_results[26] = VERIFIED;
254 expected_results[27] = VERIFIED;
255 expected_results[28] = CA_CERT_NOT_FOR_CERT_ISSUER;
256 expected_results[29] = CA_CERT_NOT_FOR_CERT_ISSUER;
257 expected_results[30] = VERIFIED;
258
259 expected_results[31] = CA_CERT_NOT_FOR_CRL_ISSUER;
260 expected_results[32] = CA_CERT_NOT_FOR_CRL_ISSUER;
261 expected_results[33] = VERIFIED;
262
263 /*
264 Policy tests: a little trickier because there are other inputs
265 which affect the result.
266
267 In the case of the tests currently in the suite, the default
268 method (with acceptable policy being "any-policy" and with no
269 explict policy required), will almost always result in a verified
270 status. This is not particularly helpful. So, we should do several
271 different tests for each test set:
272
273 1) With the user policy as any-policy and no explicit policy
274 2) With the user policy as any-policy and an explicit policy required
275 3) With the user policy as test-policy-1 (2.16.840.1.101.3.1.48.1) and
276 an explict policy required
277 4) With the user policy as either test-policy-1 or test-policy-2 and an
278 explicit policy required
279
280 This provides reasonably good coverage of the possible outcomes.
281 */
282
283 expected_results[34] = VERIFIED;
284 expected_results[35] = VERIFIED;
285 expected_results[36] = VERIFIED;
286 expected_results[37] = VERIFIED;
287 expected_results[38] = VERIFIED;
288 expected_results[39] = VERIFIED;
289 expected_results[40] = VERIFIED;
290 expected_results[41] = VERIFIED;
291 expected_results[42] = VERIFIED;
292 expected_results[43] = VERIFIED;
293 expected_results[44] = VERIFIED;
294
295 //expected_results[45] = EXPLICT_POLICY_REQUIRED;
296 //expected_results[46] = ACCEPT;
297 //expected_results[47] = EXPLICT_POLICY_REQUIRED;
298
299 expected_results[48] = VERIFIED;
300 expected_results[49] = VERIFIED;
301 expected_results[50] = VERIFIED;
302 expected_results[51] = VERIFIED;
303 expected_results[52] = VERIFIED;
304 expected_results[53] = VERIFIED;
305
306 expected_results[54] = CERT_CHAIN_TOO_LONG;
307 expected_results[55] = CERT_CHAIN_TOO_LONG;
308 expected_results[56] = VERIFIED;
309 expected_results[57] = VERIFIED;
310 expected_results[58] = CERT_CHAIN_TOO_LONG;
311 expected_results[59] = CERT_CHAIN_TOO_LONG;
312 expected_results[60] = CERT_CHAIN_TOO_LONG;
313 expected_results[61] = CERT_CHAIN_TOO_LONG;
314 expected_results[62] = VERIFIED;
315 expected_results[63] = VERIFIED;
316
317 expected_results[64] = SIGNATURE_ERROR;
318
319 /************ CHANGE OF TEST RESULT FOR TEST #65 *************************
320 I cannot figure out what exactly the problem here is supposed to be;
321 looking at it by hand, everything seems fine. If someone can explain I
322 would be happy to listen.
323 ************************************************************************/
324 expected_results[65] = VERIFIED;
325 expected_results[66] = CRL_ISSUER_NOT_FOUND;
326
327 /************ CHANGE OF TEST RESULT FOR TEST #67 *************************
328 The test docs say this should be verified. However, the problem being that
329 there is an extra CRL with an unknown issuer. Returning VERIFIED in this
330 case is obviously bad, since the user may well want to know that the CRL
331 in question has no known issuer. So we return CRL_ISSUER_NOT_FOUND instead
332 of VERIFIED. The actual certificate path of course still verifies, but
333 it's kind of an all-or-nothing testing procedure.
334 ************************************************************************/
335 expected_results[67] = CRL_ISSUER_NOT_FOUND;
336
337 expected_results[68] = CERT_IS_REVOKED;
338 expected_results[69] = CERT_IS_REVOKED;
339 expected_results[70] = CERT_IS_REVOKED;
340 expected_results[71] = CERT_IS_REVOKED;
341 expected_results[72] = CRL_HAS_EXPIRED;
342 expected_results[73] = CRL_HAS_EXPIRED;
343 expected_results[74] = VERIFIED;
344
345 /* These tests use weird CRL extensions which aren't supported yet */
346 //expected_results[75] = ;
347 //expected_results[76] = ;
348 }
349