1#!/usr/bin/env python3 2 3# Greentea host test script for Mbed TLS on-target test suite testing. 4# 5# Copyright The Mbed TLS Contributors 6# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later 7# 8# This file is provided under the Apache License 2.0, or the 9# GNU General Public License v2.0 or later. 10# 11# ********** 12# Apache License 2.0: 13# 14# Licensed under the Apache License, Version 2.0 (the "License"); you may 15# not use this file except in compliance with the License. 16# You may obtain a copy of the License at 17# 18# http://www.apache.org/licenses/LICENSE-2.0 19# 20# Unless required by applicable law or agreed to in writing, software 21# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 22# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 23# See the License for the specific language governing permissions and 24# limitations under the License. 25# 26# ********** 27# 28# ********** 29# GNU General Public License v2.0 or later: 30# 31# This program is free software; you can redistribute it and/or modify 32# it under the terms of the GNU General Public License as published by 33# the Free Software Foundation; either version 2 of the License, or 34# (at your option) any later version. 35# 36# This program is distributed in the hope that it will be useful, 37# but WITHOUT ANY WARRANTY; without even the implied warranty of 38# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 39# GNU General Public License for more details. 40# 41# You should have received a copy of the GNU General Public License along 42# with this program; if not, write to the Free Software Foundation, Inc., 43# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 44# 45# ********** 46 47 48""" 49Mbed TLS on-target test suite tests are implemented as Greentea 50tests. Greentea tests are implemented in two parts: target test and 51host test. Target test is a C application that is built for the 52target platform and executes on the target. Host test is a Python 53class derived from mbed_host_tests.BaseHostTest. Target communicates 54with the host over serial for the test data and sends back the result. 55 56Python tool mbedgt (Greentea) is responsible for flashing the test 57binary on to the target and dynamically loading this host test module. 58 59Greentea documentation can be found here: 60https://github.com/ARMmbed/greentea 61""" 62 63 64import re 65import os 66import binascii 67 68from mbed_host_tests import BaseHostTest, event_callback # pylint: disable=import-error 69 70 71class TestDataParserError(Exception): 72 """Indicates error in test data, read from .data file.""" 73 pass 74 75 76class TestDataParser: 77 """ 78 Parses test name, dependencies, test function name and test parameters 79 from the data file. 80 """ 81 82 def __init__(self): 83 """ 84 Constructor 85 """ 86 self.tests = [] 87 88 def parse(self, data_file): 89 """ 90 Data file parser. 91 92 :param data_file: Data file path 93 """ 94 with open(data_file, 'r') as data_f: 95 self.__parse(data_f) 96 97 @staticmethod 98 def __escaped_split(inp_str, split_char): 99 """ 100 Splits inp_str on split_char except when escaped. 101 102 :param inp_str: String to split 103 :param split_char: Split character 104 :return: List of splits 105 """ 106 split_colon_fn = lambda x: re.sub(r'\\' + split_char, split_char, x) 107 if len(split_char) > 1: 108 raise ValueError('Expected split character. Found string!') 109 out = list(map(split_colon_fn, re.split(r'(?<!\\)' + split_char, inp_str))) 110 out = [x for x in out if x] 111 return out 112 113 def __parse(self, data_f): 114 """ 115 Parses data file using supplied file object. 116 117 :param data_f: Data file object 118 :return: 119 """ 120 for line in data_f: 121 line = line.strip() 122 if not line: 123 continue 124 # Read test name 125 name = line 126 127 # Check dependencies 128 dependencies = [] 129 line = next(data_f).strip() 130 match = re.search('depends_on:(.*)', line) 131 if match: 132 dependencies = [int(x) for x in match.group(1).split(':')] 133 line = next(data_f).strip() 134 135 # Read test vectors 136 line = line.replace('\\n', '\n') 137 parts = self.__escaped_split(line, ':') 138 function_name = int(parts[0]) 139 args = parts[1:] 140 args_count = len(args) 141 if args_count % 2 != 0: 142 err_str_fmt = "Number of test arguments({}) should be even: {}" 143 raise TestDataParserError(err_str_fmt.format(args_count, line)) 144 grouped_args = [(args[i * 2], args[(i * 2) + 1]) 145 for i in range(int(len(args)/2))] 146 self.tests.append((name, function_name, dependencies, 147 grouped_args)) 148 149 def get_test_data(self): 150 """ 151 Returns test data. 152 """ 153 return self.tests 154 155 156class MbedTlsTest(BaseHostTest): 157 """ 158 Host test for Mbed TLS unit tests. This script is loaded at 159 run time by Greentea for executing Mbed TLS test suites. Each 160 communication from the target is received in this object as 161 an event, which is then handled by the event handler method 162 decorated by the associated event. Ex: @event_callback('GO'). 163 164 Target test sends requests for dispatching next test. It reads 165 tests from the intermediate data file and sends test function 166 identifier, dependency identifiers, expression identifiers and 167 the test data in binary form. Target test checks dependencies 168 , evaluate integer constant expressions and dispatches the test 169 function with received test parameters. After test function is 170 finished, target sends the result. This class handles the result 171 event and prints verdict in the form that Greentea understands. 172 173 """ 174 # status/error codes from suites/helpers.function 175 DEPENDENCY_SUPPORTED = 0 176 KEY_VALUE_MAPPING_FOUND = DEPENDENCY_SUPPORTED 177 DISPATCH_TEST_SUCCESS = DEPENDENCY_SUPPORTED 178 179 KEY_VALUE_MAPPING_NOT_FOUND = -1 # Expression Id not found. 180 DEPENDENCY_NOT_SUPPORTED = -2 # Dependency not supported. 181 DISPATCH_TEST_FN_NOT_FOUND = -3 # Test function not found. 182 DISPATCH_INVALID_TEST_DATA = -4 # Invalid parameter type. 183 DISPATCH_UNSUPPORTED_SUITE = -5 # Test suite not supported/enabled. 184 185 def __init__(self): 186 """ 187 Constructor initialises test index to 0. 188 """ 189 super(MbedTlsTest, self).__init__() 190 self.tests = [] 191 self.test_index = -1 192 self.dep_index = 0 193 self.suite_passed = True 194 self.error_str = dict() 195 self.error_str[self.DEPENDENCY_SUPPORTED] = \ 196 'DEPENDENCY_SUPPORTED' 197 self.error_str[self.KEY_VALUE_MAPPING_NOT_FOUND] = \ 198 'KEY_VALUE_MAPPING_NOT_FOUND' 199 self.error_str[self.DEPENDENCY_NOT_SUPPORTED] = \ 200 'DEPENDENCY_NOT_SUPPORTED' 201 self.error_str[self.DISPATCH_TEST_FN_NOT_FOUND] = \ 202 'DISPATCH_TEST_FN_NOT_FOUND' 203 self.error_str[self.DISPATCH_INVALID_TEST_DATA] = \ 204 'DISPATCH_INVALID_TEST_DATA' 205 self.error_str[self.DISPATCH_UNSUPPORTED_SUITE] = \ 206 'DISPATCH_UNSUPPORTED_SUITE' 207 208 def setup(self): 209 """ 210 Setup hook implementation. Reads test suite data file and parses out 211 tests. 212 """ 213 binary_path = self.get_config_item('image_path') 214 script_dir = os.path.split(os.path.abspath(__file__))[0] 215 suite_name = os.path.splitext(os.path.basename(binary_path))[0] 216 data_file = ".".join((suite_name, 'datax')) 217 data_file = os.path.join(script_dir, '..', 'mbedtls', 218 suite_name, data_file) 219 if os.path.exists(data_file): 220 self.log("Running tests from %s" % data_file) 221 parser = TestDataParser() 222 parser.parse(data_file) 223 self.tests = parser.get_test_data() 224 self.print_test_info() 225 else: 226 self.log("Data file not found: %s" % data_file) 227 self.notify_complete(False) 228 229 def print_test_info(self): 230 """ 231 Prints test summary read by Greentea to detect test cases. 232 """ 233 self.log('{{__testcase_count;%d}}' % len(self.tests)) 234 for name, _, _, _ in self.tests: 235 self.log('{{__testcase_name;%s}}' % name) 236 237 @staticmethod 238 def align_32bit(data_bytes): 239 """ 240 4 byte aligns input byte array. 241 242 :return: 243 """ 244 data_bytes += bytearray((4 - (len(data_bytes))) % 4) 245 246 @staticmethod 247 def hex_str_bytes(hex_str): 248 """ 249 Converts Hex string representation to byte array 250 251 :param hex_str: Hex in string format. 252 :return: Output Byte array 253 """ 254 if hex_str[0] != '"' or hex_str[len(hex_str) - 1] != '"': 255 raise TestDataParserError("HEX test parameter missing '\"':" 256 " %s" % hex_str) 257 hex_str = hex_str.strip('"') 258 if len(hex_str) % 2 != 0: 259 raise TestDataParserError("HEX parameter len should be mod of " 260 "2: %s" % hex_str) 261 262 data_bytes = binascii.unhexlify(hex_str) 263 return data_bytes 264 265 @staticmethod 266 def int32_to_big_endian_bytes(i): 267 """ 268 Coverts i to byte array in big endian format. 269 270 :param i: Input integer 271 :return: Output bytes array in big endian or network order 272 """ 273 data_bytes = bytearray([((i >> x) & 0xff) for x in [24, 16, 8, 0]]) 274 return data_bytes 275 276 def test_vector_to_bytes(self, function_id, dependencies, parameters): 277 """ 278 Converts test vector into a byte array that can be sent to the target. 279 280 :param function_id: Test Function Identifier 281 :param dependencies: Dependency list 282 :param parameters: Test function input parameters 283 :return: Byte array and its length 284 """ 285 data_bytes = bytearray([len(dependencies)]) 286 if dependencies: 287 data_bytes += bytearray(dependencies) 288 data_bytes += bytearray([function_id, len(parameters)]) 289 for typ, param in parameters: 290 if typ in ('int', 'exp'): 291 i = int(param, 0) 292 data_bytes += b'I' if typ == 'int' else b'E' 293 self.align_32bit(data_bytes) 294 data_bytes += self.int32_to_big_endian_bytes(i) 295 elif typ == 'char*': 296 param = param.strip('"') 297 i = len(param) + 1 # + 1 for null termination 298 data_bytes += b'S' 299 self.align_32bit(data_bytes) 300 data_bytes += self.int32_to_big_endian_bytes(i) 301 data_bytes += bytearray(param, encoding='ascii') 302 data_bytes += b'\0' # Null terminate 303 elif typ == 'hex': 304 binary_data = self.hex_str_bytes(param) 305 data_bytes += b'H' 306 self.align_32bit(data_bytes) 307 i = len(binary_data) 308 data_bytes += self.int32_to_big_endian_bytes(i) 309 data_bytes += binary_data 310 length = self.int32_to_big_endian_bytes(len(data_bytes)) 311 return data_bytes, length 312 313 def run_next_test(self): 314 """ 315 Fetch next test information and execute the test. 316 317 """ 318 self.test_index += 1 319 self.dep_index = 0 320 if self.test_index < len(self.tests): 321 name, function_id, dependencies, args = self.tests[self.test_index] 322 self.run_test(name, function_id, dependencies, args) 323 else: 324 self.notify_complete(self.suite_passed) 325 326 def run_test(self, name, function_id, dependencies, args): 327 """ 328 Execute the test on target by sending next test information. 329 330 :param name: Test name 331 :param function_id: function identifier 332 :param dependencies: Dependencies list 333 :param args: test parameters 334 :return: 335 """ 336 self.log("Running: %s" % name) 337 338 param_bytes, length = self.test_vector_to_bytes(function_id, 339 dependencies, args) 340 self.send_kv( 341 ''.join('{:02x}'.format(x) for x in length), 342 ''.join('{:02x}'.format(x) for x in param_bytes) 343 ) 344 345 @staticmethod 346 def get_result(value): 347 """ 348 Converts result from string type to integer 349 :param value: Result code in string 350 :return: Integer result code. Value is from the test status 351 constants defined under the MbedTlsTest class. 352 """ 353 try: 354 return int(value) 355 except ValueError: 356 ValueError("Result should return error number. " 357 "Instead received %s" % value) 358 359 @event_callback('GO') 360 def on_go(self, _key, _value, _timestamp): 361 """ 362 Sent by the target to start first test. 363 364 :param _key: Event key 365 :param _value: Value. ignored 366 :param _timestamp: Timestamp ignored. 367 :return: 368 """ 369 self.run_next_test() 370 371 @event_callback("R") 372 def on_result(self, _key, value, _timestamp): 373 """ 374 Handle result. Prints test start, finish required by Greentea 375 to detect test execution. 376 377 :param _key: Event key 378 :param value: Value. ignored 379 :param _timestamp: Timestamp ignored. 380 :return: 381 """ 382 int_val = self.get_result(value) 383 name, _, _, _ = self.tests[self.test_index] 384 self.log('{{__testcase_start;%s}}' % name) 385 self.log('{{__testcase_finish;%s;%d;%d}}' % (name, int_val == 0, 386 int_val != 0)) 387 if int_val != 0: 388 self.suite_passed = False 389 self.run_next_test() 390 391 @event_callback("F") 392 def on_failure(self, _key, value, _timestamp): 393 """ 394 Handles test execution failure. That means dependency not supported or 395 Test function not supported. Hence marking test as skipped. 396 397 :param _key: Event key 398 :param value: Value. ignored 399 :param _timestamp: Timestamp ignored. 400 :return: 401 """ 402 int_val = self.get_result(value) 403 if int_val in self.error_str: 404 err = self.error_str[int_val] 405 else: 406 err = 'Unknown error' 407 # For skip status, do not write {{__testcase_finish;...}} 408 self.log("Error: %s" % err) 409 self.run_next_test() 410