1# Greentea host test script for Mbed TLS on-target test suite testing. 2# 3# Copyright (C) 2018, Arm Limited, All Rights Reserved 4# SPDX-License-Identifier: Apache-2.0 5# 6# Licensed under the Apache License, Version 2.0 (the "License"); you may 7# not use this file except in compliance with the License. 8# You may obtain a copy of the License at 9# 10# http://www.apache.org/licenses/LICENSE-2.0 11# 12# Unless required by applicable law or agreed to in writing, software 13# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 14# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15# See the License for the specific language governing permissions and 16# limitations under the License. 17# 18# This file is part of Mbed TLS (https://tls.mbed.org) 19 20 21""" 22Mbed TLS on-target test suite tests are implemented as Greentea 23tests. Greentea tests are implemented in two parts: target test and 24host test. Target test is a C application that is built for the 25target platform and executes on the target. Host test is a Python 26class derived from mbed_host_tests.BaseHostTest. Target communicates 27with the host over serial for the test data and sends back the result. 28 29Python tool mbedgt (Greentea) is responsible for flashing the test 30binary on to the target and dynamically loading this host test module. 31 32Greentea documentation can be found here: 33https://github.com/ARMmbed/greentea 34""" 35 36 37import re 38import os 39import binascii 40 41from mbed_host_tests import BaseHostTest, event_callback # pylint: disable=import-error 42 43 44class TestDataParserError(Exception): 45 """Indicates error in test data, read from .data file.""" 46 pass 47 48 49class TestDataParser(object): 50 """ 51 Parses test name, dependencies, test function name and test parameters 52 from the data file. 53 """ 54 55 def __init__(self): 56 """ 57 Constructor 58 """ 59 self.tests = [] 60 61 def parse(self, data_file): 62 """ 63 Data file parser. 64 65 :param data_file: Data file path 66 """ 67 with open(data_file, 'r') as data_f: 68 self.__parse(data_f) 69 70 @staticmethod 71 def __escaped_split(inp_str, split_char): 72 """ 73 Splits inp_str on split_char except when escaped. 74 75 :param inp_str: String to split 76 :param split_char: Split character 77 :return: List of splits 78 """ 79 split_colon_fn = lambda x: re.sub(r'\\' + split_char, split_char, x) 80 if len(split_char) > 1: 81 raise ValueError('Expected split character. Found string!') 82 out = map(split_colon_fn, re.split(r'(?<!\\)' + split_char, inp_str)) 83 out = [x for x in out if x] 84 return out 85 86 def __parse(self, data_f): 87 """ 88 Parses data file using supplied file object. 89 90 :param data_f: Data file object 91 :return: 92 """ 93 for line in data_f: 94 line = line.strip() 95 if not line: 96 continue 97 # Read test name 98 name = line 99 100 # Check dependencies 101 dependencies = [] 102 line = data_f.next().strip() 103 match = re.search('depends_on:(.*)', line) 104 if match: 105 dependencies = [int(x) for x in match.group(1).split(':')] 106 line = data_f.next().strip() 107 108 # Read test vectors 109 line = line.replace('\\n', '\n') 110 parts = self.__escaped_split(line, ':') 111 function_name = int(parts[0]) 112 args = parts[1:] 113 args_count = len(args) 114 if args_count % 2 != 0: 115 err_str_fmt = "Number of test arguments({}) should be even: {}" 116 raise TestDataParserError(err_str_fmt.format(args_count, line)) 117 grouped_args = [(args[i * 2], args[(i * 2) + 1]) 118 for i in range(len(args)/2)] 119 self.tests.append((name, function_name, dependencies, 120 grouped_args)) 121 122 def get_test_data(self): 123 """ 124 Returns test data. 125 """ 126 return self.tests 127 128 129class MbedTlsTest(BaseHostTest): 130 """ 131 Host test for Mbed TLS unit tests. This script is loaded at 132 run time by Greentea for executing Mbed TLS test suites. Each 133 communication from the target is received in this object as 134 an event, which is then handled by the event handler method 135 decorated by the associated event. Ex: @event_callback('GO'). 136 137 Target test sends requests for dispatching next test. It reads 138 tests from the intermediate data file and sends test function 139 identifier, dependency identifiers, expression identifiers and 140 the test data in binary form. Target test checks dependencies 141 , evaluate integer constant expressions and dispatches the test 142 function with received test parameters. After test function is 143 finished, target sends the result. This class handles the result 144 event and prints verdict in the form that Greentea understands. 145 146 """ 147 # status/error codes from suites/helpers.function 148 DEPENDENCY_SUPPORTED = 0 149 KEY_VALUE_MAPPING_FOUND = DEPENDENCY_SUPPORTED 150 DISPATCH_TEST_SUCCESS = DEPENDENCY_SUPPORTED 151 152 KEY_VALUE_MAPPING_NOT_FOUND = -1 # Expression Id not found. 153 DEPENDENCY_NOT_SUPPORTED = -2 # Dependency not supported. 154 DISPATCH_TEST_FN_NOT_FOUND = -3 # Test function not found. 155 DISPATCH_INVALID_TEST_DATA = -4 # Invalid parameter type. 156 DISPATCH_UNSUPPORTED_SUITE = -5 # Test suite not supported/enabled. 157 158 def __init__(self): 159 """ 160 Constructor initialises test index to 0. 161 """ 162 super(MbedTlsTest, self).__init__() 163 self.tests = [] 164 self.test_index = -1 165 self.dep_index = 0 166 self.suite_passed = True 167 self.error_str = dict() 168 self.error_str[self.DEPENDENCY_SUPPORTED] = \ 169 'DEPENDENCY_SUPPORTED' 170 self.error_str[self.KEY_VALUE_MAPPING_NOT_FOUND] = \ 171 'KEY_VALUE_MAPPING_NOT_FOUND' 172 self.error_str[self.DEPENDENCY_NOT_SUPPORTED] = \ 173 'DEPENDENCY_NOT_SUPPORTED' 174 self.error_str[self.DISPATCH_TEST_FN_NOT_FOUND] = \ 175 'DISPATCH_TEST_FN_NOT_FOUND' 176 self.error_str[self.DISPATCH_INVALID_TEST_DATA] = \ 177 'DISPATCH_INVALID_TEST_DATA' 178 self.error_str[self.DISPATCH_UNSUPPORTED_SUITE] = \ 179 'DISPATCH_UNSUPPORTED_SUITE' 180 181 def setup(self): 182 """ 183 Setup hook implementation. Reads test suite data file and parses out 184 tests. 185 """ 186 binary_path = self.get_config_item('image_path') 187 script_dir = os.path.split(os.path.abspath(__file__))[0] 188 suite_name = os.path.splitext(os.path.basename(binary_path))[0] 189 data_file = ".".join((suite_name, 'datax')) 190 data_file = os.path.join(script_dir, '..', 'mbedtls', 191 suite_name, data_file) 192 if os.path.exists(data_file): 193 self.log("Running tests from %s" % data_file) 194 parser = TestDataParser() 195 parser.parse(data_file) 196 self.tests = parser.get_test_data() 197 self.print_test_info() 198 else: 199 self.log("Data file not found: %s" % data_file) 200 self.notify_complete(False) 201 202 def print_test_info(self): 203 """ 204 Prints test summary read by Greentea to detect test cases. 205 """ 206 self.log('{{__testcase_count;%d}}' % len(self.tests)) 207 for name, _, _, _ in self.tests: 208 self.log('{{__testcase_name;%s}}' % name) 209 210 @staticmethod 211 def align_32bit(data_bytes): 212 """ 213 4 byte aligns input byte array. 214 215 :return: 216 """ 217 data_bytes += bytearray((4 - (len(data_bytes))) % 4) 218 219 @staticmethod 220 def hex_str_bytes(hex_str): 221 """ 222 Converts Hex string representation to byte array 223 224 :param hex_str: Hex in string format. 225 :return: Output Byte array 226 """ 227 if hex_str[0] != '"' or hex_str[len(hex_str) - 1] != '"': 228 raise TestDataParserError("HEX test parameter missing '\"':" 229 " %s" % hex_str) 230 hex_str = hex_str.strip('"') 231 if len(hex_str) % 2 != 0: 232 raise TestDataParserError("HEX parameter len should be mod of " 233 "2: %s" % hex_str) 234 235 data_bytes = binascii.unhexlify(hex_str) 236 return data_bytes 237 238 @staticmethod 239 def int32_to_big_endian_bytes(i): 240 """ 241 Coverts i to byte array in big endian format. 242 243 :param i: Input integer 244 :return: Output bytes array in big endian or network order 245 """ 246 data_bytes = bytearray([((i >> x) & 0xff) for x in [24, 16, 8, 0]]) 247 return data_bytes 248 249 def test_vector_to_bytes(self, function_id, dependencies, parameters): 250 """ 251 Converts test vector into a byte array that can be sent to the target. 252 253 :param function_id: Test Function Identifier 254 :param dependencies: Dependency list 255 :param parameters: Test function input parameters 256 :return: Byte array and its length 257 """ 258 data_bytes = bytearray([len(dependencies)]) 259 if dependencies: 260 data_bytes += bytearray(dependencies) 261 data_bytes += bytearray([function_id, len(parameters)]) 262 for typ, param in parameters: 263 if typ == 'int' or typ == 'exp': 264 i = int(param) 265 data_bytes += 'I' if typ == 'int' else 'E' 266 self.align_32bit(data_bytes) 267 data_bytes += self.int32_to_big_endian_bytes(i) 268 elif typ == 'char*': 269 param = param.strip('"') 270 i = len(param) + 1 # + 1 for null termination 271 data_bytes += 'S' 272 self.align_32bit(data_bytes) 273 data_bytes += self.int32_to_big_endian_bytes(i) 274 data_bytes += bytearray(list(param)) 275 data_bytes += '\0' # Null terminate 276 elif typ == 'hex': 277 binary_data = self.hex_str_bytes(param) 278 data_bytes += 'H' 279 self.align_32bit(data_bytes) 280 i = len(binary_data) 281 data_bytes += self.int32_to_big_endian_bytes(i) 282 data_bytes += binary_data 283 length = self.int32_to_big_endian_bytes(len(data_bytes)) 284 return data_bytes, length 285 286 def run_next_test(self): 287 """ 288 Fetch next test information and execute the test. 289 290 """ 291 self.test_index += 1 292 self.dep_index = 0 293 if self.test_index < len(self.tests): 294 name, function_id, dependencies, args = self.tests[self.test_index] 295 self.run_test(name, function_id, dependencies, args) 296 else: 297 self.notify_complete(self.suite_passed) 298 299 def run_test(self, name, function_id, dependencies, args): 300 """ 301 Execute the test on target by sending next test information. 302 303 :param name: Test name 304 :param function_id: function identifier 305 :param dependencies: Dependencies list 306 :param args: test parameters 307 :return: 308 """ 309 self.log("Running: %s" % name) 310 311 param_bytes, length = self.test_vector_to_bytes(function_id, 312 dependencies, args) 313 self.send_kv(length, param_bytes) 314 315 @staticmethod 316 def get_result(value): 317 """ 318 Converts result from string type to integer 319 :param value: Result code in string 320 :return: Integer result code. Value is from the test status 321 constants defined under the MbedTlsTest class. 322 """ 323 try: 324 return int(value) 325 except ValueError: 326 ValueError("Result should return error number. " 327 "Instead received %s" % value) 328 329 @event_callback('GO') 330 def on_go(self, _key, _value, _timestamp): 331 """ 332 Sent by the target to start first test. 333 334 :param _key: Event key 335 :param _value: Value. ignored 336 :param _timestamp: Timestamp ignored. 337 :return: 338 """ 339 self.run_next_test() 340 341 @event_callback("R") 342 def on_result(self, _key, value, _timestamp): 343 """ 344 Handle result. Prints test start, finish required by Greentea 345 to detect test execution. 346 347 :param _key: Event key 348 :param value: Value. ignored 349 :param _timestamp: Timestamp ignored. 350 :return: 351 """ 352 int_val = self.get_result(value) 353 name, _, _, _ = self.tests[self.test_index] 354 self.log('{{__testcase_start;%s}}' % name) 355 self.log('{{__testcase_finish;%s;%d;%d}}' % (name, int_val == 0, 356 int_val != 0)) 357 if int_val != 0: 358 self.suite_passed = False 359 self.run_next_test() 360 361 @event_callback("F") 362 def on_failure(self, _key, value, _timestamp): 363 """ 364 Handles test execution failure. That means dependency not supported or 365 Test function not supported. Hence marking test as skipped. 366 367 :param _key: Event key 368 :param value: Value. ignored 369 :param _timestamp: Timestamp ignored. 370 :return: 371 """ 372 int_val = self.get_result(value) 373 if int_val in self.error_str: 374 err = self.error_str[int_val] 375 else: 376 err = 'Unknown error' 377 # For skip status, do not write {{__testcase_finish;...}} 378 self.log("Error: %s" % err) 379 self.run_next_test() 380