1# Licensed to the Apache Software Foundation (ASF) under one 2# or more contributor license agreements. See the NOTICE file 3# distributed with this work for additional information 4# regarding copyright ownership. The ASF licenses this file 5# to you under the Apache License, Version 2.0 (the 6# "License"); you may not use this file except in compliance 7# with the License. You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, 12# software distributed under the License is distributed on an 13# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14# KIND, either express or implied. See the License for the 15# specific language governing permissions and limitations 16# under the License. 17 18"""Test converted models layer by layer 19""" 20import argparse 21import logging 22import os 23import warnings 24 25import numpy as np 26import cv2 27import mxnet as mx 28 29logging.basicConfig(level=logging.INFO) 30 31 32def read_image(img_path, image_dims=None, mean=None): 33 """ 34 Reads an image from file path or URL, optionally resizing to given image dimensions and 35 subtracting mean. 36 :param img_path: path to file, or url to download 37 :param image_dims: image dimensions to resize to, or None 38 :param mean: mean file to subtract, or None 39 :return: loaded image, in RGB format 40 """ 41 42 import urllib 43 44 filename = img_path.split("/")[-1] 45 if img_path.startswith('http'): 46 urllib.urlretrieve(img_path, filename) 47 img = cv2.imread(filename) 48 else: 49 img = cv2.imread(img_path) 50 51 img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) 52 53 if image_dims is not None: 54 img = cv2.resize(img, image_dims) # resize to image_dims to fit model 55 img = np.rollaxis(img, 2) # change to (c, h, w) order 56 img = img[np.newaxis, :] # extend to (n, c, h, w) 57 if mean is not None: 58 mean = np.array(mean) 59 if mean.shape == (3,): 60 mean = mean[np.newaxis, :, np.newaxis, np.newaxis] # extend to (n, c, 1, 1) 61 img = img.astype(np.float32) - mean # subtract mean 62 63 return img 64 65 66def _ch_dev(arg_params, aux_params, ctx): 67 """ 68 Changes device of given mxnet arguments 69 :param arg_params: arguments 70 :param aux_params: auxiliary parameters 71 :param ctx: new device context 72 :return: arguments and auxiliary parameters on new device 73 """ 74 new_args = dict() 75 new_auxs = dict() 76 for k, v in arg_params.items(): 77 new_args[k] = v.as_in_context(ctx) 78 for k, v in aux_params.items(): 79 new_auxs[k] = v.as_in_context(ctx) 80 return new_args, new_auxs 81 82 83def convert_and_compare_caffe_to_mxnet(image_url, gpu, caffe_prototxt_path, caffe_model_path, 84 caffe_mean, mean_diff_allowed, max_diff_allowed): 85 """ 86 Run the layer comparison on a caffe model, given its prototxt, weights and mean. 87 The comparison is done by inferring on a given image using both caffe and mxnet model 88 :param image_url: image file or url to run inference on 89 :param gpu: gpu to use, -1 for cpu 90 :param caffe_prototxt_path: path to caffe prototxt 91 :param caffe_model_path: path to caffe weights 92 :param caffe_mean: path to caffe mean file 93 """ 94 95 import caffe 96 from caffe_proto_utils import read_network_dag, process_network_proto, read_caffe_mean 97 from convert_model import convert_model 98 99 if isinstance(caffe_mean, str): 100 caffe_mean = read_caffe_mean(caffe_mean) 101 elif caffe_mean is None: 102 pass 103 elif len(caffe_mean) == 3: 104 # swap channels from Caffe BGR to RGB 105 caffe_mean = caffe_mean[::-1] 106 107 # get caffe root location, this is needed to run the upgrade network utility, so we only need 108 # to support parsing of latest caffe 109 caffe_root = os.path.dirname(os.path.dirname(caffe.__path__[0])) 110 caffe_prototxt_path = process_network_proto(caffe_root, caffe_prototxt_path) 111 112 _, layer_name_to_record, top_to_layers = read_network_dag(caffe_prototxt_path) 113 114 caffe.set_mode_cpu() 115 caffe_net = caffe.Net(caffe_prototxt_path, caffe_model_path, caffe.TEST) 116 117 image_dims = tuple(caffe_net.blobs['data'].shape)[2:4] 118 119 logging.info('getting image %s', image_url) 120 img_rgb = read_image(image_url, image_dims, caffe_mean) 121 img_bgr = img_rgb[:, ::-1, :, :] 122 123 caffe_net.blobs['data'].reshape(*img_bgr.shape) 124 caffe_net.blobs['data'].data[...] = img_bgr 125 _ = caffe_net.forward() 126 127 # read sym and add all outputs 128 sym, arg_params, aux_params, _ = convert_model(caffe_prototxt_path, caffe_model_path) 129 sym = sym.get_internals() 130 131 # now mxnet 132 if gpu < 0: 133 ctx = mx.cpu(0) 134 else: 135 ctx = mx.gpu(gpu) 136 137 arg_params, aux_params = _ch_dev(arg_params, aux_params, ctx) 138 arg_params["data"] = mx.nd.array(img_rgb, ctx) 139 arg_params["prob_label"] = mx.nd.empty((1,), ctx) 140 exe = sym.bind(ctx, arg_params, args_grad=None, grad_req="null", aux_states=aux_params) 141 exe.forward(is_train=False) 142 143 compare_layers_from_nets(caffe_net, arg_params, aux_params, exe, layer_name_to_record, 144 top_to_layers, mean_diff_allowed, max_diff_allowed) 145 146 147def _bfs(root_node, process_node): 148 """ 149 Implementation of Breadth-first search (BFS) on caffe network DAG 150 :param root_node: root node of caffe network DAG 151 :param process_node: function to run on each node 152 """ 153 154 from collections import deque 155 156 seen_nodes = set() 157 next_nodes = deque() 158 159 seen_nodes.add(root_node) 160 next_nodes.append(root_node) 161 162 while next_nodes: 163 current_node = next_nodes.popleft() 164 165 # process current node 166 process_node(current_node) 167 168 for child_node in current_node.children: 169 if child_node not in seen_nodes: 170 seen_nodes.add(child_node) 171 next_nodes.append(child_node) 172 173 174def compare_layers_from_nets(caffe_net, arg_params, aux_params, exe, layer_name_to_record, 175 top_to_layers, mean_diff_allowed, max_diff_allowed): 176 """ 177 Compare layer by layer of a caffe network with mxnet network 178 :param caffe_net: loaded caffe network 179 :param arg_params: arguments 180 :param aux_params: auxiliary parameters 181 :param exe: mxnet model 182 :param layer_name_to_record: map between caffe layer and information record 183 :param top_to_layers: map between caffe blob name to layers which outputs it (including inplace) 184 :param mean_diff_allowed: mean difference allowed between caffe blob and mxnet blob 185 :param max_diff_allowed: max difference allowed between caffe blob and mxnet blob 186 """ 187 188 import re 189 190 log_format = ' {0:<40} {1:<40} {2:<8} {3:>10} {4:>10} {5:<1}' 191 192 compare_layers_from_nets.is_first_convolution = True 193 194 def _compare_blob(caf_blob, mx_blob, caf_name, mx_name, blob_type, note): 195 diff = np.abs(mx_blob - caf_blob) 196 diff_mean = diff.mean() 197 diff_max = diff.max() 198 logging.info(log_format.format(caf_name, mx_name, blob_type, '%4.5f' % diff_mean, 199 '%4.5f' % diff_max, note)) 200 assert diff_mean < mean_diff_allowed 201 assert diff_max < max_diff_allowed 202 203 def _process_layer_parameters(layer): 204 205 logging.debug('processing layer %s of type %s', layer.name, layer.type) 206 207 normalized_layer_name = re.sub('[-/]', '_', layer.name) 208 209 # handle weight and bias of convolution and fully-connected layers 210 if layer.name in caffe_net.params and layer.type in ['Convolution', 'InnerProduct', 211 'Deconvolution']: 212 213 has_bias = len(caffe_net.params[layer.name]) > 1 214 215 mx_name_weight = '{}_weight'.format(normalized_layer_name) 216 mx_beta = arg_params[mx_name_weight].asnumpy() 217 218 # first convolution should change from BGR to RGB 219 if layer.type == 'Convolution' and compare_layers_from_nets.is_first_convolution: 220 compare_layers_from_nets.is_first_convolution = False 221 222 # if RGB or RGBA 223 if mx_beta.shape[1] == 3 or mx_beta.shape[1] == 4: 224 # Swapping BGR of caffe into RGB in mxnet 225 mx_beta[:, [0, 2], :, :] = mx_beta[:, [2, 0], :, :] 226 227 caf_beta = caffe_net.params[layer.name][0].data 228 _compare_blob(caf_beta, mx_beta, layer.name, mx_name_weight, 'weight', '') 229 230 if has_bias: 231 mx_name_bias = '{}_bias'.format(normalized_layer_name) 232 mx_gamma = arg_params[mx_name_bias].asnumpy() 233 caf_gamma = caffe_net.params[layer.name][1].data 234 _compare_blob(caf_gamma, mx_gamma, layer.name, mx_name_bias, 'bias', '') 235 236 elif layer.name in caffe_net.params and layer.type == 'Scale': 237 238 if 'scale' in normalized_layer_name: 239 bn_name = normalized_layer_name.replace('scale', 'bn') 240 elif 'sc' in normalized_layer_name: 241 bn_name = normalized_layer_name.replace('sc', 'bn') 242 else: 243 assert False, 'Unknown name convention for bn/scale' 244 245 beta_name = '{}_beta'.format(bn_name) 246 gamma_name = '{}_gamma'.format(bn_name) 247 248 mx_beta = arg_params[beta_name].asnumpy() 249 caf_beta = caffe_net.params[layer.name][1].data 250 _compare_blob(caf_beta, mx_beta, layer.name, beta_name, 'mov_mean', '') 251 252 mx_gamma = arg_params[gamma_name].asnumpy() 253 caf_gamma = caffe_net.params[layer.name][0].data 254 _compare_blob(caf_gamma, mx_gamma, layer.name, gamma_name, 'mov_var', '') 255 256 elif layer.name in caffe_net.params and layer.type == 'BatchNorm': 257 258 mean_name = '{}_moving_mean'.format(normalized_layer_name) 259 var_name = '{}_moving_var'.format(normalized_layer_name) 260 261 caf_rescale_factor = caffe_net.params[layer.name][2].data 262 263 mx_mean = aux_params[mean_name].asnumpy() 264 caf_mean = caffe_net.params[layer.name][0].data / caf_rescale_factor 265 _compare_blob(caf_mean, mx_mean, layer.name, mean_name, 'mean', '') 266 267 mx_var = aux_params[var_name].asnumpy() 268 caf_var = caffe_net.params[layer.name][1].data / caf_rescale_factor 269 _compare_blob(caf_var, mx_var, layer.name, var_name, 'var', 270 'expect 1e-04 change due to cudnn eps') 271 272 elif layer.type in ['Input', 'Pooling', 'ReLU', 'Eltwise', 'Softmax', 'LRN', 'Concat', 273 'Dropout', 'Crop']: 274 # no parameters to check for these layers 275 pass 276 277 else: 278 warnings.warn('No handling for layer %s of type %s, should we ignore it?', layer.name, 279 layer.type) 280 281 282 def _process_layer_output(caffe_blob_name): 283 284 logging.debug('processing blob %s', caffe_blob_name) 285 286 # skip blobs not originating from actual layers, e.g. artificial split layers added by caffe 287 if caffe_blob_name not in top_to_layers: 288 return 289 290 caf_blob = caffe_net.blobs[caffe_blob_name].data 291 292 # data should change from BGR to RGB 293 if caffe_blob_name == 'data': 294 295 # if RGB or RGBA 296 if caf_blob.shape[1] == 3 or caf_blob.shape[1] == 4: 297 # Swapping BGR of caffe into RGB in mxnet 298 caf_blob[:, [0, 2], :, :] = caf_blob[:, [2, 0], :, :] 299 mx_name = 'data' 300 301 else: 302 # get last layer name which outputs this blob name 303 last_layer_name = top_to_layers[caffe_blob_name][-1] 304 normalized_last_layer_name = re.sub('[-/]', '_', last_layer_name) 305 mx_name = '{}_output'.format(normalized_last_layer_name) 306 if 'scale' in mx_name: 307 mx_name = mx_name.replace('scale', 'bn') 308 elif 'sc' in mx_name: 309 mx_name = mx_name.replace('sc', 'bn') 310 311 if mx_name not in exe.output_dict: 312 logging.error('mxnet blob %s is missing, time to extend the compare tool..', mx_name) 313 return 314 315 mx_blob = exe.output_dict[mx_name].asnumpy() 316 _compare_blob(caf_blob, mx_blob, caffe_blob_name, mx_name, 'output', '') 317 318 return 319 320 # check layer parameters 321 logging.info('\n***** Network Parameters '.ljust(140, '*')) 322 logging.info(log_format.format('CAFFE', 'MXNET', 'Type', 'Mean(diff)', 'Max(diff)', 'Note')) 323 first_layer_name = layer_name_to_record.keys()[0] 324 _bfs(layer_name_to_record[first_layer_name], _process_layer_parameters) 325 326 # check layer output 327 logging.info('\n***** Network Outputs '.ljust(140, '*')) 328 logging.info(log_format.format('CAFFE', 'MXNET', 'Type', 'Mean(diff)', 'Max(diff)', 'Note')) 329 for caffe_blob_name in caffe_net.blobs.keys(): 330 _process_layer_output(caffe_blob_name) 331 332 333def main(): 334 """Entrypoint for compare_layers""" 335 336 parser = argparse.ArgumentParser( 337 description='Tool for testing caffe to mxnet conversion layer by layer') 338 parser.add_argument('--image_url', type=str, 339 default='https://github.com/dmlc/web-data/raw/master/mxnet/doc/'\ 340 'tutorials/python/predict_image/cat.jpg', 341 help='input image to test inference, can be either file path or url') 342 parser.add_argument('--caffe_prototxt_path', type=str, 343 default='./model.prototxt', 344 help='path to caffe prototxt') 345 parser.add_argument('--caffe_model_path', type=str, 346 default='./model.caffemodel', 347 help='path to caffe weights') 348 parser.add_argument('--caffe_mean', type=str, 349 default='./model_mean.binaryproto', 350 help='path to caffe mean file') 351 parser.add_argument('--mean_diff_allowed', type=int, default=1e-03, 352 help='mean difference allowed between caffe blob and mxnet blob') 353 parser.add_argument('--max_diff_allowed', type=int, default=1e-01, 354 help='max difference allowed between caffe blob and mxnet blob') 355 parser.add_argument('--gpu', type=int, default=-1, help='the gpu id used for predict') 356 args = parser.parse_args() 357 convert_and_compare_caffe_to_mxnet(args.image_url, args.gpu, args.caffe_prototxt_path, 358 args.caffe_model_path, args.caffe_mean, 359 args.mean_diff_allowed, args.max_diff_allowed) 360 361if __name__ == '__main__': 362 main() 363
