1from __future__ import absolute_import
2from __future__ import division
3from __future__ import print_function
4
5import sys
6import io
7import os
8import argparse
9import yaml
10
11model_template_str = '''
12models:
13  - model:
14      name: 'vgg19'
15      source: 'tensorflow'
16      targets: ['onnx', 'cntk', 'tensorflow', 'mxnet', 'pytorch', 'coreml', 'keras', 'caffe']
17  - model:
18      name: 'inception_v1'
19      source: 'tensorflow'
20      targets: ['onnx', 'tensorflow', 'mxnet', 'pytorch', 'coreml', 'keras', 'caffe']
21  - model:
22      name: 'inception_v3'
23      source: 'tensorflow'
24      targets: ['onnx', 'cntk', 'tensorflow', 'mxnet', 'pytorch', 'coreml', 'keras', 'caffe']
25  - model:
26      name: 'resnet_v1_152'
27      source: 'tensorflow'
28      targets: ['tensorflow', 'mxnet', 'pytorch', 'coreml', 'keras', 'caffe']
29  - model:
30      name: 'resnet_v2_152'
31      source: 'tensorflow'
32      targets: ['cntk', 'tensorflow', 'mxnet', 'pytorch', 'coreml', 'keras', 'caffe']
33  - model:
34      name: 'mobilenet_v1_1.0'
35      source: 'tensorflow'
36      targets: ['onnx', 'cntk', 'tensorflow', 'mxnet', 'pytorch', 'coreml', 'keras', 'caffe']
37  - model:
38      name: 'mobilenet_v2_1.0_224'
39      source: 'tensorflow'
40      targets: ['onnx', 'cntk', 'tensorflow', 'mxnet', 'pytorch', 'coreml', 'keras', 'caffe']
41  - model:
42      name: 'nasnet-a_large'
43      source: 'tensorflow'
44      targets: ['tensorflow', 'mxnet', 'pytorch']
45  - model:
46      name: 'inception_resnet_v2'
47      source: 'tensorflow'
48      targets: ['onnx', 'tensorflow', 'mxnet', 'pytorch', 'keras', 'caffe']
49  - model:
50      name: 'facenet'
51      source: 'tensorflow'
52      targets: ['onnx', 'tensorflow', 'mxnet', 'pytorch', 'keras', 'caffe']
53  - model:
54      name: 'rnn_embedding'
55      source: 'tensorflow'
56      targets: ['cntk', 'tensorflow', 'mxnet', 'pytorch', 'keras']
57
58  - model:
59      name: 'inception_v1'
60      source: 'tensorflow_frozen'
61      targets: ['onnx', 'tensorflow', 'mxnet', 'coreml', 'keras']
62  - model:
63      name: 'inception_v3'
64      source: 'tensorflow_frozen'
65      targets: ['onnx', 'tensorflow', 'mxnet', 'coreml', 'keras']
66  - model:
67      name: 'mobilenet_v1_1.0'
68      source: 'tensorflow_frozen'
69      targets: ['onnx', 'tensorflow', 'mxnet', 'coreml', 'keras']
70  - model:
71      name: 'facenet'
72      source: 'tensorflow_frozen'
73      targets: ['onnx', 'tensorflow', 'mxnet', 'keras']
74
75  - model:
76      name: 'inception_v3'
77      source: 'cntk'
78      targets: ['onnx', 'cntk', 'tensorflow', 'mxnet']
79  - model:
80      name: 'resnet18'
81      source: 'cntk'
82      targets: ['onnx', 'cntk', 'tensorflow', 'mxnet', 'pytorch', 'coreml', 'keras', 'caffe']
83  - model:
84      name: 'resnet152'
85      source: 'cntk'
86      targets: ['onnx', 'cntk', 'tensorflow', 'mxnet', 'pytorch', 'coreml', 'keras', 'caffe']
87
88  - model:
89      name: 'vgg19'
90      source: 'mxnet'
91      targets: ['onnx', 'cntk', 'tensorflow', 'mxnet', 'pytorch', 'coreml', 'keras', 'caffe']
92  - model:
93      name: 'imagenet1k-inception-bn'
94      source: 'mxnet'
95      targets: ['onnx', 'cntk', 'tensorflow', 'mxnet', 'pytorch', 'coreml', 'keras', 'caffe']
96  - model:
97      name: 'imagenet1k-resnet-18'
98      source: 'mxnet'
99      targets: ['onnx', 'cntk', 'tensorflow', 'mxnet', 'pytorch', 'coreml', 'keras', 'caffe']
100  - model:
101      name: 'imagenet1k-resnet-152'
102      source: 'mxnet'
103      targets: ['onnx', 'cntk', 'tensorflow', 'mxnet', 'pytorch', 'coreml', 'keras', 'caffe']
104  - model:
105      name: 'squeezenet_v1.1'
106      source: 'mxnet'
107      targets: ['onnx', 'cntk', 'tensorflow', 'mxnet', 'pytorch', 'coreml', 'keras', 'caffe']
108  - model:
109      name: 'imagenet1k-resnext-101-64x4d'
110      source: 'mxnet'
111      targets: ['onnx', 'cntk', 'tensorflow', 'mxnet', 'pytorch', 'coreml', 'keras', 'caffe']
112  - model:
113      name: 'imagenet1k-resnext-50'
114      source: 'mxnet'
115      targets: ['onnx', 'cntk', 'tensorflow', 'mxnet', 'pytorch', 'coreml', 'keras', 'caffe']
116
117  - model:
118      name: 'alexnet'
119      source: 'pytorch'
120      targets: ['tensorflow', 'mxnet', 'pytorch', 'coreml', 'keras', 'caffe']
121  - model:
122      name: 'densenet201'
123      source: 'pytorch'
124      targets: ['tensorflow', 'mxnet', 'pytorch', 'coreml', 'keras', 'caffe']
125  - model:
126      name: 'inception_v3'
127      source: 'pytorch'
128      targets: ['tensorflow', 'mxnet', 'pytorch', 'coreml', 'keras', 'caffe']
129  - model:
130      name: 'vgg19'
131      source: 'pytorch'
132      targets: ['tensorflow', 'mxnet', 'pytorch', 'coreml', 'keras', 'caffe']
133  - model:
134      name: 'vgg19_bn'
135      source: 'pytorch'
136      targets: ['tensorflow', 'mxnet', 'pytorch', 'coreml', 'keras', 'caffe']
137  - model:
138      name: 'resnet152'
139      source: 'pytorch'
140      targets: ['tensorflow', 'mxnet', 'pytorch', 'coreml', 'keras', 'caffe']
141
142  - model:
143      name: 'inception_v3'
144      source: 'coreml'
145      targets: ['onnx', 'tensorflow', 'mxnet', 'pytorch', 'coreml', 'keras', 'caffe']
146  - model:
147      name: 'mobilenet'
148      source: 'coreml'
149      targets: ['onnx', 'tensorflow', 'mxnet', 'pytorch', 'coreml', 'keras', 'caffe']
150  - model:
151      name: 'resnet50'
152      source: 'coreml'
153      targets: ['onnx', 'tensorflow', 'mxnet', 'pytorch', 'coreml', 'keras', 'caffe']
154  - model:
155      name: 'tinyyolo'
156      source: 'coreml'
157      targets: ['onnx', 'tensorflow', 'mxnet', 'pytorch', 'coreml', 'keras']
158  - model:
159      name: 'vgg16'
160      source: 'coreml'
161      targets: ['onnx', 'tensorflow', 'mxnet', 'pytorch', 'coreml', 'keras', 'caffe']
162
163  - model:
164      name: 'vgg19'
165      source: 'keras'
166      targets: ['onnx', 'cntk', 'tensorflow', 'mxnet', 'pytorch', 'coreml', 'keras', 'caffe']
167  - model:
168      name: 'inception_v3'
169      source: 'keras'
170      targets: ['onnx', 'cntk', 'tensorflow', 'mxnet', 'pytorch', 'coreml', 'keras', 'caffe']
171  - model:
172      name: 'resnet50'
173      source: 'keras'
174      targets: ['onnx', 'cntk', 'tensorflow', 'mxnet', 'pytorch', 'coreml', 'keras', 'caffe']
175  - model:
176      name: 'densenet'
177      source: 'keras'
178      targets: ['onnx', 'cntk', 'tensorflow', 'mxnet', 'pytorch', 'coreml', 'keras', 'caffe']
179  - model:
180      name: 'xception'
181      source: 'keras'
182      targets: ['tensorflow', 'coreml', 'keras']
183  - model:
184      name: 'mobilenet'
185      source: 'keras'
186      targets: ['onnx', 'tensorflow', 'coreml', 'keras']
187  - model:
188      name: 'yolo2'
189      source: 'keras'
190      targets: ['onnx', 'keras']
191
192  - model:
193      name: 'alexnet'
194      source: 'caffe'
195      targets: ['onnx', 'cntk', 'tensorflow', 'mxnet', 'pytorch', 'coreml', 'caffe']
196  - model:
197      name: 'inception_v1'
198      source: 'caffe'
199      targets: ['onnx', 'cntk', 'tensorflow', 'mxnet', 'pytorch', 'coreml', 'keras', 'caffe']
200  - model:
201      name: 'inception_v4'
202      source: 'caffe'
203      targets: ['onnx', 'cntk', 'tensorflow', 'pytorch', 'coreml', 'keras']
204  - model:
205      name: 'resnet152'
206      source: 'caffe'
207      targets: ['onnx', 'cntk', 'tensorflow', 'mxnet', 'pytorch', 'coreml', 'keras', 'caffe']
208  - model:
209      name: 'squeezenet'
210      source: 'caffe'
211      targets: ['onnx', 'cntk', 'tensorflow', 'mxnet', 'pytorch', 'coreml', 'keras', 'caffe']
212  - model:
213      name: 'vgg19'
214      source: 'caffe'
215      targets: ['onnx', 'cntk', 'tensorflow', 'mxnet', 'pytorch', 'coreml', 'keras', 'caffe']
216  - model:
217      name: 'voc-fcn8s'
218      source: 'caffe'
219      targets: ['cntk', 'tensorflow', 'coreml']
220  - model:
221      name: 'voc-fcn16s'
222      source: 'caffe'
223      targets: ['cntk', 'tensorflow', 'coreml']
224  - model:
225      name: 'voc-fcn32s'
226      source: 'caffe'
227      targets: ['cntk', 'tensorflow', 'coreml']
228  - model:
229      name: 'xception'
230      source: 'caffe'
231      targets: ['onnx', 'cntk', 'tensorflow', 'mxnet', 'pytorch', 'coreml']
232
233  - model:
234      name: 'resnet50'
235      source: 'paddle'
236      targets: ['onnx']
237  - model:
238      name: 'vgg16'
239      source: 'paddle'
240      targets: ['onnx']
241
242'''
243
244code_template_str = '''
245from __future__ import absolute_import
246from __future__ import print_function
247
248import os
249from conversion_imagenet import TestModels
250from conversion_imagenet import check_env
251
252def get_test_table():
253    return {{ '{1}' :
254    {{
255        '{0}'                : [TestModels.{2}_emit]
256    }}}}
257
258
259
260def test_{1}_{2}_{3}():
261    if not check_env('{1}', '{2}', '{0}'):
262        return
263
264    test_table = get_test_table()
265    tester = TestModels(test_table)
266    tester._test_function('{1}', tester.{1}_parse)
267
268
269if __name__ == '__main__':
270    test_{1}_{2}_{3}()
271
272'''
273
274travis_template_str = '''
275sudo: required
276dist: xenial
277
278os:
279  - linux
280
281language: python
282python:
283  - "2.7"
284  - "3.5"
285
286env:
287{0}
288
289cache:
290  directories:
291    - $HOME/.cache/pip
292
293addons:
294  apt:
295    update: true
296
297before_install:
298  - sudo apt-get install -y openmpi-bin
299  - sudo apt-get install -y libprotobuf-dev libsnappy-dev libhdf5-serial-dev protobuf-compiler
300  - sudo apt-get install -y libatlas-base-dev
301  - sudo apt-get install -y libgflags-dev libgoogle-glog-dev
302  - if [ "$TEST_SOURCE_FRAMEWORK" = "caffe" ] || [ "$TEST_TARGET_FRAMEWORK" = "caffe" ]; then sudo apt-get install -y --no-install-recommends libboost-all-dev; fi
303
304install:
305  - pip install -q -r $(python requirements/select_requirements.py)
306  - pip install wget
307
308before_script:
309  - export LD_LIBRARY_PATH=$(python -c "import os; print(os.path.dirname(os.__file__) + '/site-packages/caffe/libs')"):${{LD_LIBRARY_PATH}}
310
311after_failure: true
312
313after_success: true
314
315after_script: true
316
317script: bash test.sh $TEST_SOURCE_FRAMEWORK $TEST_TARGET_FRAMEWORK $TEST_MODEL
318
319matrix:
320  fast_finish: true
321
322  allow_failures:
323    - env: TEST_SOURCE_FRAMEWORK=paddle TEST_MODEL=resnet50
324    - env: TEST_SOURCE_FRAMEWORK=paddle TEST_MODEL=vgg16
325
326notifications:
327  email:
328    on_success: never
329    on_failure: never
330
331'''
332
333
334def gen_test(output_dir, model):
335    model_name = model['name']
336    normalized_model_name = model_name.replace('.', '_')
337    normalized_model_name2 = normalized_model_name.replace('-', '_')
338    length = len(model['targets'])
339    for i in range(length):
340        test_file = os.path.join(output_dir, 'test_{0}_{1}_{2}.py'
341                    .format(model['source'], model['targets'][i], normalized_model_name))
342        with open(test_file, "w+") as f:
343            code = code_template_str.format(model_name, model['source'], model['targets'][i], normalized_model_name2)
344            f.write(code)
345
346
347def gen_tests(output_dir):
348    y = yaml.load(model_template_str)
349    length = len(y['models'])
350    for i in range(length):
351        gen_test(output_dir, y['models'][i]['model'])
352
353
354def gen_travis(output_dir):
355    y = yaml.load(model_template_str)
356    travis_file = os.path.join(output_dir, 'travis.yml')
357
358    env_str = ''
359    length = len(y['models'])
360    for i in range(length):
361        model = y['models'][i]['model']
362        model_name = model['name']
363        normalized_model_name = model_name.replace('.', '_')
364        source_framework = model['source']
365        if False:
366            env_str += '  - TEST_SOURCE_FRAMEWORK={0} TEST_MODEL={1}\n'.format(source_framework, normalized_model_name)
367        else:
368            length2 = len(model['targets'])
369            for j in range(length2):
370                target_framework = model['targets'][j]
371                env_str += '  - TEST_SOURCE_FRAMEWORK={0} TEST_TARGET_FRAMEWORK={1} TEST_MODEL={2}\n'.format(source_framework, target_framework, normalized_model_name)
372
373    with open(travis_file, "w+") as f:
374        code = travis_template_str.format(env_str)
375        f.write(code)
376
377    return
378
379
380def prepare_env(FLAGS):
381    output_dir = FLAGS.output_dir
382    if (not os.path.exists(output_dir)):
383        os.mkdir(output_dir)
384    if ((not os.path.isdir(output_dir)) or (not os.path.exists(output_dir))):
385        print('Cannot create target output directory: "{0}"'.format(output_dir))
386        return False
387    return True
388
389
390def main():
391    parser = argparse.ArgumentParser()
392    parser.add_argument('-o', '--output_dir', help='The output directory.', required=True)
393    FLAGS, unparsed = parser.parse_known_args()
394    if (not prepare_env(FLAGS)):
395        return
396
397    output_dir = FLAGS.output_dir
398    gen_travis(output_dir)
399    gen_tests(output_dir)
400
401
402if __name__ == '__main__':
403    main()
404