1# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> 2# 3# This file is part of Ansible 4# 5# Ansible is free software: you can redistribute it and/or modify 6# it under the terms of the GNU General Public License as published by 7# the Free Software Foundation, either version 3 of the License, or 8# (at your option) any later version. 9# 10# Ansible is distributed in the hope that it will be useful, 11# but WITHOUT ANY WARRANTY; without even the implied warranty of 12# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13# GNU General Public License for more details. 14# 15# You should have received a copy of the GNU General Public License 16# along with Ansible. If not, see <http://www.gnu.org/licenses/>. 17 18# Make coding more python3-ish 19from __future__ import (absolute_import, division, print_function) 20__metaclass__ = type 21 22import mock 23 24from units.compat import unittest 25from units.compat.mock import patch, MagicMock 26from ansible.errors import AnsibleError 27from ansible.executor.task_executor import TaskExecutor, remove_omit 28from ansible.plugins.loader import action_loader, lookup_loader 29from ansible.parsing.yaml.objects import AnsibleUnicode 30from ansible.utils.unsafe_proxy import AnsibleUnsafeText, AnsibleUnsafeBytes 31from ansible.module_utils.six import text_type 32 33from units.mock.loader import DictDataLoader 34 35 36class TestTaskExecutor(unittest.TestCase): 37 38 def test_task_executor_init(self): 39 fake_loader = DictDataLoader({}) 40 mock_host = MagicMock() 41 mock_task = MagicMock() 42 mock_play_context = MagicMock() 43 mock_shared_loader = MagicMock() 44 new_stdin = None 45 job_vars = dict() 46 mock_queue = MagicMock() 47 te = TaskExecutor( 48 host=mock_host, 49 task=mock_task, 50 job_vars=job_vars, 51 play_context=mock_play_context, 52 new_stdin=new_stdin, 53 loader=fake_loader, 54 shared_loader_obj=mock_shared_loader, 55 final_q=mock_queue, 56 ) 57 58 def test_task_executor_run(self): 59 fake_loader = DictDataLoader({}) 60 61 mock_host = MagicMock() 62 63 mock_task = MagicMock() 64 mock_task._role._role_path = '/path/to/role/foo' 65 66 mock_play_context = MagicMock() 67 68 mock_shared_loader = MagicMock() 69 mock_queue = MagicMock() 70 71 new_stdin = None 72 job_vars = dict() 73 74 te = TaskExecutor( 75 host=mock_host, 76 task=mock_task, 77 job_vars=job_vars, 78 play_context=mock_play_context, 79 new_stdin=new_stdin, 80 loader=fake_loader, 81 shared_loader_obj=mock_shared_loader, 82 final_q=mock_queue, 83 ) 84 85 te._get_loop_items = MagicMock(return_value=None) 86 te._execute = MagicMock(return_value=dict()) 87 res = te.run() 88 89 te._get_loop_items = MagicMock(return_value=[]) 90 res = te.run() 91 92 te._get_loop_items = MagicMock(return_value=['a', 'b', 'c']) 93 te._run_loop = MagicMock(return_value=[dict(item='a', changed=True), dict(item='b', failed=True), dict(item='c')]) 94 res = te.run() 95 96 te._get_loop_items = MagicMock(side_effect=AnsibleError("")) 97 res = te.run() 98 self.assertIn("failed", res) 99 100 def test_task_executor_run_clean_res(self): 101 te = TaskExecutor(None, MagicMock(), None, None, None, None, None, None) 102 te._get_loop_items = MagicMock(return_value=[1]) 103 te._run_loop = MagicMock( 104 return_value=[ 105 { 106 'unsafe_bytes': AnsibleUnsafeBytes(b'{{ $bar }}'), 107 'unsafe_text': AnsibleUnsafeText(u'{{ $bar }}'), 108 'bytes': b'bytes', 109 'text': u'text', 110 'int': 1, 111 } 112 ] 113 ) 114 res = te.run() 115 data = res['results'][0] 116 self.assertIsInstance(data['unsafe_bytes'], AnsibleUnsafeText) 117 self.assertIsInstance(data['unsafe_text'], AnsibleUnsafeText) 118 self.assertIsInstance(data['bytes'], text_type) 119 self.assertIsInstance(data['text'], text_type) 120 self.assertIsInstance(data['int'], int) 121 122 def test_task_executor_get_loop_items(self): 123 fake_loader = DictDataLoader({}) 124 125 mock_host = MagicMock() 126 127 mock_task = MagicMock() 128 mock_task.loop_with = 'items' 129 mock_task.loop = ['a', 'b', 'c'] 130 131 mock_play_context = MagicMock() 132 133 mock_shared_loader = MagicMock() 134 mock_shared_loader.lookup_loader = lookup_loader 135 136 new_stdin = None 137 job_vars = dict() 138 mock_queue = MagicMock() 139 140 te = TaskExecutor( 141 host=mock_host, 142 task=mock_task, 143 job_vars=job_vars, 144 play_context=mock_play_context, 145 new_stdin=new_stdin, 146 loader=fake_loader, 147 shared_loader_obj=mock_shared_loader, 148 final_q=mock_queue, 149 ) 150 151 items = te._get_loop_items() 152 self.assertEqual(items, ['a', 'b', 'c']) 153 154 def test_task_executor_run_loop(self): 155 items = ['a', 'b', 'c'] 156 157 fake_loader = DictDataLoader({}) 158 159 mock_host = MagicMock() 160 161 def _copy(exclude_parent=False, exclude_tasks=False): 162 new_item = MagicMock() 163 return new_item 164 165 mock_task = MagicMock() 166 mock_task.copy.side_effect = _copy 167 168 mock_play_context = MagicMock() 169 170 mock_shared_loader = MagicMock() 171 mock_queue = MagicMock() 172 173 new_stdin = None 174 job_vars = dict() 175 176 te = TaskExecutor( 177 host=mock_host, 178 task=mock_task, 179 job_vars=job_vars, 180 play_context=mock_play_context, 181 new_stdin=new_stdin, 182 loader=fake_loader, 183 shared_loader_obj=mock_shared_loader, 184 final_q=mock_queue, 185 ) 186 187 def _execute(variables): 188 return dict(item=variables.get('item')) 189 190 te._squash_items = MagicMock(return_value=items) 191 te._execute = MagicMock(side_effect=_execute) 192 193 res = te._run_loop(items) 194 self.assertEqual(len(res), 3) 195 196 def test_task_executor_squash_items(self): 197 items = ['a', 'b', 'c'] 198 199 fake_loader = DictDataLoader({}) 200 201 mock_host = MagicMock() 202 203 loop_var = 'item' 204 205 def _evaluate_conditional(templar, variables): 206 item = variables.get(loop_var) 207 if item == 'b': 208 return False 209 return True 210 211 mock_task = MagicMock() 212 mock_task.evaluate_conditional.side_effect = _evaluate_conditional 213 214 mock_play_context = MagicMock() 215 216 mock_shared_loader = None 217 mock_queue = MagicMock() 218 219 new_stdin = None 220 job_vars = dict(pkg_mgr='yum') 221 222 te = TaskExecutor( 223 host=mock_host, 224 task=mock_task, 225 job_vars=job_vars, 226 play_context=mock_play_context, 227 new_stdin=new_stdin, 228 loader=fake_loader, 229 shared_loader_obj=mock_shared_loader, 230 final_q=mock_queue, 231 ) 232 233 # No replacement 234 mock_task.action = 'yum' 235 new_items = te._squash_items(items=items, loop_var='item', variables=job_vars) 236 self.assertEqual(new_items, ['a', 'b', 'c']) 237 self.assertIsInstance(mock_task.args, MagicMock) 238 239 mock_task.action = 'foo' 240 mock_task.args = {'name': '{{item}}'} 241 new_items = te._squash_items(items=items, loop_var='item', variables=job_vars) 242 self.assertEqual(new_items, ['a', 'b', 'c']) 243 self.assertEqual(mock_task.args, {'name': '{{item}}'}) 244 245 mock_task.action = 'yum' 246 mock_task.args = {'name': 'static'} 247 new_items = te._squash_items(items=items, loop_var='item', variables=job_vars) 248 self.assertEqual(new_items, ['a', 'b', 'c']) 249 self.assertEqual(mock_task.args, {'name': 'static'}) 250 251 mock_task.action = 'yum' 252 mock_task.args = {'name': '{{pkg_mgr}}'} 253 new_items = te._squash_items(items=items, loop_var='item', variables=job_vars) 254 self.assertEqual(new_items, ['a', 'b', 'c']) 255 self.assertEqual(mock_task.args, {'name': '{{pkg_mgr}}'}) 256 257 mock_task.action = '{{unknown}}' 258 mock_task.args = {'name': '{{item}}'} 259 new_items = te._squash_items(items=items, loop_var='item', variables=job_vars) 260 self.assertEqual(new_items, ['a', 'b', 'c']) 261 self.assertEqual(mock_task.args, {'name': '{{item}}'}) 262 263 # Could do something like this to recover from bad deps in a package 264 job_vars = dict(pkg_mgr='yum', packages=['a', 'b']) 265 items = ['absent', 'latest'] 266 mock_task.action = 'yum' 267 mock_task.args = {'name': '{{ packages }}', 'state': '{{ item }}'} 268 new_items = te._squash_items(items=items, loop_var='item', variables=job_vars) 269 self.assertEqual(new_items, items) 270 self.assertEqual(mock_task.args, {'name': '{{ packages }}', 'state': '{{ item }}'}) 271 272 # Maybe should raise an error in this case. The user would have to specify: 273 # - yum: name="{{ packages[item] }}" 274 # with_items: 275 # - ['a', 'b'] 276 # - ['foo', 'bar'] 277 # you can't use a list as a dict key so that would probably throw 278 # an error later. If so, we can throw it now instead. 279 # Squashing in this case would not be intuitive as the user is being 280 # explicit in using each list entry as a key. 281 job_vars = dict(pkg_mgr='yum', packages={"a": "foo", "b": "bar", "foo": "baz", "bar": "quux"}) 282 items = [['a', 'b'], ['foo', 'bar']] 283 mock_task.action = 'yum' 284 mock_task.args = {'name': '{{ packages[item] }}'} 285 new_items = te._squash_items(items=items, loop_var='item', variables=job_vars) 286 self.assertEqual(new_items, items) 287 self.assertEqual(mock_task.args, {'name': '{{ packages[item] }}'}) 288 289 # Replaces 290 items = ['a', 'b', 'c'] 291 mock_task.action = 'yum' 292 mock_task.args = {'name': '{{item}}'} 293 new_items = te._squash_items(items=items, loop_var='item', variables=job_vars) 294 self.assertEqual(new_items, [['a', 'c']]) 295 self.assertEqual(mock_task.args, {'name': ['a', 'c']}) 296 297 mock_task.action = '{{pkg_mgr}}' 298 mock_task.args = {'name': '{{item}}'} 299 new_items = te._squash_items(items=items, loop_var='item', variables=job_vars) 300 self.assertEqual(new_items, [['a', 'c']]) 301 self.assertEqual(mock_task.args, {'name': ['a', 'c']}) 302 303 # New loop_var 304 mock_task.action = 'yum' 305 mock_task.args = {'name': '{{a_loop_var_item}}'} 306 mock_task.loop_control = {'loop_var': 'a_loop_var_item'} 307 loop_var = 'a_loop_var_item' 308 new_items = te._squash_items(items=items, loop_var='a_loop_var_item', variables=job_vars) 309 self.assertEqual(new_items, [['a', 'c']]) 310 self.assertEqual(mock_task.args, {'name': ['a', 'c']}) 311 loop_var = 'item' 312 313 # 314 # These are presently not optimized but could be in the future. 315 # Expected output if they were optimized is given as a comment 316 # Please move these to a different section if they are optimized 317 # 318 319 # Squashing lists 320 job_vars = dict(pkg_mgr='yum') 321 items = [['a', 'b'], ['foo', 'bar']] 322 mock_task.action = 'yum' 323 mock_task.args = {'name': '{{ item }}'} 324 new_items = te._squash_items(items=items, loop_var='item', variables=job_vars) 325 # self.assertEqual(new_items, [['a', 'b', 'foo', 'bar']]) 326 # self.assertEqual(mock_task.args, {'name': ['a', 'b', 'foo', 'bar']}) 327 self.assertEqual(new_items, items) 328 self.assertEqual(mock_task.args, {'name': '{{ item }}'}) 329 330 # Retrieving from a dict 331 items = ['a', 'b', 'foo'] 332 mock_task.action = 'yum' 333 mock_task.args = {'name': '{{ packages[item] }}'} 334 new_items = te._squash_items(items=items, loop_var='item', variables=job_vars) 335 # self.assertEqual(new_items, [['foo', 'baz']]) 336 # self.assertEqual(mock_task.args, {'name': ['foo', 'baz']}) 337 self.assertEqual(new_items, items) 338 self.assertEqual(mock_task.args, {'name': '{{ packages[item] }}'}) 339 340 # Another way to retrieve from a dict 341 job_vars = dict(pkg_mgr='yum') 342 items = [{'package': 'foo'}, {'package': 'bar'}] 343 mock_task.action = 'yum' 344 mock_task.args = {'name': '{{ item["package"] }}'} 345 new_items = te._squash_items(items=items, loop_var='item', variables=job_vars) 346 # self.assertEqual(new_items, [['foo', 'bar']]) 347 # self.assertEqual(mock_task.args, {'name': ['foo', 'bar']}) 348 self.assertEqual(new_items, items) 349 self.assertEqual(mock_task.args, {'name': '{{ item["package"] }}'}) 350 351 items = [ 352 dict(name='a', state='present'), 353 dict(name='b', state='present'), 354 dict(name='c', state='present'), 355 ] 356 mock_task.action = 'yum' 357 mock_task.args = {'name': '{{item.name}}', 'state': '{{item.state}}'} 358 new_items = te._squash_items(items=items, loop_var='item', variables=job_vars) 359 # self.assertEqual(new_items, [dict(name=['a', 'b', 'c'], state='present')]) 360 # self.assertEqual(mock_task.args, {'name': ['a', 'b', 'c'], 'state': 'present'}) 361 self.assertEqual(new_items, items) 362 self.assertEqual(mock_task.args, {'name': '{{item.name}}', 'state': '{{item.state}}'}) 363 364 items = [ 365 dict(name='a', state='present'), 366 dict(name='b', state='present'), 367 dict(name='c', state='absent'), 368 ] 369 mock_task.action = 'yum' 370 mock_task.args = {'name': '{{item.name}}', 'state': '{{item.state}}'} 371 new_items = te._squash_items(items=items, loop_var='item', variables=job_vars) 372 # self.assertEqual(new_items, [dict(name=['a', 'b'], state='present'), 373 # dict(name='c', state='absent')]) 374 # self.assertEqual(mock_task.args, {'name': '{{item.name}}', 'state': '{{item.state}}'}) 375 self.assertEqual(new_items, items) 376 self.assertEqual(mock_task.args, {'name': '{{item.name}}', 'state': '{{item.state}}'}) 377 378 def test_task_executor_get_action_handler(self): 379 te = TaskExecutor( 380 host=MagicMock(), 381 task=MagicMock(), 382 job_vars={}, 383 play_context=MagicMock(), 384 new_stdin=None, 385 loader=DictDataLoader({}), 386 shared_loader_obj=MagicMock(), 387 final_q=MagicMock(), 388 ) 389 390 action_loader = te._shared_loader_obj.action_loader 391 action_loader.has_plugin.return_value = True 392 action_loader.get.return_value = mock.sentinel.handler 393 394 mock_connection = MagicMock() 395 mock_templar = MagicMock() 396 action = 'namespace.prefix_suffix' 397 te._task.action = action 398 399 handler = te._get_action_handler(mock_connection, mock_templar) 400 401 self.assertIs(mock.sentinel.handler, handler) 402 403 action_loader.has_plugin.assert_called_once_with( 404 action, collection_list=te._task.collections) 405 406 action_loader.get.assert_called_once_with( 407 te._task.action, task=te._task, connection=mock_connection, 408 play_context=te._play_context, loader=te._loader, 409 templar=mock_templar, shared_loader_obj=te._shared_loader_obj, 410 collection_list=te._task.collections) 411 412 def test_task_executor_get_handler_prefix(self): 413 te = TaskExecutor( 414 host=MagicMock(), 415 task=MagicMock(), 416 job_vars={}, 417 play_context=MagicMock(), 418 new_stdin=None, 419 loader=DictDataLoader({}), 420 shared_loader_obj=MagicMock(), 421 final_q=MagicMock(), 422 ) 423 424 action_loader = te._shared_loader_obj.action_loader 425 action_loader.has_plugin.side_effect = [False, True] 426 action_loader.get.return_value = mock.sentinel.handler 427 action_loader.__contains__.return_value = True 428 429 mock_connection = MagicMock() 430 mock_templar = MagicMock() 431 action = 'namespace.netconf_suffix' 432 module_prefix = action.split('_')[0] 433 te._task.action = action 434 435 handler = te._get_action_handler(mock_connection, mock_templar) 436 437 self.assertIs(mock.sentinel.handler, handler) 438 action_loader.has_plugin.assert_has_calls([mock.call(action, collection_list=te._task.collections), 439 mock.call(module_prefix, collection_list=te._task.collections)]) 440 441 action_loader.get.assert_called_once_with( 442 module_prefix, task=te._task, connection=mock_connection, 443 play_context=te._play_context, loader=te._loader, 444 templar=mock_templar, shared_loader_obj=te._shared_loader_obj, 445 collection_list=te._task.collections) 446 447 def test_task_executor_get_handler_normal(self): 448 te = TaskExecutor( 449 host=MagicMock(), 450 task=MagicMock(), 451 job_vars={}, 452 play_context=MagicMock(), 453 new_stdin=None, 454 loader=DictDataLoader({}), 455 shared_loader_obj=MagicMock(), 456 final_q=MagicMock(), 457 ) 458 459 action_loader = te._shared_loader_obj.action_loader 460 action_loader.has_plugin.return_value = False 461 action_loader.get.return_value = mock.sentinel.handler 462 action_loader.__contains__.return_value = False 463 464 mock_connection = MagicMock() 465 mock_templar = MagicMock() 466 action = 'namespace.prefix_suffix' 467 module_prefix = action.split('_')[0] 468 te._task.action = action 469 handler = te._get_action_handler(mock_connection, mock_templar) 470 471 self.assertIs(mock.sentinel.handler, handler) 472 473 action_loader.has_plugin.assert_has_calls([mock.call(action, collection_list=te._task.collections), 474 mock.call(module_prefix, collection_list=te._task.collections)]) 475 476 action_loader.get.assert_called_once_with( 477 'ansible.legacy.normal', task=te._task, connection=mock_connection, 478 play_context=te._play_context, loader=te._loader, 479 templar=mock_templar, shared_loader_obj=te._shared_loader_obj, 480 collection_list=None) 481 482 def test_task_executor_execute(self): 483 fake_loader = DictDataLoader({}) 484 485 mock_host = MagicMock() 486 487 mock_task = MagicMock() 488 mock_task.args = dict() 489 mock_task.retries = 0 490 mock_task.delay = -1 491 mock_task.register = 'foo' 492 mock_task.until = None 493 mock_task.changed_when = None 494 mock_task.failed_when = None 495 mock_task.post_validate.return_value = None 496 # mock_task.async_val cannot be left unset, because on Python 3 MagicMock() 497 # > 0 raises a TypeError There are two reasons for using the value 1 498 # here: on Python 2 comparing MagicMock() > 0 returns True, and the 499 # other reason is that if I specify 0 here, the test fails. ;) 500 mock_task.async_val = 1 501 mock_task.poll = 0 502 503 mock_play_context = MagicMock() 504 mock_play_context.post_validate.return_value = None 505 mock_play_context.update_vars.return_value = None 506 507 mock_connection = MagicMock() 508 mock_connection.set_host_overrides.return_value = None 509 mock_connection._connect.return_value = None 510 511 mock_action = MagicMock() 512 mock_queue = MagicMock() 513 514 shared_loader = None 515 new_stdin = None 516 job_vars = dict(omit="XXXXXXXXXXXXXXXXXXX") 517 518 te = TaskExecutor( 519 host=mock_host, 520 task=mock_task, 521 job_vars=job_vars, 522 play_context=mock_play_context, 523 new_stdin=new_stdin, 524 loader=fake_loader, 525 shared_loader_obj=shared_loader, 526 final_q=mock_queue, 527 ) 528 529 te._get_connection = MagicMock(return_value=mock_connection) 530 te._get_action_handler = MagicMock(return_value=mock_action) 531 532 mock_action.run.return_value = dict(ansible_facts=dict()) 533 res = te._execute() 534 535 mock_task.changed_when = MagicMock(return_value=AnsibleUnicode("1 == 1")) 536 res = te._execute() 537 538 mock_task.changed_when = None 539 mock_task.failed_when = MagicMock(return_value=AnsibleUnicode("1 == 1")) 540 res = te._execute() 541 542 mock_task.failed_when = None 543 mock_task.evaluate_conditional.return_value = False 544 res = te._execute() 545 546 mock_task.evaluate_conditional.return_value = True 547 mock_task.args = dict(_raw_params='foo.yml', a='foo', b='bar') 548 mock_task.action = 'include' 549 res = te._execute() 550 551 def test_task_executor_poll_async_result(self): 552 fake_loader = DictDataLoader({}) 553 554 mock_host = MagicMock() 555 556 mock_task = MagicMock() 557 mock_task.async_val = 0.1 558 mock_task.poll = 0.05 559 560 mock_play_context = MagicMock() 561 562 mock_connection = MagicMock() 563 564 mock_action = MagicMock() 565 mock_queue = MagicMock() 566 567 shared_loader = MagicMock() 568 shared_loader.action_loader = action_loader 569 570 new_stdin = None 571 job_vars = dict(omit="XXXXXXXXXXXXXXXXXXX") 572 573 te = TaskExecutor( 574 host=mock_host, 575 task=mock_task, 576 job_vars=job_vars, 577 play_context=mock_play_context, 578 new_stdin=new_stdin, 579 loader=fake_loader, 580 shared_loader_obj=shared_loader, 581 final_q=mock_queue, 582 ) 583 584 te._connection = MagicMock() 585 586 def _get(*args, **kwargs): 587 mock_action = MagicMock() 588 mock_action.run.return_value = dict(stdout='') 589 return mock_action 590 591 # testing with some bad values in the result passed to poll async, 592 # and with a bad value returned from the mock action 593 with patch.object(action_loader, 'get', _get): 594 mock_templar = MagicMock() 595 res = te._poll_async_result(result=dict(), templar=mock_templar) 596 self.assertIn('failed', res) 597 res = te._poll_async_result(result=dict(ansible_job_id=1), templar=mock_templar) 598 self.assertIn('failed', res) 599 600 def _get(*args, **kwargs): 601 mock_action = MagicMock() 602 mock_action.run.return_value = dict(finished=1) 603 return mock_action 604 605 # now testing with good values 606 with patch.object(action_loader, 'get', _get): 607 mock_templar = MagicMock() 608 res = te._poll_async_result(result=dict(ansible_job_id=1), templar=mock_templar) 609 self.assertEqual(res, dict(finished=1)) 610 611 def test_recursive_remove_omit(self): 612 omit_token = 'POPCORN' 613 614 data = { 615 'foo': 'bar', 616 'baz': 1, 617 'qux': ['one', 'two', 'three'], 618 'subdict': { 619 'remove': 'POPCORN', 620 'keep': 'not_popcorn', 621 'subsubdict': { 622 'remove': 'POPCORN', 623 'keep': 'not_popcorn', 624 }, 625 'a_list': ['POPCORN'], 626 }, 627 'a_list': ['POPCORN'], 628 'list_of_lists': [ 629 ['some', 'thing'], 630 ], 631 'list_of_dicts': [ 632 { 633 'remove': 'POPCORN', 634 } 635 ], 636 } 637 638 expected = { 639 'foo': 'bar', 640 'baz': 1, 641 'qux': ['one', 'two', 'three'], 642 'subdict': { 643 'keep': 'not_popcorn', 644 'subsubdict': { 645 'keep': 'not_popcorn', 646 }, 647 'a_list': ['POPCORN'], 648 }, 649 'a_list': ['POPCORN'], 650 'list_of_lists': [ 651 ['some', 'thing'], 652 ], 653 'list_of_dicts': [{}], 654 } 655 656 self.assertEqual(remove_omit(data, omit_token), expected) 657