1""" 2Tests for the Openstack Cloud Provider 3""" 4 5 6import logging 7import os 8import shutil 9from time import sleep 10 11import pytest 12import salt.utils.files 13from salt.config import cloud_config, cloud_providers_config 14from salt.utils.yaml import safe_load 15from tests.support.case import ShellCase 16from tests.support.helpers import random_string 17from tests.support.paths import FILES 18from tests.support.runtests import RUNTIME_VARS 19 20TIMEOUT = 500 21 22log = logging.getLogger(__name__) 23 24 25@pytest.mark.expensive_test 26class CloudTest(ShellCase): 27 PROVIDER = "" 28 REQUIRED_PROVIDER_CONFIG_ITEMS = tuple() 29 __RE_RUN_DELAY = 30 30 __RE_TRIES = 12 31 32 @staticmethod 33 def clean_cloud_dir(tmp_dir): 34 """ 35 Clean the cloud.providers.d tmp directory 36 """ 37 # make sure old provider configs are deleted 38 if not os.path.isdir(tmp_dir): 39 return 40 for fname in os.listdir(tmp_dir): 41 os.remove(os.path.join(tmp_dir, fname)) 42 43 def query_instances(self): 44 """ 45 Standardize the data returned from a salt-cloud --query 46 """ 47 return { 48 x.strip(": ") 49 for x in self.run_cloud("--query") 50 if x.lstrip().lower().startswith("cloud-test-") 51 } 52 53 def _instance_exists(self, instance_name=None, query=None): 54 """ 55 :param instance_name: The name of the instance to check for in salt-cloud. 56 For example this is may used when a test temporarily renames an instance 57 :param query: The result of a salt-cloud --query run outside of this function 58 """ 59 if not instance_name: 60 instance_name = self.instance_name 61 if not query: 62 query = self.query_instances() 63 64 log.debug('Checking for "%s" in %s', instance_name, query) 65 if isinstance(query, set): 66 return instance_name in query 67 return any(instance_name == q.strip(": ") for q in query) 68 69 def assertInstanceExists(self, creation_ret=None, instance_name=None): 70 """ 71 :param instance_name: Override the checked instance name, otherwise the class default will be used. 72 :param creation_ret: The return value from the run_cloud() function that created the instance 73 """ 74 if not instance_name: 75 instance_name = self.instance_name 76 77 # If it exists but doesn't show up in the creation_ret, there was probably an error during creation 78 if creation_ret: 79 self.assertIn( 80 instance_name, 81 [i.strip(": ") for i in creation_ret], 82 "An error occured during instance creation: |\n\t{}\n\t|".format( 83 "\n\t".join(creation_ret) 84 ), 85 ) 86 else: 87 # Verify that the instance exists via query 88 query = self.query_instances() 89 for tries in range(self.__RE_TRIES): 90 if self._instance_exists(instance_name, query): 91 log.debug( 92 'Instance "%s" reported after %s seconds', 93 instance_name, 94 tries * self.__RE_RUN_DELAY, 95 ) 96 break 97 else: 98 sleep(self.__RE_RUN_DELAY) 99 query = self.query_instances() 100 101 # Assert that the last query was successful 102 self.assertTrue( 103 self._instance_exists(instance_name, query), 104 'Instance "{}" was not created successfully: {}'.format( 105 self.instance_name, ", ".join(query) 106 ), 107 ) 108 109 log.debug('Instance exists and was created: "%s"', instance_name) 110 111 def assertDestroyInstance(self, instance_name=None, timeout=None): 112 if timeout is None: 113 timeout = TIMEOUT 114 if not instance_name: 115 instance_name = self.instance_name 116 log.debug('Deleting instance "%s"', instance_name) 117 delete_str = self.run_cloud( 118 "-d {} --assume-yes --out=yaml".format(instance_name), timeout=timeout 119 ) 120 if delete_str: 121 delete = safe_load("\n".join(delete_str)) 122 self.assertIn(self.profile_str, delete) 123 self.assertIn(self.PROVIDER, delete[self.profile_str]) 124 self.assertIn(instance_name, delete[self.profile_str][self.PROVIDER]) 125 126 delete_status = delete[self.profile_str][self.PROVIDER][instance_name] 127 if isinstance(delete_status, str): 128 self.assertEqual(delete_status, "True") 129 return 130 elif isinstance(delete_status, dict): 131 current_state = delete_status.get("currentState") 132 if current_state: 133 if current_state.get("ACTION"): 134 self.assertIn(".delete", current_state.get("ACTION")) 135 return 136 else: 137 self.assertEqual(current_state.get("name"), "shutting-down") 138 return 139 # It's not clear from the delete string that deletion was successful, ask salt-cloud after a delay 140 query = self.query_instances() 141 # some instances take a while to report their destruction 142 for tries in range(6): 143 if self._instance_exists(query=query): 144 sleep(30) 145 log.debug( 146 'Instance "%s" still found in query after %s tries: %s', 147 instance_name, 148 tries, 149 query, 150 ) 151 query = self.query_instances() 152 # The last query should have been successful 153 self.assertNotIn(instance_name, self.query_instances()) 154 155 @property 156 def instance_name(self): 157 if not hasattr(self, "_instance_name"): 158 # Create the cloud instance name to be used throughout the tests 159 subclass = self.__class__.__name__.strip("Test") 160 # Use the first three letters of the subclass, fill with '-' if too short 161 self._instance_name = random_string( 162 "cloud-test-{:-<3}-".format(subclass[:3]), uppercase=False 163 ).lower() 164 return self._instance_name 165 166 @property 167 def providers(self): 168 if not hasattr(self, "_providers"): 169 self._providers = self.run_cloud("--list-providers") 170 return self._providers 171 172 @property 173 def provider_config(self): 174 if not hasattr(self, "_provider_config"): 175 self._provider_config = cloud_providers_config( 176 os.path.join( 177 RUNTIME_VARS.TMP_CONF_DIR, 178 "cloud.providers.d", 179 self.PROVIDER + ".conf", 180 ) 181 ) 182 return self._provider_config[self.profile_str][self.PROVIDER] 183 184 @property 185 def config(self): 186 if not hasattr(self, "_config"): 187 self._config = cloud_config( 188 os.path.join( 189 RUNTIME_VARS.TMP_CONF_DIR, 190 "cloud.profiles.d", 191 self.PROVIDER + ".conf", 192 ) 193 ) 194 return self._config 195 196 @property 197 def profile_str(self): 198 return self.PROVIDER + "-config" 199 200 def add_profile_config(self, name, data, conf, new_profile): 201 """ 202 copy the current profile and add a new profile in the same file 203 """ 204 conf_path = os.path.join(RUNTIME_VARS.TMP_CONF_DIR, "cloud.profiles.d", conf) 205 with salt.utils.files.fopen(conf_path, "r") as fp: 206 conf = safe_load(fp) 207 conf[new_profile] = conf[name].copy() 208 conf[new_profile].update(data) 209 with salt.utils.files.fopen(conf_path, "w") as fp: 210 salt.utils.yaml.safe_dump(conf, fp) 211 212 def setUp(self): 213 """ 214 Sets up the test requirements. In child classes, define PROVIDER and REQUIRED_PROVIDER_CONFIG_ITEMS or this will fail 215 """ 216 super().setUp() 217 218 if not self.PROVIDER: 219 self.fail("A PROVIDER must be defined for this test") 220 221 # check if appropriate cloud provider and profile files are present 222 if self.profile_str + ":" not in self.providers: 223 self.skipTest( 224 "Configuration file for {0} was not found. Check {0}.conf files " 225 "in tests/integration/files/conf/cloud.*.d/ to run these tests.".format( 226 self.PROVIDER 227 ) 228 ) 229 230 missing_conf_item = [] 231 for att in self.REQUIRED_PROVIDER_CONFIG_ITEMS: 232 if not self.provider_config.get(att): 233 missing_conf_item.append(att) 234 235 if missing_conf_item: 236 self.skipTest( 237 "Conf items are missing that must be provided to run these tests: {}".format( 238 ", ".join(missing_conf_item) 239 ) 240 + "\nCheck tests/integration/files/conf/cloud.providers.d/{}.conf".format( 241 self.PROVIDER 242 ) 243 ) 244 245 def _alt_names(self): 246 """ 247 Check for an instances created alongside this test's instance that weren't cleaned up 248 """ 249 query = self.query_instances() 250 instances = set() 251 for q in query: 252 # Verify but this is a new name and not a shutting down ec2 instance 253 if q.startswith(self.instance_name) and not q.split("-")[-1].startswith( 254 "DEL" 255 ): 256 instances.add(q) 257 log.debug( 258 'Adding "%s" to the set of instances that needs to be deleted', q 259 ) 260 return instances 261 262 def _ensure_deletion(self, instance_name=None): 263 """ 264 Make sure that the instance absolutely gets deleted, but fail the test if it happens in the tearDown 265 :return True if an instance was deleted, False if no instance was deleted; and a message 266 """ 267 destroyed = False 268 if not instance_name: 269 instance_name = self.instance_name 270 271 if self._instance_exists(instance_name): 272 for tries in range(3): 273 try: 274 self.assertDestroyInstance(instance_name) 275 return ( 276 False, 277 'The instance "{}" was deleted during the tearDown, not the' 278 " test.".format(instance_name), 279 ) 280 except AssertionError as e: 281 log.error( 282 'Failed to delete instance "%s". Tries: %s\n%s', 283 instance_name, 284 tries, 285 str(e), 286 ) 287 if not self._instance_exists(): 288 destroyed = True 289 break 290 else: 291 sleep(30) 292 293 if not destroyed: 294 # Destroying instances in the tearDown is a contingency, not the way things should work by default. 295 return ( 296 False, 297 'The Instance "{}" was not deleted after multiple attempts'.format( 298 instance_name 299 ), 300 ) 301 302 return ( 303 True, 304 'The instance "{}" cleaned up properly after the test'.format( 305 instance_name 306 ), 307 ) 308 309 def tearDown(self): 310 """ 311 Clean up after tests, If the instance still exists for any reason, delete it. 312 Instances should be destroyed before the tearDown, assertDestroyInstance() should be called exactly 313 one time in a test for each instance created. This is a failSafe and something went wrong 314 if the tearDown is where an instance is destroyed. 315 """ 316 success = True 317 fail_messages = [] 318 alt_names = self._alt_names() 319 for instance in alt_names: 320 alt_destroyed, alt_destroy_message = self._ensure_deletion(instance) 321 if not alt_destroyed: 322 success = False 323 fail_messages.append(alt_destroy_message) 324 log.error( 325 'Failed to destroy instance "%s": %s', instance, alt_destroy_message 326 ) 327 self.assertTrue(success, "\n".join(fail_messages)) 328 self.assertFalse( 329 alt_names, "Cleanup should happen in the test, not the TearDown" 330 ) 331 332 @classmethod 333 def tearDownClass(cls): 334 cls.clean_cloud_dir(cls.tmp_provider_dir) 335 336 @classmethod 337 def setUpClass(cls): 338 # clean up before setup 339 cls.tmp_provider_dir = os.path.join( 340 RUNTIME_VARS.TMP_CONF_DIR, "cloud.providers.d" 341 ) 342 cls.clean_cloud_dir(cls.tmp_provider_dir) 343 344 # add the provider config for only the cloud we are testing 345 provider_file = cls.PROVIDER + ".conf" 346 shutil.copyfile( 347 os.path.join( 348 os.path.join(FILES, "conf", "cloud.providers.d"), provider_file 349 ), 350 os.path.join(os.path.join(cls.tmp_provider_dir, provider_file)), 351 ) 352