1#    Licensed under the Apache License, Version 2.0 (the "License"); you may
2#    not use this file except in compliance with the License. You may obtain
3#    a copy of the License at
4#
5#         http://www.apache.org/licenses/LICENSE-2.0
6#
7#    Unless required by applicable law or agreed to in writing, software
8#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
9#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
10#    License for the specific language governing permissions and limitations
11#    under the License.
12
13import os
14import time
15
16from cinderclient.v2 import client as cinderclient
17import fixtures
18from glanceclient import client as glanceclient
19from keystoneauth1.exceptions import discovery as discovery_exc
20from keystoneauth1 import identity
21from keystoneauth1 import session as ksession
22from keystoneclient import client as keystoneclient
23from keystoneclient import discover as keystone_discover
24from neutronclient.v2_0 import client as neutronclient
25import openstack.config
26import openstack.config.exceptions
27from oslo_utils import uuidutils
28import tempest.lib.cli.base
29import testtools
30
31import novaclient
32import novaclient.api_versions
33from novaclient import base
34import novaclient.client
35from novaclient.v2 import networks
36import novaclient.v2.shell
37
38BOOT_IS_COMPLETE = ("login as 'cirros' user. default password: "
39                    "'gocubsgo'. use 'sudo' for root.")
40
41
42def is_keystone_version_available(session, version):
43    """Given a (major, minor) pair, check if the API version is enabled."""
44
45    d = keystone_discover.Discover(session)
46    try:
47        d.create_client(version)
48    except (discovery_exc.DiscoveryFailure, discovery_exc.VersionNotAvailable):
49        return False
50    else:
51        return True
52
53
54# The following are simple filter functions that filter our available
55# image / flavor list so that they can be used in standard testing.
56def pick_flavor(flavors):
57    """Given a flavor list pick a reasonable one."""
58    for flavor_priority in ('m1.nano', 'm1.micro', 'm1.tiny', 'm1.small'):
59        for flavor in flavors:
60            if flavor.name == flavor_priority:
61                return flavor
62    raise NoFlavorException()
63
64
65def pick_image(images):
66    firstImage = None
67    for image in images:
68        firstImage = firstImage or image
69        if image.name.startswith('cirros') and (
70                image.name.endswith('-uec') or
71                image.name.endswith('-disk.img')):
72            return image
73
74    # We didn't find the specific cirros image we'd like to use, so just use
75    # the first available.
76    if firstImage:
77        return firstImage
78
79    raise NoImageException()
80
81
82def pick_network(networks):
83    network_name = os.environ.get('OS_NOVACLIENT_NETWORK')
84    if network_name:
85        for network in networks:
86            if network.name == network_name:
87                return network
88        raise NoNetworkException()
89    return networks[0]
90
91
92class NoImageException(Exception):
93    """We couldn't find an acceptable image."""
94    pass
95
96
97class NoFlavorException(Exception):
98    """We couldn't find an acceptable flavor."""
99    pass
100
101
102class NoNetworkException(Exception):
103    """We couldn't find an acceptable network."""
104    pass
105
106
107class NoCloudConfigException(Exception):
108    """We couldn't find a cloud configuration."""
109    pass
110
111
112CACHE = {}
113
114
115class ClientTestBase(testtools.TestCase):
116    """Base test class for read only python-novaclient commands.
117
118    This is a first pass at a simple read only python-novaclient test. This
119    only exercises client commands that are read only.
120
121    This should test commands:
122    * as a regular user
123    * as a admin user
124    * with and without optional parameters
125    * initially just check return codes, and later test command outputs
126
127    """
128    COMPUTE_API_VERSION = None
129
130    log_format = ('%(asctime)s %(process)d %(levelname)-8s '
131                  '[%(name)s] %(message)s')
132
133    def setUp(self):
134        super(ClientTestBase, self).setUp()
135
136        test_timeout = os.environ.get('OS_TEST_TIMEOUT', 0)
137        try:
138            test_timeout = int(test_timeout)
139        except ValueError:
140            test_timeout = 0
141        if test_timeout > 0:
142            self.useFixture(fixtures.Timeout(test_timeout, gentle=True))
143
144        if (os.environ.get('OS_STDOUT_CAPTURE') == 'True' or
145                os.environ.get('OS_STDOUT_CAPTURE') == '1'):
146            stdout = self.useFixture(fixtures.StringStream('stdout')).stream
147            self.useFixture(fixtures.MonkeyPatch('sys.stdout', stdout))
148        if (os.environ.get('OS_STDERR_CAPTURE') == 'True' or
149                os.environ.get('OS_STDERR_CAPTURE') == '1'):
150            stderr = self.useFixture(fixtures.StringStream('stderr')).stream
151            self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr))
152
153        if (os.environ.get('OS_LOG_CAPTURE') != 'False' and
154                os.environ.get('OS_LOG_CAPTURE') != '0'):
155            self.useFixture(fixtures.LoggerFixture(nuke_handlers=False,
156                                                   format=self.log_format,
157                                                   level=None))
158
159        # Collecting of credentials:
160        #
161        # Grab the cloud config from a user's clouds.yaml file.
162        # First look for a functional_admin cloud, as this is a cloud
163        # that the user may have defined for functional testing that has
164        # admin credentials.
165        # If that is not found, get the devstack config and override the
166        # username and project_name to be admin so that admin credentials
167        # will be used.
168        #
169        # Finally, fall back to looking for environment variables to support
170        # existing users running these the old way. We should deprecate that
171        # as tox 2.0 blanks out environment.
172        #
173        # TODO(sdague): while we collect this information in
174        # tempest-lib, we do it in a way that's not available for top
175        # level tests. Long term this probably needs to be in the base
176        # class.
177        openstack_config = openstack.config.OpenStackConfig()
178        try:
179            cloud_config = openstack_config.get_one_cloud('functional_admin')
180        except openstack.config.exceptions.OpenStackConfigException:
181            try:
182                cloud_config = openstack_config.get_one_cloud(
183                    'devstack', auth=dict(
184                        username='admin', project_name='admin'))
185            except openstack.config.exceptions.OpenStackConfigException:
186                try:
187                    cloud_config = openstack_config.get_one_cloud('envvars')
188                except openstack.config.exceptions.OpenStackConfigException:
189                    cloud_config = None
190
191        if cloud_config is None:
192            raise NoCloudConfigException(
193                "Could not find a cloud named functional_admin or a cloud"
194                " named devstack. Please check your clouds.yaml file and"
195                " try again.")
196        auth_info = cloud_config.config['auth']
197
198        user = auth_info['username']
199        passwd = auth_info['password']
200        self.project_name = auth_info['project_name']
201        auth_url = auth_info['auth_url']
202        user_domain_id = auth_info['user_domain_id']
203        self.project_domain_id = auth_info['project_domain_id']
204
205        if 'insecure' in cloud_config.config:
206            self.insecure = cloud_config.config['insecure']
207        else:
208            self.insecure = False
209
210        auth = identity.Password(username=user,
211                                 password=passwd,
212                                 project_name=self.project_name,
213                                 auth_url=auth_url,
214                                 project_domain_id=self.project_domain_id,
215                                 user_domain_id=user_domain_id)
216        session = ksession.Session(auth=auth, verify=(not self.insecure))
217
218        self.client = self._get_novaclient(session)
219
220        self.glance = glanceclient.Client('2', session=session)
221
222        # pick some reasonable flavor / image combo
223        if "flavor" not in CACHE:
224            CACHE["flavor"] = pick_flavor(self.client.flavors.list())
225        if "image" not in CACHE:
226            CACHE["image"] = pick_image(self.glance.images.list())
227        self.flavor = CACHE["flavor"]
228        self.image = CACHE["image"]
229
230        if "network" not in CACHE:
231            # Get the networks from neutron.
232            neutron = neutronclient.Client(session=session)
233            neutron_networks = neutron.list_networks()['networks']
234            # Convert the neutron dicts to Network objects.
235            nets = []
236            for network in neutron_networks:
237                nets.append(networks.Network(
238                    networks.NeutronManager, network))
239            # Keep track of whether or not there are multiple networks
240            # available to the given tenant because if so, a specific
241            # network ID has to be passed in on server create requests
242            # otherwise the server POST will fail with a 409.
243            CACHE['multiple_networks'] = len(nets) > 1
244            CACHE["network"] = pick_network(nets)
245        self.network = CACHE["network"]
246        self.multiple_networks = CACHE['multiple_networks']
247
248        # create a CLI client in case we'd like to do CLI
249        # testing. tempest.lib does this really weird thing where it
250        # builds a giant factory of all the CLIs that it knows
251        # about. Eventually that should really be unwound into
252        # something more sensible.
253        cli_dir = os.environ.get(
254            'OS_NOVACLIENT_EXEC_DIR',
255            os.path.join(os.path.abspath('.'), '.tox/functional/bin'))
256
257        self.cli_clients = tempest.lib.cli.base.CLIClient(
258            username=user,
259            password=passwd,
260            tenant_name=self.project_name,
261            uri=auth_url,
262            cli_dir=cli_dir,
263            insecure=self.insecure)
264
265        self.keystone = keystoneclient.Client(session=session,
266                                              username=user,
267                                              password=passwd)
268        self.cinder = cinderclient.Client(auth=auth, session=session)
269
270    def _get_novaclient(self, session):
271        nc = novaclient.client.Client("2", session=session)
272
273        if self.COMPUTE_API_VERSION:
274            if "min_api_version" not in CACHE:
275                # Obtain supported versions by API side
276                v = nc.versions.get_current()
277                if not hasattr(v, 'version') or not v.version:
278                    # API doesn't support microversions
279                    CACHE["min_api_version"] = (
280                        novaclient.api_versions.APIVersion("2.0"))
281                    CACHE["max_api_version"] = (
282                        novaclient.api_versions.APIVersion("2.0"))
283                else:
284                    CACHE["min_api_version"] = (
285                        novaclient.api_versions.APIVersion(v.min_version))
286                    CACHE["max_api_version"] = (
287                        novaclient.api_versions.APIVersion(v.version))
288
289            if self.COMPUTE_API_VERSION == "2.latest":
290                requested_version = min(novaclient.API_MAX_VERSION,
291                                        CACHE["max_api_version"])
292            else:
293                requested_version = novaclient.api_versions.APIVersion(
294                    self.COMPUTE_API_VERSION)
295
296            if not requested_version.matches(CACHE["min_api_version"],
297                                             CACHE["max_api_version"]):
298                msg = ("%s is not supported by Nova-API. Supported version" %
299                       self.COMPUTE_API_VERSION)
300                if CACHE["min_api_version"] == CACHE["max_api_version"]:
301                    msg += ": %s" % CACHE["min_api_version"].get_string()
302                else:
303                    msg += "s: %s - %s" % (
304                        CACHE["min_api_version"].get_string(),
305                        CACHE["max_api_version"].get_string())
306                self.skipTest(msg)
307
308            nc.api_version = requested_version
309        return nc
310
311    def nova(self, action, flags='', params='', fail_ok=False,
312             endpoint_type='publicURL', merge_stderr=False):
313        if self.COMPUTE_API_VERSION:
314            flags += " --os-compute-api-version %s " % self.COMPUTE_API_VERSION
315        return self.cli_clients.nova(action, flags, params, fail_ok,
316                                     endpoint_type, merge_stderr)
317
318    def wait_for_volume_status(self, volume, status, timeout=60,
319                               poll_interval=1):
320        """Wait until volume reaches given status.
321
322        :param volume: volume resource
323        :param status: expected status of volume
324        :param timeout: timeout in seconds
325        :param poll_interval: poll interval in seconds
326        """
327        start_time = time.time()
328        while time.time() - start_time < timeout:
329            volume = self.cinder.volumes.get(volume.id)
330            if volume.status == status:
331                break
332            time.sleep(poll_interval)
333        else:
334            self.fail("Volume %s did not reach status %s after %d s"
335                      % (volume.id, status, timeout))
336
337    def wait_for_server_os_boot(self, server_id, timeout=300,
338                                poll_interval=1):
339        """Wait until instance's operating system  is completely booted.
340
341        :param server_id: uuid4 id of given instance
342        :param timeout: timeout in seconds
343        :param poll_interval: poll interval in seconds
344        """
345        start_time = time.time()
346        console = None
347        while time.time() - start_time < timeout:
348            console = self.nova('console-log %s ' % server_id)
349            if BOOT_IS_COMPLETE in console:
350                break
351            time.sleep(poll_interval)
352        else:
353            self.fail("Server %s did not boot after %d s.\nConsole:\n%s"
354                      % (server_id, timeout, console))
355
356    def wait_for_resource_delete(self, resource, manager,
357                                 timeout=60, poll_interval=1):
358        """Wait until getting the resource raises NotFound exception.
359
360        :param resource: Resource object.
361        :param manager: Manager object with get method.
362        :param timeout: timeout in seconds
363        :param poll_interval: poll interval in seconds
364        """
365        start_time = time.time()
366        while time.time() - start_time < timeout:
367            try:
368                manager.get(resource)
369            except Exception as e:
370                if getattr(e, "http_status", None) == 404:
371                    break
372                else:
373                    raise
374            time.sleep(poll_interval)
375        else:
376            self.fail("The resource '%s' still exists." % base.getid(resource))
377
378    def name_generate(self):
379        """Generate randomized name for some entity."""
380        # NOTE(andreykurilin): name_generator method is used for various
381        #   resources (servers, flavors, volumes, keystone users, etc).
382        #   Since the length of name has limits we cannot use the whole UUID,
383        #   so the first 8 chars is taken from it.
384        #   Based on the fact that the new name includes class and method
385        #   names, 8 chars of uuid should be enough to prevent any conflicts,
386        #   even if the single test will be launched in parallel thousand times
387        return "%(prefix)s-%(test_cls)s-%(test_name)s" % {
388            "prefix": uuidutils.generate_uuid()[:8],
389            "test_cls": self.__class__.__name__,
390            "test_name": self.id().rsplit(".", 1)[-1]
391        }
392
393    def _get_value_from_the_table(self, table, key):
394        """Parses table to get desired value.
395
396        EXAMPLE of the table:
397        # +-------------+----------------------------------+
398        # |   Property  |              Value               |
399        # +-------------+----------------------------------+
400        # | description |                                  |
401        # |   enabled   |               True               |
402        # |      id     | 582df899eabc47018c96713c2f7196ba |
403        # |     name    |              admin               |
404        # +-------------+----------------------------------+
405        """
406        lines = table.split("\n")
407        for line in lines:
408            if "|" in line:
409                l_property, l_value = line.split("|")[1:3]
410                if l_property.strip() == key:
411                    return l_value.strip()
412        raise ValueError("Property '%s' is missing from the table:\n%s" %
413                         (key, table))
414
415    def _get_column_value_from_single_row_table(self, table, column):
416        """Get the value for the column in the single-row table
417
418        Example table:
419
420        +----------+-------------+----------+----------+
421        | address  | cidr        | hostname | host     |
422        +----------+-------------+----------+----------+
423        | 10.0.0.3 | 10.0.0.0/24 | test     | myhost   |
424        +----------+-------------+----------+----------+
425
426        :param table: newline-separated table with |-separated cells
427        :param column: name of the column to look for
428        :raises: ValueError if the column value is not found
429        """
430        lines = table.split("\n")
431        # Determine the column header index first.
432        column_index = -1
433        for line in lines:
434            if "|" in line:
435                if column_index == -1:
436                    headers = line.split("|")[1:-1]
437                    for index, header in enumerate(headers):
438                        if header.strip() == column:
439                            column_index = index
440                            break
441                else:
442                    # We expect a single-row table so we should be able to get
443                    # the value now using the column index.
444                    return line.split("|")[1:-1][column_index].strip()
445
446        raise ValueError("Unable to find value for column '%s'." % column)
447
448    def _get_list_of_values_from_single_column_table(self, table, column):
449        """Get the list of values for the column in the single-column table
450
451        Example table:
452
453        +------+
454        | Tags |
455        +------+
456        | tag1 |
457        | tag2 |
458        +------+
459
460        :param table: newline-separated table with |-separated cells
461        :param column: name of the column to look for
462        :raises: ValueError if the single column has some other name
463        """
464        lines = table.split("\n")
465        column_name = None
466        values = []
467        for line in lines:
468            if "|" in line:
469                if not column_name:
470                    column_name = line.split("|")[1].strip()
471                    if column_name != column:
472                        raise ValueError(
473                            "The table has no column %(expected)s "
474                            "but has column %(actual)s." % {
475                                'expected': column, 'actual': column_name})
476                else:
477                    values.append(line.split("|")[1].strip())
478        return values
479
480    def _create_server(self, name=None, flavor=None, with_network=True,
481                       add_cleanup=True, **kwargs):
482        name = name or self.name_generate()
483        if with_network:
484            nics = [{"net-id": self.network.id}]
485        else:
486            nics = None
487        flavor = flavor or self.flavor
488        server = self.client.servers.create(name, self.image, flavor,
489                                            nics=nics, **kwargs)
490        if add_cleanup:
491            self.addCleanup(server.delete)
492        novaclient.v2.shell._poll_for_status(
493            self.client.servers.get, server.id,
494            'building', ['active'])
495        return server
496
497    def _wait_for_state_change(self, server_id, status):
498        novaclient.v2.shell._poll_for_status(
499            self.client.servers.get, server_id, None, [status],
500            show_progress=False, poll_period=1, silent=True)
501
502    def _get_project_id(self, name):
503        """Obtain project id by project name."""
504        if self.keystone.version == "v3":
505            project = self.keystone.projects.find(name=name)
506        else:
507            project = self.keystone.tenants.find(name=name)
508        return project.id
509
510    def _cleanup_server(self, server_id):
511        """Deletes a server and waits for it to be gone."""
512        self.client.servers.delete(server_id)
513        self.wait_for_resource_delete(server_id, self.client.servers)
514
515    def _get_absolute_limits(self):
516        """Returns the absolute limits (quota usage) including reserved quota
517        usage for the given tenant running the test.
518
519        :return: A dict where the key is the limit (or usage) and value.
520        """
521        # The absolute limits are returned in a generator so convert to a dict.
522        return {limit.name: limit.value
523                for limit in self.client.limits.get(reserved=True).absolute}
524
525    def _pick_alternate_flavor(self):
526        """Given the flavor picked in the base class setup, this finds the
527        opposite flavor to use for a resize test. For example, if m1.nano is
528        the flavor, then use m1.micro, but those are only available if Tempest
529        is configured. If m1.tiny, then use m1.small.
530        """
531        flavor_name = self.flavor.name
532        if flavor_name == 'm1.nano':
533            # This is an upsize test.
534            return 'm1.micro'
535        if flavor_name == 'm1.micro':
536            # This is a downsize test.
537            return 'm1.nano'
538        if flavor_name == 'm1.tiny':
539            # This is an upsize test.
540            return 'm1.small'
541        if flavor_name == 'm1.small':
542            # This is a downsize test.
543            return 'm1.tiny'
544        self.fail('Unable to find alternate for flavor: %s' % flavor_name)
545
546
547class TenantTestBase(ClientTestBase):
548    """Base test class for additional tenant and user creation which
549    could be required in various test scenarios
550    """
551
552    def setUp(self):
553        super(TenantTestBase, self).setUp()
554        user_name = uuidutils.generate_uuid()
555        project_name = uuidutils.generate_uuid()
556        password = 'password'
557
558        if self.keystone.version == "v3":
559            project = self.keystone.projects.create(project_name,
560                                                    self.project_domain_id)
561            self.project_id = project.id
562            self.addCleanup(self.keystone.projects.delete, self.project_id)
563
564            self.user_id = self.keystone.users.create(
565                name=user_name, password=password,
566                default_project=self.project_id).id
567
568            for role in self.keystone.roles.list():
569                if "member" in role.name.lower():
570                    self.keystone.roles.grant(role.id, user=self.user_id,
571                                              project=self.project_id)
572                    break
573        else:
574            project = self.keystone.tenants.create(project_name)
575            self.project_id = project.id
576            self.addCleanup(self.keystone.tenants.delete, self.project_id)
577
578            self.user_id = self.keystone.users.create(
579                user_name, password, tenant_id=self.project_id).id
580
581        self.addCleanup(self.keystone.users.delete, self.user_id)
582        self.cli_clients_2 = tempest.lib.cli.base.CLIClient(
583            username=user_name,
584            password=password,
585            tenant_name=project_name,
586            uri=self.cli_clients.uri,
587            cli_dir=self.cli_clients.cli_dir,
588            insecure=self.insecure)
589
590    def another_nova(self, action, flags='', params='', fail_ok=False,
591                     endpoint_type='publicURL', merge_stderr=False):
592        flags += " --os-compute-api-version %s " % self.COMPUTE_API_VERSION
593        return self.cli_clients_2.nova(action, flags, params, fail_ok,
594                                       endpoint_type, merge_stderr)
595