1"""CloudStack plugin for integration tests."""
2from __future__ import (absolute_import, division, print_function)
3__metaclass__ = type
4
5import json
6import os
7import re
8import time
9
10from . import (
11    CloudProvider,
12    CloudEnvironment,
13    CloudEnvironmentConfig,
14)
15
16from ..util import (
17    find_executable,
18    ApplicationError,
19    display,
20    SubprocessError,
21    ConfigParser,
22)
23
24from ..http import (
25    HttpClient,
26    HttpError,
27    urlparse,
28)
29
30from ..docker_util import (
31    docker_run,
32    docker_rm,
33    docker_inspect,
34    docker_pull,
35    docker_network_inspect,
36    docker_exec,
37    get_docker_container_id,
38    get_docker_preferred_network_name,
39    get_docker_hostname,
40    is_docker_user_defined_network,
41)
42
43
44class CsCloudProvider(CloudProvider):
45    """CloudStack cloud provider plugin. Sets up cloud resources before delegation."""
46    DOCKER_SIMULATOR_NAME = 'cloudstack-sim'
47
48    def __init__(self, args):
49        """
50        :type args: TestConfig
51        """
52        super(CsCloudProvider, self).__init__(args)
53
54        # The simulator must be pinned to a specific version to guarantee CI passes with the version used.
55        self.image = 'quay.io/ansible/cloudstack-test-container:1.2.0'
56        self.container_name = ''
57        self.endpoint = ''
58        self.host = ''
59        self.port = 0
60
61    def filter(self, targets, exclude):
62        """Filter out the cloud tests when the necessary config and resources are not available.
63        :type targets: tuple[TestTarget]
64        :type exclude: list[str]
65        """
66        if os.path.isfile(self.config_static_path):
67            return
68
69        docker = find_executable('docker', required=False)
70
71        if docker:
72            return
73
74        skip = 'cloud/%s/' % self.platform
75        skipped = [target.name for target in targets if skip in target.aliases]
76
77        if skipped:
78            exclude.append(skip)
79            display.warning('Excluding tests marked "%s" which require the "docker" command or config (see "%s"): %s'
80                            % (skip.rstrip('/'), self.config_template_path, ', '.join(skipped)))
81
82    def setup(self):
83        """Setup the cloud resource before delegation and register a cleanup callback."""
84        super(CsCloudProvider, self).setup()
85
86        if self._use_static_config():
87            self._setup_static()
88        else:
89            self._setup_dynamic()
90
91    def get_remote_ssh_options(self):
92        """Get any additional options needed when delegating tests to a remote instance via SSH.
93        :rtype: list[str]
94        """
95        if self.managed:
96            return ['-R', '8888:%s:8888' % get_docker_hostname()]
97
98        return []
99
100    def get_docker_run_options(self):
101        """Get any additional options needed when delegating tests to a docker container.
102        :rtype: list[str]
103        """
104        network = get_docker_preferred_network_name(self.args)
105
106        if self.managed and not is_docker_user_defined_network(network):
107            return ['--link', self.DOCKER_SIMULATOR_NAME]
108
109        return []
110
111    def cleanup(self):
112        """Clean up the cloud resource and any temporary configuration files after tests complete."""
113        if self.container_name:
114            if self.ci_provider.code:
115                docker_rm(self.args, self.container_name)
116            elif not self.args.explain:
117                display.notice('Remember to run `docker rm -f %s` when finished testing.' % self.container_name)
118
119        super(CsCloudProvider, self).cleanup()
120
121    def _setup_static(self):
122        """Configure CloudStack tests for use with static configuration."""
123        parser = ConfigParser()
124        parser.read(self.config_static_path)
125
126        self.endpoint = parser.get('cloudstack', 'endpoint')
127
128        parts = urlparse(self.endpoint)
129
130        self.host = parts.hostname
131
132        if not self.host:
133            raise ApplicationError('Could not determine host from endpoint: %s' % self.endpoint)
134
135        if parts.port:
136            self.port = parts.port
137        elif parts.scheme == 'http':
138            self.port = 80
139        elif parts.scheme == 'https':
140            self.port = 443
141        else:
142            raise ApplicationError('Could not determine port from endpoint: %s' % self.endpoint)
143
144        display.info('Read cs host "%s" and port %d from config: %s' % (self.host, self.port, self.config_static_path), verbosity=1)
145
146        self._wait_for_service()
147
148    def _setup_dynamic(self):
149        """Create a CloudStack simulator using docker."""
150        config = self._read_config_template()
151
152        self.container_name = self.DOCKER_SIMULATOR_NAME
153
154        results = docker_inspect(self.args, self.container_name)
155
156        if results and not results[0]['State']['Running']:
157            docker_rm(self.args, self.container_name)
158            results = []
159
160        if results:
161            display.info('Using the existing CloudStack simulator docker container.', verbosity=1)
162        else:
163            display.info('Starting a new CloudStack simulator docker container.', verbosity=1)
164            docker_pull(self.args, self.image)
165            docker_run(self.args, self.image, ['-d', '-p', '8888:8888', '--name', self.container_name])
166
167            # apply work-around for OverlayFS issue
168            # https://github.com/docker/for-linux/issues/72#issuecomment-319904698
169            docker_exec(self.args, self.container_name, ['find', '/var/lib/mysql', '-type', 'f', '-exec', 'touch', '{}', ';'])
170
171            if not self.args.explain:
172                display.notice('The CloudStack simulator will probably be ready in 2 - 4 minutes.')
173
174        container_id = get_docker_container_id()
175
176        if container_id:
177            self.host = self._get_simulator_address()
178            display.info('Found CloudStack simulator container address: %s' % self.host, verbosity=1)
179        else:
180            self.host = get_docker_hostname()
181
182        self.port = 8888
183        self.endpoint = 'http://%s:%d' % (self.host, self.port)
184
185        self._wait_for_service()
186
187        if self.args.explain:
188            values = dict(
189                HOST=self.host,
190                PORT=str(self.port),
191            )
192        else:
193            credentials = self._get_credentials()
194
195            if self.args.docker:
196                host = self.DOCKER_SIMULATOR_NAME
197            elif self.args.remote:
198                host = 'localhost'
199            else:
200                host = self.host
201
202            values = dict(
203                HOST=host,
204                PORT=str(self.port),
205                KEY=credentials['apikey'],
206                SECRET=credentials['secretkey'],
207            )
208
209            display.sensitive.add(values['SECRET'])
210
211        config = self._populate_config_template(config, values)
212
213        self._write_config(config)
214
215    def _get_simulator_address(self):
216        current_network = get_docker_preferred_network_name(self.args)
217        networks = docker_network_inspect(self.args, current_network)
218
219        try:
220            network = [network for network in networks if network['Name'] == current_network][0]
221            containers = network['Containers']
222            container = [containers[container] for container in containers if containers[container]['Name'] == self.DOCKER_SIMULATOR_NAME][0]
223            return re.sub(r'/[0-9]+$', '', container['IPv4Address'])
224        except Exception:
225            display.error('Failed to process the following docker network inspect output:\n%s' %
226                          json.dumps(networks, indent=4, sort_keys=True))
227            raise
228
229    def _wait_for_service(self):
230        """Wait for the CloudStack service endpoint to accept connections."""
231        if self.args.explain:
232            return
233
234        client = HttpClient(self.args, always=True)
235        endpoint = self.endpoint
236
237        for _iteration in range(1, 30):
238            display.info('Waiting for CloudStack service: %s' % endpoint, verbosity=1)
239
240            try:
241                client.get(endpoint)
242                return
243            except SubprocessError:
244                pass
245
246            time.sleep(10)
247
248        raise ApplicationError('Timeout waiting for CloudStack service.')
249
250    def _get_credentials(self):
251        """Wait for the CloudStack simulator to return credentials.
252        :rtype: dict[str, str]
253        """
254        client = HttpClient(self.args, always=True)
255        endpoint = '%s/admin.json' % self.endpoint
256
257        for _iteration in range(1, 30):
258            display.info('Waiting for CloudStack credentials: %s' % endpoint, verbosity=1)
259
260            response = client.get(endpoint)
261
262            if response.status_code == 200:
263                try:
264                    return response.json()
265                except HttpError as ex:
266                    display.error(ex)
267
268            time.sleep(10)
269
270        raise ApplicationError('Timeout waiting for CloudStack credentials.')
271
272
273class CsCloudEnvironment(CloudEnvironment):
274    """CloudStack cloud environment plugin. Updates integration test environment after delegation."""
275    def get_environment_config(self):
276        """
277        :rtype: CloudEnvironmentConfig
278        """
279        parser = ConfigParser()
280        parser.read(self.config_path)
281
282        config = dict(parser.items('default'))
283
284        env_vars = dict(
285            CLOUDSTACK_ENDPOINT=config['endpoint'],
286            CLOUDSTACK_KEY=config['key'],
287            CLOUDSTACK_SECRET=config['secret'],
288            CLOUDSTACK_TIMEOUT=config['timeout'],
289        )
290
291        display.sensitive.add(env_vars['CLOUDSTACK_SECRET'])
292
293        ansible_vars = dict(
294            cs_resource_prefix=self.resource_prefix,
295        )
296
297        return CloudEnvironmentConfig(
298            env_vars=env_vars,
299            ansible_vars=ansible_vars,
300        )
301