1# Copyright 2017 Datera
2# All Rights Reserved.
3#
4#    Licensed under the Apache License, Version 2.0 (the "License"); you may
5#    not use this file except in compliance with the License. You may obtain
6#    a copy of the License at
7#
8#         http://www.apache.org/licenses/LICENSE-2.0
9#
10#    Unless required by applicable law or agreed to in writing, software
11#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13#    License for the specific language governing permissions and limitations
14#    under the License.
15
16import functools
17import json
18import re
19import six
20import time
21import types
22import uuid
23
24import eventlet
25import requests
26
27from oslo_log import log as logging
28from six.moves import http_client
29
30from cinder import context
31from cinder import exception
32from cinder.i18n import _
33from cinder.volume import qos_specs
34from cinder.volume import volume_types
35
36
37LOG = logging.getLogger(__name__)
38OS_PREFIX = "OS-"
39UNMANAGE_PREFIX = "UNMANAGED-"
40
41# Taken from this SO post :
42# http://stackoverflow.com/a/18516125
43# Using old-style string formatting because of the nature of the regex
44# conflicting with new-style curly braces
45UUID4_STR_RE = ("%s[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[89ab]"
46                "[a-f0-9]{3}-?[a-f0-9]{12}")
47UUID4_RE = re.compile(UUID4_STR_RE % OS_PREFIX)
48
49# Recursive dict to assemble basic url structure for the most common
50# API URL endpoints. Most others are constructed from these
51URL_TEMPLATES = {
52    'ai': lambda: 'app_instances',
53    'ai_inst': lambda: (URL_TEMPLATES['ai']() + '/{}'),
54    'si': lambda: (URL_TEMPLATES['ai_inst']() + '/storage_instances'),
55    'si_inst': lambda storage_name: (
56        (URL_TEMPLATES['si']() + '/{}').format(
57            '{}', storage_name)),
58    'vol': lambda storage_name: (
59        (URL_TEMPLATES['si_inst'](storage_name) + '/volumes')),
60    'vol_inst': lambda storage_name, volume_name: (
61        (URL_TEMPLATES['vol'](storage_name) + '/{}').format(
62            '{}', volume_name)),
63    'at': lambda: 'app_templates/{}'}
64
65DEFAULT_SI_SLEEP = 1
66DEFAULT_SI_SLEEP_API_2 = 5
67DEFAULT_SNAP_SLEEP = 1
68INITIATOR_GROUP_PREFIX = "IG-"
69API_VERSIONS = ["2", "2.1"]
70API_TIMEOUT = 20
71
72###############
73# METADATA KEYS
74###############
75
76M_TYPE = 'cinder_volume_type'
77M_CALL = 'cinder_calls'
78M_CLONE = 'cinder_clone_from'
79M_MANAGED = 'cinder_managed'
80
81M_KEYS = [M_TYPE, M_CALL, M_CLONE, M_MANAGED]
82
83
84def _get_name(name):
85    return "".join((OS_PREFIX, name))
86
87
88def _get_unmanaged(name):
89    return "".join((UNMANAGE_PREFIX, name))
90
91
92def _authenticated(func):
93    """Ensure the driver is authenticated to make a request.
94
95    In do_setup() we fetch an auth token and store it. If that expires when
96    we do API request, we'll fetch a new one.
97    """
98    @functools.wraps(func)
99    def func_wrapper(driver, *args, **kwargs):
100        try:
101            return func(driver, *args, **kwargs)
102        except exception.NotAuthorized:
103            # Prevent recursion loop. After the driver arg is the
104            # resource_type arg from _issue_api_request(). If attempt to
105            # login failed, we should just give up.
106            if args[0] == 'login':
107                raise
108
109            # Token might've expired, get a new one, try again.
110            driver.login()
111            return func(driver, *args, **kwargs)
112    return func_wrapper
113
114
115def _api_lookup(func):
116    """Perform a dynamic API implementation lookup for a call
117
118    Naming convention follows this pattern:
119
120        # original_func(args) --> _original_func_X_?Y?(args)
121        # where X and Y are the major and minor versions of the latest
122        # supported API version
123
124        # From the Datera box we've determined that it supports API
125        # versions ['2', '2.1']
126        # This is the original function call
127        @_api_lookup
128        def original_func(arg1, arg2):
129            print("I'm a shim, this won't get executed!")
130            pass
131
132        # This is the function that is actually called after determining
133        # the correct API version to use
134        def _original_func_2_1(arg1, arg2):
135            some_version_2_1_implementation_here()
136
137        # This is the function that would be called if the previous function
138        # did not exist:
139        def _original_func_2(arg1, arg2):
140            some_version_2_implementation_here()
141
142        # This function would NOT be called, because the connected Datera box
143        # does not support the 1.5 version of the API
144        def _original_func_1_5(arg1, arg2):
145            some_version_1_5_implementation_here()
146    """
147    @functools.wraps(func)
148    def wrapper(*args, **kwargs):
149        obj = args[0]
150        api_versions = _get_supported_api_versions(obj)
151        api_version = None
152        index = -1
153        while True:
154            try:
155                api_version = api_versions[index]
156            except (IndexError, KeyError):
157                msg = _("No compatible API version found for this product: "
158                        "api_versions -> %(api_version)s, %(func)s")
159                LOG.error(msg, api_version=api_version, func=func)
160                raise exception.DateraAPIException(msg % {
161                    'api_version': api_version, 'func': func})
162            # Py27
163            try:
164                name = "_" + "_".join(
165                    (func.func_name, api_version.replace(".", "_")))
166            # Py3+
167            except AttributeError:
168                name = "_" + "_".join(
169                    (func.__name__, api_version.replace(".", "_")))
170            try:
171                if obj.do_profile:
172                    LOG.info("Trying method: %s", name)
173                    call_id = uuid.uuid4()
174                    LOG.debug("Profiling method: %s, id %s", name, call_id)
175                    t1 = time.time()
176                    obj.thread_local.trace_id = call_id
177                result = getattr(obj, name)(*args[1:], **kwargs)
178                if obj.do_profile:
179                    t2 = time.time()
180                    timedelta = round(t2 - t1, 3)
181                    LOG.debug("Profile for method %s, id %s: %ss",
182                              name, call_id, timedelta)
183                return result
184            except AttributeError as e:
185                # If we find the attribute name in the error message
186                # then we continue otherwise, raise to prevent masking
187                # errors
188                if name not in six.text_type(e):
189                    raise
190                else:
191                    LOG.info(e)
192                    index -= 1
193            except exception.DateraAPIException as e:
194                if "UnsupportedVersionError" in six.text_type(e):
195                    index -= 1
196                else:
197                    raise
198
199    return wrapper
200
201
202def _get_supported_api_versions(driver):
203    t = time.time()
204    if driver.api_cache and driver.api_timeout - t < API_TIMEOUT:
205        return driver.api_cache
206    driver.api_timeout = t + API_TIMEOUT
207    results = []
208    host = driver.configuration.san_ip
209    port = driver.configuration.datera_api_port
210    client_cert = driver.configuration.driver_client_cert
211    client_cert_key = driver.configuration.driver_client_cert_key
212    cert_data = None
213    header = {'Content-Type': 'application/json; charset=utf-8',
214              'Datera-Driver': 'OpenStack-Cinder-{}'.format(driver.VERSION)}
215    protocol = 'http'
216    if client_cert:
217        protocol = 'https'
218        cert_data = (client_cert, client_cert_key)
219    try:
220        url = '%s://%s:%s/api_versions' % (protocol, host, port)
221        resp = driver._request(url, "get", None, header, cert_data)
222        data = resp.json()
223        results = [elem.strip("v") for elem in data['api_versions']]
224    except (exception.DateraAPIException, KeyError):
225        # Fallback to pre-endpoint logic
226        for version in API_VERSIONS[0:-1]:
227            url = '%s://%s:%s/v%s' % (protocol, host, port, version)
228            resp = driver._request(url, "get", None, header, cert_data)
229            if ("api_req" in resp.json() or
230                    str(resp.json().get("code")) == "99"):
231                results.append(version)
232            else:
233                LOG.error("No supported API versions available, "
234                          "Please upgrade your Datera EDF software")
235    return results
236
237
238def _get_volume_type_obj(driver, resource):
239    type_id = resource.get('volume_type_id', None)
240    # Handle case of volume with no type.  We still want the
241    # specified defaults from above
242    if type_id:
243        ctxt = context.get_admin_context()
244        volume_type = volume_types.get_volume_type(ctxt, type_id)
245    else:
246        volume_type = None
247    return volume_type
248
249
250def _get_policies_for_resource(driver, resource):
251    """Get extra_specs and qos_specs of a volume_type.
252
253    This fetches the scoped keys from the volume type. Anything set from
254     qos_specs will override key/values set from extra_specs.
255    """
256    volume_type = driver._get_volume_type_obj(resource)
257    # Handle case of volume with no type.  We still want the
258    # specified defaults from above
259    if volume_type:
260        specs = volume_type.get('extra_specs')
261    else:
262        specs = {}
263
264    # Set defaults:
265    policies = {k.lstrip('DF:'): str(v['default']) for (k, v)
266                in driver._init_vendor_properties()[0].items()}
267
268    if volume_type:
269        # Populate updated value
270        for key, value in specs.items():
271            if ':' in key:
272                fields = key.split(':')
273                key = fields[1]
274                policies[key] = value
275
276        qos_specs_id = volume_type.get('qos_specs_id')
277        if qos_specs_id is not None:
278            ctxt = context.get_admin_context()
279            qos_kvs = qos_specs.get_qos_specs(ctxt, qos_specs_id)['specs']
280            if qos_kvs:
281                policies.update(qos_kvs)
282    # Cast everything except booleans int that can be cast
283    for k, v in policies.items():
284        # Handle String Boolean case
285        if v == 'True' or v == 'False':
286            policies[k] = policies[k] == 'True'
287            continue
288        # Int cast
289        try:
290            policies[k] = int(v)
291        except ValueError:
292            pass
293    return policies
294
295
296# ================
297# = API Requests =
298# ================
299
300def _request(driver, connection_string, method, payload, header, cert_data):
301    LOG.debug("Endpoint for Datera API call: %s", connection_string)
302    LOG.debug("Payload for Datera API call: %s", payload)
303    try:
304        response = getattr(requests, method)(connection_string,
305                                             data=payload, headers=header,
306                                             verify=False, cert=cert_data)
307        return response
308    except requests.exceptions.RequestException as ex:
309        msg = _(
310            'Failed to make a request to Datera cluster endpoint due '
311            'to the following reason: %s') % six.text_type(
312            ex.message)
313        LOG.error(msg)
314        raise exception.DateraAPIException(msg)
315
316
317def _raise_response(driver, response):
318    msg = _('Request to Datera cluster returned bad status:'
319            ' %(status)s | %(reason)s') % {
320                'status': response.status_code,
321                'reason': response.reason}
322    LOG.error(msg)
323    raise exception.DateraAPIException(msg)
324
325
326def _handle_bad_status(driver,
327                       response,
328                       connection_string,
329                       method,
330                       payload,
331                       header,
332                       cert_data,
333                       sensitive=False,
334                       conflict_ok=False):
335    if (response.status_code == http_client.BAD_REQUEST and
336            connection_string.endswith("api_versions")):
337        # Raise the exception, but don't log any error.  We'll just fall
338        # back to the old style of determining API version.  We make this
339        # request a lot, so logging it is just noise
340        raise exception.DateraAPIException
341    if response.status_code == http_client.NOT_FOUND:
342        raise exception.NotFound(response.json()['message'])
343    elif response.status_code in [http_client.FORBIDDEN,
344                                  http_client.UNAUTHORIZED]:
345        raise exception.NotAuthorized()
346    elif response.status_code == http_client.CONFLICT and conflict_ok:
347        # Don't raise, because we're expecting a conflict
348        pass
349    elif response.status_code == http_client.SERVICE_UNAVAILABLE:
350        current_retry = 0
351        while current_retry <= driver.retry_attempts:
352            LOG.debug("Datera 503 response, trying request again")
353            eventlet.sleep(driver.interval)
354            resp = driver._request(connection_string,
355                                   method,
356                                   payload,
357                                   header,
358                                   cert_data)
359            if resp.ok:
360                return response.json()
361            elif resp.status_code != http_client.SERVICE_UNAVAILABLE:
362                driver._raise_response(resp)
363    else:
364        driver._raise_response(response)
365
366
367@_authenticated
368def _issue_api_request(driver, resource_url, method='get', body=None,
369                       sensitive=False, conflict_ok=False,
370                       api_version='2', tenant=None):
371    """All API requests to Datera cluster go through this method.
372
373    :param resource_url: the url of the resource
374    :param method: the request verb
375    :param body: a dict with options for the action_type
376    :param sensitive: Bool, whether request should be obscured from logs
377    :param conflict_ok: Bool, True to suppress ConflictError exceptions
378    during this request
379    :param api_version: The Datera api version for the request
380    :param tenant: The tenant header value for the request (only applicable
381    to 2.1 product versions and later)
382    :returns: a dict of the response from the Datera cluster
383    """
384    host = driver.configuration.san_ip
385    port = driver.configuration.datera_api_port
386    api_token = driver.datera_api_token
387
388    payload = json.dumps(body, ensure_ascii=False)
389    payload.encode('utf-8')
390
391    header = {'Content-Type': 'application/json; charset=utf-8'}
392    header.update(driver.HEADER_DATA)
393
394    protocol = 'http'
395    if driver.configuration.driver_use_ssl:
396        protocol = 'https'
397
398    if api_token:
399        header['Auth-Token'] = api_token
400
401    if tenant == "all":
402        header['tenant'] = tenant
403    elif tenant and '/root' not in tenant:
404        header['tenant'] = "".join(("/root/", tenant))
405    elif tenant and '/root' in tenant:
406        header['tenant'] = tenant
407    elif driver.tenant_id and driver.tenant_id.lower() != "map":
408        header['tenant'] = driver.tenant_id
409
410    client_cert = driver.configuration.driver_client_cert
411    client_cert_key = driver.configuration.driver_client_cert_key
412    cert_data = None
413
414    if client_cert:
415        protocol = 'https'
416        cert_data = (client_cert, client_cert_key)
417
418    connection_string = '%s://%s:%s/v%s/%s' % (protocol, host, port,
419                                               api_version, resource_url)
420
421    request_id = uuid.uuid4()
422
423    if driver.do_profile:
424        t1 = time.time()
425    if not sensitive:
426        LOG.debug("\nDatera Trace ID: %(tid)s\n"
427                  "Datera Request ID: %(rid)s\n"
428                  "Datera Request URL: /v%(api)s/%(url)s\n"
429                  "Datera Request Method: %(method)s\n"
430                  "Datera Request Payload: %(payload)s\n"
431                  "Datera Request Headers: %(header)s\n",
432                  {'tid': driver.thread_local.trace_id,
433                   'rid': request_id,
434                   'api': api_version,
435                   'url': resource_url,
436                   'method': method,
437                   'payload': payload,
438                   'header': header})
439    response = driver._request(connection_string,
440                               method,
441                               payload,
442                               header,
443                               cert_data)
444
445    data = response.json()
446
447    timedelta = "Profiling disabled"
448    if driver.do_profile:
449        t2 = time.time()
450        timedelta = round(t2 - t1, 3)
451    if not sensitive:
452        LOG.debug("\nDatera Trace ID: %(tid)s\n"
453                  "Datera Response ID: %(rid)s\n"
454                  "Datera Response TimeDelta: %(delta)ss\n"
455                  "Datera Response URL: %(url)s\n"
456                  "Datera Response Payload: %(payload)s\n"
457                  "Datera Response Object: %(obj)s\n",
458                  {'tid': driver.thread_local.trace_id,
459                   'rid': request_id,
460                   'delta': timedelta,
461                   'url': response.url,
462                   'payload': payload,
463                   'obj': vars(response)})
464    if not response.ok:
465        driver._handle_bad_status(response,
466                                  connection_string,
467                                  method,
468                                  payload,
469                                  header,
470                                  cert_data,
471                                  conflict_ok=conflict_ok)
472
473    return data
474
475
476def register_driver(driver):
477    for func in [_get_supported_api_versions,
478                 _get_volume_type_obj,
479                 _get_policies_for_resource,
480                 _request,
481                 _raise_response,
482                 _handle_bad_status,
483                 _issue_api_request]:
484        # PY27
485
486        f = types.MethodType(func, driver)
487        try:
488            setattr(driver, func.func_name, f)
489        # PY3+
490        except AttributeError:
491            setattr(driver, func.__name__, f)
492