1# Copyright (C) 2010-2020 by the Free Software Foundation, Inc. 2# 3# This file is part of GNU Mailman. 4# 5# GNU Mailman is free software: you can redistribute it and/or modify it under 6# the terms of the GNU General Public License as published by the Free 7# Software Foundation, either version 3 of the License, or (at your option) 8# any later version. 9# 10# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT 11# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 13# more details. 14# 15# You should have received a copy of the GNU General Public License along with 16# GNU Mailman. If not, see <https://www.gnu.org/licenses/>. 17 18"""Web service helpers.""" 19 20import json 21import types 22import falcon 23import hashlib 24 25from contextlib import suppress 26from datetime import datetime, timedelta 27from email.header import Header 28from email.message import Message 29from enum import Enum 30from lazr.config import as_boolean 31from mailman.config import config 32from pprint import pformat 33from public import public 34 35 36CONTENT_TYPE_JSON = 'application/json; charset=UTF-8' 37CONTENT_TYPE_TEXT_PLAIN = 'text/plain' 38 39 40class ExtendedEncoder(json.JSONEncoder): 41 """An extended JSON encoder which knows about other data types.""" 42 43 def default(self, obj): 44 if isinstance(obj, datetime): 45 return obj.isoformat() 46 elif isinstance(obj, timedelta): 47 # as_timedelta() does not recognize microseconds, so convert these 48 # to floating seconds, but only if there are any seconds. 49 if obj.seconds > 0 or obj.microseconds > 0: 50 seconds = obj.seconds + obj.microseconds / 1000000.0 51 return '{}d{}s'.format(obj.days, seconds) 52 return '{}d'.format(obj.days) 53 elif isinstance(obj, Enum): 54 # It's up to the decoding validator to associate this name with 55 # the right Enum class. 56 return obj.name 57 elif isinstance(obj, bytes): 58 return bytes_to_str(obj) 59 elif isinstance(obj, Message): 60 return obj.as_string() 61 elif isinstance(obj, Header): 62 return str(obj) 63 return super().default(obj) 64 65 66def bytes_to_str(value): 67 # Convert a string to unicode when the encoding is not declared. 68 if not isinstance(value, bytes): 69 return value 70 for encoding in ('ascii', 'utf-8', 'raw_unicode_escape'): 71 with suppress(UnicodeDecodeError): 72 return value.decode(encoding) 73 74 75@public 76def etag(resource): 77 """Calculate the etag and return a JSON representation. 78 79 The input is a dictionary representing the resource. This 80 dictionary must not contain an `http_etag` key. This function 81 calculates the etag by using the sha1 hexdigest of the 82 pretty-printed (and thus key-sorted and predictable) representation 83 of the dictionary. It then inserts this value under the `http_etag` 84 key, and returns the JSON representation of the modified dictionary. 85 86 :param resource: The original resource representation. 87 :type resource: dictionary 88 :return: JSON representation of the modified dictionary. 89 :rtype string 90 """ 91 assert 'http_etag' not in resource, 'Resource already etagged' 92 # Calculate the tag from a predictable (i.e. sorted) representation of the 93 # dictionary. The actual details aren't so important. pformat() is 94 # guaranteed to sort the keys, however it returns a str and the hash 95 # library requires a bytes. Use the safest possible encoding. 96 hashfood = pformat(resource).encode('raw-unicode-escape') 97 etag = hashlib.sha1(hashfood).hexdigest() 98 resource['http_etag'] = '"{}"'.format(etag) 99 return json.dumps(resource, cls=ExtendedEncoder, 100 sort_keys=as_boolean(config.devmode.enabled)) 101 102 103@public 104class CollectionMixin: 105 """Mixin class for common collection-ish things.""" 106 107 def _resource_as_dict(self, resource): 108 """Return the dictionary representation of a resource. 109 110 This must be implemented by subclasses. 111 112 :param resource: The resource object. 113 :type resource: object 114 :return: The representation of the resource. 115 :rtype: dict 116 """ 117 raise NotImplementedError 118 119 def _resource_as_json(self, resource): 120 """Return the JSON formatted representation of the resource.""" 121 resource = self._resource_as_dict(resource) 122 assert resource is not None, resource 123 return etag(resource) 124 125 def _get_collection(self, request): 126 """Return the collection as a sequence. 127 128 The returned value must support the collections.abc.Sequence 129 API. This method must be implemented by subclasses. 130 131 :param request: An http request. 132 :return: The collection 133 :rtype: collections.abc.Sequence 134 """ 135 raise NotImplementedError 136 137 def _paginate(self, request, collection): 138 """Method to paginate through collection result lists. 139 140 Use this to return only a slice of a collection, specified in 141 the request itself. The request should use query parameters 142 `count` and `page` to specify the slice they want. The slice 143 will start at index ``(page - 1) * count`` and end (exclusive) 144 at ``(page * count)``. 145 """ 146 # Allow falcon's HTTPBadRequest exceptions to percolate up. They'll 147 # get turned into HTTP 400 errors. 148 count = request.get_param_as_int('count') 149 page = request.get_param_as_int('page') 150 total_size = len(collection) 151 if count is None and page is None: 152 return 0, total_size, collection 153 # TODO(maxking): Count and page should be positive integers. Once 154 # falcon 2.0.0 is out and we can jump to it, we can remove this logic 155 # and use `min_value` parameter in request.get_param_as_int. 156 if count < 0: 157 raise falcon.HTTPInvalidParam( 158 count, 'count should be a positive integer.') 159 if page < 1: 160 raise falcon.HTTPInvalidParam( 161 page, 'page should be greater than 0.') 162 list_start = (page - 1) * count 163 list_end = page * count 164 return list_start, total_size, collection[list_start:list_end] 165 166 def _make_collection(self, request): 167 """Provide the collection to the REST layer.""" 168 start, total_size, collection = self._paginate( 169 request, self._get_collection(request)) 170 result = dict(start=start, total_size=total_size) 171 if len(collection) != 0: 172 entries = [self._resource_as_dict(resource) 173 for resource in collection] 174 assert None not in entries, entries 175 # Tag the resources but use the dictionaries. 176 [etag(resource) for resource in entries] 177 # Create the collection resource 178 result['entries'] = entries 179 return result 180 181 182@public 183class GetterSetter: 184 """Get and set attributes on an object. 185 186 Most attributes are fairly simple - a getattr() or setattr() on the object 187 does the trick, with the appropriate encoding or decoding on the way in 188 and out. Encoding doesn't happen here though; the standard JSON library 189 handles most types, but see ExtendedEncoder for additional support. 190 191 Others are more complicated since they aren't kept in the model as direct 192 columns in the database. These will use subclasses of this base class. 193 Read-only attributes will have a decoder which always raises ValueError. 194 """ 195 196 def __init__(self, decoder=None): 197 """Create a getter/setter for a specific attribute. 198 199 :param decoder: The callable for decoding a web request value string 200 into the specific data type needed by the object's attribute. Use 201 None to indicate a read-only attribute. The callable should raise 202 ValueError when the web request value cannot be converted. 203 :type decoder: callable 204 """ 205 self.decoder = decoder 206 207 def get(self, obj, attribute): 208 """Return the named object attribute value. 209 210 :param obj: The object to access. 211 :type obj: object 212 :param attribute: The attribute name. 213 :type attribute: string 214 :return: The attribute value, ready for JSON encoding. 215 :rtype: object 216 """ 217 value = getattr(obj, attribute) 218 # If the attribute is a generator type, return a list instead. 219 if isinstance(value, types.GeneratorType): 220 value = list(value) 221 return value 222 223 def put(self, obj, attribute, value): 224 """Set the named object attribute value. 225 226 :param obj: The object to change. 227 :type obj: object 228 :param attribute: The attribute name. 229 :type attribute: string 230 :param value: The new value for the attribute. 231 """ 232 setattr(obj, attribute, value) 233 234 def __call__(self, value): 235 """Convert the value to its internal format. 236 237 :param value: The web request value to convert. 238 :type value: string 239 :return: The converted value. 240 :rtype: object 241 """ 242 if self.decoder is None: 243 return value 244 return self.decoder(value) 245 246 247# Falcon REST framework add-ons. 248 249@public 250def child(matcher=None): 251 def decorator(func): 252 if matcher is None: 253 func.__matcher__ = func.__name__ 254 else: 255 func.__matcher__ = matcher 256 return func 257 return decorator 258 259 260@public 261class ChildError: 262 def __init__(self, status): 263 self._status = status 264 265 def _oops(self, request, response): 266 raise falcon.HTTPError(self._status, None) 267 268 on_get = _oops 269 on_post = _oops 270 on_put = _oops 271 on_patch = _oops 272 on_delete = _oops 273 274 275@public 276class BadRequest(ChildError): 277 def __init__(self): 278 super().__init__(falcon.HTTP_400) 279 280 281@public 282class NotFound(ChildError): 283 def __init__(self): 284 super().__init__(falcon.HTTP_404) 285 286 287@public 288def okay(response, body=None): 289 response.status = falcon.HTTP_200 290 if body is not None: 291 response.body = body 292 293 294@public 295def no_content(response): 296 response.status = falcon.HTTP_204 297 298 299@public 300def not_found(response, body='404 Not Found'): 301 response.status = falcon.HTTP_404 302 response.content_type = CONTENT_TYPE_JSON 303 if isinstance(body, bytes): 304 body = body.decode() 305 if body is not None: 306 response.body = falcon.HTTPNotFound(description=body).to_json() 307 308 309@public 310def accepted(response, body=None): 311 response.status = falcon.HTTP_202 312 if body is not None: 313 response.body = body 314 315 316@public 317def bad_request(response, body='400 Bad Request'): 318 response.status = falcon.HTTP_400 319 response.content_type = CONTENT_TYPE_JSON 320 if isinstance(body, bytes): 321 body = body.decode() 322 if body is not None: 323 response.body = falcon.HTTPBadRequest(description=body).to_json() 324 325 326@public 327def created(response, location): 328 response.status = falcon.HTTP_201 329 response.location = location 330 331 332@public 333def conflict(response, body='409 Conflict'): 334 response.status = falcon.HTTP_409 335 response.content_type = CONTENT_TYPE_JSON 336 if isinstance(body, bytes): 337 body = body.decode() 338 if body is not None: 339 response.body = falcon.HTTPConflict(description=body).to_json() 340 341 342@public 343def forbidden(response, body='403 Forbidden'): 344 response.status = falcon.HTTP_403 345 response.content_type = CONTENT_TYPE_JSON 346 if isinstance(body, bytes): 347 body = body.decode() 348 if body is not None: 349 response.body = falcon.HTTPForbidden(description=body).to_json() 350 351 352@public 353def get_request_params(request): 354 """Return the request items based on the content-type header""" 355 # maxking: If there is a requirement for a new media type in future, The 356 # way to handle that would be add a new media type handler in Falcon's API 357 # class and then use the items returned by that handler here to return the 358 # values to validators in the form of a dictionary. 359 360 # We parse the request based on the content type. Falcon has a default 361 # JSONHandler handler to parse json media type, so we can just do 362 # `request.media` to return the request params passed as json body. 363 if (request.content_type and 364 request.content_type.startswith('application/json')): 365 return request.media or dict() 366 # request.params returns the parameters passed as URL form encoded. 367 return request.params or dict() 368