1import argparse 2import warnings 3from shlex import split 4from http.cookies import SimpleCookie 5from urllib.parse import urlparse 6 7from w3lib.http import basic_auth_header 8 9 10class CurlParser(argparse.ArgumentParser): 11 def error(self, message): 12 error_msg = f'There was an error parsing the curl command: {message}' 13 raise ValueError(error_msg) 14 15 16curl_parser = CurlParser() 17curl_parser.add_argument('url') 18curl_parser.add_argument('-H', '--header', dest='headers', action='append') 19curl_parser.add_argument('-X', '--request', dest='method') 20curl_parser.add_argument('-d', '--data', '--data-raw', dest='data') 21curl_parser.add_argument('-u', '--user', dest='auth') 22 23 24safe_to_ignore_arguments = [ 25 ['--compressed'], 26 # `--compressed` argument is not safe to ignore, but it's included here 27 # because the `HttpCompressionMiddleware` is enabled by default 28 ['-s', '--silent'], 29 ['-v', '--verbose'], 30 ['-#', '--progress-bar'] 31] 32 33for argument in safe_to_ignore_arguments: 34 curl_parser.add_argument(*argument, action='store_true') 35 36 37def _parse_headers_and_cookies(parsed_args): 38 headers = [] 39 cookies = {} 40 for header in parsed_args.headers or (): 41 name, val = header.split(':', 1) 42 name = name.strip() 43 val = val.strip() 44 if name.title() == 'Cookie': 45 for name, morsel in SimpleCookie(val).items(): 46 cookies[name] = morsel.value 47 else: 48 headers.append((name, val)) 49 50 if parsed_args.auth: 51 user, password = parsed_args.auth.split(':', 1) 52 headers.append(('Authorization', basic_auth_header(user, password))) 53 54 return headers, cookies 55 56 57def curl_to_request_kwargs(curl_command, ignore_unknown_options=True): 58 """Convert a cURL command syntax to Request kwargs. 59 60 :param str curl_command: string containing the curl command 61 :param bool ignore_unknown_options: If true, only a warning is emitted when 62 cURL options are unknown. Otherwise 63 raises an error. (default: True) 64 :return: dictionary of Request kwargs 65 """ 66 67 curl_args = split(curl_command) 68 69 if curl_args[0] != 'curl': 70 raise ValueError('A curl command must start with "curl"') 71 72 parsed_args, argv = curl_parser.parse_known_args(curl_args[1:]) 73 74 if argv: 75 msg = f'Unrecognized options: {", ".join(argv)}' 76 if ignore_unknown_options: 77 warnings.warn(msg) 78 else: 79 raise ValueError(msg) 80 81 url = parsed_args.url 82 83 # curl automatically prepends 'http' if the scheme is missing, but Request 84 # needs the scheme to work 85 parsed_url = urlparse(url) 86 if not parsed_url.scheme: 87 url = 'http://' + url 88 89 method = parsed_args.method or 'GET' 90 91 result = {'method': method.upper(), 'url': url} 92 93 headers, cookies = _parse_headers_and_cookies(parsed_args) 94 95 if headers: 96 result['headers'] = headers 97 if cookies: 98 result['cookies'] = cookies 99 if parsed_args.data: 100 result['body'] = parsed_args.data 101 if not parsed_args.method: 102 # if the "data" is specified but the "method" is not specified, 103 # the default method is 'POST' 104 result['method'] = 'POST' 105 106 return result 107