1# -*- coding: utf-8 -*- 2import os 3import csv 4import hashlib 5import hmac 6import re 7import inspect 8 9from io import StringIO 10from codecs import getincrementalencoder 11 12from searx import logger 13 14 15VALID_LANGUAGE_CODE = re.compile(r'^[a-z]{2,3}(-[a-zA-Z]{2})?$') 16 17logger = logger.getChild('webutils') 18 19 20class UnicodeWriter: 21 """ 22 A CSV writer which will write rows to CSV file "f", 23 which is encoded in the given encoding. 24 """ 25 26 def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds): 27 # Redirect output to a queue 28 self.queue = StringIO() 29 self.writer = csv.writer(self.queue, dialect=dialect, **kwds) 30 self.stream = f 31 self.encoder = getincrementalencoder(encoding)() 32 33 def writerow(self, row): 34 self.writer.writerow(row) 35 # Fetch UTF-8 output from the queue ... 36 data = self.queue.getvalue() 37 data = data.strip('\x00') 38 # ... and reencode it into the target encoding 39 data = self.encoder.encode(data) 40 # write to the target stream 41 self.stream.write(data.decode()) 42 # empty queue 43 self.queue.truncate(0) 44 45 def writerows(self, rows): 46 for row in rows: 47 self.writerow(row) 48 49 50def get_resources_directory(searx_directory, subdirectory, resources_directory): 51 if not resources_directory: 52 resources_directory = os.path.join(searx_directory, subdirectory) 53 if not os.path.isdir(resources_directory): 54 raise Exception(resources_directory + " is not a directory") 55 return resources_directory 56 57 58def get_themes(templates_path): 59 """Returns available themes list.""" 60 themes = os.listdir(templates_path) 61 if '__common__' in themes: 62 themes.remove('__common__') 63 return themes 64 65 66def get_static_files(static_path): 67 static_files = set() 68 static_path_length = len(static_path) + 1 69 for directory, _, files in os.walk(static_path): 70 for filename in files: 71 f = os.path.join(directory[static_path_length:], filename) 72 static_files.add(f) 73 return static_files 74 75 76def get_result_templates(templates_path): 77 result_templates = set() 78 templates_path_length = len(templates_path) + 1 79 for directory, _, files in os.walk(templates_path): 80 if directory.endswith('result_templates'): 81 for filename in files: 82 f = os.path.join(directory[templates_path_length:], filename) 83 result_templates.add(f) 84 return result_templates 85 86 87def new_hmac(secret_key, url): 88 try: 89 secret_key_bytes = bytes(secret_key, 'utf-8') 90 except TypeError as err: 91 if isinstance(secret_key, bytes): 92 secret_key_bytes = secret_key 93 else: 94 raise err 95 return hmac.new(secret_key_bytes, url, hashlib.sha256).hexdigest() 96 97 98def prettify_url(url, max_length=74): 99 if len(url) > max_length: 100 chunk_len = int(max_length / 2 + 1) 101 return '{0}[...]{1}'.format(url[:chunk_len], url[-chunk_len:]) 102 else: 103 return url 104 105 106def highlight_content(content, query): 107 108 if not content: 109 return None 110 # ignoring html contents 111 # TODO better html content detection 112 if content.find('<') != -1: 113 return content 114 115 if content.lower().find(query.lower()) > -1: 116 query_regex = '({0})'.format(re.escape(query)) 117 content = re.sub(query_regex, '<span class="highlight">\\1</span>', 118 content, flags=re.I | re.U) 119 else: 120 regex_parts = [] 121 for chunk in query.split(): 122 chunk = chunk.replace('"', '') 123 if len(chunk) == 0: 124 continue 125 elif len(chunk) == 1: 126 regex_parts.append('\\W+{0}\\W+'.format(re.escape(chunk))) 127 else: 128 regex_parts.append('{0}'.format(re.escape(chunk))) 129 query_regex = '({0})'.format('|'.join(regex_parts)) 130 content = re.sub(query_regex, '<span class="highlight">\\1</span>', 131 content, flags=re.I | re.U) 132 133 return content 134 135 136def is_flask_run_cmdline(): 137 """Check if the application was started using "flask run" command line 138 139 Inspect the callstack. 140 See https://github.com/pallets/flask/blob/master/src/flask/__main__.py 141 142 Returns: 143 bool: True if the application was started using "flask run". 144 """ 145 frames = inspect.stack() 146 if len(frames) < 2: 147 return False 148 return frames[-2].filename.endswith('flask/cli.py') 149