1import copy 2import json 3import os 4import time 5from os.path import isdir 6 7import fasteners 8 9from conans.errors import ConanException 10from conans.model.ref import ConanFileReference, PackageReference 11from conans.util.files import md5sum, sha1sum 12from conans.util.log import logger 13 14 15# FIXME: Conan 2.0 the traces should have all the revisions information also. 16 17TRACER_ACTIONS = ["UPLOADED_RECIPE", "UPLOADED_PACKAGE", 18 "DOWNLOADED_RECIPE", "DOWNLOADED_RECIPE_SOURCES", "DOWNLOADED_PACKAGE", 19 "PACKAGE_BUILT_FROM_SOURCES", 20 "GOT_RECIPE_FROM_LOCAL_CACHE", "GOT_PACKAGE_FROM_LOCAL_CACHE", 21 "REST_API_CALL", "COMMAND", 22 "EXCEPTION", 23 "DOWNLOAD", 24 "UNZIP", "ZIP"] 25 26MASKED_FIELD = "**********" 27 28 29def _validate_action(action_name): 30 if action_name not in TRACER_ACTIONS: 31 raise ConanException("Unknown action %s" % action_name) 32 33 34def _get_tracer_file(): 35 """ 36 If CONAN_TRACE_FILE is a file in an existing dir will log to it creating the file if needed 37 Otherwise won't log anything 38 """ 39 trace_path = os.environ.get("CONAN_TRACE_FILE", None) 40 if trace_path is not None: 41 if not os.path.isabs(trace_path): 42 raise ConanException("Bad CONAN_TRACE_FILE value. The specified " 43 "path has to be an absolute path to a file.") 44 if not os.path.exists(os.path.dirname(trace_path)): 45 raise ConanException("Bad CONAN_TRACE_FILE value. The specified " 46 "path doesn't exist: '%s'" % os.path.dirname(trace_path)) 47 if isdir(trace_path): 48 raise ConanException("CONAN_TRACE_FILE is a directory. Please, specify a file path") 49 return trace_path 50 51 52def _append_to_log(obj): 53 """Add a new line to the log file locking the file to protect concurrent access""" 54 if _get_tracer_file(): 55 filepath = _get_tracer_file() 56 with fasteners.InterProcessLock(filepath + ".lock", logger=logger): 57 with open(filepath, "a") as logfile: 58 logfile.write(json.dumps(obj, sort_keys=True) + "\n") 59 60 61def _append_action(action_name, props): 62 """Validate the action_name and append to logs""" 63 _validate_action(action_name) 64 props["_action"] = action_name 65 props["time"] = time.time() 66 _append_to_log(props) 67 68 69# ############## LOG METHODS ###################### 70 71def _file_document(name, path): 72 return {"name": name, "path": path, "md5": md5sum(path), "sha1": sha1sum(path)} 73 74 75def log_recipe_upload(ref, duration, files_uploaded, remote_name): 76 files_uploaded = files_uploaded or {} 77 files_uploaded = [_file_document(name, path) for name, path in files_uploaded.items()] 78 _append_action("UPLOADED_RECIPE", {"_id": repr(ref.copy_clear_rev()), 79 "duration": duration, 80 "files": files_uploaded, 81 "remote": remote_name}) 82 83 84def log_package_upload(pref, duration, files_uploaded, remote): 85 """files_uploaded is a dict with relative path as keys and abs path as values""" 86 files_uploaded = files_uploaded or {} 87 files_uploaded = [_file_document(name, path) for name, path in files_uploaded.items()] 88 _append_action("UPLOADED_PACKAGE", {"_id": repr(pref.copy_clear_revs()), 89 "duration": duration, 90 "files": files_uploaded, 91 "remote": remote.name}) 92 93 94def log_recipe_download(ref, duration, remote_name, files_downloaded): 95 assert(isinstance(ref, ConanFileReference)) 96 files_downloaded = files_downloaded or {} 97 files_downloaded = [_file_document(name, path) for name, path in files_downloaded.items()] 98 _append_action("DOWNLOADED_RECIPE", {"_id": repr(ref.copy_clear_rev()), 99 "duration": duration, 100 "remote": remote_name, 101 "files": files_downloaded}) 102 103 104def log_recipe_sources_download(ref, duration, remote_name, files_downloaded): 105 assert(isinstance(ref, ConanFileReference)) 106 files_downloaded = files_downloaded or {} 107 files_downloaded = [_file_document(name, path) for name, path in files_downloaded.items()] 108 _append_action("DOWNLOADED_RECIPE_SOURCES", {"_id": repr(ref.copy_clear_rev()), 109 "duration": duration, 110 "remote": remote_name, 111 "files": files_downloaded}) 112 113 114def log_package_download(pref, duration, remote, files_downloaded): 115 files_downloaded = files_downloaded or {} 116 files_downloaded = [_file_document(name, path) for name, path in files_downloaded.items()] 117 _append_action("DOWNLOADED_PACKAGE", {"_id": repr(pref.copy_clear_revs()), 118 "duration": duration, 119 "remote": remote.name, 120 "files": files_downloaded}) 121 122 123def log_recipe_got_from_local_cache(ref): 124 assert(isinstance(ref, ConanFileReference)) 125 _append_action("GOT_RECIPE_FROM_LOCAL_CACHE", {"_id": repr(ref.copy_clear_rev())}) 126 127 128def log_package_got_from_local_cache(pref): 129 assert(isinstance(pref, PackageReference)) 130 _append_action("GOT_PACKAGE_FROM_LOCAL_CACHE", {"_id": repr(pref.copy_clear_revs())}) 131 132 133def log_package_built(pref, duration, log_run=None): 134 assert(isinstance(pref, PackageReference)) 135 _append_action("PACKAGE_BUILT_FROM_SOURCES", 136 {"_id": repr(pref.copy_clear_revs()), "duration": duration, "log": log_run}) 137 138 139def log_client_rest_api_call(url, method, duration, headers): 140 headers = copy.copy(headers) 141 if "Authorization" in headers: 142 headers["Authorization"] = MASKED_FIELD 143 if "X-Client-Anonymous-Id" in headers: 144 headers["X-Client-Anonymous-Id"] = MASKED_FIELD 145 if "signature=" in url: 146 url = url.split("signature=")[0] + "signature=%s" % MASKED_FIELD 147 _append_action("REST_API_CALL", {"method": method, "url": url, 148 "duration": duration, "headers": headers}) 149 150 151def log_command(name, parameters): 152 if name == "authenticate" and "password" in parameters: 153 parameters = copy.copy(parameters) # Ensure we don't alter any app object like args 154 parameters["password"] = MASKED_FIELD 155 _append_action("COMMAND", {"name": name, "parameters": parameters}) 156 logger.debug("CONAN_API: %s(%s)" % (name, ",".join("%s=%s" % (k, v) 157 for k, v in parameters.items()))) 158 159 160def log_exception(exc, message): 161 _append_action("EXCEPTION", {"class": str(exc.__class__.__name__), "message": message}) 162 163 164def log_download(url, duration): 165 _append_action("DOWNLOAD", {"url": url, "duration": duration}) 166 167 168def log_uncompressed_file(src_path, duration, dest_folder): 169 _append_action("UNZIP", {"src": src_path, "dst": dest_folder, "duration": duration}) 170 171 172def log_compressed_files(files, duration, tgz_path): 173 files = files or {} 174 files_compressed = [_file_document(name, path) for name, path in files.items()] 175 _append_action("ZIP", {"src": files_compressed, "dst": tgz_path, "duration": duration}) 176