1""" 2Module to hold the logger instances themselves. 3""" 4 5from __future__ import absolute_import 6 7import logging 8import sys 9 10from . import buildlogger 11from . import formatters 12from .. import errors 13 14_DEFAULT_FORMAT = "[%(name)s] %(message)s" 15 16EXECUTOR_LOGGER_NAME = "executor" 17FIXTURE_LOGGER_NAME = "fixture" 18TESTS_LOGGER_NAME = "tests" 19 20EXECUTOR_LOGGER = None 21 22 23def _build_logger_server(logging_config): 24 """Create and return a new BuildloggerServer if "buildlogger" is configured as 25 one of the handler class in the configuration, return None otherwise. 26 """ 27 for logger_name in (FIXTURE_LOGGER_NAME, TESTS_LOGGER_NAME): 28 logger_info = logging_config[logger_name] 29 for handler_info in logger_info["handlers"]: 30 if handler_info["class"] == "buildlogger": 31 return buildlogger.BuildloggerServer() 32 33 34def configure_loggers(logging_config): 35 buildlogger.BUILDLOGGER_FALLBACK = BaseLogger("buildlogger") 36 # The 'buildlogger' prefix is not added to the fallback logger since the prefix of the original 37 # logger will be there as part of the logged message. 38 buildlogger.BUILDLOGGER_FALLBACK.addHandler( 39 _fallback_buildlogger_handler(include_logger_name=False)) 40 build_logger_server = _build_logger_server(logging_config) 41 fixture_logger = FixtureRootLogger(logging_config, build_logger_server) 42 tests_logger = TestsRootLogger(logging_config, build_logger_server) 43 global EXECUTOR_LOGGER 44 EXECUTOR_LOGGER = ExecutorRootLogger(logging_config, build_logger_server, 45 fixture_logger, tests_logger) 46 47 48class BaseLogger(logging.Logger): 49 """Base class for the custom loggers used in this library. 50 51 Custom loggers share access to the logging configuration and provide methods 52 to create other loggers. 53 """ 54 def __init__(self, name, logging_config=None, build_logger_server=None, parent=None): 55 """Initialize a BaseLogger. 56 57 :param name: the logger name. 58 :param logging_config: the logging configuration. 59 :param build_logger_server: the build logger server (e.g. logkeeper). 60 :param parent: the parent logger. 61 """ 62 logging.Logger.__init__(self, name, level=logging.DEBUG) 63 self._logging_config = logging_config 64 self._build_logger_server = build_logger_server 65 if parent: 66 self.parent = parent 67 self.propagate = True 68 69 @property 70 def build_logger_server(self): 71 """The configured BuildloggerServer instance, or None.""" 72 if self._build_logger_server: 73 return self._build_logger_server 74 elif self.parent: 75 # Fetching the value from parent 76 return getattr(self.parent, "build_logger_server", None) 77 return None 78 79 @property 80 def logging_config(self): 81 """The logging configuration.""" 82 if self._logging_config: 83 return self._logging_config 84 elif self.parent: 85 # Fetching the value from parent 86 return getattr(self.parent, "logging_config", None) 87 return None 88 89 @staticmethod 90 def get_formatter(logger_info): 91 log_format = logger_info.get("format", _DEFAULT_FORMAT) 92 return formatters.ISO8601Formatter(fmt=log_format) 93 94 95class RootLogger(BaseLogger): 96 """A custom class for top-level loggers (executor, fixture, tests).""" 97 def __init__(self, name, logging_config, build_logger_server): 98 """Initialize a RootLogger. 99 100 :param name: the logger name. 101 :param logging_config: the logging configuration. 102 :param build_logger_server: the build logger server, if one is configured. 103 """ 104 BaseLogger.__init__(self, name, logging_config, build_logger_server) 105 self._configure() 106 107 def _configure(self): 108 if self.name not in self.logging_config: 109 raise ValueError("Logging configuration should contain the %s component", self.name) 110 logger_info = self.logging_config[self.name] 111 formatter = self.get_formatter(logger_info) 112 113 for handler_info in logger_info.get("handlers", []): 114 self._add_handler(handler_info, formatter) 115 116 def _add_handler(self, handler_info, formatter): 117 handler_class = handler_info["class"] 118 if handler_class == "logging.FileHandler": 119 handler = logging.FileHandler(filename=handler_info["filename"], 120 mode=handler_info.get("mode", "w")) 121 elif handler_class == "logging.NullHandler": 122 handler = logging.NullHandler() 123 elif handler_class == "logging.StreamHandler": 124 handler = logging.StreamHandler(sys.stdout) 125 elif handler_class == "buildlogger": 126 return # Buildlogger handlers are applied when creating specific child loggers 127 else: 128 raise ValueError("Unknown handler class '%s'" % handler_class) 129 handler.setFormatter(formatter) 130 self.addHandler(handler) 131 132 133class ExecutorRootLogger(RootLogger): 134 """Class for the "executor" top-level logger.""" 135 def __init__(self, logging_config, build_logger_server, fixture_root_logger, tests_root_logger): 136 """Initialize an ExecutorRootLogger.""" 137 RootLogger.__init__(self, EXECUTOR_LOGGER_NAME, logging_config, build_logger_server) 138 self.fixture_root_logger = fixture_root_logger 139 self.tests_root_logger = tests_root_logger 140 141 def new_resmoke_logger(self): 142 """Create a child logger of this logger with the name "resmoke".""" 143 return BaseLogger("resmoke", parent=self) 144 145 def new_job_logger(self, test_kind, job_num): 146 """Create a new child JobLogger.""" 147 return JobLogger(test_kind, job_num, self, self.fixture_root_logger) 148 149 def new_testqueue_logger(self, test_kind): 150 """Create a new TestQueueLogger that will be a child of the "tests" root logger.""" 151 return TestQueueLogger(test_kind, self.tests_root_logger) 152 153 def new_hook_logger(self, behavior_class, fixture_logger): 154 """Create a new child hook logger.""" 155 return HookLogger(behavior_class, fixture_logger, self.tests_root_logger) 156 157 158class JobLogger(BaseLogger): 159 def __init__(self, test_kind, job_num, parent, fixture_root_logger): 160 """Initialize a JobLogger. 161 162 :param test_kind: the test kind (e.g. js_test, db_test, etc.). 163 :param job_num: a job number. 164 :param fixture_root_logger: the root logger for the fixture logs. 165 """ 166 name = "executor:%s:job%d" % (test_kind, job_num) 167 BaseLogger.__init__(self, name, parent=parent) 168 self.job_num = job_num 169 self.fixture_root_logger = fixture_root_logger 170 if self.build_logger_server: 171 # If we're configured to log messages to the buildlogger server, then request a new 172 # build_id for this job. 173 self.build_id = self.build_logger_server.new_build_id("job%d" % job_num) 174 if not self.build_id: 175 buildlogger.set_log_output_incomplete() 176 raise errors.LoggerRuntimeConfigError( 177 "Encountered an error configuring buildlogger for job #{:d}: Failed to get a" 178 " new build_id".format(job_num)) 179 180 url = self.build_logger_server.get_build_log_url(self.build_id) 181 parent.info("Writing output of job #%d to %s.", job_num, url) 182 else: 183 self.build_id = None 184 185 def new_fixture_logger(self, fixture_class): 186 """Create a new fixture logger that will be a child of the "fixture" root logger.""" 187 return FixtureLogger(fixture_class, self.job_num, self.build_id, self.fixture_root_logger) 188 189 def new_test_logger(self, test_shortname, test_basename, command, parent): 190 """Create a new test logger that will be a child of the given parent.""" 191 if self.build_id: 192 # If we're configured to log messages to the buildlogger server, then request a new 193 # test_id for this test. 194 test_id = self.build_logger_server.new_test_id(self.build_id, test_basename, command) 195 if not test_id: 196 buildlogger.set_log_output_incomplete() 197 raise errors.LoggerRuntimeConfigError( 198 "Encountered an error configuring buildlogger for test {}: Failed to get a new" 199 " test_id".format(test_basename)) 200 201 url = self.build_logger_server.get_test_log_url(self.build_id, test_id) 202 self.info("Writing output of %s to %s.", test_basename, url) 203 return TestLogger(test_shortname, parent, self.build_id, test_id, url) 204 205 return TestLogger(test_shortname, parent) 206 207 208class TestLogger(BaseLogger): 209 def __init__(self, test_name, parent, build_id=None, test_id=None, url=None): 210 """Initialize a TestLogger. 211 212 :param test_name: the test name. 213 :param parent: the parent logger. 214 :param build_id: the build logger build id. 215 :param test_id: the build logger test id. 216 :param url: the build logger URL endpoint for the test. 217 """ 218 name = "%s:%s" % (parent.name, test_name) 219 BaseLogger.__init__(self, name, parent=parent) 220 self.url_endpoint = url 221 self._add_build_logger_handler(build_id, test_id) 222 223 def _add_build_logger_handler(self, build_id, test_id): 224 logger_info = self.logging_config[TESTS_LOGGER_NAME] 225 handler_info = _get_buildlogger_handler_info(logger_info) 226 if handler_info is not None: 227 handler = self.build_logger_server.get_test_handler(build_id, test_id, handler_info) 228 handler.setFormatter(self.get_formatter(logger_info)) 229 self.addHandler(handler) 230 231 def new_test_thread_logger(self, test_kind, thread_id): 232 """Create a new child test thread logger.""" 233 return BaseLogger("%s:%s" % (test_kind, thread_id), parent=self) 234 235 236class FixtureRootLogger(RootLogger): 237 """Class for the "fixture" top-level logger.""" 238 def __init__(self, logging_config, build_logger_server): 239 """Initialize a FixtureRootLogger. 240 241 :param logging_config: the logging configuration. 242 :param build_logger_server: the build logger server, if one is configured. 243 """ 244 RootLogger.__init__(self, FIXTURE_LOGGER_NAME, logging_config, build_logger_server) 245 246 247class FixtureLogger(BaseLogger): 248 def __init__(self, fixture_class, job_num, build_id, fixture_root_logger): 249 """Initialize a FixtureLogger. 250 251 :param fixture_class: the name of the fixture class. 252 :param job_num: the number of the job the fixture is running on. 253 :param build_id: the build logger build id, if any. 254 :param fixture_root_logger: the root logger for the fixture logs. 255 """ 256 BaseLogger.__init__(self, "%s:job%d" % (fixture_class, job_num), parent=fixture_root_logger) 257 self.fixture_class = fixture_class 258 self.job_num = job_num 259 self._add_build_logger_handler(build_id) 260 261 def _add_build_logger_handler(self, build_id): 262 logger_info = self.logging_config[FIXTURE_LOGGER_NAME] 263 handler_info = _get_buildlogger_handler_info(logger_info) 264 if handler_info is not None: 265 handler = self.build_logger_server.get_global_handler(build_id, handler_info) 266 handler.setFormatter(self.get_formatter(logger_info)) 267 self.addHandler(handler) 268 269 def new_fixture_node_logger(self, node_name): 270 """Create a new child FixtureNodeLogger.""" 271 return FixtureNodeLogger(self.fixture_class, self.job_num, node_name, self) 272 273 274class FixtureNodeLogger(BaseLogger): 275 def __init__(self, fixture_class, job_num, node_name, fixture_logger): 276 """Initialize a FixtureNodeLogger. 277 278 :param fixture_class: the name of the fixture implementation class. 279 :param job_num: the number of the job the fixture is running on. 280 :param node_name: the node display name. 281 :param fixture_logger: the parent fixture logger. 282 """ 283 BaseLogger.__init__(self, "%s:job%d:%s" % (fixture_class, job_num, node_name), 284 parent=fixture_logger) 285 self.fixture_class = fixture_class 286 self.job_num = job_num 287 self.node_name = node_name 288 289 def new_fixture_node_logger(self, node_name): 290 """Create a new child FixtureNodeLogger.""" 291 return FixtureNodeLogger(self.fixture_class, self.job_num, 292 "%s:%s" % (self.node_name, node_name), self) 293 294 295class TestsRootLogger(RootLogger): 296 """Class for the "tests" top-level logger.""" 297 def __init__(self, logging_config, build_logger_server): 298 """Initialize a TestsRootLogger. 299 300 :param logging_config: the logging configuration. 301 :param build_logger_server: the build logger server, if one is configured. 302 """ 303 RootLogger.__init__(self, TESTS_LOGGER_NAME, logging_config, build_logger_server) 304 305 306class TestQueueLogger(BaseLogger): 307 def __init__(self, test_kind, tests_root_logger): 308 """Initialize a TestQueueLogger. 309 310 :param test_kind: the test kind (e.g. js_test, db_test, cpp_unit_test, etc.). 311 :param tests_root_logger: the root logger for the tests logs. 312 """ 313 BaseLogger.__init__(self, test_kind, parent=tests_root_logger) 314 315 316class HookLogger(BaseLogger): 317 def __init__(self, behavior_class, fixture_logger, tests_root_logger): 318 """Initialize a HookLogger. 319 320 :param behavior_class: the hook's name (e.g. CheckReplDBHash, ValidateCollections, etc.). 321 :param fixture_logger: the logger for the fixtures logs. 322 :param tests_root_logger: the root logger for the tests logs. 323 """ 324 logger_name = "{}:job{:d}".format(behavior_class, fixture_logger.job_num) 325 BaseLogger.__init__(self, logger_name, parent=fixture_logger) 326 327 self.test_case_logger = BaseLogger(logger_name, parent=tests_root_logger) 328 329 330# Util methods 331 332def _fallback_buildlogger_handler(include_logger_name=True): 333 """ 334 Returns a handler that writes to stderr. 335 """ 336 if include_logger_name: 337 log_format = "[fallback] [%(name)s] %(message)s" 338 else: 339 log_format = "[fallback] %(message)s" 340 formatter = formatters.ISO8601Formatter(fmt=log_format) 341 342 handler = logging.StreamHandler(sys.stderr) 343 handler.setFormatter(formatter) 344 345 return handler 346 347 348def _get_buildlogger_handler_info(logger_info): 349 """ 350 Returns the buildlogger handler information if it exists, and None 351 otherwise. 352 """ 353 for handler_info in logger_info["handlers"]: 354 handler_info = handler_info.copy() 355 if handler_info.pop("class") == "buildlogger": 356 return handler_info 357 return None 358