1"""Reverter class saves configuration checkpoints and allows for recovery.""" 2import csv 3import glob 4import logging 5import shutil 6import time 7import traceback 8from typing import Iterable 9from typing import List 10from typing import Set 11from typing import TextIO 12from typing import Tuple 13 14from certbot import configuration 15from certbot import errors 16from certbot import util 17from certbot._internal import constants 18from certbot.compat import filesystem 19from certbot.compat import os 20 21logger = logging.getLogger(__name__) 22 23 24class Reverter: 25 """Reverter Class - save and revert configuration checkpoints. 26 27 This class can be used by the plugins, especially Installers, to 28 undo changes made to the user's system. Modifications to files and 29 commands to do undo actions taken by the plugin should be registered 30 with this class before the action is taken. 31 32 Once a change has been registered with this class, there are three 33 states the change can be in. First, the change can be a temporary 34 change. This should be used for changes that will soon be reverted, 35 such as config changes for the purpose of solving a challenge. 36 Changes are added to this state through calls to 37 :func:`~add_to_temp_checkpoint` and reverted when 38 :func:`~revert_temporary_config` or :func:`~recovery_routine` is 39 called. 40 41 The second state a change can be in is in progress. These changes 42 are not temporary, however, they also have not been finalized in a 43 checkpoint. A change must become in progress before it can be 44 finalized. Changes are added to this state through calls to 45 :func:`~add_to_checkpoint` and reverted when 46 :func:`~recovery_routine` is called. 47 48 The last state a change can be in is finalized in a checkpoint. A 49 change is put into this state by first becoming an in progress 50 change and then calling :func:`~finalize_checkpoint`. Changes 51 in this state can be reverted through calls to 52 :func:`~rollback_checkpoints`. 53 54 As a final note, creating new files and registering undo commands 55 are handled specially and use the methods 56 :func:`~register_file_creation` and :func:`~register_undo_command` 57 respectively. Both of these methods can be used to create either 58 temporary or in progress changes. 59 60 .. note:: Consider moving everything over to CSV format. 61 62 :param config: Configuration. 63 :type config: :class:`certbot.configuration.NamespaceConfig` 64 65 """ 66 def __init__(self, config: configuration.NamespaceConfig) -> None: 67 self.config = config 68 69 util.make_or_verify_dir( 70 config.backup_dir, constants.CONFIG_DIRS_MODE, self.config.strict_permissions) 71 72 def revert_temporary_config(self) -> None: 73 """Reload users original configuration files after a temporary save. 74 75 This function should reinstall the users original configuration files 76 for all saves with temporary=True 77 78 :raises .ReverterError: when unable to revert config 79 80 """ 81 if os.path.isdir(self.config.temp_checkpoint_dir): 82 try: 83 self._recover_checkpoint(self.config.temp_checkpoint_dir) 84 except errors.ReverterError: 85 # We have a partial or incomplete recovery 86 logger.critical( 87 "Incomplete or failed recovery for %s", 88 self.config.temp_checkpoint_dir, 89 ) 90 raise errors.ReverterError("Unable to revert temporary config") 91 92 def rollback_checkpoints(self, rollback: int = 1) -> None: 93 """Revert 'rollback' number of configuration checkpoints. 94 95 :param int rollback: Number of checkpoints to reverse. A str num will be 96 cast to an integer. So "2" is also acceptable. 97 98 :raises .ReverterError: 99 if there is a problem with the input or if the function is 100 unable to correctly revert the configuration checkpoints 101 102 """ 103 try: 104 rollback = int(rollback) 105 except ValueError: 106 logger.error("Rollback argument must be a positive integer") 107 raise errors.ReverterError("Invalid Input") 108 # Sanity check input 109 if rollback < 0: 110 logger.error("Rollback argument must be a positive integer") 111 raise errors.ReverterError("Invalid Input") 112 113 backups = os.listdir(self.config.backup_dir) 114 backups.sort() 115 116 if not backups: 117 logger.warning( 118 "Certbot hasn't modified your configuration, so rollback " 119 "isn't available.") 120 elif len(backups) < rollback: 121 logger.warning("Unable to rollback %d checkpoints, only %d exist", 122 rollback, len(backups)) 123 124 while rollback > 0 and backups: 125 cp_dir = os.path.join(self.config.backup_dir, backups.pop()) 126 try: 127 self._recover_checkpoint(cp_dir) 128 except errors.ReverterError: 129 logger.critical("Failed to load checkpoint during rollback") 130 raise errors.ReverterError( 131 "Unable to load checkpoint during rollback") 132 rollback -= 1 133 134 def add_to_temp_checkpoint(self, save_files: Set[str], save_notes: str) -> None: 135 """Add files to temporary checkpoint. 136 137 :param set save_files: set of filepaths to save 138 :param str save_notes: notes about changes during the save 139 140 """ 141 self._add_to_checkpoint_dir( 142 self.config.temp_checkpoint_dir, save_files, save_notes) 143 144 def add_to_checkpoint(self, save_files: Set[str], save_notes: str) -> None: 145 """Add files to a permanent checkpoint. 146 147 :param set save_files: set of filepaths to save 148 :param str save_notes: notes about changes during the save 149 150 """ 151 # Check to make sure we are not overwriting a temp file 152 self._check_tempfile_saves(save_files) 153 self._add_to_checkpoint_dir( 154 self.config.in_progress_dir, save_files, save_notes) 155 156 def _add_to_checkpoint_dir(self, cp_dir: str, save_files: Set[str], save_notes: str) -> None: 157 """Add save files to checkpoint directory. 158 159 :param str cp_dir: Checkpoint directory filepath 160 :param set save_files: set of files to save 161 :param str save_notes: notes about changes made during the save 162 163 :raises IOError: if unable to open cp_dir + FILEPATHS file 164 :raises .ReverterError: if unable to add checkpoint 165 166 """ 167 util.make_or_verify_dir( 168 cp_dir, constants.CONFIG_DIRS_MODE, self.config.strict_permissions) 169 170 op_fd, existing_filepaths = self._read_and_append( 171 os.path.join(cp_dir, "FILEPATHS")) 172 173 idx = len(existing_filepaths) 174 175 for filename in save_files: 176 # No need to copy/index already existing files 177 # The oldest copy already exists in the directory... 178 if filename not in existing_filepaths: 179 # Tag files with index so multiple files can 180 # have the same filename 181 logger.debug("Creating backup of %s", filename) 182 try: 183 shutil.copy2(filename, os.path.join( 184 cp_dir, os.path.basename(filename) + "_" + str(idx))) 185 op_fd.write('{0}\n'.format(filename)) 186 # https://stackoverflow.com/questions/4726260/effective-use-of-python-shutil-copy2 187 except IOError: 188 op_fd.close() 189 logger.error( 190 "Unable to add file %s to checkpoint %s", 191 filename, cp_dir) 192 raise errors.ReverterError( 193 "Unable to add file {0} to checkpoint " 194 "{1}".format(filename, cp_dir)) 195 idx += 1 196 op_fd.close() 197 198 with open(os.path.join(cp_dir, "CHANGES_SINCE"), "a") as notes_fd: 199 notes_fd.write(save_notes) 200 201 def _read_and_append(self, filepath: str) -> Tuple[TextIO, List[str]]: 202 """Reads the file lines and returns a file obj. 203 204 Read the file returning the lines, and a pointer to the end of the file. 205 206 """ 207 # pylint: disable=consider-using-with 208 # Open up filepath differently depending on if it already exists 209 if os.path.isfile(filepath): 210 op_fd = open(filepath, "r+") 211 lines = op_fd.read().splitlines() 212 else: 213 lines = [] 214 op_fd = open(filepath, "w") 215 216 return op_fd, lines 217 218 def _recover_checkpoint(self, cp_dir: str) -> None: 219 """Recover a specific checkpoint. 220 221 Recover a specific checkpoint provided by cp_dir 222 Note: this function does not reload augeas. 223 224 :param str cp_dir: checkpoint directory file path 225 226 :raises errors.ReverterError: If unable to recover checkpoint 227 228 """ 229 # Undo all commands 230 if os.path.isfile(os.path.join(cp_dir, "COMMANDS")): 231 self._run_undo_commands(os.path.join(cp_dir, "COMMANDS")) 232 # Revert all changed files 233 if os.path.isfile(os.path.join(cp_dir, "FILEPATHS")): 234 try: 235 with open(os.path.join(cp_dir, "FILEPATHS")) as paths_fd: 236 filepaths = paths_fd.read().splitlines() 237 for idx, path in enumerate(filepaths): 238 shutil.copy2(os.path.join( 239 cp_dir, 240 os.path.basename(path) + "_" + str(idx)), path) 241 except (IOError, OSError): 242 # This file is required in all checkpoints. 243 logger.error("Unable to recover files from %s", cp_dir) 244 raise errors.ReverterError( 245 "Unable to recover files from %s" % cp_dir) 246 247 # Remove any newly added files if they exist 248 self._remove_contained_files(os.path.join(cp_dir, "NEW_FILES")) 249 250 try: 251 shutil.rmtree(cp_dir) 252 except OSError: 253 logger.error("Unable to remove directory: %s", cp_dir) 254 raise errors.ReverterError( 255 "Unable to remove directory: %s" % cp_dir) 256 257 def _run_undo_commands(self, filepath: str) -> None: 258 """Run all commands in a file.""" 259 # NOTE: csv module uses native strings. That is unicode on Python 3 260 # It is strongly advised to set newline = '' on Python 3 with CSV, 261 # and it fixes problems on Windows. 262 kwargs = {'newline': ''} 263 with open(filepath, 'r', **kwargs) as csvfile: # type: ignore 264 csvreader = csv.reader(csvfile) 265 for command in reversed(list(csvreader)): 266 try: 267 util.run_script(command) 268 except errors.SubprocessError: 269 logger.error( 270 "Unable to run undo command: %s", " ".join(command)) 271 272 def _check_tempfile_saves(self, save_files: Set[str]) -> None: 273 """Verify save isn't overwriting any temporary files. 274 275 :param set save_files: Set of files about to be saved. 276 277 :raises certbot.errors.ReverterError: 278 when save is attempting to overwrite a temporary file. 279 280 """ 281 protected_files = [] 282 283 # Get temp modified files 284 temp_path = os.path.join(self.config.temp_checkpoint_dir, "FILEPATHS") 285 if os.path.isfile(temp_path): 286 with open(temp_path, "r") as protected_fd: 287 protected_files.extend(protected_fd.read().splitlines()) 288 289 # Get temp new files 290 new_path = os.path.join(self.config.temp_checkpoint_dir, "NEW_FILES") 291 if os.path.isfile(new_path): 292 with open(new_path, "r") as protected_fd: 293 protected_files.extend(protected_fd.read().splitlines()) 294 295 # Verify no save_file is in protected_files 296 for filename in protected_files: 297 if filename in save_files: 298 raise errors.ReverterError( 299 "Attempting to overwrite challenge " 300 "file - %s" % filename) 301 302 def register_file_creation(self, temporary: bool, *files: str) -> None: 303 r"""Register the creation of all files during certbot execution. 304 305 Call this method before writing to the file to make sure that the 306 file will be cleaned up if the program exits unexpectedly. 307 (Before a save occurs) 308 309 :param bool temporary: If the file creation registry is for 310 a temp or permanent save. 311 :param \*files: file paths (str) to be registered 312 313 :raises certbot.errors.ReverterError: If 314 call does not contain necessary parameters or if the file creation 315 is unable to be registered. 316 317 """ 318 # Make sure some files are provided... as this is an error 319 # Made this mistake in my initial implementation of apache.dvsni.py 320 if not files: 321 raise errors.ReverterError("Forgot to provide files to registration call") 322 323 cp_dir = self._get_cp_dir(temporary) 324 325 # Append all new files (that aren't already registered) 326 new_fd = None 327 try: 328 new_fd, ex_files = self._read_and_append(os.path.join(cp_dir, "NEW_FILES")) 329 330 for path in files: 331 if path not in ex_files: 332 new_fd.write("{0}\n".format(path)) 333 except (IOError, OSError): 334 logger.error("Unable to register file creation(s) - %s", files) 335 raise errors.ReverterError( 336 "Unable to register file creation(s) - {0}".format(files)) 337 finally: 338 if new_fd is not None: 339 new_fd.close() 340 341 def register_undo_command(self, temporary: bool, command: Iterable[str]) -> None: 342 """Register a command to be run to undo actions taken. 343 344 .. warning:: This function does not enforce order of operations in terms 345 of file modification vs. command registration. All undo commands 346 are run first before all normal files are reverted to their previous 347 state. If you need to maintain strict order, you may create 348 checkpoints before and after the the command registration. This 349 function may be improved in the future based on demand. 350 351 :param bool temporary: Whether the command should be saved in the 352 IN_PROGRESS or TEMPORARY checkpoints. 353 :param command: Command to be run. 354 :type command: list of str 355 356 """ 357 commands_fp = os.path.join(self._get_cp_dir(temporary), "COMMANDS") 358 # It is strongly advised to set newline = '' on Python 3 with CSV, 359 # and it fixes problems on Windows. 360 kwargs = {'newline': ''} 361 try: 362 mode = "a" if os.path.isfile(commands_fp) else "w" 363 with open(commands_fp, mode, **kwargs) as f: # type: ignore 364 csvwriter = csv.writer(f) 365 csvwriter.writerow(command) 366 except (IOError, OSError): 367 logger.error("Unable to register undo command") 368 raise errors.ReverterError( 369 "Unable to register undo command.") 370 371 def _get_cp_dir(self, temporary: bool) -> str: 372 """Return the proper reverter directory.""" 373 if temporary: 374 cp_dir = self.config.temp_checkpoint_dir 375 else: 376 cp_dir = self.config.in_progress_dir 377 378 util.make_or_verify_dir( 379 cp_dir, constants.CONFIG_DIRS_MODE, self.config.strict_permissions) 380 381 return cp_dir 382 383 def recovery_routine(self) -> None: 384 """Revert configuration to most recent finalized checkpoint. 385 386 Remove all changes (temporary and permanent) that have not been 387 finalized. This is useful to protect against crashes and other 388 execution interruptions. 389 390 :raises .errors.ReverterError: If unable to recover the configuration 391 392 """ 393 # First, any changes found in NamespaceConfig.temp_checkpoint_dir are removed, 394 # then IN_PROGRESS changes are removed The order is important. 395 # IN_PROGRESS is unable to add files that are already added by a TEMP 396 # change. Thus TEMP must be rolled back first because that will be the 397 # 'latest' occurrence of the file. 398 self.revert_temporary_config() 399 if os.path.isdir(self.config.in_progress_dir): 400 try: 401 self._recover_checkpoint(self.config.in_progress_dir) 402 except errors.ReverterError: 403 # We have a partial or incomplete recovery 404 logger.critical("Incomplete or failed recovery for IN_PROGRESS " 405 "checkpoint - %s", 406 self.config.in_progress_dir) 407 raise errors.ReverterError( 408 "Incomplete or failed recovery for IN_PROGRESS checkpoint " 409 "- %s" % self.config.in_progress_dir) 410 411 def _remove_contained_files(self, file_list: str) -> bool: 412 """Erase all files contained within file_list. 413 414 :param str file_list: file containing list of file paths to be deleted 415 416 :returns: Success 417 :rtype: bool 418 419 :raises certbot.errors.ReverterError: If 420 all files within file_list cannot be removed 421 422 """ 423 # Check to see that file exists to differentiate can't find file_list 424 # and can't remove filepaths within file_list errors. 425 if not os.path.isfile(file_list): 426 return False 427 try: 428 with open(file_list, "r") as list_fd: 429 filepaths = list_fd.read().splitlines() 430 for path in filepaths: 431 # Files are registered before they are added... so 432 # check to see if file exists first 433 if os.path.lexists(path): 434 os.remove(path) 435 else: 436 logger.warning( 437 "File: %s - Could not be found to be deleted\n" 438 " - Certbot probably shut down unexpectedly", 439 path) 440 except (IOError, OSError): 441 logger.critical( 442 "Unable to remove filepaths contained within %s", file_list) 443 raise errors.ReverterError( 444 "Unable to remove filepaths contained within " 445 "{0}".format(file_list)) 446 447 return True 448 449 def finalize_checkpoint(self, title: str) -> None: 450 """Finalize the checkpoint. 451 452 Timestamps and permanently saves all changes made through the use 453 of :func:`~add_to_checkpoint` and :func:`~register_file_creation` 454 455 :param str title: Title describing checkpoint 456 457 :raises certbot.errors.ReverterError: when the 458 checkpoint is not able to be finalized. 459 460 """ 461 # Check to make sure an "in progress" directory exists 462 if not os.path.isdir(self.config.in_progress_dir): 463 return 464 465 changes_since_path = os.path.join(self.config.in_progress_dir, "CHANGES_SINCE") 466 changes_since_tmp_path = os.path.join(self.config.in_progress_dir, "CHANGES_SINCE.tmp") 467 468 if not os.path.exists(changes_since_path): 469 logger.info("Rollback checkpoint is empty (no changes made?)") 470 with open(changes_since_path, 'w') as f: 471 f.write("No changes\n") 472 473 # Add title to self.config.in_progress_dir CHANGES_SINCE 474 try: 475 with open(changes_since_tmp_path, "w") as changes_tmp: 476 changes_tmp.write("-- %s --\n" % title) 477 with open(changes_since_path, "r") as changes_orig: 478 changes_tmp.write(changes_orig.read()) 479 480 # Move self.config.in_progress_dir to Backups directory 481 shutil.move(changes_since_tmp_path, changes_since_path) 482 except (IOError, OSError): 483 logger.error("Unable to finalize checkpoint - adding title") 484 logger.debug("Exception was:\n%s", traceback.format_exc()) 485 raise errors.ReverterError("Unable to add title") 486 487 # rename the directory as a timestamp 488 self._timestamp_progress_dir() 489 490 def _checkpoint_timestamp(self) -> str: 491 "Determine the timestamp of the checkpoint, enforcing monotonicity." 492 timestamp = str(time.time()) 493 others = glob.glob(os.path.join(self.config.backup_dir, "[0-9]*")) 494 others = [os.path.basename(d) for d in others] 495 others.append(timestamp) 496 others.sort() 497 if others[-1] != timestamp: 498 timetravel = str(float(others[-1]) + 1) 499 logger.warning("Current timestamp %s does not correspond to newest reverter " 500 "checkpoint; your clock probably jumped. Time travelling to %s", 501 timestamp, timetravel) 502 timestamp = timetravel 503 elif len(others) > 1 and others[-2] == timestamp: 504 # It is possible if the checkpoints are made extremely quickly 505 # that will result in a name collision. 506 logger.debug("Race condition with timestamp %s, incrementing by 0.01", timestamp) 507 timetravel = str(float(others[-1]) + 0.01) 508 timestamp = timetravel 509 return timestamp 510 511 def _timestamp_progress_dir(self) -> None: 512 """Timestamp the checkpoint.""" 513 # It is possible save checkpoints faster than 1 per second resulting in 514 # collisions in the naming convention. 515 516 for _ in range(2): 517 timestamp = self._checkpoint_timestamp() 518 final_dir = os.path.join(self.config.backup_dir, timestamp) 519 try: 520 filesystem.replace(self.config.in_progress_dir, final_dir) 521 return 522 except OSError: 523 logger.warning("Unexpected race condition, retrying (%s)", timestamp) 524 525 # After 10 attempts... something is probably wrong here... 526 logger.error( 527 "Unable to finalize checkpoint, %s -> %s", 528 self.config.in_progress_dir, final_dir) 529 raise errors.ReverterError( 530 "Unable to finalize checkpoint renaming") 531