1#!/usr/bin/env python3 2 3# CDDL HEADER START 4# 5# The contents of this file are subject to the terms of the 6# Common Development and Distribution License (the "License"). 7# You may not use this file except in compliance with the License. 8# 9# See LICENSE.txt included in this distribution for the specific 10# language governing permissions and limitations under the License. 11# 12# When distributing Covered Code, include this CDDL HEADER in each 13# file and include the License file at LICENSE.txt. 14# If applicable, add the following below this CDDL HEADER, with the 15# fields enclosed by brackets "[]" replaced with your own identifying 16# information: Portions Copyright [yyyy] [name of copyright owner] 17# 18# CDDL HEADER END 19 20# 21# Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. 22# 23 24""" 25 This script is wrapper of commands to add/remove project or refresh 26 configuration using read-only configuration. 27""" 28 29import argparse 30import io 31import shutil 32import sys 33import tempfile 34from os import path 35 36from filelock import Timeout, FileLock 37 38from .utils.command import Command 39from .utils.log import get_console_logger, get_class_basename, \ 40 fatal 41from .utils.opengrok import get_configuration, set_configuration, \ 42 add_project, delete_project, get_config_value 43from .utils.parsers import get_base_parser 44from .utils.utils import get_command, is_web_uri 45from .utils.exitvals import ( 46 FAILURE_EXITVAL, 47 SUCCESS_EXITVAL 48) 49 50MAJOR_VERSION = sys.version_info[0] 51if (MAJOR_VERSION < 3): 52 print("Need Python 3, you are running {}".format(MAJOR_VERSION)) 53 sys.exit(1) 54 55__version__ = "0.4" 56 57 58def exec_command(doit, logger, cmd, msg): 59 """ 60 Execute given command and return its output. 61 Exit the program on failure. 62 """ 63 cmd = Command(cmd, logger=logger, redirect_stderr=False) 64 if not doit: 65 logger.info(cmd) 66 return 67 cmd.execute() 68 if cmd.getstate() != Command.FINISHED \ 69 or cmd.getretcode() != SUCCESS_EXITVAL: 70 logger.error(msg) 71 logger.error("Standard output: {}".format(cmd.getoutput())) 72 logger.error("Error output: {}".format(cmd.geterroutput())) 73 sys.exit(FAILURE_EXITVAL) 74 75 logger.debug(cmd.geterroutputstr()) 76 77 return cmd.getoutput() 78 79 80def get_config_file(basedir): 81 """ 82 Return configuration file in basedir 83 """ 84 85 return path.join(basedir, "etc", "configuration.xml") 86 87 88def install_config(doit, logger, src, dst): 89 """ 90 Copy the data of src to dst. Exit on failure. 91 """ 92 if not doit: 93 logger.debug("Not copying {} to {}".format(src, dst)) 94 return 95 96 # 97 # Copy the file so that close() triggered unlink() 98 # does not fail. 99 # 100 logger.debug("Copying {} to {}".format(src, dst)) 101 try: 102 shutil.copyfile(src, dst) 103 except PermissionError: 104 logger.error('Failed to copy {} to {} (permissions)'. 105 format(src, dst)) 106 sys.exit(FAILURE_EXITVAL) 107 except OSError: 108 logger.error('Failed to copy {} to {} (I/O)'. 109 format(src, dst)) 110 sys.exit(FAILURE_EXITVAL) 111 112 113def config_refresh(doit, logger, basedir, uri, configmerge, jar_file, 114 roconfig, java): 115 """ 116 Refresh current configuration file with configuration retrieved 117 from webapp. If roconfig is not None, the current config is merged with 118 readonly configuration first. 119 120 The merge of the current config from the webapp with the read-only config 121 is done as a workaround for https://github.com/oracle/opengrok/issues/2002 122 """ 123 124 main_config = get_config_file(basedir) 125 if not path.isfile(main_config): 126 logger.error("file {} does not exist".format(main_config)) 127 sys.exit(FAILURE_EXITVAL) 128 129 if doit: 130 current_config = get_configuration(logger, uri) 131 if not current_config: 132 sys.exit(FAILURE_EXITVAL) 133 else: 134 current_config = None 135 136 with tempfile.NamedTemporaryFile() as fcur: 137 logger.debug("Temporary file for current config: {}".format(fcur.name)) 138 if doit: 139 fcur.write(bytearray(''.join(current_config), "UTF-8")) 140 fcur.flush() 141 142 if not roconfig: 143 logger.info('Refreshing configuration') 144 install_config(doit, logger, fcur.name, main_config) 145 else: 146 logger.info('Refreshing configuration ' 147 '(merging with read-only config)') 148 configmerge_cmd = configmerge 149 configmerge_cmd.extend(['-a', jar_file, roconfig, fcur.name]) 150 if java: 151 configmerge_cmd.append('-j') 152 configmerge_cmd.append(java) 153 merged_config = exec_command(doit, logger, 154 configmerge_cmd, 155 "cannot merge configuration") 156 with tempfile.NamedTemporaryFile() as fmerged: 157 logger.debug("Temporary file for merged config: {}". 158 format(fmerged.name)) 159 if doit: 160 fmerged.write(bytearray(''.join(merged_config), "UTF-8")) 161 fmerged.flush() 162 install_config(doit, logger, fmerged.name, main_config) 163 164 165def project_add(doit, logger, project, uri): 166 """ 167 Adds a project to configuration. Works in multiple steps: 168 169 1. add the project to configuration 170 2. refresh on disk configuration 171 """ 172 173 logger.info("Adding project {}".format(project)) 174 175 if doit: 176 add_project(logger, project, uri) 177 178 179def project_delete(logger, project, uri, doit=True, deletesource=False): 180 """ 181 Delete the project for configuration and all its data. 182 Works in multiple steps: 183 184 1. delete the project from configuration and its indexed data 185 2. refresh on disk configuration 186 3. delete the source code for the project 187 """ 188 189 # Be extra careful as we will be recursively removing directory structure. 190 if not project or len(project) == 0: 191 raise Exception("invalid call to project_delete(): missing project") 192 193 logger.info("Deleting project {} and its index data".format(project)) 194 195 if doit: 196 delete_project(logger, project, uri) 197 198 if deletesource: 199 src_root = get_config_value(logger, 'sourceRoot', uri) 200 if not src_root or len(src_root) == 0: 201 raise Exception("source root empty") 202 logger.debug("Source root = {}".format(src_root)) 203 sourcedir = path.join(src_root, project) 204 logger.debug("Removing directory tree {}".format(sourcedir)) 205 if doit: 206 logger.info("Removing source code under {}".format(sourcedir)) 207 shutil.rmtree(sourcedir) 208 209 210def main(): 211 parser = argparse.ArgumentParser(description='project management.', 212 formatter_class=argparse. 213 ArgumentDefaultsHelpFormatter, 214 parents=[get_base_parser( 215 tool_version=__version__) 216 ]) 217 218 parser.add_argument('-b', '--base', default="/var/opengrok", 219 help='OpenGrok instance base directory') 220 parser.add_argument('-R', '--roconfig', 221 help='OpenGrok read-only configuration file') 222 parser.add_argument('-U', '--uri', default='http://localhost:8080/source', 223 help='URI of the webapp with context path') 224 parser.add_argument('-c', '--configmerge', 225 help='path to the ConfigMerge binary') 226 parser.add_argument('--java', help='Path to java binary ' 227 '(needed for config merge program)') 228 parser.add_argument('-j', '--jar', help='Path to jar archive to run') 229 parser.add_argument('-u', '--upload', action='store_true', 230 help='Upload configuration at the end') 231 parser.add_argument('-n', '--noop', action='store_true', default=False, 232 help='Do not run any commands or modify any config' 233 ', just report. Usually implies ' 234 'the --debug option.') 235 parser.add_argument('-N', '--nosourcedelete', action='store_true', 236 default=False, help='Do not delete source code when ' 237 'deleting a project') 238 239 group = parser.add_mutually_exclusive_group() 240 group.add_argument('-a', '--add', metavar='project', nargs='+', 241 help='Add project (assumes its source is available ' 242 'under source root') 243 group.add_argument('-d', '--delete', metavar='project', nargs='+', 244 help='Delete project and its data and source code') 245 group.add_argument('-r', '--refresh', action='store_true', 246 help='Refresh configuration. If read-only ' 247 'configuration is supplied, it is merged ' 248 'with current ' 249 'configuration.') 250 251 try: 252 args = parser.parse_args() 253 except ValueError as e: 254 fatal(e) 255 256 doit = not args.noop 257 configmerge = None 258 259 # 260 # Setup logger as a first thing after parsing arguments so that it can be 261 # used through the rest of the program. 262 # 263 logger = get_console_logger(get_class_basename(), args.loglevel) 264 265 if args.nosourcedelete and not args.delete: 266 logger.error("The no source delete option is only valid for delete") 267 sys.exit(FAILURE_EXITVAL) 268 269 # Set the base directory 270 if args.base: 271 if path.isdir(args.base): 272 logger.debug("Using {} as instance base". 273 format(args.base)) 274 else: 275 logger.error("Not a directory: {}\n" 276 "Set the base directory with the --base option." 277 .format(args.base)) 278 sys.exit(FAILURE_EXITVAL) 279 280 # If read-only configuration file is specified, this means read-only 281 # configuration will need to be merged with active webapp configuration. 282 # This requires config merge tool to be run so couple of other things 283 # need to be checked. 284 if args.roconfig: 285 if path.isfile(args.roconfig): 286 logger.debug("Using {} as read-only config".format(args.roconfig)) 287 else: 288 logger.error("File {} does not exist".format(args.roconfig)) 289 sys.exit(FAILURE_EXITVAL) 290 291 configmerge_file = get_command(logger, args.configmerge, 292 "opengrok-config-merge") 293 if configmerge_file is None: 294 logger.error("Use the --configmerge option to specify the path to" 295 "the config merge script") 296 sys.exit(FAILURE_EXITVAL) 297 298 configmerge = [configmerge_file] 299 if args.loglevel: 300 configmerge.append('-l') 301 configmerge.append(str(args.loglevel)) 302 303 if args.jar is None: 304 logger.error('jar file needed for config merge tool, ' 305 'use --jar to specify one') 306 sys.exit(FAILURE_EXITVAL) 307 308 uri = args.uri 309 if not is_web_uri(uri): 310 logger.error("Not a URI: {}".format(uri)) 311 sys.exit(FAILURE_EXITVAL) 312 logger.debug("web application URI = {}".format(uri)) 313 314 lock = FileLock(path.join(tempfile.gettempdir(), 315 path.basename(sys.argv[0]) + ".lock")) 316 try: 317 with lock.acquire(timeout=0): 318 if args.add: 319 for proj in args.add: 320 project_add(doit=doit, logger=logger, 321 project=proj, 322 uri=uri) 323 324 config_refresh(doit=doit, logger=logger, 325 basedir=args.base, 326 uri=uri, 327 configmerge=configmerge, 328 jar_file=args.jar, 329 roconfig=args.roconfig, 330 java=args.java) 331 elif args.delete: 332 for proj in args.delete: 333 project_delete(logger=logger, 334 project=proj, 335 uri=uri, doit=doit, 336 deletesource=not args.nosourcedelete) 337 338 config_refresh(doit=doit, logger=logger, 339 basedir=args.base, 340 uri=uri, 341 configmerge=configmerge, 342 jar_file=args.jar, 343 roconfig=args.roconfig, 344 java=args.java) 345 elif args.refresh: 346 config_refresh(doit=doit, logger=logger, 347 basedir=args.base, 348 uri=uri, 349 configmerge=configmerge, 350 jar_file=args.jar, 351 roconfig=args.roconfig, 352 java=args.java) 353 else: 354 parser.print_help() 355 sys.exit(FAILURE_EXITVAL) 356 357 if args.upload: 358 main_config = get_config_file(basedir=args.base) 359 if path.isfile(main_config): 360 if doit: 361 with io.open(main_config, mode='r', 362 encoding="utf-8") as config_file: 363 config_data = config_file.read().encode("utf-8") 364 if not set_configuration(logger, 365 config_data, uri): 366 sys.exit(FAILURE_EXITVAL) 367 else: 368 logger.error("file {} does not exist".format(main_config)) 369 sys.exit(FAILURE_EXITVAL) 370 except Timeout: 371 logger.warning("Already running, exiting.") 372 sys.exit(FAILURE_EXITVAL) 373 374 375if __name__ == '__main__': 376 main() 377