1#!/usr/bin/env python3
2
3# CDDL HEADER START
4#
5# The contents of this file are subject to the terms of the
6# Common Development and Distribution License (the "License").
7# You may not use this file except in compliance with the License.
8#
9# See LICENSE.txt included in this distribution for the specific
10# language governing permissions and limitations under the License.
11#
12# When distributing Covered Code, include this CDDL HEADER in each
13# file and include the License file at LICENSE.txt.
14# If applicable, add the following below this CDDL HEADER, with the
15# fields enclosed by brackets "[]" replaced with your own identifying
16# information: Portions Copyright [yyyy] [name of copyright owner]
17#
18# CDDL HEADER END
19
20#
21# Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
22#
23
24"""
25    This script is wrapper of commands to add/remove project or refresh
26    configuration using read-only configuration.
27"""
28
29import argparse
30import io
31import shutil
32import sys
33import tempfile
34from os import path
35
36from filelock import Timeout, FileLock
37
38from .utils.command import Command
39from .utils.log import get_console_logger, get_class_basename, \
40    fatal
41from .utils.opengrok import get_configuration, set_configuration, \
42    add_project, delete_project, get_config_value
43from .utils.parsers import get_base_parser
44from .utils.utils import get_command, is_web_uri
45from .utils.exitvals import (
46    FAILURE_EXITVAL,
47    SUCCESS_EXITVAL
48)
49
50MAJOR_VERSION = sys.version_info[0]
51if (MAJOR_VERSION < 3):
52    print("Need Python 3, you are running {}".format(MAJOR_VERSION))
53    sys.exit(1)
54
55__version__ = "0.4"
56
57
58def exec_command(doit, logger, cmd, msg):
59    """
60    Execute given command and return its output.
61    Exit the program on failure.
62    """
63    cmd = Command(cmd, logger=logger, redirect_stderr=False)
64    if not doit:
65        logger.info(cmd)
66        return
67    cmd.execute()
68    if cmd.getstate() != Command.FINISHED \
69            or cmd.getretcode() != SUCCESS_EXITVAL:
70        logger.error(msg)
71        logger.error("Standard output: {}".format(cmd.getoutput()))
72        logger.error("Error output: {}".format(cmd.geterroutput()))
73        sys.exit(FAILURE_EXITVAL)
74
75    logger.debug(cmd.geterroutputstr())
76
77    return cmd.getoutput()
78
79
80def get_config_file(basedir):
81    """
82    Return configuration file in basedir
83    """
84
85    return path.join(basedir, "etc", "configuration.xml")
86
87
88def install_config(doit, logger, src, dst):
89    """
90    Copy the data of src to dst. Exit on failure.
91    """
92    if not doit:
93        logger.debug("Not copying {} to {}".format(src, dst))
94        return
95
96    #
97    # Copy the file so that close() triggered unlink()
98    # does not fail.
99    #
100    logger.debug("Copying {} to {}".format(src, dst))
101    try:
102        shutil.copyfile(src, dst)
103    except PermissionError:
104        logger.error('Failed to copy {} to {} (permissions)'.
105                     format(src, dst))
106        sys.exit(FAILURE_EXITVAL)
107    except OSError:
108        logger.error('Failed to copy {} to {} (I/O)'.
109                     format(src, dst))
110        sys.exit(FAILURE_EXITVAL)
111
112
113def config_refresh(doit, logger, basedir, uri, configmerge, jar_file,
114                   roconfig, java):
115    """
116    Refresh current configuration file with configuration retrieved
117    from webapp. If roconfig is not None, the current config is merged with
118    readonly configuration first.
119
120    The merge of the current config from the webapp with the read-only config
121    is done as a workaround for https://github.com/oracle/opengrok/issues/2002
122    """
123
124    main_config = get_config_file(basedir)
125    if not path.isfile(main_config):
126        logger.error("file {} does not exist".format(main_config))
127        sys.exit(FAILURE_EXITVAL)
128
129    if doit:
130        current_config = get_configuration(logger, uri)
131        if not current_config:
132            sys.exit(FAILURE_EXITVAL)
133    else:
134        current_config = None
135
136    with tempfile.NamedTemporaryFile() as fcur:
137        logger.debug("Temporary file for current config: {}".format(fcur.name))
138        if doit:
139            fcur.write(bytearray(''.join(current_config), "UTF-8"))
140            fcur.flush()
141
142        if not roconfig:
143            logger.info('Refreshing configuration')
144            install_config(doit, logger, fcur.name, main_config)
145        else:
146            logger.info('Refreshing configuration '
147                        '(merging with read-only config)')
148            configmerge_cmd = configmerge
149            configmerge_cmd.extend(['-a', jar_file, roconfig, fcur.name])
150            if java:
151                configmerge_cmd.append('-j')
152                configmerge_cmd.append(java)
153            merged_config = exec_command(doit, logger,
154                                         configmerge_cmd,
155                                         "cannot merge configuration")
156            with tempfile.NamedTemporaryFile() as fmerged:
157                logger.debug("Temporary file for merged config: {}".
158                             format(fmerged.name))
159                if doit:
160                    fmerged.write(bytearray(''.join(merged_config), "UTF-8"))
161                    fmerged.flush()
162                    install_config(doit, logger, fmerged.name, main_config)
163
164
165def project_add(doit, logger, project, uri):
166    """
167    Adds a project to configuration. Works in multiple steps:
168
169      1. add the project to configuration
170      2. refresh on disk configuration
171    """
172
173    logger.info("Adding project {}".format(project))
174
175    if doit:
176        add_project(logger, project, uri)
177
178
179def project_delete(logger, project, uri, doit=True, deletesource=False):
180    """
181    Delete the project for configuration and all its data.
182    Works in multiple steps:
183
184      1. delete the project from configuration and its indexed data
185      2. refresh on disk configuration
186      3. delete the source code for the project
187    """
188
189    # Be extra careful as we will be recursively removing directory structure.
190    if not project or len(project) == 0:
191        raise Exception("invalid call to project_delete(): missing project")
192
193    logger.info("Deleting project {} and its index data".format(project))
194
195    if doit:
196        delete_project(logger, project, uri)
197
198    if deletesource:
199        src_root = get_config_value(logger, 'sourceRoot', uri)
200        if not src_root or len(src_root) == 0:
201            raise Exception("source root empty")
202        logger.debug("Source root = {}".format(src_root))
203        sourcedir = path.join(src_root, project)
204        logger.debug("Removing directory tree {}".format(sourcedir))
205        if doit:
206            logger.info("Removing source code under {}".format(sourcedir))
207            shutil.rmtree(sourcedir)
208
209
210def main():
211    parser = argparse.ArgumentParser(description='project management.',
212                                     formatter_class=argparse.
213                                     ArgumentDefaultsHelpFormatter,
214                                     parents=[get_base_parser(
215                                         tool_version=__version__)
216                                     ])
217
218    parser.add_argument('-b', '--base', default="/var/opengrok",
219                        help='OpenGrok instance base directory')
220    parser.add_argument('-R', '--roconfig',
221                        help='OpenGrok read-only configuration file')
222    parser.add_argument('-U', '--uri', default='http://localhost:8080/source',
223                        help='URI of the webapp with context path')
224    parser.add_argument('-c', '--configmerge',
225                        help='path to the ConfigMerge binary')
226    parser.add_argument('--java', help='Path to java binary '
227                                       '(needed for config merge program)')
228    parser.add_argument('-j', '--jar', help='Path to jar archive to run')
229    parser.add_argument('-u', '--upload', action='store_true',
230                        help='Upload configuration at the end')
231    parser.add_argument('-n', '--noop', action='store_true', default=False,
232                        help='Do not run any commands or modify any config'
233                             ', just report. Usually implies '
234                             'the --debug option.')
235    parser.add_argument('-N', '--nosourcedelete', action='store_true',
236                        default=False, help='Do not delete source code when '
237                                            'deleting a project')
238
239    group = parser.add_mutually_exclusive_group()
240    group.add_argument('-a', '--add', metavar='project', nargs='+',
241                       help='Add project (assumes its source is available '
242                            'under source root')
243    group.add_argument('-d', '--delete', metavar='project', nargs='+',
244                       help='Delete project and its data and source code')
245    group.add_argument('-r', '--refresh', action='store_true',
246                       help='Refresh configuration. If read-only '
247                            'configuration is supplied, it is merged '
248                            'with current '
249                            'configuration.')
250
251    try:
252        args = parser.parse_args()
253    except ValueError as e:
254        fatal(e)
255
256    doit = not args.noop
257    configmerge = None
258
259    #
260    # Setup logger as a first thing after parsing arguments so that it can be
261    # used through the rest of the program.
262    #
263    logger = get_console_logger(get_class_basename(), args.loglevel)
264
265    if args.nosourcedelete and not args.delete:
266        logger.error("The no source delete option is only valid for delete")
267        sys.exit(FAILURE_EXITVAL)
268
269    # Set the base directory
270    if args.base:
271        if path.isdir(args.base):
272            logger.debug("Using {} as instance base".
273                         format(args.base))
274        else:
275            logger.error("Not a directory: {}\n"
276                         "Set the base directory with the --base option."
277                         .format(args.base))
278            sys.exit(FAILURE_EXITVAL)
279
280    # If read-only configuration file is specified, this means read-only
281    # configuration will need to be merged with active webapp configuration.
282    # This requires config merge tool to be run so couple of other things
283    # need to be checked.
284    if args.roconfig:
285        if path.isfile(args.roconfig):
286            logger.debug("Using {} as read-only config".format(args.roconfig))
287        else:
288            logger.error("File {} does not exist".format(args.roconfig))
289            sys.exit(FAILURE_EXITVAL)
290
291        configmerge_file = get_command(logger, args.configmerge,
292                                       "opengrok-config-merge")
293        if configmerge_file is None:
294            logger.error("Use the --configmerge option to specify the path to"
295                         "the config merge script")
296            sys.exit(FAILURE_EXITVAL)
297
298        configmerge = [configmerge_file]
299        if args.loglevel:
300            configmerge.append('-l')
301            configmerge.append(str(args.loglevel))
302
303        if args.jar is None:
304            logger.error('jar file needed for config merge tool, '
305                         'use --jar to specify one')
306            sys.exit(FAILURE_EXITVAL)
307
308    uri = args.uri
309    if not is_web_uri(uri):
310        logger.error("Not a URI: {}".format(uri))
311        sys.exit(FAILURE_EXITVAL)
312    logger.debug("web application URI = {}".format(uri))
313
314    lock = FileLock(path.join(tempfile.gettempdir(),
315                              path.basename(sys.argv[0]) + ".lock"))
316    try:
317        with lock.acquire(timeout=0):
318            if args.add:
319                for proj in args.add:
320                    project_add(doit=doit, logger=logger,
321                                project=proj,
322                                uri=uri)
323
324                config_refresh(doit=doit, logger=logger,
325                               basedir=args.base,
326                               uri=uri,
327                               configmerge=configmerge,
328                               jar_file=args.jar,
329                               roconfig=args.roconfig,
330                               java=args.java)
331            elif args.delete:
332                for proj in args.delete:
333                    project_delete(logger=logger,
334                                   project=proj,
335                                   uri=uri, doit=doit,
336                                   deletesource=not args.nosourcedelete)
337
338                config_refresh(doit=doit, logger=logger,
339                               basedir=args.base,
340                               uri=uri,
341                               configmerge=configmerge,
342                               jar_file=args.jar,
343                               roconfig=args.roconfig,
344                               java=args.java)
345            elif args.refresh:
346                config_refresh(doit=doit, logger=logger,
347                               basedir=args.base,
348                               uri=uri,
349                               configmerge=configmerge,
350                               jar_file=args.jar,
351                               roconfig=args.roconfig,
352                               java=args.java)
353            else:
354                parser.print_help()
355                sys.exit(FAILURE_EXITVAL)
356
357            if args.upload:
358                main_config = get_config_file(basedir=args.base)
359                if path.isfile(main_config):
360                    if doit:
361                        with io.open(main_config, mode='r',
362                                     encoding="utf-8") as config_file:
363                            config_data = config_file.read().encode("utf-8")
364                            if not set_configuration(logger,
365                                                     config_data, uri):
366                                sys.exit(FAILURE_EXITVAL)
367                else:
368                    logger.error("file {} does not exist".format(main_config))
369                    sys.exit(FAILURE_EXITVAL)
370    except Timeout:
371        logger.warning("Already running, exiting.")
372        sys.exit(FAILURE_EXITVAL)
373
374
375if __name__ == '__main__':
376    main()
377