1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * See LICENSE.txt included in this distribution for the specific 9 * language governing permissions and limitations under the License. 10 * 11 * When distributing Covered Code, include this CDDL HEADER in each 12 * file and include the License file at LICENSE.txt. 13 * If applicable, add the following below this CDDL HEADER, with the 14 * fields enclosed by brackets "[]" replaced with your own identifying 15 * information: Portions Copyright [yyyy] [name of copyright owner] 16 * 17 * CDDL HEADER END 18 */ 19 20 /* 21 * Copyright (c) 2006, 2019, Oracle and/or its affiliates. All rights reserved. 22 * Portions Copyright (c) 2017-2020, Chris Fraire <cfraire@me.com>. 23 */ 24 package org.opengrok.indexer.configuration; 25 26 import static org.opengrok.indexer.configuration.Configuration.makeXMLStringAsConfiguration; 27 28 import java.io.File; 29 import java.io.FileNotFoundException; 30 import java.io.IOException; 31 import java.nio.file.Path; 32 import java.util.ArrayList; 33 import java.util.Arrays; 34 import java.util.Collections; 35 import java.util.Date; 36 import java.util.HashSet; 37 import java.util.List; 38 import java.util.Map; 39 import java.util.Set; 40 import java.util.SortedSet; 41 import java.util.TreeSet; 42 import java.util.concurrent.ConcurrentHashMap; 43 import java.util.concurrent.ExecutorService; 44 import java.util.concurrent.Executors; 45 import java.util.concurrent.ThreadFactory; 46 import java.util.concurrent.TimeUnit; 47 import java.util.function.Function; 48 import java.util.logging.Level; 49 import java.util.logging.Logger; 50 import java.util.stream.Collectors; 51 import javax.ws.rs.client.ClientBuilder; 52 import javax.ws.rs.client.Entity; 53 import javax.ws.rs.core.Response; 54 import org.apache.lucene.index.IndexReader; 55 import org.apache.lucene.index.MultiReader; 56 import org.apache.lucene.search.SearcherManager; 57 import org.apache.lucene.store.AlreadyClosedException; 58 import org.apache.lucene.store.Directory; 59 import org.apache.lucene.store.FSDirectory; 60 import org.apache.lucene.util.NamedThreadFactory; 61 import org.opengrok.indexer.authorization.AuthorizationFramework; 62 import org.opengrok.indexer.authorization.AuthorizationStack; 63 import org.opengrok.indexer.history.HistoryGuru; 64 import org.opengrok.indexer.history.RepositoryInfo; 65 import org.opengrok.indexer.index.Filter; 66 import org.opengrok.indexer.index.IgnoredNames; 67 import org.opengrok.indexer.index.IndexDatabase; 68 import org.opengrok.indexer.index.IndexerParallelizer; 69 import org.opengrok.indexer.logger.LoggerFactory; 70 import org.opengrok.indexer.util.CloseableReentrantReadWriteLock; 71 import org.opengrok.indexer.util.CtagsUtil; 72 import org.opengrok.indexer.util.ForbiddenSymlinkException; 73 import org.opengrok.indexer.util.LazilyInstantiate; 74 import org.opengrok.indexer.util.PathUtils; 75 import org.opengrok.indexer.util.ResourceLock; 76 import org.opengrok.indexer.web.Prefix; 77 import org.opengrok.indexer.web.Statistics; 78 import org.opengrok.indexer.web.Util; 79 import org.opengrok.indexer.web.messages.Message; 80 import org.opengrok.indexer.web.messages.MessagesContainer; 81 import org.opengrok.indexer.web.messages.MessagesContainer.AcceptedMessage; 82 83 /** 84 * The RuntimeEnvironment class is used as a placeholder for the current 85 * configuration this execution context (classloader) is using. 86 */ 87 public final class RuntimeEnvironment { 88 89 private static final Logger LOGGER = LoggerFactory.getLogger(RuntimeEnvironment.class); 90 91 private static final String URL_PREFIX = "/source" + Prefix.SEARCH_R + "?"; 92 93 private Configuration configuration; 94 private final CloseableReentrantReadWriteLock configLock; 95 private final LazilyInstantiate<IndexerParallelizer> lzIndexerParallelizer; 96 private final LazilyInstantiate<ExecutorService> lzSearchExecutor; 97 private final LazilyInstantiate<ExecutorService> lzRevisionExecutor; 98 private static final RuntimeEnvironment instance = new RuntimeEnvironment(); 99 100 private final Map<Project, List<RepositoryInfo>> repository_map = new ConcurrentHashMap<>(); 101 private final Map<String, SearcherManager> searcherManagerMap = new ConcurrentHashMap<>(); 102 103 private String configURI; 104 private Statistics statistics = new Statistics(); 105 IncludeFiles includeFiles = new IncludeFiles(); 106 private final MessagesContainer messagesContainer = new MessagesContainer(); 107 108 private static final IndexTimestamp indexTime = new IndexTimestamp(); 109 110 /** 111 * Stores a transient value when 112 * {@link #setCtags(java.lang.String)} is called -- i.e. the 113 * value is not mediated to {@link Configuration}. 114 */ 115 private String ctags; 116 /** 117 * Stores a transient value when 118 * {@link #setMandoc(java.lang.String)} is called -- i.e. the 119 * value is not mediated to {@link Configuration}. 120 */ 121 private String mandoc; 122 123 private transient File dtagsEftar = null; 124 125 private transient volatile Boolean ctagsFound; 126 private final transient Set<String> ctagsLanguages = new HashSet<>(); 127 128 public WatchDogService watchDog; 129 130 /** 131 * Creates a new instance of RuntimeEnvironment. Private to ensure a 132 * singleton anti-pattern. 133 */ RuntimeEnvironment()134 private RuntimeEnvironment() { 135 configuration = new Configuration(); 136 configLock = new CloseableReentrantReadWriteLock(); 137 watchDog = new WatchDogService(); 138 lzIndexerParallelizer = LazilyInstantiate.using(() -> 139 new IndexerParallelizer(this)); 140 lzSearchExecutor = LazilyInstantiate.using(() -> newSearchExecutor()); 141 lzRevisionExecutor = LazilyInstantiate.using(() -> newRevisionExecutor()); 142 } 143 144 // Instance of authorization framework and its lock. 145 private AuthorizationFramework authFramework; 146 private final Object authFrameworkLock = new Object(); 147 148 /** Gets the thread pool used for multi-project searches. */ getSearchExecutor()149 public ExecutorService getSearchExecutor() { 150 return lzSearchExecutor.get(); 151 } 152 newSearchExecutor()153 private ExecutorService newSearchExecutor() { 154 return Executors.newFixedThreadPool( 155 this.getMaxSearchThreadCount(), 156 new ThreadFactory() { 157 @Override 158 public Thread newThread(Runnable runnable) { 159 Thread thread = Executors.defaultThreadFactory().newThread(runnable); 160 thread.setName("search-" + thread.getId()); 161 return thread; 162 } 163 }); 164 } 165 166 public ExecutorService getRevisionExecutor() { 167 return lzRevisionExecutor.get(); 168 } 169 170 private ExecutorService newRevisionExecutor() { 171 return Executors.newFixedThreadPool(this.getMaxRevisionThreadCount(), 172 new NamedThreadFactory("get-revision")); 173 } 174 175 public void shutdownRevisionExecutor() throws InterruptedException { 176 getRevisionExecutor().shutdownNow(); 177 getRevisionExecutor().awaitTermination(getCommandTimeout(), TimeUnit.SECONDS); 178 } 179 180 /** 181 * Get the one and only instance of the RuntimeEnvironment. 182 * 183 * @return the one and only instance of the RuntimeEnvironment 184 */ 185 public static RuntimeEnvironment getInstance() { 186 return instance; 187 } 188 189 public IndexerParallelizer getIndexerParallelizer() { 190 return lzIndexerParallelizer.get(); 191 } 192 193 private String getCanonicalPath(String s) { 194 if (s == null) { 195 return null; 196 } 197 try { 198 File file = new File(s); 199 if (!file.exists()) { 200 return s; 201 } 202 return file.getCanonicalPath(); 203 } catch (IOException ex) { 204 LOGGER.log(Level.SEVERE, "Failed to get canonical path", ex); 205 return s; 206 } 207 } 208 209 public int getScanningDepth() { 210 return syncReadConfiguration(Configuration::getScanningDepth); 211 } 212 213 public void setScanningDepth(int scanningDepth) { 214 syncWriteConfiguration(scanningDepth, Configuration::setScanningDepth); 215 } 216 217 public int getCommandTimeout() { 218 return syncReadConfiguration(Configuration::getCommandTimeout); 219 } 220 221 public void setCommandTimeout(int commandTimeout) { 222 syncWriteConfiguration(commandTimeout, Configuration::setCommandTimeout); 223 } 224 225 public int getInteractiveCommandTimeout() { 226 return syncReadConfiguration(Configuration::getInteractiveCommandTimeout); 227 } 228 229 public void setInteractiveCommandTimeout(int interactiveCommandTimeout) { 230 syncWriteConfiguration(interactiveCommandTimeout, 231 Configuration::setInteractiveCommandTimeout); 232 } 233 234 public long getCtagsTimeout() { 235 return syncReadConfiguration(Configuration::getCtagsTimeout); 236 } 237 238 public void setCtagsTimeout(long ctagsTimeout) { 239 syncWriteConfiguration(ctagsTimeout, Configuration::setCtagsTimeout); 240 } 241 242 public Statistics getStatistics() { 243 return statistics; 244 } 245 246 public void setStatistics(Statistics statistics) { 247 this.statistics = statistics; 248 } 249 250 public void setLastEditedDisplayMode(boolean lastEditedDisplayMode) { 251 syncWriteConfiguration(lastEditedDisplayMode, Configuration::setLastEditedDisplayMode); 252 } 253 254 public boolean isLastEditedDisplayMode() { 255 return syncReadConfiguration(Configuration::isLastEditedDisplayMode); 256 } 257 258 /** 259 * Get the path to the where the web application includes are stored. 260 * 261 * @return the path to the web application include files 262 */ 263 public String getIncludeRootPath() { 264 return syncReadConfiguration(Configuration::getIncludeRoot); 265 } 266 267 /** 268 * Set include root path. 269 * @param includeRoot path 270 */ 271 public void setIncludeRoot(String includeRoot) { 272 syncWriteConfiguration(getCanonicalPath(includeRoot), Configuration::setIncludeRoot); 273 } 274 275 /** 276 * Get the path to the where the index database is stored. 277 * 278 * @return the path to the index database 279 */ 280 public String getDataRootPath() { 281 return syncReadConfiguration(Configuration::getDataRoot); 282 } 283 284 /** 285 * Get a file representing the index database. 286 * 287 * @return the index database 288 */ 289 public File getDataRootFile() { 290 File ret = null; 291 String file = getDataRootPath(); 292 if (file != null) { 293 ret = new File(file); 294 } 295 296 return ret; 297 } 298 299 /** 300 * Set the path to where the index database is stored. 301 * 302 * @param dataRoot the index database 303 */ 304 public void setDataRoot(String dataRoot) { 305 syncWriteConfiguration(getCanonicalPath(dataRoot), Configuration::setDataRoot); 306 } 307 308 /** 309 * Get the path to where the sources are located. 310 * 311 * @return path to where the sources are located 312 */ 313 public String getSourceRootPath() { 314 return syncReadConfiguration(Configuration::getSourceRoot); 315 } 316 317 /** 318 * Get a file representing the directory where the sources are located. 319 * 320 * @return A file representing the directory where the sources are located 321 */ 322 public File getSourceRootFile() { 323 File ret = null; 324 String file = getSourceRootPath(); 325 if (file != null) { 326 ret = new File(file); 327 } 328 329 return ret; 330 } 331 332 /** 333 * Specify the source root. 334 * 335 * @param sourceRoot the location of the sources 336 */ 337 public void setSourceRoot(String sourceRoot) { 338 syncWriteConfiguration(getCanonicalPath(sourceRoot), Configuration::setSourceRoot); 339 } 340 341 /** 342 * Returns a path relative to source root. This would just be a simple 343 * substring operation, except we need to support symlinks outside the 344 * source root. 345 * 346 * @param file A file to resolve 347 * @return Path relative to source root 348 * @throws IOException If an IO error occurs 349 * @throws FileNotFoundException if the file is not relative to source root 350 * or if {@code sourceRoot} is not defined 351 * @throws ForbiddenSymlinkException if symbolic-link checking encounters 352 * an ineligible link 353 */ 354 public String getPathRelativeToSourceRoot(File file) 355 throws IOException, ForbiddenSymlinkException { 356 String sourceRoot = getSourceRootPath(); 357 if (sourceRoot == null) { 358 throw new FileNotFoundException("sourceRoot is not defined"); 359 } 360 361 String maybeRelPath = PathUtils.getRelativeToCanonical(file.getPath(), 362 sourceRoot, getAllowedSymlinks(), getCanonicalRoots()); 363 File maybeRelFile = new File(maybeRelPath); 364 if (!maybeRelFile.isAbsolute()) { 365 /* 366 * N.b. OpenGrok has a weird convention that source-root "relative" 367 * paths must start with a '/' as they are elsewhere directly 368 * appended to getSourceRootPath() and also stored as such. 369 */ 370 maybeRelPath = File.separator + maybeRelPath; 371 return maybeRelPath; 372 } 373 374 throw new FileNotFoundException("Failed to resolve [" + file.getPath() 375 + "] relative to source root [" + sourceRoot + "]"); 376 } 377 378 /** 379 * Do we have any projects ? 380 * 381 * @return true if we have projects 382 */ 383 public boolean hasProjects() { 384 return (this.isProjectsEnabled() && getProjects().size() > 0); 385 } 386 387 /** 388 * Get list of projects. 389 * 390 * @return a list containing all of the projects 391 */ 392 public List<Project> getProjectList() { 393 return new ArrayList<>(getProjects().values()); 394 } 395 396 /** 397 * Get project map. 398 * 399 * @return a Map with all of the projects 400 */ 401 public Map<String, Project> getProjects() { 402 return syncReadConfiguration(Configuration::getProjects); 403 } 404 405 /** 406 * Get names of all projects. 407 * 408 * @return a list containing names of all projects. 409 */ 410 public List<String> getProjectNames() { 411 return getProjectList().stream().map(Project::getName).collect(Collectors.toList()); 412 } 413 414 /** 415 * Set the list of the projects. 416 * 417 * @param projects the map of projects to use 418 */ 419 public void setProjects(Map<String, Project> projects) { 420 syncWriteConfiguration(projects, (c, p) -> { 421 if (p != null) { 422 populateGroups(getGroups(), new TreeSet<>(p.values())); 423 } 424 c.setProjects(p); 425 }); 426 } 427 428 /** 429 * Do we have groups? 430 * 431 * @return true if we have groups 432 */ 433 public boolean hasGroups() { 434 return (getGroups() != null && !getGroups().isEmpty()); 435 } 436 437 /** 438 * Get all of the groups. 439 * 440 * @return a set containing all of the groups (may be null) 441 */ 442 public Set<Group> getGroups() { 443 return syncReadConfiguration(Configuration::getGroups); 444 } 445 446 /** 447 * Set the list of the groups. 448 * 449 * @param groups the set of groups to use 450 */ 451 public void setGroups(Set<Group> groups) { 452 syncWriteConfiguration(groups, (c, g) -> { 453 populateGroups(g, new TreeSet<>(getProjects().values())); 454 c.setGroups(g); 455 }); 456 } 457 458 /** 459 * Returns constructed project - repositories map. 460 * 461 * @return the map 462 * @see #generateProjectRepositoriesMap 463 */ 464 public Map<Project, List<RepositoryInfo>> getProjectRepositoriesMap() { 465 return repository_map; 466 } 467 468 /** 469 * Gets a static placeholder for the web application context name that is 470 * translated to the true servlet {@code contextPath} on demand. 471 * @return {@code "/source"} + {@link Prefix#SEARCH_R} + {@code "?"} 472 */ 473 public String getUrlPrefix() { 474 return URL_PREFIX; 475 } 476 477 /** 478 * Gets the name of the ctags program to use: either the last value passed 479 * successfully to {@link #setCtags(java.lang.String)}, or 480 * {@link Configuration#getCtags()}, or the system property for 481 * {@code "org.opengrok.indexer.analysis.Ctags"}, or "ctags" as a 482 * default. 483 * @return a defined value 484 */ 485 public String getCtags() { 486 if (ctags != null) { 487 return ctags; 488 } 489 490 String value = syncReadConfiguration(Configuration::getCtags); 491 return value != null ? value : 492 System.getProperty(CtagsUtil.SYSTEM_CTAGS_PROPERTY, "ctags"); 493 } 494 495 /** 496 * Sets the name of the ctags program to use, or resets to use the fallbacks 497 * documented for {@link #getCtags()}. 498 * <p> 499 * N.b. the value is not mediated to {@link Configuration}. 500 * 501 * @param ctags a defined value or {@code null} to reset to use the 502 * {@link Configuration#getCtags()} fallbacks 503 * @see #getCtags() 504 */ 505 public void setCtags(String ctags) { 506 this.ctags = ctags; 507 } 508 509 /** 510 * Gets the name of the mandoc program to use: either the last value passed 511 * successfully to {@link #setMandoc(java.lang.String)}, or 512 * {@link Configuration#getMandoc()}, or the system property for 513 * {@code "org.opengrok.indexer.analysis.Mandoc"}, or {@code null} as a 514 * default. 515 * @return a defined instance or {@code null} 516 */ 517 public String getMandoc() { 518 if (mandoc != null) { 519 return mandoc; 520 } 521 522 String value = syncReadConfiguration(Configuration::getMandoc); 523 return value != null ? value : 524 System.getProperty("org.opengrok.indexer.analysis.Mandoc"); 525 } 526 527 /** 528 * Sets the name of the mandoc program to use, or resets to use the 529 * fallbacks documented for {@link #getMandoc()}. 530 * <p> 531 * N.b. the value is not mediated to {@link Configuration}. 532 * 533 * @param value a defined value or {@code null} to reset to use the 534 * {@link Configuration#getMandoc()} fallbacks 535 * @see #getMandoc() 536 */ 537 public void setMandoc(String value) { 538 this.mandoc = value; 539 } 540 541 public int getCachePages() { 542 return syncReadConfiguration(Configuration::getCachePages); 543 } 544 545 public void setCachePages(int cachePages) { 546 syncWriteConfiguration(cachePages, Configuration::setCachePages); 547 } 548 549 public int getHitsPerPage() { 550 return syncReadConfiguration(Configuration::getHitsPerPage); 551 } 552 553 public void setHitsPerPage(int hitsPerPage) { 554 syncWriteConfiguration(hitsPerPage, Configuration::setHitsPerPage); 555 } 556 557 /** 558 * Validate that there is a Universal ctags program. 559 * 560 * @return true if success, false otherwise 561 */ 562 public boolean validateUniversalCtags() { 563 if (ctagsFound == null) { 564 String ctagsBinary = getCtags(); 565 try (ResourceLock resourceLock = configLock.writeLockAsResource()) { 566 //noinspection ConstantConditions to avoid warning of no reference to auto-closeable 567 assert resourceLock != null; 568 if (ctagsFound == null) { 569 ctagsFound = CtagsUtil.validate(ctagsBinary); 570 if (ctagsFound) { 571 List<String> languages = CtagsUtil.getLanguages(ctagsBinary); 572 if (languages != null) { 573 ctagsLanguages.addAll(languages); 574 } 575 } 576 } 577 } 578 } 579 return ctagsFound; 580 } 581 582 /** 583 * Gets the base set of supported Ctags languages. 584 * @return a defined set which may be empty if 585 * {@link #validateUniversalCtags()} has not yet been called or if the call 586 * fails 587 */ 588 public Set<String> getCtagsLanguages() { 589 return Collections.unmodifiableSet(ctagsLanguages); 590 } 591 592 /** 593 * Get the max time a SCM operation may use to avoid being cached. 594 * 595 * @return the max time 596 */ 597 public int getHistoryReaderTimeLimit() { 598 return syncReadConfiguration(Configuration::getHistoryCacheTime); 599 } 600 601 /** 602 * Specify the maximum time a SCM operation should take before it will be 603 * cached (in ms). 604 * 605 * @param historyCacheTime the max time in ms before it is cached 606 */ 607 public void setHistoryReaderTimeLimit(int historyCacheTime) { 608 syncWriteConfiguration(historyCacheTime, Configuration::setHistoryCacheTime); 609 } 610 611 /** 612 * Is history cache currently enabled? 613 * 614 * @return true if history cache is enabled 615 */ 616 public boolean useHistoryCache() { 617 return syncReadConfiguration(Configuration::isHistoryCache); 618 } 619 620 /** 621 * Specify if we should use history cache or not. 622 * 623 * @param useHistoryCache set false if you do not want to use history cache 624 */ 625 public void setUseHistoryCache(boolean useHistoryCache) { 626 syncWriteConfiguration(useHistoryCache, Configuration::setHistoryCache); 627 } 628 629 /** 630 * Should we generate HTML or not during the indexing phase. 631 * 632 * @return true if HTML should be generated during the indexing phase 633 */ 634 public boolean isGenerateHtml() { 635 return syncReadConfiguration(Configuration::isGenerateHtml); 636 } 637 638 /** 639 * Specify if we should generate HTML or not during the indexing phase. 640 * 641 * @param generateHtml set this to true to pregenerate HTML 642 */ 643 public void setGenerateHtml(boolean generateHtml) { 644 syncWriteConfiguration(generateHtml, Configuration::setGenerateHtml); 645 } 646 647 /** 648 * Set if we should compress the xref files or not. 649 * 650 * @param compressXref set to true if the generated html files should be 651 * compressed 652 */ 653 public void setCompressXref(boolean compressXref) { 654 syncWriteConfiguration(compressXref, Configuration::setCompressXref); 655 } 656 657 /** 658 * Are we using compressed HTML files? 659 * 660 * @return {@code true} if the html-files should be compressed. 661 */ 662 public boolean isCompressXref() { 663 return syncReadConfiguration(Configuration::isCompressXref); 664 } 665 666 public boolean isQuickContextScan() { 667 return syncReadConfiguration(Configuration::isQuickContextScan); 668 } 669 670 public void setQuickContextScan(boolean quickContextScan) { 671 syncWriteConfiguration(quickContextScan, Configuration::setQuickContextScan); 672 } 673 674 public List<RepositoryInfo> getRepositories() { 675 return syncReadConfiguration(Configuration::getRepositories); 676 } 677 678 /** 679 * Set the list of repositories. 680 * 681 * @param repositories the repositories to use 682 */ 683 public void setRepositories(List<RepositoryInfo> repositories) { 684 syncWriteConfiguration(repositories, Configuration::setRepositories); 685 } 686 687 public void removeRepositories() { 688 syncWriteConfiguration(null, Configuration::setRepositories); 689 } 690 691 /** 692 * Search through the directory for repositories and use the result to replace 693 * the lists of repositories in both RuntimeEnvironment/Configuration and HistoryGuru. 694 * 695 * @param dir the directories to start the search in 696 */ 697 public void setRepositories(String... dir) { 698 List<RepositoryInfo> repos = new ArrayList<>(HistoryGuru.getInstance(). 699 addRepositories(Arrays.stream(dir).map(File::new).toArray(File[]::new), 700 getIgnoredNames())); 701 setRepositories(repos); 702 } 703 704 /** 705 * Add repositories to the list. 706 * @param repositories list of repositories 707 */ 708 public void addRepositories(List<RepositoryInfo> repositories) { 709 syncWriteConfiguration(repositories, Configuration::addRepositories); 710 } 711 712 /** 713 * Set the specified projects as default in the configuration. 714 * This method should be called only after projects were discovered and became part of the configuration, 715 * i.e. after {@link org.opengrok.indexer.index.Indexer#prepareIndexer} was called. 716 * 717 * @param defaultProjects The default project to use 718 * @see #setDefaultProjects 719 */ 720 public void setDefaultProjectsFromNames(Set<String> defaultProjects) { 721 if (defaultProjects != null && !defaultProjects.isEmpty()) { 722 Set<Project> projects = new TreeSet<>(); 723 for (String projectPath : defaultProjects) { 724 if (projectPath.equals("__all__")) { 725 projects.addAll(getProjects().values()); 726 break; 727 } 728 for (Project p : getProjectList()) { 729 if (p.getPath().equals(Util.fixPathIfWindows(projectPath))) { 730 projects.add(p); 731 break; 732 } 733 } 734 } 735 if (!projects.isEmpty()) { 736 setDefaultProjects(projects); 737 } 738 } 739 } 740 741 /** 742 * Set the projects that are specified to be the default projects to use. 743 * The default projects are the projects you will search (from the web 744 * application) if the page request didn't contain the cookie.. 745 * 746 * @param defaultProjects The default project to use 747 */ 748 public void setDefaultProjects(Set<Project> defaultProjects) { 749 syncWriteConfiguration(defaultProjects, Configuration::setDefaultProjects); 750 } 751 752 /** 753 * Get the projects that are specified to be the default projects to use. 754 * The default projects are the projects you will search (from the web 755 * application) if the page request didn't contain the cookie.. 756 * 757 * @return the default projects (may be null if not specified) 758 */ 759 public Set<Project> getDefaultProjects() { 760 Set<Project> projects = syncReadConfiguration(Configuration::getDefaultProjects); 761 if (projects == null) { 762 return null; 763 } 764 return Collections.unmodifiableSet(projects); 765 } 766 767 /** 768 * 769 * @return at what size (in MB) we should flush the buffer 770 */ 771 public double getRamBufferSize() { 772 return syncReadConfiguration(Configuration::getRamBufferSize); 773 } 774 775 /** 776 * Set the size of buffer which will determine when the docs are flushed to 777 * disk. Specify size in MB please. 16MB is default note that this is per 778 * thread (lucene uses 8 threads by default in 4.x) 779 * 780 * @param ramBufferSize the size(in MB) when we should flush the docs 781 */ 782 public void setRamBufferSize(double ramBufferSize) { 783 syncWriteConfiguration(ramBufferSize, Configuration::setRamBufferSize); 784 } 785 786 public void setPluginDirectory(String pluginDirectory) { 787 syncWriteConfiguration(pluginDirectory, Configuration::setPluginDirectory); 788 } 789 790 public String getPluginDirectory() { 791 return syncReadConfiguration(Configuration::getPluginDirectory); 792 } 793 794 public boolean isAuthorizationWatchdog() { 795 return syncReadConfiguration(Configuration::isAuthorizationWatchdogEnabled); 796 } 797 798 public void setAuthorizationWatchdog(boolean authorizationWatchdogEnabled) { 799 syncWriteConfiguration(authorizationWatchdogEnabled, 800 Configuration::setAuthorizationWatchdogEnabled); 801 } 802 803 public AuthorizationStack getPluginStack() { 804 return syncReadConfiguration(Configuration::getPluginStack); 805 } 806 807 public void setPluginStack(AuthorizationStack pluginStack) { 808 syncWriteConfiguration(pluginStack, Configuration::setPluginStack); 809 } 810 811 /** 812 * Is the progress print flag turned on? 813 * 814 * @return true if we can print per project progress % 815 */ 816 public boolean isPrintProgress() { 817 return syncReadConfiguration(Configuration::isPrintProgress); 818 } 819 820 /** 821 * Set the printing of progress % flag (user convenience). 822 * 823 * @param printProgress new value 824 */ 825 public void setPrintProgress(boolean printProgress) { 826 syncWriteConfiguration(printProgress, Configuration::setPrintProgress); 827 } 828 829 /** 830 * Specify if a search may start with a wildcard. Note that queries that 831 * start with a wildcard will give a significant impact on the search 832 * performance. 833 * 834 * @param allowLeadingWildcard set to true to activate (disabled by default) 835 */ 836 public void setAllowLeadingWildcard(boolean allowLeadingWildcard) { 837 syncWriteConfiguration(allowLeadingWildcard, Configuration::setAllowLeadingWildcard); 838 } 839 840 /** 841 * Is leading wildcards allowed? 842 * 843 * @return true if a search may start with a wildcard 844 */ 845 public boolean isAllowLeadingWildcard() { 846 return syncReadConfiguration(Configuration::isAllowLeadingWildcard); 847 } 848 849 public IgnoredNames getIgnoredNames() { 850 return syncReadConfiguration(Configuration::getIgnoredNames); 851 } 852 853 public void setIgnoredNames(IgnoredNames ignoredNames) { 854 syncWriteConfiguration(ignoredNames, Configuration::setIgnoredNames); 855 } 856 857 public Filter getIncludedNames() { 858 return syncReadConfiguration(Configuration::getIncludedNames); 859 } 860 861 public void setIncludedNames(Filter includedNames) { 862 syncWriteConfiguration(includedNames, Configuration::setIncludedNames); 863 } 864 865 /** 866 * Returns the user page for the history listing. 867 * 868 * @return the URL string fragment preceeding the username 869 */ 870 public String getUserPage() { 871 return syncReadConfiguration(Configuration::getUserPage); 872 } 873 874 /** 875 * Get the client command to use to access the repository for the given 876 * fully qualified classname. 877 * 878 * @param clazzName name of the targeting class 879 * @return {@code null} if not yet set, the client command otherwise. 880 */ 881 public String getRepoCmd(String clazzName) { 882 return syncReadConfiguration(c -> c.getRepoCmd(clazzName)); 883 } 884 885 /** 886 * Set the client command to use to access the repository for the given 887 * fully qualified classname. 888 * 889 * @param clazzName name of the targeting class. If {@code null} this method 890 * does nothing. 891 * @param cmd the client command to use. If {@code null} the corresponding 892 * entry for the given clazzName get removed. 893 * @return the client command previously set, which might be {@code null}. 894 */ 895 public String setRepoCmd(String clazzName, String cmd) { 896 syncWriteConfiguration(null, (c, ignored) -> c.setRepoCmd(clazzName, cmd)); 897 return cmd; 898 } 899 900 public void setRepoCmds(Map<String, String> cmds) { 901 syncWriteConfiguration(cmds, Configuration::setCmds); 902 } 903 904 /** 905 * Sets the user page for the history listing. 906 * 907 * @param userPage the URL fragment preceeding the username from history 908 */ 909 public void setUserPage(String userPage) { 910 syncWriteConfiguration(userPage, Configuration::setUserPage); 911 } 912 913 /** 914 * Returns the user page suffix for the history listing. 915 * 916 * @return the URL string fragment following the username 917 */ 918 public String getUserPageSuffix() { 919 return syncReadConfiguration(Configuration::getUserPageSuffix); 920 } 921 922 /** 923 * Sets the user page suffix for the history listing. 924 * 925 * @param userPageSuffix the URL fragment following the username from 926 * history 927 */ 928 public void setUserPageSuffix(String userPageSuffix) { 929 syncWriteConfiguration(userPageSuffix, Configuration::setUserPageSuffix); 930 } 931 932 /** 933 * Returns the bug page for the history listing. 934 * 935 * @return the URL string fragment preceeding the bug ID 936 */ 937 public String getBugPage() { 938 return syncReadConfiguration(Configuration::getBugPage); 939 } 940 941 /** 942 * Sets the bug page for the history listing. 943 * 944 * @param bugPage the URL fragment preceeding the bug ID 945 */ 946 public void setBugPage(String bugPage) { 947 syncWriteConfiguration(bugPage, Configuration::setBugPage); 948 } 949 950 /** 951 * Returns the bug regex for the history listing. 952 * 953 * @return the regex that is looked for in history comments 954 */ 955 public String getBugPattern() { 956 return syncReadConfiguration(Configuration::getBugPattern); 957 } 958 959 /** 960 * Sets the bug regex for the history listing. 961 * 962 * @param bugPattern the regex to search history comments 963 */ 964 public void setBugPattern(String bugPattern) { 965 syncWriteConfiguration(bugPattern, Configuration::setBugPattern); 966 } 967 968 /** 969 * Returns the review(ARC) page for the history listing. 970 * 971 * @return the URL string fragment preceeding the review page ID 972 */ 973 public String getReviewPage() { 974 return syncReadConfiguration(Configuration::getReviewPage); 975 } 976 977 /** 978 * Sets the review(ARC) page for the history listing. 979 * 980 * @param reviewPage the URL fragment preceeding the review page ID 981 */ 982 public void setReviewPage(String reviewPage) { 983 syncWriteConfiguration(reviewPage, Configuration::setReviewPage); 984 } 985 986 /** 987 * Returns the review(ARC) regex for the history listing. 988 * 989 * @return the regex that is looked for in history comments 990 */ 991 public String getReviewPattern() { 992 return syncReadConfiguration(Configuration::getReviewPattern); 993 } 994 995 /** 996 * Sets the review(ARC) regex for the history listing. 997 * 998 * @param reviewPattern the regex to search history comments 999 */ 1000 public void setReviewPattern(String reviewPattern) { 1001 syncWriteConfiguration(reviewPattern, Configuration::setReviewPattern); 1002 } 1003 1004 public String getWebappLAF() { 1005 return syncReadConfiguration(Configuration::getWebappLAF); 1006 } 1007 1008 public void setWebappLAF(String webappLAF) { 1009 syncWriteConfiguration(webappLAF, Configuration::setWebappLAF); 1010 } 1011 1012 /** 1013 * Gets a value indicating if the web app should run ctags as necessary. 1014 * @return the value of {@link Configuration#isWebappCtags()} 1015 */ 1016 public boolean isWebappCtags() { 1017 return syncReadConfiguration(Configuration::isWebappCtags); 1018 } 1019 1020 public Configuration.RemoteSCM getRemoteScmSupported() { 1021 return syncReadConfiguration(Configuration::getRemoteScmSupported); 1022 } 1023 1024 public void setRemoteScmSupported(Configuration.RemoteSCM remoteScmSupported) { 1025 syncWriteConfiguration(remoteScmSupported, Configuration::setRemoteScmSupported); 1026 } 1027 1028 public boolean isOptimizeDatabase() { 1029 return syncReadConfiguration(Configuration::isOptimizeDatabase); 1030 } 1031 1032 public void setOptimizeDatabase(boolean optimizeDatabase) { 1033 syncWriteConfiguration(optimizeDatabase, Configuration::setOptimizeDatabase); 1034 } 1035 1036 public LuceneLockName getLuceneLocking() { 1037 return syncReadConfiguration(Configuration::getLuceneLocking); 1038 } 1039 1040 public boolean isIndexVersionedFilesOnly() { 1041 return syncReadConfiguration(Configuration::isIndexVersionedFilesOnly); 1042 } 1043 1044 public void setIndexVersionedFilesOnly(boolean indexVersionedFilesOnly) { 1045 syncWriteConfiguration(indexVersionedFilesOnly, Configuration::setIndexVersionedFilesOnly); 1046 } 1047 1048 /** 1049 * Gets the value of {@link Configuration#getIndexingParallelism()} -- or 1050 * if zero, then as a default gets the number of available processors. 1051 * @return a natural number >= 1 1052 */ 1053 public int getIndexingParallelism() { 1054 int parallelism = syncReadConfiguration(Configuration::getIndexingParallelism); 1055 return parallelism < 1 ? Runtime.getRuntime().availableProcessors() : 1056 parallelism; 1057 } 1058 1059 /** 1060 * Gets the value of {@link Configuration#getHistoryParallelism()} -- or 1061 * if zero, then as a default gets the number of available processors. 1062 * @return a natural number >= 1 1063 */ 1064 public int getHistoryParallelism() { 1065 int parallelism = syncReadConfiguration(Configuration::getHistoryParallelism); 1066 return parallelism < 1 ? Runtime.getRuntime().availableProcessors() : 1067 parallelism; 1068 } 1069 1070 /** 1071 * Gets the value of {@link Configuration#getHistoryRenamedParallelism()} -- or 1072 * if zero, then as a default gets the number of available processors. 1073 * @return a natural number >= 1 1074 */ 1075 public int getHistoryRenamedParallelism() { 1076 int parallelism = syncReadConfiguration(Configuration::getHistoryRenamedParallelism); 1077 return parallelism < 1 ? Runtime.getRuntime().availableProcessors() : 1078 parallelism; 1079 } 1080 1081 public boolean isTagsEnabled() { 1082 return syncReadConfiguration(Configuration::isTagsEnabled); 1083 } 1084 1085 public void setTagsEnabled(boolean tagsEnabled) { 1086 syncWriteConfiguration(tagsEnabled, Configuration::setTagsEnabled); 1087 } 1088 1089 public boolean isScopesEnabled() { 1090 return syncReadConfiguration(Configuration::isScopesEnabled); 1091 } 1092 1093 public void setScopesEnabled(boolean scopesEnabled) { 1094 syncWriteConfiguration(scopesEnabled, Configuration::setScopesEnabled); 1095 } 1096 1097 public boolean isProjectsEnabled() { 1098 return syncReadConfiguration(Configuration::isProjectsEnabled); 1099 } 1100 1101 public void setProjectsEnabled(boolean projectsEnabled) { 1102 syncWriteConfiguration(projectsEnabled, Configuration::setProjectsEnabled); 1103 } 1104 1105 public boolean isFoldingEnabled() { 1106 return syncReadConfiguration(Configuration::isFoldingEnabled); 1107 } 1108 1109 public void setFoldingEnabled(boolean foldingEnabled) { 1110 syncWriteConfiguration(foldingEnabled, Configuration::setFoldingEnabled); 1111 } 1112 1113 public Date getDateForLastIndexRun() { 1114 return indexTime.getDateForLastIndexRun(); 1115 } 1116 1117 public String getCTagsExtraOptionsFile() { 1118 return syncReadConfiguration(Configuration::getCTagsExtraOptionsFile); 1119 } 1120 1121 public void setCTagsExtraOptionsFile(String ctagsExtraOptionsFile) { 1122 syncWriteConfiguration(ctagsExtraOptionsFile, Configuration::setCTagsExtraOptionsFile); 1123 } 1124 1125 public Set<String> getAllowedSymlinks() { 1126 return syncReadConfiguration(Configuration::getAllowedSymlinks); 1127 } 1128 1129 public void setAllowedSymlinks(Set<String> allowedSymlinks) { 1130 syncWriteConfiguration(allowedSymlinks, Configuration::setAllowedSymlinks); 1131 } 1132 1133 public Set<String> getCanonicalRoots() { 1134 return syncReadConfiguration(Configuration::getCanonicalRoots); 1135 } 1136 1137 public void setCanonicalRoots(Set<String> canonicalRoots) { 1138 syncWriteConfiguration(canonicalRoots, Configuration::setCanonicalRoots); 1139 } 1140 1141 /** 1142 * Return whether e-mail addresses should be obfuscated in the xref. 1143 * @return if we obfuscate emails 1144 */ 1145 public boolean isObfuscatingEMailAddresses() { 1146 return syncReadConfiguration(Configuration::isObfuscatingEMailAddresses); 1147 } 1148 1149 /** 1150 * Set whether e-mail addresses should be obfuscated in the xref. 1151 * @param obfuscatingEMailAddresses should we obfuscate emails? 1152 */ 1153 public void setObfuscatingEMailAddresses(boolean obfuscatingEMailAddresses) { 1154 syncWriteConfiguration(obfuscatingEMailAddresses, 1155 Configuration::setObfuscatingEMailAddresses); 1156 } 1157 1158 /** 1159 * Should status.jsp print internal settings, like paths and database URLs? 1160 * 1161 * @return {@code true} if status.jsp should show the configuration, 1162 * {@code false} otherwise 1163 */ 1164 public boolean isChattyStatusPage() { 1165 return syncReadConfiguration(Configuration::isChattyStatusPage); 1166 } 1167 1168 /** 1169 * Set whether status.jsp should print internal settings. 1170 * 1171 * @param chattyStatusPage {@code true} if internal settings should be printed, 1172 * {@code false} otherwise 1173 */ 1174 public void setChattyStatusPage(boolean chattyStatusPage) { 1175 syncWriteConfiguration(chattyStatusPage, Configuration::setChattyStatusPage); 1176 } 1177 1178 public void setFetchHistoryWhenNotInCache(boolean fetchHistoryWhenNotInCache) { 1179 syncWriteConfiguration(fetchHistoryWhenNotInCache, 1180 Configuration::setFetchHistoryWhenNotInCache); 1181 } 1182 1183 public boolean isFetchHistoryWhenNotInCache() { 1184 return syncReadConfiguration(Configuration::isFetchHistoryWhenNotInCache); 1185 } 1186 1187 public boolean isHistoryCache() { 1188 return syncReadConfiguration(Configuration::isHistoryCache); 1189 } 1190 1191 public void setHandleHistoryOfRenamedFiles(boolean handleHistoryOfRenamedFiles) { 1192 syncWriteConfiguration(handleHistoryOfRenamedFiles, 1193 Configuration::setHandleHistoryOfRenamedFiles); 1194 } 1195 1196 public boolean isHandleHistoryOfRenamedFiles() { 1197 return syncReadConfiguration(Configuration::isHandleHistoryOfRenamedFiles); 1198 } 1199 1200 public void setNavigateWindowEnabled(boolean navigateWindowEnabled) { 1201 syncWriteConfiguration(navigateWindowEnabled, Configuration::setNavigateWindowEnabled); 1202 } 1203 1204 public boolean isNavigateWindowEnabled() { 1205 return syncReadConfiguration(Configuration::isNavigateWindowEnabled); 1206 } 1207 1208 public void setRevisionMessageCollapseThreshold(int revisionMessageCollapseThreshold) { 1209 syncWriteConfiguration(revisionMessageCollapseThreshold, 1210 Configuration::setRevisionMessageCollapseThreshold); 1211 } 1212 1213 public int getRevisionMessageCollapseThreshold() { 1214 return syncReadConfiguration(Configuration::getRevisionMessageCollapseThreshold); 1215 } 1216 1217 public void setMaxSearchThreadCount(int maxSearchThreadCount) { 1218 syncWriteConfiguration(maxSearchThreadCount, Configuration::setMaxSearchThreadCount); 1219 } 1220 1221 public int getMaxSearchThreadCount() { 1222 return syncReadConfiguration(Configuration::getMaxSearchThreadCount); 1223 } 1224 1225 public void setMaxRevisionThreadCount(int maxRevisionThreadCount) { 1226 syncWriteConfiguration(maxRevisionThreadCount, Configuration::setMaxRevisionThreadCount); 1227 } 1228 1229 public int getMaxRevisionThreadCount() { 1230 return syncReadConfiguration(Configuration::getMaxRevisionThreadCount); 1231 } 1232 1233 public int getCurrentIndexedCollapseThreshold() { 1234 return syncReadConfiguration(Configuration::getCurrentIndexedCollapseThreshold); 1235 } 1236 1237 public void setCurrentIndexedCollapseThreshold(int currentIndexedCollapseThreshold) { 1238 syncWriteConfiguration(currentIndexedCollapseThreshold, 1239 Configuration::setCurrentIndexedCollapseThreshold); 1240 } 1241 1242 public int getGroupsCollapseThreshold() { 1243 return syncReadConfiguration(Configuration::getGroupsCollapseThreshold); 1244 } 1245 1246 // The URI is not necessary to be present in the configuration 1247 // (so that when -U option of the indexer is omitted, the config will not 1248 // be sent to the webapp) so store it only in the RuntimeEnvironment. 1249 public void setConfigURI(String host) { 1250 configURI = host; 1251 } 1252 1253 public String getConfigURI() { 1254 return configURI; 1255 } 1256 1257 public boolean isHistoryEnabled() { 1258 return syncReadConfiguration(Configuration::isHistoryEnabled); 1259 } 1260 1261 public void setHistoryEnabled(boolean historyEnabled) { 1262 syncWriteConfiguration(historyEnabled, Configuration::setHistoryEnabled); 1263 } 1264 1265 public boolean getDisplayRepositories() { 1266 return syncReadConfiguration(Configuration::getDisplayRepositories); 1267 } 1268 1269 public void setDisplayRepositories(boolean displayRepositories) { 1270 syncWriteConfiguration(displayRepositories, Configuration::setDisplayRepositories); 1271 } 1272 1273 public boolean getListDirsFirst() { 1274 return syncReadConfiguration(Configuration::getListDirsFirst); 1275 } 1276 1277 public void setListDirsFirst(boolean listDirsFirst) { 1278 syncWriteConfiguration(listDirsFirst, Configuration::setListDirsFirst); 1279 } 1280 1281 public void setTabSize(int tabSize) { 1282 syncWriteConfiguration(tabSize, Configuration::setTabSize); 1283 } 1284 1285 public int getTabSize() { 1286 return syncReadConfiguration(Configuration::getTabSize); 1287 } 1288 1289 /** 1290 * Gets the total number of context lines per file to show. 1291 * @return a value greater than zero 1292 */ 1293 public short getContextLimit() { 1294 return syncReadConfiguration(Configuration::getContextLimit); 1295 } 1296 1297 /** 1298 * Gets the number of context lines to show before or after any match. 1299 * @return a value greater than or equal to zero 1300 */ 1301 public short getContextSurround() { 1302 return syncReadConfiguration(Configuration::getContextSurround); 1303 } 1304 1305 public Set<String> getDisabledRepositories() { 1306 return syncReadConfiguration(Configuration::getDisabledRepositories); 1307 } 1308 1309 public void setDisabledRepositories(Set<String> disabledRepositories) { 1310 syncWriteConfiguration(disabledRepositories, Configuration::setDisabledRepositories); 1311 } 1312 1313 /** 1314 * Read an configuration file and set it as the current configuration. 1315 * 1316 * @param file the file to read 1317 * @throws IOException if an error occurs 1318 */ 1319 public void readConfiguration(File file) throws IOException { 1320 // The following method handles the locking. 1321 setConfiguration(Configuration.read(file)); 1322 } 1323 1324 /** 1325 * Read configuration from a file and put it into effect. 1326 * @param file the file to read 1327 * @param interactive true if run in interactive mode 1328 * @throws IOException I/O 1329 */ 1330 public void readConfiguration(File file, boolean interactive) throws IOException { 1331 // The following method handles the locking. 1332 setConfiguration(Configuration.read(file), null, interactive); 1333 } 1334 1335 /** 1336 * Write the current configuration to a file. 1337 * 1338 * @param file the file to write the configuration into 1339 * @throws IOException if an error occurs 1340 */ 1341 public void writeConfiguration(File file) throws IOException { 1342 try (ResourceLock resourceLock = configLock.readLockAsResource()) { 1343 //noinspection ConstantConditions to avoid warning of no reference to auto-closeable 1344 assert resourceLock != null; 1345 configuration.write(file); 1346 } 1347 } 1348 1349 public String getConfigurationXML() { 1350 return syncReadConfiguration(Configuration::getXMLRepresentationAsString); 1351 } 1352 1353 /** 1354 * Write the current configuration to a socket. 1355 * 1356 * @param host the host address to receive the configuration 1357 * @throws IOException if an error occurs 1358 */ 1359 public void writeConfiguration(String host) throws IOException { 1360 String configXML = syncReadConfiguration(Configuration::getXMLRepresentationAsString); 1361 1362 Response r = ClientBuilder.newClient() 1363 .target(host) 1364 .path("api") 1365 .path("v1") 1366 .path("configuration") 1367 .queryParam("reindex", true) 1368 .request() 1369 .put(Entity.xml(configXML)); 1370 1371 if (r.getStatusInfo().getFamily() != Response.Status.Family.SUCCESSFUL) { 1372 throw new IOException(r.toString()); 1373 } 1374 } 1375 1376 /** 1377 * Send message to webapp to refresh SearcherManagers for given projects. 1378 * This is used for partial reindex. 1379 * 1380 * @param subFiles list of directories to refresh corresponding SearcherManagers 1381 * @param host the host address to receive the configuration 1382 */ 1383 public void signalTorefreshSearcherManagers(List<String> subFiles, String host) { 1384 // subFile entries start with path separator so get basename 1385 // to convert them to project names. 1386 1387 subFiles.stream().map(proj -> new File(proj).getName()).forEach(project -> { 1388 Response r = ClientBuilder.newClient() 1389 .target(host) 1390 .path("api") 1391 .path("v1") 1392 .path("system") 1393 .path("refresh") 1394 .request() 1395 .put(Entity.text(project)); 1396 1397 if (r.getStatusInfo().getFamily() != Response.Status.Family.SUCCESSFUL) { 1398 LOGGER.log(Level.WARNING, "Could not refresh search manager for {0}", project); 1399 } 1400 }); 1401 } 1402 1403 /** 1404 * Generate a TreeMap of projects with corresponding repository information. 1405 * <p> 1406 * Project with some repository information is considered as a repository 1407 * otherwise it is just a simple project. 1408 */ 1409 private void generateProjectRepositoriesMap() throws IOException { 1410 repository_map.clear(); 1411 for (RepositoryInfo r : getRepositories()) { 1412 Project proj; 1413 String repoPath; 1414 try { 1415 repoPath = getPathRelativeToSourceRoot(new File(r.getDirectoryName())); 1416 } catch (ForbiddenSymlinkException e) { 1417 LOGGER.log(Level.FINER, e.getMessage()); 1418 continue; 1419 } 1420 1421 if ((proj = Project.getProject(repoPath)) != null) { 1422 List<RepositoryInfo> values = repository_map.computeIfAbsent(proj, k -> new ArrayList<>()); 1423 // the map is held under the lock because the next call to 1424 // values.add(r) which should not be called from multiple threads at the same time 1425 values.add(r); 1426 } 1427 } 1428 } 1429 1430 /** 1431 * Classifies projects and puts them in their groups. 1432 * <p> 1433 * If any of the groups contain some projects or repositories already, 1434 * these get discarded. 1435 * 1436 * @param groups set of groups to be filled with matching projects 1437 * @param projects projects to classify 1438 */ 1439 public void populateGroups(Set<Group> groups, Set<Project> projects) { 1440 if (projects == null || groups == null) { 1441 return; 1442 } 1443 1444 // clear the groups first if they had something in them 1445 for (Group group : groups) { 1446 group.getRepositories().clear(); 1447 group.getProjects().clear(); 1448 } 1449 1450 // now fill the groups with appropriate projects 1451 for (Project project : projects) { 1452 // clear the project's groups 1453 project.getGroups().clear(); 1454 1455 // filter projects only to groups which match project's name 1456 Set<Group> copy = Group.matching(project, groups); 1457 1458 // add project to the groups 1459 for (Group group : copy) { 1460 if (repository_map.get(project) == null) { 1461 group.addProject(project); 1462 } else { 1463 group.addRepository(project); 1464 } 1465 project.addGroup(group); 1466 } 1467 } 1468 } 1469 1470 /** 1471 * Sets the configuration and performs necessary actions. 1472 * 1473 * Mainly it classifies the projects in their groups and generates project - 1474 * repositories map 1475 * 1476 * @param configuration what configuration to use 1477 */ 1478 public void setConfiguration(Configuration configuration) { 1479 setConfiguration(configuration, null, false); 1480 } 1481 1482 /** 1483 * Sets the configuration and performs necessary actions. 1484 * @param configuration new configuration 1485 * @param interactive true if in interactive mode 1486 */ 1487 public void setConfiguration(Configuration configuration, boolean interactive) { 1488 setConfiguration(configuration, null, interactive); 1489 } 1490 1491 /** 1492 * Sets the configuration and performs necessary actions. 1493 * 1494 * @param configuration new configuration 1495 * @param subFileList list of repositories 1496 * @param interactive true if in interactive mode 1497 */ 1498 public synchronized void setConfiguration(Configuration configuration, List<String> subFileList, boolean interactive) { 1499 try (ResourceLock resourceLock = configLock.writeLockAsResource()) { 1500 //noinspection ConstantConditions to avoid warning of no reference to auto-closeable 1501 assert resourceLock != null; 1502 this.configuration = configuration; 1503 } 1504 1505 // HistoryGuru constructor needs environment properties so no locking is done here. 1506 HistoryGuru histGuru = HistoryGuru.getInstance(); 1507 1508 // Set the working repositories in HistoryGuru. 1509 if (subFileList != null) { 1510 histGuru.invalidateRepositories( 1511 getRepositories(), subFileList, interactive); 1512 } else { 1513 histGuru.invalidateRepositories(getRepositories(), 1514 interactive); 1515 } 1516 1517 // The invalidation of repositories above might have excluded some 1518 // repositories in HistoryGuru so the configuration needs to reflect that. 1519 setRepositories(new ArrayList<>(histGuru.getRepositories())); 1520 1521 // generate repository map is dependent on getRepositories() 1522 try { 1523 generateProjectRepositoriesMap(); 1524 } catch (IOException ex) { 1525 LOGGER.log(Level.SEVERE, "Cannot generate project - repository map", ex); 1526 } 1527 1528 // populate groups is dependent on repositories map 1529 populateGroups(getGroups(), new TreeSet<>(getProjects().values())); 1530 1531 includeFiles.reloadIncludeFiles(); 1532 } 1533 1534 public IncludeFiles getIncludeFiles() { 1535 return includeFiles; 1536 } 1537 1538 public String getStatisticsFilePath() { 1539 return syncReadConfiguration(Configuration::getStatisticsFilePath); 1540 } 1541 1542 public void setStatisticsFilePath(String statisticsFilePath) { 1543 syncWriteConfiguration(statisticsFilePath, Configuration::setStatisticsFilePath); 1544 } 1545 1546 /** 1547 * Return the authorization framework used in this environment. 1548 * 1549 * @return the framework 1550 */ 1551 public AuthorizationFramework getAuthorizationFramework() { 1552 synchronized (authFrameworkLock) { 1553 if (authFramework == null) { 1554 authFramework = new AuthorizationFramework(getPluginDirectory(), getPluginStack()); 1555 } 1556 return authFramework; 1557 } 1558 } 1559 1560 /** 1561 * Set the authorization framework for this environment. Unload all 1562 * previously load plugins. 1563 * 1564 * @param fw the new framework 1565 */ 1566 public void setAuthorizationFramework(AuthorizationFramework fw) { 1567 synchronized (authFrameworkLock) { 1568 if (this.authFramework != null) { 1569 this.authFramework.removeAll(); 1570 } 1571 this.authFramework = fw; 1572 } 1573 } 1574 1575 /** 1576 * Re-apply the configuration. 1577 * @param reindex is the message result of reindex 1578 * @param interactive true if in interactive mode 1579 */ 1580 public void applyConfig(boolean reindex, boolean interactive) { 1581 applyConfig(configuration, reindex, interactive); 1582 } 1583 1584 /** 1585 * Set configuration from a message. The message could have come from the 1586 * Indexer (in which case some extra work is needed) or is it just a request 1587 * to set new configuration in place. 1588 * 1589 * @param configuration XML configuration 1590 * @param reindex is the message result of reindex 1591 * @param interactive true if in interactive mode 1592 * @see #applyConfig(org.opengrok.indexer.configuration.Configuration, 1593 * boolean, boolean) applyConfig(config, reindex, interactive) 1594 */ 1595 public void applyConfig(String configuration, boolean reindex, boolean interactive) { 1596 Configuration config; 1597 try { 1598 config = makeXMLStringAsConfiguration(configuration); 1599 } catch (IOException ex) { 1600 LOGGER.log(Level.WARNING, "Configuration decoding failed", ex); 1601 return; 1602 } 1603 1604 applyConfig(config, reindex, interactive); 1605 } 1606 1607 /** 1608 * Set configuration from the incoming parameter. The configuration could 1609 * have come from the Indexer (in which case some extra work is needed) or 1610 * is it just a request to set new configuration in place. 1611 * 1612 * @param config the incoming configuration 1613 * @param reindex is the message result of reindex 1614 * @param interactive true if in interactive mode 1615 */ 1616 public void applyConfig(Configuration config, boolean reindex, boolean interactive) { 1617 setConfiguration(config, interactive); 1618 LOGGER.log(Level.INFO, "Configuration updated"); 1619 1620 if (reindex) { 1621 // We are assuming that each update of configuration means reindex. If dedicated thread is introduced 1622 // in the future solely for the purpose of getting the event of reindex, the 2 calls below should 1623 // be moved there. 1624 refreshSearcherManagerMap(); 1625 maybeRefreshIndexSearchers(); 1626 // Force timestamp to update itself upon new config arrival. 1627 refreshDateForLastIndexRun(); 1628 } 1629 1630 // start/stop the watchdog if necessary 1631 if (isAuthorizationWatchdog() && getPluginDirectory() != null) { 1632 watchDog.start(new File(getPluginDirectory())); 1633 } else { 1634 watchDog.stop(); 1635 } 1636 1637 // set the new plugin directory and reload the authorization framework 1638 getAuthorizationFramework().setPluginDirectory(getPluginDirectory()); 1639 getAuthorizationFramework().setStack(getPluginStack()); 1640 getAuthorizationFramework().reload(); 1641 1642 messagesContainer.setMessageLimit(getMessageLimit()); 1643 } 1644 1645 public void setIndexTimestamp() throws IOException { 1646 indexTime.stamp(); 1647 } 1648 1649 public void refreshDateForLastIndexRun() { 1650 indexTime.refreshDateForLastIndexRun(); 1651 } 1652 1653 private void maybeRefreshSearcherManager(SearcherManager sm) { 1654 try { 1655 sm.maybeRefresh(); 1656 } catch (AlreadyClosedException ex) { 1657 // This is a case of removed project. See refreshSearcherManagerMap() for details. 1658 } catch (IOException ex) { 1659 LOGGER.log(Level.SEVERE, "maybeRefresh failed", ex); 1660 } 1661 } 1662 1663 public void maybeRefreshIndexSearchers(Iterable<String> projects) { 1664 for (String proj : projects) { 1665 if (searcherManagerMap.containsKey(proj)) { 1666 maybeRefreshSearcherManager(searcherManagerMap.get(proj)); 1667 } 1668 } 1669 } 1670 1671 public void maybeRefreshIndexSearchers() { 1672 for (Map.Entry<String, SearcherManager> entry : searcherManagerMap.entrySet()) { 1673 maybeRefreshSearcherManager(entry.getValue()); 1674 } 1675 } 1676 1677 /** 1678 * Get IndexSearcher for given project. 1679 * Each IndexSearcher is born from a SearcherManager object. There is one SearcherManager for every project. 1680 * This schema makes it possible to reuse IndexSearcher/IndexReader objects so the heavy lifting 1681 * (esp. system calls) performed in FSDirectory and DirectoryReader happens only once for a project. 1682 * The caller has to make sure that the IndexSearcher is returned back 1683 * to the SearcherManager. This is done with returnIndexSearcher(). 1684 * The return of the IndexSearcher should happen only after the search result data are read fully. 1685 * 1686 * @param projectName project 1687 * @return SearcherManager for given project 1688 * @throws IOException I/O exception 1689 */ 1690 public SuperIndexSearcher getIndexSearcher(String projectName) throws IOException { 1691 SearcherManager mgr = searcherManagerMap.get(projectName); 1692 SuperIndexSearcher searcher; 1693 1694 if (mgr == null) { 1695 File indexDir = new File(getDataRootPath(), IndexDatabase.INDEX_DIR); 1696 Directory dir = FSDirectory.open(new File(indexDir, projectName).toPath()); 1697 mgr = new SearcherManager(dir, new ThreadpoolSearcherFactory()); 1698 searcherManagerMap.put(projectName, mgr); 1699 searcher = (SuperIndexSearcher) mgr.acquire(); 1700 searcher.setSearcherManager(mgr); 1701 } else { 1702 searcher = (SuperIndexSearcher) mgr.acquire(); 1703 searcher.setSearcherManager(mgr); 1704 } 1705 1706 return searcher; 1707 } 1708 1709 /** 1710 * After new configuration is put into place, the set of projects might 1711 * change so we go through the SearcherManager objects and close those where 1712 * the corresponding project is no longer present. 1713 */ 1714 public void refreshSearcherManagerMap() { 1715 ArrayList<String> toRemove = new ArrayList<>(); 1716 1717 for (Map.Entry<String, SearcherManager> entry : searcherManagerMap.entrySet()) { 1718 // If a project is gone, close the corresponding SearcherManager 1719 // so that it cannot produce new IndexSearcher objects. 1720 if (!getProjectNames().contains(entry.getKey())) { 1721 try { 1722 LOGGER.log(Level.FINE, 1723 "closing SearcherManager for project" + entry.getKey()); 1724 entry.getValue().close(); 1725 } catch (IOException ex) { 1726 LOGGER.log(Level.SEVERE, 1727 "cannot close SearcherManager for project" + entry.getKey(), ex); 1728 } 1729 toRemove.add(entry.getKey()); 1730 } 1731 } 1732 1733 for (String proj : toRemove) { 1734 searcherManagerMap.remove(proj); 1735 } 1736 } 1737 1738 /** 1739 * Return collection of IndexReader objects as MultiReader object 1740 * for given list of projects. 1741 * The caller is responsible for releasing the IndexSearcher objects 1742 * so we add them to the map. 1743 * 1744 * @param projects list of projects 1745 * @param searcherList each SuperIndexSearcher produced will be put into this list 1746 * @return MultiReader for the projects 1747 */ 1748 public MultiReader getMultiReader(SortedSet<String> projects, 1749 ArrayList<SuperIndexSearcher> searcherList) { 1750 1751 IndexReader[] subreaders = new IndexReader[projects.size()]; 1752 int ii = 0; 1753 1754 // TODO might need to rewrite to Project instead of String, need changes in projects.jspf too. 1755 for (String proj : projects) { 1756 try { 1757 SuperIndexSearcher searcher = getIndexSearcher(proj); 1758 subreaders[ii++] = searcher.getIndexReader(); 1759 searcherList.add(searcher); 1760 } catch (IOException | NullPointerException ex) { 1761 LOGGER.log(Level.SEVERE, 1762 "cannot get IndexReader for project " + proj, ex); 1763 return null; 1764 } 1765 } 1766 MultiReader multiReader = null; 1767 try { 1768 multiReader = new MultiReader(subreaders, true); 1769 } catch (IOException ex) { 1770 LOGGER.log(Level.SEVERE, 1771 "cannot construct MultiReader for set of projects", ex); 1772 } 1773 return multiReader; 1774 } 1775 1776 public void startExpirationTimer() { 1777 messagesContainer.setMessageLimit(getMessageLimit()); 1778 messagesContainer.startExpirationTimer(); 1779 } 1780 1781 public void stopExpirationTimer() { 1782 messagesContainer.stopExpirationTimer(); 1783 } 1784 1785 /** 1786 * Get the default set of messages for the main tag. 1787 * 1788 * @return set of messages 1789 */ 1790 public SortedSet<AcceptedMessage> getMessages() { 1791 return messagesContainer.getMessages(); 1792 } 1793 1794 /** 1795 * Get the set of messages for the arbitrary tag. 1796 * 1797 * @param tag the message tag 1798 * @return set of messages 1799 */ 1800 public SortedSet<AcceptedMessage> getMessages(final String tag) { 1801 return messagesContainer.getMessages(tag); 1802 } 1803 1804 /** 1805 * Add a message to the application. 1806 * Also schedules a expiration timer to remove this message after its expiration. 1807 * 1808 * @param message the message 1809 */ 1810 public void addMessage(final Message message) { 1811 messagesContainer.addMessage(message); 1812 } 1813 1814 /** 1815 * Remove all messages containing at least one of the tags. 1816 * @param tags set of tags 1817 * @param text message text (can be null, empty) 1818 */ 1819 public void removeAnyMessage(final Set<String> tags, final String text) { 1820 messagesContainer.removeAnyMessage(tags, text); 1821 } 1822 1823 /** 1824 * @return all messages regardless their tag 1825 */ 1826 public Set<AcceptedMessage> getAllMessages() { 1827 return messagesContainer.getAllMessages(); 1828 } 1829 1830 public Path getDtagsEftarPath() { 1831 return syncReadConfiguration(Configuration::getDtagsEftarPath); 1832 } 1833 1834 /** 1835 * Get the eftar file, which contains definition tags for path descriptions. 1836 * 1837 * @return {@code null} if there is no such file, the file otherwise. 1838 */ 1839 public File getDtagsEftar() { 1840 if (dtagsEftar == null) { 1841 File tmp = getDtagsEftarPath().toFile(); 1842 if (tmp.canRead()) { 1843 dtagsEftar = tmp; 1844 } 1845 } 1846 return dtagsEftar; 1847 } 1848 1849 public SuggesterConfig getSuggesterConfig() { 1850 return syncReadConfiguration(Configuration::getSuggesterConfig); 1851 } 1852 1853 public void setSuggesterConfig(SuggesterConfig suggesterConfig) { 1854 syncWriteConfiguration(suggesterConfig, Configuration::setSuggesterConfig); 1855 } 1856 1857 /** 1858 * Applies the specified function to the runtime configuration, after having 1859 * obtained the configuration read-lock (and releasing afterward). 1860 * @param function a defined function 1861 * @param <R> the type of the result of the function 1862 * @return the function result 1863 */ 1864 public <R> R syncReadConfiguration(Function<Configuration, R> function) { 1865 try (ResourceLock resourceLock = configLock.readLockAsResource()) { 1866 //noinspection ConstantConditions to avoid warning of no reference to auto-closeable 1867 assert resourceLock != null; 1868 return function.apply(configuration); 1869 } 1870 } 1871 1872 /** 1873 * Performs the specified operation which is provided the runtime 1874 * configuration and the specified argument, after first having obtained the 1875 * configuration write-lock (and releasing afterward). 1876 * @param <V> the type of the input to the operation 1877 * @param v the input argument 1878 * @param consumer a defined consumer 1879 */ 1880 public <V> void syncWriteConfiguration(V v, ConfigurationValueConsumer<V> consumer) { 1881 try (ResourceLock resourceLock = configLock.writeLockAsResource()) { 1882 //noinspection ConstantConditions to avoid warning of no reference to auto-closeable 1883 assert resourceLock != null; 1884 consumer.accept(configuration, v); 1885 } 1886 } 1887 1888 private int getMessageLimit() { 1889 return syncReadConfiguration(Configuration::getMessageLimit); 1890 } 1891 } 1892