1 /******************************************************************************* 2 * Copyright (c) 2009, 2017 IBM Corporation and others. 3 * 4 * This program and the accompanying materials 5 * are made available under the terms of the Eclipse Public License 2.0 6 * which accompanies this distribution, and is available at 7 * https://www.eclipse.org/legal/epl-2.0/ 8 * 9 * SPDX-License-Identifier: EPL-2.0 10 * 11 * Contributors: 12 * IBM Corporation - initial API and implementation 13 *******************************************************************************/ 14 package org.eclipse.equinox.p2.internal.repository.tools; 15 16 import java.io.File; 17 import java.net.URI; 18 import java.net.URISyntaxException; 19 import java.util.*; 20 import org.eclipse.core.runtime.*; 21 import org.eclipse.equinox.app.IApplication; 22 import org.eclipse.equinox.app.IApplicationContext; 23 import org.eclipse.equinox.internal.p2.core.helpers.LogHelper; 24 import org.eclipse.equinox.internal.p2.director.PermissiveSlicer; 25 import org.eclipse.equinox.internal.p2.repository.Transport; 26 import org.eclipse.equinox.internal.p2.repository.helpers.RepositoryHelper; 27 import org.eclipse.equinox.p2.core.ProvisionException; 28 import org.eclipse.equinox.p2.engine.*; 29 import org.eclipse.equinox.p2.internal.repository.comparator.MD5ArtifactComparator; 30 import org.eclipse.equinox.p2.internal.repository.mirroring.*; 31 import org.eclipse.equinox.p2.metadata.*; 32 import org.eclipse.equinox.p2.planner.IPlanner; 33 import org.eclipse.equinox.p2.planner.IProfileChangeRequest; 34 import org.eclipse.equinox.p2.query.*; 35 import org.eclipse.equinox.p2.repository.artifact.IArtifactDescriptor; 36 import org.eclipse.equinox.p2.repository.artifact.IArtifactRepository; 37 import org.eclipse.equinox.p2.repository.metadata.IMetadataRepository; 38 import org.eclipse.osgi.util.NLS; 39 40 public class MirrorApplication extends AbstractApplication implements IApplication, IExecutableExtension { 41 private static final String LOG_ROOT = "p2.mirror"; //$NON-NLS-1$ 42 private static final String MIRROR_MODE = "metadataOrArtifacts"; //$NON-NLS-1$ 43 44 protected SlicingOptions slicingOptions = new SlicingOptions(); 45 46 private URI baseline; 47 private String comparatorID; 48 private IQuery<IArtifactDescriptor> compareExclusions = null; 49 private boolean compare = false; 50 private boolean failOnError = true; 51 private boolean raw = true; 52 private boolean verbose = false; 53 private boolean validate = false; 54 private boolean mirrorReferences = true; 55 private String metadataOrArtifacts = null; 56 private String[] rootIUs = null; 57 private boolean includePacked = true; 58 private boolean mirrorProperties = false; 59 60 private File mirrorLogFile; // file to log mirror output to (optional) 61 private File comparatorLogFile; // file to comparator output to (optional) 62 private IArtifactMirrorLog mirrorLog; 63 private IArtifactMirrorLog comparatorLog; 64 65 /** 66 * Convert a list of tokens into an array. The list separator has to be 67 * specified. 68 */ getArrayArgsFromString(String list, String separator)69 public static String[] getArrayArgsFromString(String list, String separator) { 70 if (list == null || list.trim().equals("")) //$NON-NLS-1$ 71 return new String[0]; 72 List<String> result = new ArrayList<>(); 73 for (StringTokenizer tokens = new StringTokenizer(list, separator); tokens.hasMoreTokens();) { 74 String token = tokens.nextToken().trim(); 75 if (!token.equals("")) { //$NON-NLS-1$ 76 if ((token.indexOf('[') >= 0 || token.indexOf('(') >= 0) && tokens.hasMoreTokens()) 77 result.add(token + separator + tokens.nextToken()); 78 else 79 result.add(token); 80 } 81 } 82 return result.toArray(new String[result.size()]); 83 } 84 85 @Override start(IApplicationContext context)86 public Object start(IApplicationContext context) throws Exception { 87 Map<?, ?> args = context.getArguments(); 88 initializeFromArguments((String[]) args.get(IApplicationContext.APPLICATION_ARGS)); 89 run(null); 90 return IApplication.EXIT_OK; 91 } 92 93 @Override stop()94 public void stop() { 95 // TODO Auto-generated method stub 96 97 } 98 99 /* 100 * The old "org.eclipse.equinox.p2.artifact.repository.mirrorApplication" application only does artifacts 101 * Similary, "org.eclipse.equinox.p2.metadata.repository.mirrorApplication" only does metadata 102 */ 103 @Override setInitializationData(IConfigurationElement config, String propertyName, Object data)104 public void setInitializationData(IConfigurationElement config, String propertyName, Object data) { 105 if (data instanceof Map<?, ?> && ((Map<?, ?>) data).containsKey(MIRROR_MODE)) { 106 metadataOrArtifacts = (String) ((Map<?, ?>) data).get(MIRROR_MODE); 107 } 108 } 109 initializeFromArguments(String[] args)110 public void initializeFromArguments(String[] args) throws Exception { 111 if (args == null) 112 return; 113 114 File comparatorLogLocation = null; 115 File mirrorLogLocation = null; 116 117 RepositoryDescriptor destination = new RepositoryDescriptor(); 118 RepositoryDescriptor sourceRepo = new RepositoryDescriptor(); 119 if (metadataOrArtifacts != null) { 120 destination.setKind(metadataOrArtifacts); 121 sourceRepo.setKind(metadataOrArtifacts); 122 } 123 124 addDestination(destination); 125 addSource(sourceRepo); 126 127 for (int i = 0; i < args.length; i++) { 128 // check for args without parameters (i.e., a flag arg) 129 if (args[i].equalsIgnoreCase("-raw")) //$NON-NLS-1$ 130 raw = true; 131 else if (args[i].equalsIgnoreCase("-ignoreErrors")) //$NON-NLS-1$ 132 failOnError = false; 133 else if (args[i].equalsIgnoreCase("-verbose")) //$NON-NLS-1$ 134 verbose = true; 135 else if (args[i].equalsIgnoreCase("-compare")) //$NON-NLS-1$ 136 compare = true; 137 else if (args[i].equalsIgnoreCase("-validate")) //$NON-NLS-1$ 138 validate = true; 139 else if (args[i].equalsIgnoreCase("-references")) //$NON-NLS-1$ 140 mirrorReferences = true; 141 else if (args[i].equalsIgnoreCase("-properties")) //$NON-NLS-1$ 142 mirrorProperties = true; 143 144 // check for args with parameters. If we are at the last argument or 145 // if the next one has a '-' as the first character, then we can't have 146 // an arg with a param so continue. 147 if (i == args.length - 1 || args[i + 1].startsWith("-")) //$NON-NLS-1$ 148 continue; 149 150 String arg = args[++i]; 151 152 if (args[i - 1].equalsIgnoreCase("-comparator")) //$NON-NLS-1$ 153 comparatorID = arg; 154 else if (args[i - 1].equalsIgnoreCase("-comparatorLog")) //$NON-NLS-1$ 155 comparatorLogLocation = new File(arg); 156 else if (args[i - 1].equalsIgnoreCase("-destinationName")) //$NON-NLS-1$ 157 destination.setName(arg); 158 else if (args[i - 1].equalsIgnoreCase("-writeMode")) { //$NON-NLS-1$ 159 if (args[i].equalsIgnoreCase("clean")) //$NON-NLS-1$ 160 destination.setAppend(false); 161 } else if (args[i - 1].equalsIgnoreCase("-log")) { //$NON-NLS-1$ 162 mirrorLogLocation = new File(arg); 163 } else if (args[i - 1].equalsIgnoreCase("-roots")) { //$NON-NLS-1$ 164 rootIUs = getArrayArgsFromString(arg, ","); //$NON-NLS-1$ 165 } else if (args[i - 1].equalsIgnoreCase("-references")) {//$NON-NLS-1$ 166 mirrorReferences = Boolean.parseBoolean(args[i]); 167 } else { 168 try { 169 if (args[i - 1].equalsIgnoreCase("-source")) { //$NON-NLS-1$ 170 URI uri = RepositoryHelper.localRepoURIHelper(URIUtil.fromString(arg)); 171 sourceRepo.setLocation(uri); 172 destination.setFormat(uri); 173 } else if (args[i - 1].equalsIgnoreCase("-destination")) //$NON-NLS-1$ 174 destination.setLocation(RepositoryHelper.localRepoURIHelper(URIUtil.fromString(arg))); 175 else if (args[i - 1].equalsIgnoreCase("-compareAgainst")) { //$NON-NLS-1$ 176 baseline = RepositoryHelper.localRepoURIHelper(URIUtil.fromString(arg)); 177 compare = true; 178 } 179 } catch (URISyntaxException e) { 180 throw new IllegalArgumentException(NLS.bind(Messages.ProcessRepo_location_not_url, arg)); 181 } 182 } 183 } 184 185 // Create logs 186 if (mirrorLogLocation != null) 187 mirrorLog = getLog(mirrorLogLocation, "p2.artifact.mirror"); //$NON-NLS-1$ 188 if (comparatorLogLocation != null && comparatorID != null) 189 comparatorLog = getLog(comparatorLogLocation, comparatorID); 190 } 191 192 @Override run(IProgressMonitor monitor)193 public IStatus run(IProgressMonitor monitor) throws ProvisionException { 194 IStatus mirrorStatus = Status.OK_STATUS; 195 try { 196 initializeRepos(new NullProgressMonitor()); 197 initializeLogs(); 198 validate(); 199 initializeIUs(); 200 IQueryable<IInstallableUnit> slice = slice(new NullProgressMonitor()); 201 if (destinationArtifactRepository != null) { 202 mirrorStatus = mirrorArtifacts(slice, new NullProgressMonitor()); 203 if (failOnError && mirrorStatus.getSeverity() == IStatus.ERROR) 204 return mirrorStatus; 205 } 206 if (destinationMetadataRepository != null) 207 mirrorMetadata(slice, new NullProgressMonitor()); 208 } finally { 209 finalizeRepositories(); 210 finalizeLogs(); 211 } 212 if (mirrorStatus.isOK()) 213 return Status.OK_STATUS; 214 return mirrorStatus; 215 } 216 mirrorArtifacts(IQueryable<IInstallableUnit> slice, IProgressMonitor monitor)217 private IStatus mirrorArtifacts(IQueryable<IInstallableUnit> slice, IProgressMonitor monitor) { 218 Mirroring mirror = getMirroring(slice, monitor); 219 220 IStatus result = mirror.run(failOnError, verbose); 221 222 if (mirrorLog != null) 223 mirrorLog.log(result); 224 else 225 LogHelper.log(result); 226 return result; 227 } 228 getMirroring(IQueryable<IInstallableUnit> slice, IProgressMonitor monitor)229 protected Mirroring getMirroring(IQueryable<IInstallableUnit> slice, IProgressMonitor monitor) { 230 // Obtain ArtifactKeys from IUs 231 IQueryResult<IInstallableUnit> ius = slice.query(QueryUtil.createIUAnyQuery(), monitor); 232 boolean iusSpecified = !ius.isEmpty(); // call before ius.iterator() to avoid bug 420318 233 ArrayList<IArtifactKey> keys = new ArrayList<>(); 234 for (IInstallableUnit iu : ius) { 235 keys.addAll(iu.getArtifacts()); 236 } 237 238 Mirroring mirror = new Mirroring(getCompositeArtifactRepository(), destinationArtifactRepository, raw); 239 mirror.setCompare(compare); 240 mirror.setComparatorId(comparatorID == null ? MD5ArtifactComparator.MD5_COMPARATOR_ID : comparatorID); 241 mirror.setBaseline(initializeBaseline()); 242 mirror.setValidate(validate); 243 mirror.setCompareExclusions(compareExclusions); 244 mirror.setTransport((Transport) agent.getService(Transport.SERVICE_NAME)); 245 mirror.setIncludePacked(includePacked); 246 mirror.setMirrorProperties(mirrorProperties); 247 248 // If IUs have been specified then only they should be mirrored, otherwise mirror everything. 249 if (iusSpecified) 250 mirror.setArtifactKeys(keys.toArray(new IArtifactKey[keys.size()])); 251 252 if (comparatorLog != null) 253 mirror.setComparatorLog(comparatorLog); 254 return mirror; 255 } 256 initializeBaseline()257 private IArtifactRepository initializeBaseline() { 258 if (baseline == null) 259 return null; 260 try { 261 return addRepository(getArtifactRepositoryManager(), baseline, 0, null); 262 } catch (ProvisionException e) { 263 if (mirrorLog != null && e.getStatus() != null) 264 mirrorLog.log(e.getStatus()); 265 return null; 266 } 267 } 268 mirrorMetadata(IQueryable<IInstallableUnit> slice, IProgressMonitor monitor)269 private void mirrorMetadata(IQueryable<IInstallableUnit> slice, IProgressMonitor monitor) { 270 IQueryResult<IInstallableUnit> allIUs = slice.query(QueryUtil.createIUAnyQuery(), monitor); 271 destinationMetadataRepository.addInstallableUnits(allIUs.toUnmodifiableSet()); 272 if (mirrorReferences) 273 destinationMetadataRepository.addReferences(getCompositeMetadataRepository().getReferences()); 274 } 275 276 /* 277 * Ensure all mandatory parameters have been set. Throw an exception if there 278 * are any missing. We don't require the user to specify the artifact repository here, 279 * we will default to the ones already registered in the manager. (callers are free 280 * to add more if they wish) 281 */ validate()282 private void validate() throws ProvisionException { 283 if (sourceRepositories.isEmpty()) 284 throw new ProvisionException(Messages.MirrorApplication_set_source_repositories); 285 if (!hasArtifactSources() && destinationArtifactRepository != null) 286 throw new ProvisionException(Messages.MirrorApplication_artifactDestinationNoSource); 287 if (!hasMetadataSources() && destinationMetadataRepository != null) 288 throw new ProvisionException(Messages.MirrorApplication_metadataDestinationNoSource); 289 } 290 291 /* 292 * If no IUs have been specified we want to mirror them all 293 */ initializeIUs()294 private void initializeIUs() throws ProvisionException { 295 IMetadataRepository metadataRepo = getCompositeMetadataRepository(); 296 297 if (rootIUs != null) { 298 sourceIUs = new ArrayList<>(); 299 for (String rootIU : rootIUs) { 300 String[] segments = getArrayArgsFromString(rootIU, "/"); //$NON-NLS-1$ 301 VersionRange range = segments.length > 1 ? VersionRange.create(segments[1]) : null; 302 Iterator<IInstallableUnit> queryResult = metadataRepo.query(QueryUtil.createIUQuery(segments[0], range), null).iterator(); 303 while (queryResult.hasNext()) 304 sourceIUs.add(queryResult.next()); 305 } 306 } else if (sourceIUs == null || sourceIUs.isEmpty()) { 307 sourceIUs = new ArrayList<>(); 308 Iterator<IInstallableUnit> queryResult = metadataRepo.query(QueryUtil.createIUAnyQuery(), null).iterator(); 309 while (queryResult.hasNext()) 310 sourceIUs.add(queryResult.next()); 311 /* old metadata mirroring app did not throw an exception here */ 312 if (sourceIUs.size() == 0 && destinationMetadataRepository != null && metadataOrArtifacts == null) 313 throw new ProvisionException(Messages.MirrorApplication_no_IUs); 314 } 315 } 316 317 /* 318 * Initialize logs, if applicable 319 */ initializeLogs()320 private void initializeLogs() { 321 if (compare && comparatorLogFile != null) 322 comparatorLog = getLog(comparatorLogFile, comparatorID); 323 if (mirrorLog == null && mirrorLogFile != null) 324 mirrorLog = getLog(mirrorLogFile, LOG_ROOT); 325 } 326 327 /* 328 * Finalize logs, if applicable 329 */ finalizeLogs()330 private void finalizeLogs() { 331 if (comparatorLog != null) 332 comparatorLog.close(); 333 if (mirrorLog != null) 334 mirrorLog.close(); 335 } 336 337 /* 338 * Get the log for a location 339 */ getLog(File location, String root)340 private IArtifactMirrorLog getLog(File location, String root) { 341 String absolutePath = location.getAbsolutePath(); 342 if (absolutePath.toLowerCase().endsWith(".xml")) //$NON-NLS-1$ 343 return new XMLMirrorLog(absolutePath, 0, root); 344 return new FileMirrorLog(absolutePath, 0, root); 345 } 346 performResolution(IProgressMonitor monitor)347 private IQueryable<IInstallableUnit> performResolution(IProgressMonitor monitor) throws ProvisionException { 348 IProfileRegistry registry = Activator.getProfileRegistry(); 349 String profileId = "MirrorApplication-" + System.currentTimeMillis(); //$NON-NLS-1$ 350 IProfile profile = registry.addProfile(profileId, slicingOptions.getFilter()); 351 IPlanner planner = (IPlanner) Activator.getAgent().getService(IPlanner.SERVICE_NAME); 352 if (planner == null) 353 throw new IllegalStateException(); 354 IProfileChangeRequest pcr = planner.createChangeRequest(profile); 355 pcr.addAll(sourceIUs); 356 IProvisioningPlan plan = planner.getProvisioningPlan(pcr, null, monitor); 357 registry.removeProfile(profileId); 358 @SuppressWarnings("unchecked") 359 IQueryable<IInstallableUnit>[] arr = new IQueryable[plan.getInstallerPlan() == null ? 1 : 2]; 360 arr[0] = plan.getAdditions(); 361 if (plan.getInstallerPlan() != null) 362 arr[1] = plan.getInstallerPlan().getAdditions(); 363 return new CompoundQueryable<>(arr); 364 } 365 slice(IProgressMonitor monitor)366 private IQueryable<IInstallableUnit> slice(IProgressMonitor monitor) throws ProvisionException { 367 if (slicingOptions == null) 368 slicingOptions = new SlicingOptions(); 369 if (slicingOptions.getInstallTimeLikeResolution()) 370 return performResolution(monitor); 371 372 PermissiveSlicer slicer = new PermissiveSlicer(getCompositeMetadataRepository(), slicingOptions.getFilter(), slicingOptions.includeOptionalDependencies(), slicingOptions.isEverythingGreedy(), slicingOptions.forceFilterTo(), slicingOptions.considerStrictDependencyOnly(), slicingOptions.followOnlyFilteredRequirements()); 373 IQueryable<IInstallableUnit> slice = slicer.slice(sourceIUs.toArray(new IInstallableUnit[sourceIUs.size()]), monitor); 374 375 if (slice != null && slicingOptions.latestVersionOnly()) { 376 IQueryResult<IInstallableUnit> queryResult = slice.query(QueryUtil.createLatestIUQuery(), monitor); 377 slice = queryResult; 378 } 379 if (slicer.getStatus().getSeverity() != IStatus.OK && mirrorLog != null) { 380 mirrorLog.log(slicer.getStatus()); 381 } 382 if (slice == null) { 383 throw new ProvisionException(slicer.getStatus()); 384 } 385 return slice; 386 } 387 setSlicingOptions(SlicingOptions options)388 public void setSlicingOptions(SlicingOptions options) { 389 slicingOptions = options; 390 } 391 392 /* 393 * Set the location of the baseline repository. (used in comparison) 394 */ setBaseline(URI baseline)395 public void setBaseline(URI baseline) { 396 this.baseline = baseline; 397 compare = true; 398 } 399 400 /* 401 * Set the identifier of the comparator to use. 402 */ setComparatorID(String value)403 public void setComparatorID(String value) { 404 comparatorID = value; 405 compare = true; 406 } 407 408 /* 409 * Set whether or not the application should be calling a comparator when mirroring. 410 */ setCompare(boolean value)411 public void setCompare(boolean value) { 412 compare = value; 413 } 414 415 /* 416 * Set whether or not we should ignore errors when running the mirror application. 417 */ setIgnoreErrors(boolean value)418 public void setIgnoreErrors(boolean value) { 419 failOnError = !value; 420 } 421 422 /* 423 * Set whether or not the the artifacts are raw. 424 */ setRaw(boolean value)425 public void setRaw(boolean value) { 426 raw = value; 427 } 428 429 /* 430 * Set whether or not the mirror application should be run in verbose mode. 431 */ setVerbose(boolean value)432 public void setVerbose(boolean value) { 433 verbose = value; 434 } 435 436 /* 437 * Set the location of the log for comparator output 438 */ setComparatorLog(File comparatorLog)439 public void setComparatorLog(File comparatorLog) { 440 this.comparatorLogFile = comparatorLog; 441 } 442 443 /* 444 * Set the location of the log for mirroring. 445 */ setLog(File mirrorLog)446 public void setLog(File mirrorLog) { 447 this.mirrorLogFile = mirrorLog; 448 } 449 450 /* 451 * Set the ArtifactMirror log 452 */ setLog(IArtifactMirrorLog log)453 public void setLog(IArtifactMirrorLog log) { 454 mirrorLog = log; 455 } 456 457 /* 458 * Set if the artifact mirror should be validated 459 */ setValidate(boolean value)460 public void setValidate(boolean value) { 461 validate = value; 462 } 463 464 /* 465 * Set if references should be mirrored 466 */ setReferences(boolean flag)467 public void setReferences(boolean flag) { 468 mirrorReferences = flag; 469 } 470 setComparatorExclusions(IQuery<IArtifactDescriptor> exclusions)471 public void setComparatorExclusions(IQuery<IArtifactDescriptor> exclusions) { 472 compareExclusions = exclusions; 473 } 474 setIncludePacked(boolean includePacked)475 public void setIncludePacked(boolean includePacked) { 476 this.includePacked = includePacked; 477 } 478 setMirrorProperties(boolean mirrorProperties)479 public void setMirrorProperties(boolean mirrorProperties) { 480 this.mirrorProperties = mirrorProperties; 481 } 482 } 483