1 /*******************************************************************************
2  * Copyright (c) 2009, 2017 IBM Corporation and others.
3  *
4  * This program and the accompanying materials
5  * are made available under the terms of the Eclipse Public License 2.0
6  * which accompanies this distribution, and is available at
7  * https://www.eclipse.org/legal/epl-2.0/
8  *
9  * SPDX-License-Identifier: EPL-2.0
10  *
11  * Contributors:
12  *     IBM Corporation - initial API and implementation
13  *******************************************************************************/
14 package org.eclipse.equinox.p2.internal.repository.tools;
15 
16 import java.io.File;
17 import java.net.URI;
18 import java.net.URISyntaxException;
19 import java.util.*;
20 import org.eclipse.core.runtime.*;
21 import org.eclipse.equinox.app.IApplication;
22 import org.eclipse.equinox.app.IApplicationContext;
23 import org.eclipse.equinox.internal.p2.core.helpers.LogHelper;
24 import org.eclipse.equinox.internal.p2.director.PermissiveSlicer;
25 import org.eclipse.equinox.internal.p2.repository.Transport;
26 import org.eclipse.equinox.internal.p2.repository.helpers.RepositoryHelper;
27 import org.eclipse.equinox.p2.core.ProvisionException;
28 import org.eclipse.equinox.p2.engine.*;
29 import org.eclipse.equinox.p2.internal.repository.comparator.MD5ArtifactComparator;
30 import org.eclipse.equinox.p2.internal.repository.mirroring.*;
31 import org.eclipse.equinox.p2.metadata.*;
32 import org.eclipse.equinox.p2.planner.IPlanner;
33 import org.eclipse.equinox.p2.planner.IProfileChangeRequest;
34 import org.eclipse.equinox.p2.query.*;
35 import org.eclipse.equinox.p2.repository.artifact.IArtifactDescriptor;
36 import org.eclipse.equinox.p2.repository.artifact.IArtifactRepository;
37 import org.eclipse.equinox.p2.repository.metadata.IMetadataRepository;
38 import org.eclipse.osgi.util.NLS;
39 
40 public class MirrorApplication extends AbstractApplication implements IApplication, IExecutableExtension {
41 	private static final String LOG_ROOT = "p2.mirror"; //$NON-NLS-1$
42 	private static final String MIRROR_MODE = "metadataOrArtifacts"; //$NON-NLS-1$
43 
44 	protected SlicingOptions slicingOptions = new SlicingOptions();
45 
46 	private URI baseline;
47 	private String comparatorID;
48 	private IQuery<IArtifactDescriptor> compareExclusions = null;
49 	private boolean compare = false;
50 	private boolean failOnError = true;
51 	private boolean raw = true;
52 	private boolean verbose = false;
53 	private boolean validate = false;
54 	private boolean mirrorReferences = true;
55 	private String metadataOrArtifacts = null;
56 	private String[] rootIUs = null;
57 	private boolean includePacked = true;
58 	private boolean mirrorProperties = false;
59 
60 	private File mirrorLogFile; // file to log mirror output to (optional)
61 	private File comparatorLogFile; // file to comparator output to (optional)
62 	private IArtifactMirrorLog mirrorLog;
63 	private IArtifactMirrorLog comparatorLog;
64 
65 	/**
66 	 * Convert a list of tokens into an array. The list separator has to be
67 	 * specified.
68 	 */
getArrayArgsFromString(String list, String separator)69 	public static String[] getArrayArgsFromString(String list, String separator) {
70 		if (list == null || list.trim().equals("")) //$NON-NLS-1$
71 			return new String[0];
72 		List<String> result = new ArrayList<>();
73 		for (StringTokenizer tokens = new StringTokenizer(list, separator); tokens.hasMoreTokens();) {
74 			String token = tokens.nextToken().trim();
75 			if (!token.equals("")) { //$NON-NLS-1$
76 				if ((token.indexOf('[') >= 0 || token.indexOf('(') >= 0) && tokens.hasMoreTokens())
77 					result.add(token + separator + tokens.nextToken());
78 				else
79 					result.add(token);
80 			}
81 		}
82 		return result.toArray(new String[result.size()]);
83 	}
84 
85 	@Override
start(IApplicationContext context)86 	public Object start(IApplicationContext context) throws Exception {
87 		Map<?, ?> args = context.getArguments();
88 		initializeFromArguments((String[]) args.get(IApplicationContext.APPLICATION_ARGS));
89 		run(null);
90 		return IApplication.EXIT_OK;
91 	}
92 
93 	@Override
stop()94 	public void stop() {
95 		// TODO Auto-generated method stub
96 
97 	}
98 
99 	/*
100 	 * The old "org.eclipse.equinox.p2.artifact.repository.mirrorApplication" application only does artifacts
101 	 * Similary, "org.eclipse.equinox.p2.metadata.repository.mirrorApplication" only does metadata
102 	 */
103 	@Override
setInitializationData(IConfigurationElement config, String propertyName, Object data)104 	public void setInitializationData(IConfigurationElement config, String propertyName, Object data) {
105 		if (data instanceof Map<?, ?> && ((Map<?, ?>) data).containsKey(MIRROR_MODE)) {
106 			metadataOrArtifacts = (String) ((Map<?, ?>) data).get(MIRROR_MODE);
107 		}
108 	}
109 
initializeFromArguments(String[] args)110 	public void initializeFromArguments(String[] args) throws Exception {
111 		if (args == null)
112 			return;
113 
114 		File comparatorLogLocation = null;
115 		File mirrorLogLocation = null;
116 
117 		RepositoryDescriptor destination = new RepositoryDescriptor();
118 		RepositoryDescriptor sourceRepo = new RepositoryDescriptor();
119 		if (metadataOrArtifacts != null) {
120 			destination.setKind(metadataOrArtifacts);
121 			sourceRepo.setKind(metadataOrArtifacts);
122 		}
123 
124 		addDestination(destination);
125 		addSource(sourceRepo);
126 
127 		for (int i = 0; i < args.length; i++) {
128 			// check for args without parameters (i.e., a flag arg)
129 			if (args[i].equalsIgnoreCase("-raw")) //$NON-NLS-1$
130 				raw = true;
131 			else if (args[i].equalsIgnoreCase("-ignoreErrors")) //$NON-NLS-1$
132 				failOnError = false;
133 			else if (args[i].equalsIgnoreCase("-verbose")) //$NON-NLS-1$
134 				verbose = true;
135 			else if (args[i].equalsIgnoreCase("-compare")) //$NON-NLS-1$
136 				compare = true;
137 			else if (args[i].equalsIgnoreCase("-validate")) //$NON-NLS-1$
138 				validate = true;
139 			else if (args[i].equalsIgnoreCase("-references")) //$NON-NLS-1$
140 				mirrorReferences = true;
141 			else if (args[i].equalsIgnoreCase("-properties")) //$NON-NLS-1$
142 				mirrorProperties = true;
143 
144 			// check for args with parameters. If we are at the last argument or
145 			// if the next one has a '-' as the first character, then we can't have
146 			// an arg with a param so continue.
147 			if (i == args.length - 1 || args[i + 1].startsWith("-")) //$NON-NLS-1$
148 				continue;
149 
150 			String arg = args[++i];
151 
152 			if (args[i - 1].equalsIgnoreCase("-comparator")) //$NON-NLS-1$
153 				comparatorID = arg;
154 			else if (args[i - 1].equalsIgnoreCase("-comparatorLog")) //$NON-NLS-1$
155 				comparatorLogLocation = new File(arg);
156 			else if (args[i - 1].equalsIgnoreCase("-destinationName")) //$NON-NLS-1$
157 				destination.setName(arg);
158 			else if (args[i - 1].equalsIgnoreCase("-writeMode")) { //$NON-NLS-1$
159 				if (args[i].equalsIgnoreCase("clean")) //$NON-NLS-1$
160 					destination.setAppend(false);
161 			} else if (args[i - 1].equalsIgnoreCase("-log")) { //$NON-NLS-1$
162 				mirrorLogLocation = new File(arg);
163 			} else if (args[i - 1].equalsIgnoreCase("-roots")) { //$NON-NLS-1$
164 				rootIUs = getArrayArgsFromString(arg, ","); //$NON-NLS-1$
165 			} else if (args[i - 1].equalsIgnoreCase("-references")) {//$NON-NLS-1$
166 				mirrorReferences = Boolean.parseBoolean(args[i]);
167 			} else {
168 				try {
169 					if (args[i - 1].equalsIgnoreCase("-source")) { //$NON-NLS-1$
170 						URI uri = RepositoryHelper.localRepoURIHelper(URIUtil.fromString(arg));
171 						sourceRepo.setLocation(uri);
172 						destination.setFormat(uri);
173 					} else if (args[i - 1].equalsIgnoreCase("-destination")) //$NON-NLS-1$
174 						destination.setLocation(RepositoryHelper.localRepoURIHelper(URIUtil.fromString(arg)));
175 					else if (args[i - 1].equalsIgnoreCase("-compareAgainst")) { //$NON-NLS-1$
176 						baseline = RepositoryHelper.localRepoURIHelper(URIUtil.fromString(arg));
177 						compare = true;
178 					}
179 				} catch (URISyntaxException e) {
180 					throw new IllegalArgumentException(NLS.bind(Messages.ProcessRepo_location_not_url, arg));
181 				}
182 			}
183 		}
184 
185 		// Create logs
186 		if (mirrorLogLocation != null)
187 			mirrorLog = getLog(mirrorLogLocation, "p2.artifact.mirror"); //$NON-NLS-1$
188 		if (comparatorLogLocation != null && comparatorID != null)
189 			comparatorLog = getLog(comparatorLogLocation, comparatorID);
190 	}
191 
192 	@Override
run(IProgressMonitor monitor)193 	public IStatus run(IProgressMonitor monitor) throws ProvisionException {
194 		IStatus mirrorStatus = Status.OK_STATUS;
195 		try {
196 			initializeRepos(new NullProgressMonitor());
197 			initializeLogs();
198 			validate();
199 			initializeIUs();
200 			IQueryable<IInstallableUnit> slice = slice(new NullProgressMonitor());
201 			if (destinationArtifactRepository != null) {
202 				mirrorStatus = mirrorArtifacts(slice, new NullProgressMonitor());
203 				if (failOnError && mirrorStatus.getSeverity() == IStatus.ERROR)
204 					return mirrorStatus;
205 			}
206 			if (destinationMetadataRepository != null)
207 				mirrorMetadata(slice, new NullProgressMonitor());
208 		} finally {
209 			finalizeRepositories();
210 			finalizeLogs();
211 		}
212 		if (mirrorStatus.isOK())
213 			return Status.OK_STATUS;
214 		return mirrorStatus;
215 	}
216 
mirrorArtifacts(IQueryable<IInstallableUnit> slice, IProgressMonitor monitor)217 	private IStatus mirrorArtifacts(IQueryable<IInstallableUnit> slice, IProgressMonitor monitor) {
218 		Mirroring mirror = getMirroring(slice, monitor);
219 
220 		IStatus result = mirror.run(failOnError, verbose);
221 
222 		if (mirrorLog != null)
223 			mirrorLog.log(result);
224 		else
225 			LogHelper.log(result);
226 		return result;
227 	}
228 
getMirroring(IQueryable<IInstallableUnit> slice, IProgressMonitor monitor)229 	protected Mirroring getMirroring(IQueryable<IInstallableUnit> slice, IProgressMonitor monitor) {
230 		// Obtain ArtifactKeys from IUs
231 		IQueryResult<IInstallableUnit> ius = slice.query(QueryUtil.createIUAnyQuery(), monitor);
232 		boolean iusSpecified = !ius.isEmpty(); // call before ius.iterator() to avoid bug 420318
233 		ArrayList<IArtifactKey> keys = new ArrayList<>();
234 		for (IInstallableUnit iu : ius) {
235 			keys.addAll(iu.getArtifacts());
236 		}
237 
238 		Mirroring mirror = new Mirroring(getCompositeArtifactRepository(), destinationArtifactRepository, raw);
239 		mirror.setCompare(compare);
240 		mirror.setComparatorId(comparatorID == null ? MD5ArtifactComparator.MD5_COMPARATOR_ID : comparatorID);
241 		mirror.setBaseline(initializeBaseline());
242 		mirror.setValidate(validate);
243 		mirror.setCompareExclusions(compareExclusions);
244 		mirror.setTransport((Transport) agent.getService(Transport.SERVICE_NAME));
245 		mirror.setIncludePacked(includePacked);
246 		mirror.setMirrorProperties(mirrorProperties);
247 
248 		// If IUs have been specified then only they should be mirrored, otherwise mirror everything.
249 		if (iusSpecified)
250 			mirror.setArtifactKeys(keys.toArray(new IArtifactKey[keys.size()]));
251 
252 		if (comparatorLog != null)
253 			mirror.setComparatorLog(comparatorLog);
254 		return mirror;
255 	}
256 
initializeBaseline()257 	private IArtifactRepository initializeBaseline() {
258 		if (baseline == null)
259 			return null;
260 		try {
261 			return addRepository(getArtifactRepositoryManager(), baseline, 0, null);
262 		} catch (ProvisionException e) {
263 			if (mirrorLog != null && e.getStatus() != null)
264 				mirrorLog.log(e.getStatus());
265 			return null;
266 		}
267 	}
268 
mirrorMetadata(IQueryable<IInstallableUnit> slice, IProgressMonitor monitor)269 	private void mirrorMetadata(IQueryable<IInstallableUnit> slice, IProgressMonitor monitor) {
270 		IQueryResult<IInstallableUnit> allIUs = slice.query(QueryUtil.createIUAnyQuery(), monitor);
271 		destinationMetadataRepository.addInstallableUnits(allIUs.toUnmodifiableSet());
272 		if (mirrorReferences)
273 			destinationMetadataRepository.addReferences(getCompositeMetadataRepository().getReferences());
274 	}
275 
276 	/*
277 	 * Ensure all mandatory parameters have been set. Throw an exception if there
278 	 * are any missing. We don't require the user to specify the artifact repository here,
279 	 * we will default to the ones already registered in the manager. (callers are free
280 	 * to add more if they wish)
281 	 */
validate()282 	private void validate() throws ProvisionException {
283 		if (sourceRepositories.isEmpty())
284 			throw new ProvisionException(Messages.MirrorApplication_set_source_repositories);
285 		if (!hasArtifactSources() && destinationArtifactRepository != null)
286 			throw new ProvisionException(Messages.MirrorApplication_artifactDestinationNoSource);
287 		if (!hasMetadataSources() && destinationMetadataRepository != null)
288 			throw new ProvisionException(Messages.MirrorApplication_metadataDestinationNoSource);
289 	}
290 
291 	/*
292 	 * If no IUs have been specified we want to mirror them all
293 	 */
initializeIUs()294 	private void initializeIUs() throws ProvisionException {
295 		IMetadataRepository metadataRepo = getCompositeMetadataRepository();
296 
297 		if (rootIUs != null) {
298 			sourceIUs = new ArrayList<>();
299 			for (String rootIU : rootIUs) {
300 				String[] segments = getArrayArgsFromString(rootIU, "/"); //$NON-NLS-1$
301 				VersionRange range = segments.length > 1 ? VersionRange.create(segments[1]) : null;
302 				Iterator<IInstallableUnit> queryResult = metadataRepo.query(QueryUtil.createIUQuery(segments[0], range), null).iterator();
303 				while (queryResult.hasNext())
304 					sourceIUs.add(queryResult.next());
305 			}
306 		} else if (sourceIUs == null || sourceIUs.isEmpty()) {
307 			sourceIUs = new ArrayList<>();
308 			Iterator<IInstallableUnit> queryResult = metadataRepo.query(QueryUtil.createIUAnyQuery(), null).iterator();
309 			while (queryResult.hasNext())
310 				sourceIUs.add(queryResult.next());
311 			/* old metadata mirroring app did not throw an exception here */
312 			if (sourceIUs.size() == 0 && destinationMetadataRepository != null && metadataOrArtifacts == null)
313 				throw new ProvisionException(Messages.MirrorApplication_no_IUs);
314 		}
315 	}
316 
317 	/*
318 	 * Initialize logs, if applicable
319 	 */
initializeLogs()320 	private void initializeLogs() {
321 		if (compare && comparatorLogFile != null)
322 			comparatorLog = getLog(comparatorLogFile, comparatorID);
323 		if (mirrorLog == null && mirrorLogFile != null)
324 			mirrorLog = getLog(mirrorLogFile, LOG_ROOT);
325 	}
326 
327 	/*
328 	 * Finalize logs, if applicable
329 	 */
finalizeLogs()330 	private void finalizeLogs() {
331 		if (comparatorLog != null)
332 			comparatorLog.close();
333 		if (mirrorLog != null)
334 			mirrorLog.close();
335 	}
336 
337 	/*
338 	 * Get the log for a location
339 	 */
getLog(File location, String root)340 	private IArtifactMirrorLog getLog(File location, String root) {
341 		String absolutePath = location.getAbsolutePath();
342 		if (absolutePath.toLowerCase().endsWith(".xml")) //$NON-NLS-1$
343 			return new XMLMirrorLog(absolutePath, 0, root);
344 		return new FileMirrorLog(absolutePath, 0, root);
345 	}
346 
performResolution(IProgressMonitor monitor)347 	private IQueryable<IInstallableUnit> performResolution(IProgressMonitor monitor) throws ProvisionException {
348 		IProfileRegistry registry = Activator.getProfileRegistry();
349 		String profileId = "MirrorApplication-" + System.currentTimeMillis(); //$NON-NLS-1$
350 		IProfile profile = registry.addProfile(profileId, slicingOptions.getFilter());
351 		IPlanner planner = (IPlanner) Activator.getAgent().getService(IPlanner.SERVICE_NAME);
352 		if (planner == null)
353 			throw new IllegalStateException();
354 		IProfileChangeRequest pcr = planner.createChangeRequest(profile);
355 		pcr.addAll(sourceIUs);
356 		IProvisioningPlan plan = planner.getProvisioningPlan(pcr, null, monitor);
357 		registry.removeProfile(profileId);
358 		@SuppressWarnings("unchecked")
359 		IQueryable<IInstallableUnit>[] arr = new IQueryable[plan.getInstallerPlan() == null ? 1 : 2];
360 		arr[0] = plan.getAdditions();
361 		if (plan.getInstallerPlan() != null)
362 			arr[1] = plan.getInstallerPlan().getAdditions();
363 		return new CompoundQueryable<>(arr);
364 	}
365 
slice(IProgressMonitor monitor)366 	private IQueryable<IInstallableUnit> slice(IProgressMonitor monitor) throws ProvisionException {
367 		if (slicingOptions == null)
368 			slicingOptions = new SlicingOptions();
369 		if (slicingOptions.getInstallTimeLikeResolution())
370 			return performResolution(monitor);
371 
372 		PermissiveSlicer slicer = new PermissiveSlicer(getCompositeMetadataRepository(), slicingOptions.getFilter(), slicingOptions.includeOptionalDependencies(), slicingOptions.isEverythingGreedy(), slicingOptions.forceFilterTo(), slicingOptions.considerStrictDependencyOnly(), slicingOptions.followOnlyFilteredRequirements());
373 		IQueryable<IInstallableUnit> slice = slicer.slice(sourceIUs.toArray(new IInstallableUnit[sourceIUs.size()]), monitor);
374 
375 		if (slice != null && slicingOptions.latestVersionOnly()) {
376 			IQueryResult<IInstallableUnit> queryResult = slice.query(QueryUtil.createLatestIUQuery(), monitor);
377 			slice = queryResult;
378 		}
379 		if (slicer.getStatus().getSeverity() != IStatus.OK && mirrorLog != null) {
380 			mirrorLog.log(slicer.getStatus());
381 		}
382 		if (slice == null) {
383 			throw new ProvisionException(slicer.getStatus());
384 		}
385 		return slice;
386 	}
387 
setSlicingOptions(SlicingOptions options)388 	public void setSlicingOptions(SlicingOptions options) {
389 		slicingOptions = options;
390 	}
391 
392 	/*
393 	 * Set the location of the baseline repository. (used in comparison)
394 	 */
setBaseline(URI baseline)395 	public void setBaseline(URI baseline) {
396 		this.baseline = baseline;
397 		compare = true;
398 	}
399 
400 	/*
401 	 * Set the identifier of the comparator to use.
402 	 */
setComparatorID(String value)403 	public void setComparatorID(String value) {
404 		comparatorID = value;
405 		compare = true;
406 	}
407 
408 	/*
409 	 * Set whether or not the application should be calling a comparator when mirroring.
410 	 */
setCompare(boolean value)411 	public void setCompare(boolean value) {
412 		compare = value;
413 	}
414 
415 	/*
416 	 * Set whether or not we should ignore errors when running the mirror application.
417 	 */
setIgnoreErrors(boolean value)418 	public void setIgnoreErrors(boolean value) {
419 		failOnError = !value;
420 	}
421 
422 	/*
423 	 * Set whether or not the the artifacts are raw.
424 	 */
setRaw(boolean value)425 	public void setRaw(boolean value) {
426 		raw = value;
427 	}
428 
429 	/*
430 	 * Set whether or not the mirror application should be run in verbose mode.
431 	 */
setVerbose(boolean value)432 	public void setVerbose(boolean value) {
433 		verbose = value;
434 	}
435 
436 	/*
437 	 * Set the location of the log for comparator output
438 	 */
setComparatorLog(File comparatorLog)439 	public void setComparatorLog(File comparatorLog) {
440 		this.comparatorLogFile = comparatorLog;
441 	}
442 
443 	/*
444 	 * Set the location of the log for mirroring.
445 	 */
setLog(File mirrorLog)446 	public void setLog(File mirrorLog) {
447 		this.mirrorLogFile = mirrorLog;
448 	}
449 
450 	/*
451 	 * Set the ArtifactMirror log
452 	 */
setLog(IArtifactMirrorLog log)453 	public void setLog(IArtifactMirrorLog log) {
454 		mirrorLog = log;
455 	}
456 
457 	/*
458 	 * Set if the artifact mirror should be validated
459 	 */
setValidate(boolean value)460 	public void setValidate(boolean value) {
461 		validate = value;
462 	}
463 
464 	/*
465 	 * Set if references should be mirrored
466 	 */
setReferences(boolean flag)467 	public void setReferences(boolean flag) {
468 		mirrorReferences = flag;
469 	}
470 
setComparatorExclusions(IQuery<IArtifactDescriptor> exclusions)471 	public void setComparatorExclusions(IQuery<IArtifactDescriptor> exclusions) {
472 		compareExclusions = exclusions;
473 	}
474 
setIncludePacked(boolean includePacked)475 	public void setIncludePacked(boolean includePacked) {
476 		this.includePacked = includePacked;
477 	}
478 
setMirrorProperties(boolean mirrorProperties)479 	public void setMirrorProperties(boolean mirrorProperties) {
480 		this.mirrorProperties = mirrorProperties;
481 	}
482 }
483