1 /******************************************************************************* 2 * Copyright (c) 2005, 2020 IBM Corporation and others. 3 * 4 * This program and the accompanying materials 5 * are made available under the terms of the Eclipse Public License 2.0 6 * which accompanies this distribution, and is available at 7 * https://www.eclipse.org/legal/epl-2.0/ 8 * 9 * SPDX-License-Identifier: EPL-2.0 10 * 11 * Contributors: 12 * IBM Corporation - initial API and implementation 13 *******************************************************************************/ 14 package org.eclipse.equinox.p2.tests.artifact.repository; 15 16 import java.io.File; 17 import java.io.IOException; 18 import java.io.OutputStream; 19 import java.lang.reflect.Field; 20 import java.net.URI; 21 import java.net.URISyntaxException; 22 import java.util.HashMap; 23 import java.util.LinkedList; 24 import java.util.Map; 25 import java.util.Queue; 26 import javax.xml.parsers.DocumentBuilder; 27 import javax.xml.parsers.DocumentBuilderFactory; 28 import org.eclipse.core.runtime.IProgressMonitor; 29 import org.eclipse.core.runtime.IStatus; 30 import org.eclipse.core.runtime.NullProgressMonitor; 31 import org.eclipse.core.runtime.Status; 32 import org.eclipse.core.runtime.URIUtil; 33 import org.eclipse.equinox.internal.p2.artifact.repository.MirrorRequest; 34 import org.eclipse.equinox.internal.p2.artifact.repository.MirrorSelector; 35 import org.eclipse.equinox.internal.p2.artifact.repository.simple.SimpleArtifactRepository; 36 import org.eclipse.equinox.internal.p2.metadata.ArtifactKey; 37 import org.eclipse.equinox.internal.p2.repository.Transport; 38 import org.eclipse.equinox.p2.core.ProvisionException; 39 import org.eclipse.equinox.p2.metadata.IArtifactKey; 40 import org.eclipse.equinox.p2.metadata.Version; 41 import org.eclipse.equinox.p2.query.IQuery; 42 import org.eclipse.equinox.p2.query.IQueryResult; 43 import org.eclipse.equinox.p2.query.IQueryable; 44 import org.eclipse.equinox.p2.repository.IRepository; 45 import org.eclipse.equinox.p2.repository.artifact.ArtifactKeyQuery; 46 import org.eclipse.equinox.p2.repository.artifact.IArtifactDescriptor; 47 import org.eclipse.equinox.p2.repository.artifact.IArtifactRepository; 48 import org.eclipse.equinox.p2.repository.artifact.IArtifactRepositoryManager; 49 import org.eclipse.equinox.p2.repository.artifact.IArtifactRequest; 50 import org.eclipse.equinox.p2.repository.artifact.spi.AbstractArtifactRepository; 51 import org.eclipse.equinox.p2.repository.spi.AbstractRepository; 52 import org.eclipse.equinox.p2.tests.AbstractProvisioningTest; 53 import org.eclipse.equinox.p2.tests.AbstractWrappedArtifactRepository; 54 import org.w3c.dom.Document; 55 import org.w3c.dom.Element; 56 import org.w3c.dom.NodeList; 57 58 public class MirrorRequestTest extends AbstractProvisioningTest { 59 private static final String testDataLocation = "testData/artifactRepo/emptyJarRepo"; 60 File targetLocation; 61 IArtifactRepository targetRepository, sourceRepository; 62 URI destination, failedOptimized, pakedRepositoryLocation; 63 boolean isJava14; 64 65 @Override setUp()66 public void setUp() throws Exception { 67 super.setUp(); 68 targetLocation = File.createTempFile("target", ".repo"); 69 targetLocation.delete(); 70 targetLocation.mkdirs(); 71 targetRepository = new SimpleArtifactRepository(getAgent(), "TargetRepo", targetLocation.toURI(), null); 72 73 IArtifactRepositoryManager mgr = getArtifactRepositoryManager(); 74 sourceRepository = mgr.loadRepository((getTestData("EmptyJar repo", testDataLocation).toURI()), null); 75 failedOptimized = URIUtil.toJarURI(getTestData("Error loading test data", "testData/mirror/invalidPackedMissingCanonical.zip").toURI(), null); 76 pakedRepositoryLocation = getTestData("Error loading packed repository", "testData/mirror/mirrorPackedRepo").toURI(); 77 destination = getTempFolder().toURI(); 78 isJava14 = System.getProperty("java.specification.version").compareTo("14") >= 0 ? true : false; //$NON-NLS-1$ //$NON-NLS-2$ 79 } 80 81 @Override tearDown()82 protected void tearDown() throws Exception { 83 getArtifactRepositoryManager().removeRepository(destination); 84 getArtifactRepositoryManager().removeRepository(failedOptimized); 85 getArtifactRepositoryManager().removeRepository(targetLocation.toURI()); 86 getArtifactRepositoryManager().removeRepository(pakedRepositoryLocation); 87 AbstractProvisioningTest.delete(targetLocation); 88 delete(new File(destination)); 89 super.tearDown(); 90 } 91 testInvalidZipFileInTheSource()92 public void testInvalidZipFileInTheSource() { 93 IArtifactKey key = new ArtifactKey("org.eclipse.update.feature", "HelloWorldFeature", Version.createOSGi(1, 0, 0)); 94 Map<String, String> targetProperties = new HashMap<>(); 95 targetProperties.put("artifact.folder", "true"); 96 MirrorRequest request = new MirrorRequest(key, targetRepository, null, targetProperties, getAgent().getService(Transport.class)); 97 request.perform(sourceRepository, new NullProgressMonitor()); 98 99 assertTrue(request.getResult().matches(IStatus.ERROR)); 100 assertTrue(request.getResult().getException() instanceof IOException); 101 } 102 testMissingArtifact()103 public void testMissingArtifact() { 104 IArtifactKey key = new ArtifactKey("org.eclipse.update.feature", "Missing", Version.createOSGi(1, 0, 0)); 105 Map<String, String> targetProperties = new HashMap<>(); 106 targetProperties.put("artifact.folder", "true"); 107 MirrorRequest request = new MirrorRequest(key, targetRepository, null, targetProperties, getTransport()); 108 request.perform(sourceRepository, new NullProgressMonitor()); 109 110 assertTrue(request.getResult().matches(IStatus.ERROR)); 111 } 112 113 // Test that if MirrorRequest fails to download a packed artifact it attempts the canonical version testFailToCanonical()114 public void testFailToCanonical() { 115 RemoteRepo src = new RemoteRepo((SimpleArtifactRepository) sourceRepository); 116 117 IArtifactKey key = new ArtifactKey("test.txt", "fail_to_canonical", Version.parseVersion("1.0.0")); 118 MirrorRequest request = new MirrorRequest(key, targetRepository, null, null, getTransport()); 119 request.perform(src, new NullProgressMonitor()); 120 121 assertTrue(request.getResult().toString(), request.getResult().isOK()); 122 assertTrue(String.format("Target does not contain artifact %s", key), targetRepository.contains(key)); 123 if (isJava14) { 124 // Pack200 tools are gone in Java14+ thus pack.gz file is not downloaded 125 assertEquals("Exact number of downloads", 1, src.downloadCount); 126 } else { 127 assertEquals("Exact number of downloads", 2, src.downloadCount); 128 } 129 } 130 131 /** 132 * Same as {@link #testFailToCanonical()} but with 3 mirrors: 133 * <ul> 134 * <li><code>mirror-one</code>, which is unreachable</li> 135 * <li><code>mirror-two</code>, which has an invalid optimized artifact and causes processing step to fail</li> 136 * <li>original repository, which has a valid canonical artifact</li> 137 * </ul> 138 * 139 */ testFailToCanonicalWithMirrors()140 public void testFailToCanonicalWithMirrors() { 141 OrderedMirrorSelector selector = new OrderedMirrorSelector(sourceRepository); 142 try { 143 RemoteRepo src = new RemoteRepo((SimpleArtifactRepository) sourceRepository); 144 145 IArtifactKey key = new ArtifactKey("test.txt", "fail_to_canonical", Version.parseVersion("1.0.0")); 146 MirrorRequest request = new MirrorRequest(key, targetRepository, null, null, getTransport()); 147 request.perform(src, new NullProgressMonitor()); 148 149 assertTrue(request.getResult().toString(), request.getResult().isOK()); 150 assertTrue(String.format("Target does not contain artifact %s", key), targetRepository.contains(key)); 151 assertEquals("Exact number of downloads", 3, src.downloadCount); 152 assertEquals("All mirrors utilized", selector.mirrors.length, selector.index); 153 } finally { 154 selector.clearSelector(); 155 } 156 } 157 158 // Test that SimpleArtifactRepository & MirrorRequest use mirrors in the event of a failure. testMirrorFailOver()159 public void testMirrorFailOver() { 160 OrderedMirrorSelector selector = new OrderedMirrorSelector(sourceRepository); 161 try { 162 // call test 163 IArtifactKey key = new ArtifactKey("test.txt", "HelloWorldText", Version.parseVersion("1.0.0")); 164 MirrorRequest request = new MirrorRequest(key, targetRepository, null, null, getTransport()); 165 request.perform(sourceRepository, new NullProgressMonitor()); 166 167 // The download succeeded 168 assertTrue(request.getResult().toString(), request.getResult().isOK()); 169 // All available mirrors used 170 assertEquals("All mirrors utilized", selector.mirrors.length, selector.index); 171 } finally { 172 selector.clearSelector(); 173 } 174 } 175 176 /* 177 * Test that the expected Status level is returned when a mirror fails from packed to canonical 178 */ testStatusFromFailover()179 public void testStatusFromFailover() { 180 StatusSequenceRepository source = null; 181 LinkedList<IStatus> seq = new LinkedList<>(); 182 try { 183 source = new StatusSequenceRepository(getArtifactRepositoryManager().loadRepository(pakedRepositoryLocation, new NullProgressMonitor())); 184 185 } catch (ProvisionException e) { 186 fail("Failed to load source repository"); 187 } 188 // Set status sequence, actual Statuses added later 189 source.setSequence(seq); 190 // Grab an ArtifactKey to mirror, doesn't matter which 191 IQueryResult<IArtifactKey> keys = source.query(ArtifactKeyQuery.ALL_KEYS, null); 192 assertTrue("Unable to obtain artifact keys", keys != null && !keys.isEmpty()); 193 194 IArtifactKey key = keys.iterator().next(); 195 MirrorRequest req = new MirrorRequest(key, targetRepository, null, null, getTransport()); 196 197 // Set Status sequence 198 seq.add(new Status(IStatus.ERROR, "Activator", "Message")); 199 seq.add(new Status(IStatus.WARNING, "Activator", "Message")); 200 req.perform(source, new NullProgressMonitor()); 201 202 if (isJava14) { 203 // packed artifact is ignored as Java 14 removed pack200 204 assertEquals("Expected ERROR status", IStatus.ERROR, req.getResult().getSeverity()); 205 } else { 206 assertEquals("Expected WARNING status", IStatus.WARNING, req.getResult().getSeverity()); 207 } 208 209 // Remove key from repo so the same one can be used 210 targetRepository.removeDescriptor(key, new NullProgressMonitor()); 211 // Set Status sequence 212 req = new MirrorRequest(key, targetRepository, null, null, getTransport()); 213 214 seq.add(new Status(IStatus.WARNING, "Activator", "Message")); 215 seq.add(new Status(IStatus.INFO, "Activator", "Message")); 216 req.perform(source, new NullProgressMonitor()); 217 218 if (isJava14) { 219 // packed artifact is ignored as Java 14 removed pack200 220 assertEquals("Expected WARNING status", IStatus.WARNING, req.getResult().getSeverity()); 221 } else { 222 assertEquals("Expected INFO status", IStatus.INFO, req.getResult().getSeverity()); 223 } 224 225 // Remove key from repo so the same one can be used 226 targetRepository.removeDescriptor(key, new NullProgressMonitor()); 227 // Set Status sequence 228 req = new MirrorRequest(key, targetRepository, null, null, getTransport()); 229 230 seq.add(new Status(IStatus.INFO, "Activator", "Message")); 231 req.perform(source, new NullProgressMonitor()); 232 if (isJava14) { 233 // packed artifact is ignored as Java 14 removed pack200 234 assertEquals("Expected WARNING status", IStatus.WARNING, req.getResult().getSeverity()); 235 } else { 236 assertEquals("Expected OK status", IStatus.OK, req.getResult().getSeverity()); 237 } 238 } 239 240 /* 241 * 242 */ testFailedOptimizedMissingCanonical()243 public void testFailedOptimizedMissingCanonical() { 244 if (isJava14) { 245 // Java 14 doesn't have pack/unpack tools 246 return; 247 } 248 249 try { 250 IArtifactRepository source = new AbstractWrappedArtifactRepository(getArtifactRepositoryManager().loadRepository(failedOptimized, new NullProgressMonitor())) { 251 @Override 252 public URI getLocation() { 253 try { 254 return new URI("http://nowhere"); 255 } catch (URISyntaxException e) { 256 fail("Failed to create URI", e); 257 return null; 258 } 259 } 260 }; 261 IArtifactRepository target = getArtifactRepositoryManager().createRepository(destination, "Destination", IArtifactRepositoryManager.TYPE_SIMPLE_REPOSITORY, null); 262 263 IArtifactKey key = new ArtifactKey("osgi.bundle", "org.eclipse.ve.jfc", Version.parseVersion("1.4.0.HEAD")); 264 MirrorRequest req = new MirrorRequest(key, target, null, null, getTransport()); 265 266 req.perform(source, new NullProgressMonitor()); 267 IStatus result = req.getResult(); 268 assertTrue("MirrorRequest should have failed", result.matches(IStatus.ERROR)); 269 assertEquals("Result should contain two failures", 2, result.getChildren().length); 270 assertStatusContains("Return status does not contain Signature Verification failure", result, "Invalid content:"); 271 assertStatusContains("Return status does not contain Missing Artifact status", result, "Artifact not found:"); 272 } catch (ProvisionException e) { 273 fail("Failed to load repositories", e); 274 } 275 } 276 assertStatusContains(String message, IStatus status, String statusString)277 protected static void assertStatusContains(String message, IStatus status, String statusString) { 278 if (!statusContains(status, statusString)) 279 fail(message); 280 } 281 282 class StatusSequenceRepository extends AbstractWrappedArtifactRepository { 283 Queue<IStatus> sequence; 284 StatusSequenceRepository(IArtifactRepository repo)285 public StatusSequenceRepository(IArtifactRepository repo) { 286 super(repo); 287 } 288 289 @Override getLocation()290 public URI getLocation() { 291 // Lie about the location so packed files are used 292 try { 293 return new URI("http://somewhere"); 294 } catch (URISyntaxException e) { 295 return null; 296 } 297 } 298 299 @Override getArtifact(IArtifactDescriptor descriptor, OutputStream destination, IProgressMonitor monitor)300 public IStatus getArtifact(IArtifactDescriptor descriptor, OutputStream destination, IProgressMonitor monitor) { 301 try { 302 destination.write(new byte[] {1, 1, 2}); 303 } catch (Exception e) { 304 fail("Failed to write to stream", e); 305 } 306 if (sequence.isEmpty()) 307 return Status.OK_STATUS; 308 return sequence.remove(); 309 } 310 setSequence(Queue<IStatus> queue)311 public void setSequence(Queue<IStatus> queue) { 312 sequence = queue; 313 } 314 } 315 statusContains(IStatus status, String statusString)316 private static boolean statusContains(IStatus status, String statusString) { 317 if (status.getMessage().indexOf(statusString) != -1) 318 return true; 319 if (!status.isMultiStatus()) 320 return false; 321 322 IStatus[] children = status.getChildren(); 323 for (IStatus child : children) { 324 if (statusContains(child, statusString)) { 325 return true; 326 } 327 } 328 329 return false; 330 } 331 332 // Repository which misleads about its location 333 protected class RemoteRepo extends AbstractArtifactRepository { 334 SimpleArtifactRepository delegate; 335 int downloadCount = 0; 336 RemoteRepo(SimpleArtifactRepository repo)337 RemoteRepo(SimpleArtifactRepository repo) { 338 super(getAgent(), repo.getName(), repo.getType(), repo.getVersion(), repo.getLocation(), repo.getDescription(), repo.getProvider(), repo.getProperties()); 339 delegate = repo; 340 } 341 342 @Override getLocation()343 public synchronized URI getLocation() { 344 try { 345 return new URI("http://test/"); 346 } catch (URISyntaxException e) { 347 // Should never happen, but we'll fail anyway 348 fail("URI creation failed", e); 349 return null; 350 } 351 } 352 353 @Override contains(IArtifactDescriptor descriptor)354 public boolean contains(IArtifactDescriptor descriptor) { 355 return delegate.contains(descriptor); 356 } 357 358 @Override contains(IArtifactKey key)359 public boolean contains(IArtifactKey key) { 360 return delegate.contains(key); 361 } 362 363 @Override getArtifact(IArtifactDescriptor descriptor, OutputStream destination, IProgressMonitor monitor)364 public IStatus getArtifact(IArtifactDescriptor descriptor, OutputStream destination, IProgressMonitor monitor) { 365 downloadCount++; 366 return delegate.getArtifact(descriptor, destination, monitor); 367 } 368 369 @Override getArtifactDescriptors(IArtifactKey key)370 public IArtifactDescriptor[] getArtifactDescriptors(IArtifactKey key) { 371 return delegate.getArtifactDescriptors(key); 372 } 373 374 @Override getArtifacts(IArtifactRequest[] requests, IProgressMonitor monitor)375 public IStatus getArtifacts(IArtifactRequest[] requests, IProgressMonitor monitor) { 376 return delegate.getArtifacts(requests, monitor); 377 } 378 379 @Override getOutputStream(IArtifactDescriptor descriptor)380 public OutputStream getOutputStream(IArtifactDescriptor descriptor) throws ProvisionException { 381 return delegate.getOutputStream(descriptor); 382 } 383 384 @Override getRawArtifact(IArtifactDescriptor descriptor, OutputStream destination, IProgressMonitor monitor)385 public IStatus getRawArtifact(IArtifactDescriptor descriptor, OutputStream destination, IProgressMonitor monitor) { 386 return delegate.getRawArtifact(descriptor, destination, monitor); 387 } 388 389 @Override descriptorQueryable()390 public IQueryable<IArtifactDescriptor> descriptorQueryable() { 391 return delegate.descriptorQueryable(); 392 } 393 394 @Override query(IQuery<IArtifactKey> query, IProgressMonitor monitor)395 public IQueryResult<IArtifactKey> query(IQuery<IArtifactKey> query, IProgressMonitor monitor) { 396 return delegate.query(query, monitor); 397 } 398 } 399 400 /* 401 * Special mirror selector for testing which chooses mirrors in order 402 */ 403 protected class OrderedMirrorSelector extends MirrorSelector { 404 private URI repoLocation; 405 int index = 0; 406 MirrorInfo[] mirrors; 407 IArtifactRepository repo; 408 MirrorSelector oldSelector = null; 409 OrderedMirrorSelector(IArtifactRepository repo)410 OrderedMirrorSelector(IArtifactRepository repo) { 411 super(repo, getTransport()); 412 this.repo = repo; 413 // Setting this property forces SimpleArtifactRepository to use mirrors despite being a local repo 414 // Alternatively we could use reflect to change "location" of the repo 415 repo.setProperty(SimpleArtifactRepository.PROP_FORCE_THREADING, String.valueOf(true)); 416 setSelector(); 417 getRepoLocation(); 418 mirrors = computeMirrors("file:///" + getTestData("Mirror Location", testDataLocation + '/' + repo.getProperties().get(IRepository.PROP_MIRRORS_URL)).toString().replace('\\', '/')); 419 } 420 421 // Hijack the source repository's MirrorSelector setSelector()422 private void setSelector() { 423 Field mirrorField = null; 424 try { 425 mirrorField = SimpleArtifactRepository.class.getDeclaredField("mirrors"); 426 mirrorField.setAccessible(true); 427 oldSelector = (MirrorSelector) mirrorField.get(repo); // Store the old value so we can restore it 428 mirrorField.set(repo, this); 429 } catch (Exception e) { 430 fail("0.2", e); 431 } 432 } 433 434 // Clear the mirror selector we place on the repository clearSelector()435 public void clearSelector() { 436 if (repo == null) { 437 return; 438 } 439 repo.setProperty(SimpleArtifactRepository.PROP_FORCE_THREADING, String.valueOf(false)); 440 Field mirrorField = null; 441 try { 442 mirrorField = SimpleArtifactRepository.class.getDeclaredField("mirrors"); 443 mirrorField.setAccessible(true); 444 mirrorField.set(repo, oldSelector); 445 } catch (Exception e) { 446 fail("0.2", e); 447 } 448 } 449 450 // Overridden to prevent mirror sorting 451 @Override reportResult(String toDownload, IStatus result)452 public synchronized void reportResult(String toDownload, IStatus result) { 453 return; 454 } 455 456 // We want to test each mirror once. 457 @Override hasValidMirror()458 public synchronized boolean hasValidMirror() { 459 return mirrors != null && index < mirrors.length; 460 } 461 462 @Override getMirrorLocation(URI inputLocation, IProgressMonitor monitor)463 public synchronized URI getMirrorLocation(URI inputLocation, IProgressMonitor monitor) { 464 return URIUtil.append(nextMirror(), repoLocation.relativize(inputLocation).getPath()); 465 } 466 nextMirror()467 private URI nextMirror() { 468 Field mirrorLocation = null; 469 try { 470 mirrorLocation = MirrorInfo.class.getDeclaredField("locationString"); 471 mirrorLocation.setAccessible(true); 472 473 return URIUtil.makeAbsolute(new URI((String) mirrorLocation.get(mirrors[index++])), repoLocation); 474 } catch (Exception e) { 475 fail(Double.toString(0.4 + index), e); 476 return null; 477 } 478 } 479 getRepoLocation()480 private synchronized void getRepoLocation() { 481 Field locationField = null; 482 try { 483 locationField = AbstractRepository.class.getDeclaredField("location"); 484 locationField.setAccessible(true); 485 repoLocation = (URI) locationField.get(repo); 486 } catch (Exception e) { 487 fail("0.3", e); 488 } 489 } 490 computeMirrors(String mirrorsURL)491 private MirrorInfo[] computeMirrors(String mirrorsURL) { 492 // Copied & modified from MirrorSelector 493 try { 494 DocumentBuilderFactory domFactory = DocumentBuilderFactory.newInstance(); 495 DocumentBuilder builder = domFactory.newDocumentBuilder(); 496 Document document = builder.parse(mirrorsURL); 497 if (document == null) 498 return null; 499 NodeList mirrorNodes = document.getElementsByTagName("mirror"); //$NON-NLS-1$ 500 int mirrorCount = mirrorNodes.getLength(); 501 MirrorInfo[] infos = new MirrorInfo[mirrorCount + 1]; 502 for (int i = 0; i < mirrorCount; i++) { 503 Element mirrorNode = (Element) mirrorNodes.item(i); 504 String infoURL = mirrorNode.getAttribute("url"); //$NON-NLS-1$ 505 infos[i] = new MirrorInfo(infoURL, i); 506 } 507 //p2: add the base site as the last resort mirror so we can track download speed and failure rate 508 infos[mirrorCount] = new MirrorInfo(repoLocation.toString(), mirrorCount); 509 return infos; 510 } catch (Exception e) { 511 // log if absolute url 512 if (mirrorsURL != null && (mirrorsURL.startsWith("http://") //$NON-NLS-1$ 513 || mirrorsURL.startsWith("https://") //$NON-NLS-1$ 514 || mirrorsURL.startsWith("file://") //$NON-NLS-1$ 515 || mirrorsURL.startsWith("ftp://") //$NON-NLS-1$ 516 || mirrorsURL.startsWith("jar://"))) //$NON-NLS-1$ 517 fail("Error processing mirrors URL: " + mirrorsURL, e); //$NON-NLS-1$ 518 return null; 519 } 520 } 521 } 522 } 523