1/* 2 * Licensed to the Apache Software Foundation (ASF) under one or more 3 * contributor license agreements. See the NOTICE file distributed with 4 * this work for additional information regarding copyright ownership. 5 * The ASF licenses this file to You under the Apache License, Version 2.0 6 * (the "License"); you may not use this file except in compliance with 7 * the License. You may obtain a copy of the License at 8 * 9 * http://www.apache.org/licenses/LICENSE-2.0 10 * 11 * Unless required by applicable law or agreed to in writing, software 12 * distributed under the License is distributed on an "AS IS" BASIS, 13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 * See the License for the specific language governing permissions and 15 * limitations under the License. 16 */ 17 18package org.apache.spark.deploy 19 20import java.io.{File, OutputStream, PrintStream} 21import java.net.URI 22import java.util.jar.{JarFile, Manifest} 23import java.util.jar.Attributes.Name 24import java.util.zip.ZipFile 25 26import scala.collection.JavaConverters._ 27import scala.collection.mutable.ArrayBuffer 28 29import com.google.common.io.Files 30import org.apache.commons.io.FileUtils 31import org.scalatest.BeforeAndAfterEach 32 33import org.apache.spark.SparkFunSuite 34import org.apache.spark.api.r.RUtils 35import org.apache.spark.deploy.SparkSubmitUtils.MavenCoordinate 36import org.apache.spark.util.ResetSystemProperties 37 38class RPackageUtilsSuite 39 extends SparkFunSuite 40 with BeforeAndAfterEach 41 with ResetSystemProperties { 42 43 private val main = MavenCoordinate("a", "b", "c") 44 private val dep1 = MavenCoordinate("a", "dep1", "c") 45 private val dep2 = MavenCoordinate("a", "dep2", "d") 46 47 private def getJarPath(coord: MavenCoordinate, repo: File): File = { 48 new File(IvyTestUtils.pathFromCoordinate(coord, repo, "jar", useIvyLayout = false), 49 IvyTestUtils.artifactName(coord, useIvyLayout = false, ".jar")) 50 } 51 52 private val lineBuffer = ArrayBuffer[String]() 53 54 private val noOpOutputStream = new OutputStream { 55 def write(b: Int) = {} 56 } 57 58 /** Simple PrintStream that reads data into a buffer */ 59 private class BufferPrintStream extends PrintStream(noOpOutputStream) { 60 // scalastyle:off println 61 override def println(line: String) { 62 // scalastyle:on println 63 lineBuffer += line 64 } 65 } 66 67 override def beforeEach(): Unit = { 68 super.beforeEach() 69 System.setProperty("spark.testing", "true") 70 lineBuffer.clear() 71 } 72 73 test("pick which jars to unpack using the manifest") { 74 val deps = Seq(dep1, dep2).mkString(",") 75 IvyTestUtils.withRepository(main, Some(deps), None, withR = true) { repo => 76 val jars = Seq(main, dep1, dep2).map(c => new JarFile(getJarPath(c, new File(new URI(repo))))) 77 assert(RPackageUtils.checkManifestForR(jars(0)), "should have R code") 78 assert(!RPackageUtils.checkManifestForR(jars(1)), "should not have R code") 79 assert(!RPackageUtils.checkManifestForR(jars(2)), "should not have R code") 80 } 81 } 82 83 test("build an R package from a jar end to end") { 84 assume(RUtils.isRInstalled, "R isn't installed on this machine.") 85 val deps = Seq(dep1, dep2).mkString(",") 86 IvyTestUtils.withRepository(main, Some(deps), None, withR = true) { repo => 87 val jars = Seq(main, dep1, dep2).map { c => 88 getJarPath(c, new File(new URI(repo))) 89 }.mkString(",") 90 RPackageUtils.checkAndBuildRPackage(jars, new BufferPrintStream, verbose = true) 91 val firstJar = jars.substring(0, jars.indexOf(",")) 92 val output = lineBuffer.mkString("\n") 93 assert(output.contains("Building R package")) 94 assert(output.contains("Extracting")) 95 assert(output.contains(s"$firstJar contains R source code. Now installing package.")) 96 assert(output.contains("doesn't contain R source code, skipping...")) 97 } 98 } 99 100 test("jars that don't exist are skipped and print warning") { 101 assume(RUtils.isRInstalled, "R isn't installed on this machine.") 102 val deps = Seq(dep1, dep2).mkString(",") 103 IvyTestUtils.withRepository(main, Some(deps), None, withR = true) { repo => 104 val jars = Seq(main, dep1, dep2).map { c => 105 getJarPath(c, new File(new URI(repo))) + "dummy" 106 }.mkString(",") 107 RPackageUtils.checkAndBuildRPackage(jars, new BufferPrintStream, verbose = true) 108 val individualJars = jars.split(",") 109 val output = lineBuffer.mkString("\n") 110 individualJars.foreach { jarFile => 111 assert(output.contains(s"$jarFile")) 112 } 113 } 114 } 115 116 test("faulty R package shows documentation") { 117 assume(RUtils.isRInstalled, "R isn't installed on this machine.") 118 IvyTestUtils.withRepository(main, None, None) { repo => 119 val manifest = new Manifest 120 val attr = manifest.getMainAttributes 121 attr.put(Name.MANIFEST_VERSION, "1.0") 122 attr.put(new Name("Spark-HasRPackage"), "true") 123 val jar = IvyTestUtils.packJar(new File(new URI(repo)), dep1, Nil, 124 useIvyLayout = false, withR = false, Some(manifest)) 125 RPackageUtils.checkAndBuildRPackage(jar.getAbsolutePath, new BufferPrintStream, 126 verbose = true) 127 val output = lineBuffer.mkString("\n") 128 assert(output.contains(RPackageUtils.RJarDoc)) 129 } 130 } 131 132 test("SparkR zipping works properly") { 133 val tempDir = Files.createTempDir() 134 try { 135 IvyTestUtils.writeFile(tempDir, "test.R", "abc") 136 val fakeSparkRDir = new File(tempDir, "SparkR") 137 assert(fakeSparkRDir.mkdirs()) 138 IvyTestUtils.writeFile(fakeSparkRDir, "abc.R", "abc") 139 IvyTestUtils.writeFile(fakeSparkRDir, "DESCRIPTION", "abc") 140 IvyTestUtils.writeFile(tempDir, "package.zip", "abc") // fake zip file :) 141 val fakePackageDir = new File(tempDir, "packageTest") 142 assert(fakePackageDir.mkdirs()) 143 IvyTestUtils.writeFile(fakePackageDir, "def.R", "abc") 144 IvyTestUtils.writeFile(fakePackageDir, "DESCRIPTION", "abc") 145 val finalZip = RPackageUtils.zipRLibraries(tempDir, "sparkr.zip") 146 assert(finalZip.exists()) 147 val entries = new ZipFile(finalZip).entries().asScala.map(_.getName).toSeq 148 assert(entries.contains("/test.R")) 149 assert(entries.contains("/SparkR/abc.R")) 150 assert(entries.contains("/SparkR/DESCRIPTION")) 151 assert(!entries.contains("/package.zip")) 152 assert(entries.contains("/packageTest/def.R")) 153 assert(entries.contains("/packageTest/DESCRIPTION")) 154 } finally { 155 FileUtils.deleteDirectory(tempDir) 156 } 157 } 158} 159