1/*
2 * Licensed to the Apache Software Foundation (ASF) under one or more
3 * contributor license agreements.  See the NOTICE file distributed with
4 * this work for additional information regarding copyright ownership.
5 * The ASF licenses this file to You under the Apache License, Version 2.0
6 * (the "License"); you may not use this file except in compliance with
7 * the License.  You may obtain a copy of the License at
8 *
9 *    http://www.apache.org/licenses/LICENSE-2.0
10 *
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
16 */
17
18package org.apache.spark.deploy
19
20import java.io.{File, OutputStream, PrintStream}
21import java.net.URI
22import java.util.jar.{JarFile, Manifest}
23import java.util.jar.Attributes.Name
24import java.util.zip.ZipFile
25
26import scala.collection.JavaConverters._
27import scala.collection.mutable.ArrayBuffer
28
29import com.google.common.io.Files
30import org.apache.commons.io.FileUtils
31import org.scalatest.BeforeAndAfterEach
32
33import org.apache.spark.SparkFunSuite
34import org.apache.spark.api.r.RUtils
35import org.apache.spark.deploy.SparkSubmitUtils.MavenCoordinate
36import org.apache.spark.util.ResetSystemProperties
37
38class RPackageUtilsSuite
39  extends SparkFunSuite
40  with BeforeAndAfterEach
41  with ResetSystemProperties {
42
43  private val main = MavenCoordinate("a", "b", "c")
44  private val dep1 = MavenCoordinate("a", "dep1", "c")
45  private val dep2 = MavenCoordinate("a", "dep2", "d")
46
47  private def getJarPath(coord: MavenCoordinate, repo: File): File = {
48    new File(IvyTestUtils.pathFromCoordinate(coord, repo, "jar", useIvyLayout = false),
49      IvyTestUtils.artifactName(coord, useIvyLayout = false, ".jar"))
50  }
51
52  private val lineBuffer = ArrayBuffer[String]()
53
54  private val noOpOutputStream = new OutputStream {
55    def write(b: Int) = {}
56  }
57
58  /** Simple PrintStream that reads data into a buffer */
59  private class BufferPrintStream extends PrintStream(noOpOutputStream) {
60    // scalastyle:off println
61    override def println(line: String) {
62    // scalastyle:on println
63      lineBuffer += line
64    }
65  }
66
67  override def beforeEach(): Unit = {
68    super.beforeEach()
69    System.setProperty("spark.testing", "true")
70    lineBuffer.clear()
71  }
72
73  test("pick which jars to unpack using the manifest") {
74    val deps = Seq(dep1, dep2).mkString(",")
75    IvyTestUtils.withRepository(main, Some(deps), None, withR = true) { repo =>
76      val jars = Seq(main, dep1, dep2).map(c => new JarFile(getJarPath(c, new File(new URI(repo)))))
77      assert(RPackageUtils.checkManifestForR(jars(0)), "should have R code")
78      assert(!RPackageUtils.checkManifestForR(jars(1)), "should not have R code")
79      assert(!RPackageUtils.checkManifestForR(jars(2)), "should not have R code")
80    }
81  }
82
83  test("build an R package from a jar end to end") {
84    assume(RUtils.isRInstalled, "R isn't installed on this machine.")
85    val deps = Seq(dep1, dep2).mkString(",")
86    IvyTestUtils.withRepository(main, Some(deps), None, withR = true) { repo =>
87      val jars = Seq(main, dep1, dep2).map { c =>
88        getJarPath(c, new File(new URI(repo)))
89      }.mkString(",")
90      RPackageUtils.checkAndBuildRPackage(jars, new BufferPrintStream, verbose = true)
91      val firstJar = jars.substring(0, jars.indexOf(","))
92      val output = lineBuffer.mkString("\n")
93      assert(output.contains("Building R package"))
94      assert(output.contains("Extracting"))
95      assert(output.contains(s"$firstJar contains R source code. Now installing package."))
96      assert(output.contains("doesn't contain R source code, skipping..."))
97    }
98  }
99
100  test("jars that don't exist are skipped and print warning") {
101    assume(RUtils.isRInstalled, "R isn't installed on this machine.")
102    val deps = Seq(dep1, dep2).mkString(",")
103    IvyTestUtils.withRepository(main, Some(deps), None, withR = true) { repo =>
104      val jars = Seq(main, dep1, dep2).map { c =>
105        getJarPath(c, new File(new URI(repo))) + "dummy"
106      }.mkString(",")
107      RPackageUtils.checkAndBuildRPackage(jars, new BufferPrintStream, verbose = true)
108      val individualJars = jars.split(",")
109      val output = lineBuffer.mkString("\n")
110      individualJars.foreach { jarFile =>
111        assert(output.contains(s"$jarFile"))
112      }
113    }
114  }
115
116  test("faulty R package shows documentation") {
117    assume(RUtils.isRInstalled, "R isn't installed on this machine.")
118    IvyTestUtils.withRepository(main, None, None) { repo =>
119      val manifest = new Manifest
120      val attr = manifest.getMainAttributes
121      attr.put(Name.MANIFEST_VERSION, "1.0")
122      attr.put(new Name("Spark-HasRPackage"), "true")
123      val jar = IvyTestUtils.packJar(new File(new URI(repo)), dep1, Nil,
124        useIvyLayout = false, withR = false, Some(manifest))
125      RPackageUtils.checkAndBuildRPackage(jar.getAbsolutePath, new BufferPrintStream,
126        verbose = true)
127      val output = lineBuffer.mkString("\n")
128      assert(output.contains(RPackageUtils.RJarDoc))
129    }
130  }
131
132  test("SparkR zipping works properly") {
133    val tempDir = Files.createTempDir()
134    try {
135      IvyTestUtils.writeFile(tempDir, "test.R", "abc")
136      val fakeSparkRDir = new File(tempDir, "SparkR")
137      assert(fakeSparkRDir.mkdirs())
138      IvyTestUtils.writeFile(fakeSparkRDir, "abc.R", "abc")
139      IvyTestUtils.writeFile(fakeSparkRDir, "DESCRIPTION", "abc")
140      IvyTestUtils.writeFile(tempDir, "package.zip", "abc") // fake zip file :)
141      val fakePackageDir = new File(tempDir, "packageTest")
142      assert(fakePackageDir.mkdirs())
143      IvyTestUtils.writeFile(fakePackageDir, "def.R", "abc")
144      IvyTestUtils.writeFile(fakePackageDir, "DESCRIPTION", "abc")
145      val finalZip = RPackageUtils.zipRLibraries(tempDir, "sparkr.zip")
146      assert(finalZip.exists())
147      val entries = new ZipFile(finalZip).entries().asScala.map(_.getName).toSeq
148      assert(entries.contains("/test.R"))
149      assert(entries.contains("/SparkR/abc.R"))
150      assert(entries.contains("/SparkR/DESCRIPTION"))
151      assert(!entries.contains("/package.zip"))
152      assert(entries.contains("/packageTest/def.R"))
153      assert(entries.contains("/packageTest/DESCRIPTION"))
154    } finally {
155      FileUtils.deleteDirectory(tempDir)
156    }
157  }
158}
159