1/*
2 * Licensed to the Apache Software Foundation (ASF) under one or more
3 * contributor license agreements.  See the NOTICE file distributed with
4 * this work for additional information regarding copyright ownership.
5 * The ASF licenses this file to You under the Apache License, Version 2.0
6 * (the "License"); you may not use this file except in compliance with
7 * the License.  You may obtain a copy of the License at
8 *
9 *    http://www.apache.org/licenses/LICENSE-2.0
10 *
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
16 */
17
18package org.apache.spark.api.python
19
20import java.io.File
21import java.util.{List => JList}
22
23import scala.collection.JavaConverters._
24import scala.collection.mutable.ArrayBuffer
25
26import org.apache.spark.SparkContext
27import org.apache.spark.api.java.{JavaRDD, JavaSparkContext}
28
29private[spark] object PythonUtils {
30  /** Get the PYTHONPATH for PySpark, either from SPARK_HOME, if it is set, or from our JAR */
31  def sparkPythonPath: String = {
32    val pythonPath = new ArrayBuffer[String]
33    for (sparkHome <- sys.env.get("SPARK_HOME")) {
34      pythonPath += Seq(sparkHome, "python", "lib", "pyspark.zip").mkString(File.separator)
35      pythonPath += Seq(sparkHome, "python", "lib", "py4j-0.10.4-src.zip").mkString(File.separator)
36    }
37    pythonPath ++= SparkContext.jarOfObject(this)
38    pythonPath.mkString(File.pathSeparator)
39  }
40
41  /** Merge PYTHONPATHS with the appropriate separator. Ignores blank strings. */
42  def mergePythonPaths(paths: String*): String = {
43    paths.filter(_ != "").mkString(File.pathSeparator)
44  }
45
46  def generateRDDWithNull(sc: JavaSparkContext): JavaRDD[String] = {
47    sc.parallelize(List("a", null, "b"))
48  }
49
50  /**
51   * Convert list of T into seq of T (for calling API with varargs)
52   */
53  def toSeq[T](vs: JList[T]): Seq[T] = {
54    vs.asScala
55  }
56
57  /**
58   * Convert list of T into a (Scala) List of T
59   */
60  def toList[T](vs: JList[T]): List[T] = {
61    vs.asScala.toList
62  }
63
64  /**
65   * Convert list of T into array of T (for calling API with array)
66   */
67  def toArray[T](vs: JList[T]): Array[T] = {
68    vs.toArray().asInstanceOf[Array[T]]
69  }
70
71  /**
72   * Convert java map of K, V into Map of K, V (for calling API with varargs)
73   */
74  def toScalaMap[K, V](jm: java.util.Map[K, V]): Map[K, V] = {
75    jm.asScala.toMap
76  }
77}
78