1/* 2 * Licensed to the Apache Software Foundation (ASF) under one or more 3 * contributor license agreements. See the NOTICE file distributed with 4 * this work for additional information regarding copyright ownership. 5 * The ASF licenses this file to You under the Apache License, Version 2.0 6 * (the "License"); you may not use this file except in compliance with 7 * the License. You may obtain a copy of the License at 8 * 9 * http://www.apache.org/licenses/LICENSE-2.0 10 * 11 * Unless required by applicable law or agreed to in writing, software 12 * distributed under the License is distributed on an "AS IS" BASIS, 13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 * See the License for the specific language governing permissions and 15 * limitations under the License. 16 */ 17 18package org.apache.spark.sql.catalyst.encoders 19 20import java.util.concurrent.ConcurrentMap 21 22import com.google.common.collect.MapMaker 23 24import org.apache.spark.util.Utils 25 26object OuterScopes { 27 @transient 28 lazy val outerScopes: ConcurrentMap[String, AnyRef] = 29 new MapMaker().weakValues().makeMap() 30 31 /** 32 * Adds a new outer scope to this context that can be used when instantiating an `inner class` 33 * during deserialization. Inner classes are created when a case class is defined in the 34 * Spark REPL and registering the outer scope that this class was defined in allows us to create 35 * new instances on the spark executors. In normal use, users should not need to call this 36 * function. 37 * 38 * Warning: this function operates on the assumption that there is only ever one instance of any 39 * given wrapper class. 40 */ 41 def addOuterScope(outer: AnyRef): Unit = { 42 outerScopes.putIfAbsent(outer.getClass.getName, outer) 43 } 44 45 /** 46 * Returns a function which can get the outer scope for the given inner class. By using function 47 * as return type, we can delay the process of getting outer pointer to execution time, which is 48 * useful for inner class defined in REPL. 49 */ 50 def getOuterScope(innerCls: Class[_]): () => AnyRef = { 51 assert(innerCls.isMemberClass) 52 val outerClassName = innerCls.getDeclaringClass.getName 53 val outer = outerScopes.get(outerClassName) 54 if (outer == null) { 55 outerClassName match { 56 // If the outer class is generated by REPL, users don't need to register it as it has 57 // only one instance and there is a way to retrieve it: get the `$read` object, call the 58 // `INSTANCE()` method to get the single instance of class `$read`. Then call `$iw()` 59 // method multiply times to get the single instance of the inner most `$iw` class. 60 case REPLClass(baseClassName) => 61 () => { 62 val objClass = Utils.classForName(baseClassName + "$") 63 val objInstance = objClass.getField("MODULE$").get(null) 64 val baseInstance = objClass.getMethod("INSTANCE").invoke(objInstance) 65 val baseClass = Utils.classForName(baseClassName) 66 67 var getter = iwGetter(baseClass) 68 var obj = baseInstance 69 while (getter != null) { 70 obj = getter.invoke(obj) 71 getter = iwGetter(getter.getReturnType) 72 } 73 74 if (obj == null) { 75 throw new RuntimeException(s"Failed to get outer pointer for ${innerCls.getName}") 76 } 77 78 outerScopes.putIfAbsent(outerClassName, obj) 79 obj 80 } 81 case _ => null 82 } 83 } else { 84 () => outer 85 } 86 } 87 88 private def iwGetter(cls: Class[_]) = { 89 try { 90 cls.getMethod("$iw") 91 } catch { 92 case _: NoSuchMethodException => null 93 } 94 } 95 96 // The format of REPL generated wrapper class's name, e.g. `$line12.$read$$iw$$iw` 97 private[this] val REPLClass = """^(\$line(?:\d+)\.\$read)(?:\$\$iw)+$""".r 98} 99