SparkExecutionContextMock

Instance Constructors

new SparkExecutionContextMock(chorusAPICallerMock: ChorusAPICaller)

Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def appendPath(path: String): OutputStream

Append contents to the given HDFS path.
Append contents to the given HDFS path.
path
The HDFS path that we want to append to.
returns
OutputStream corresponding to the path.

Definition Classes
SparkExecutionContextMock → SparkExecutionContext
final def asInstanceOf[T0]: T0

Definition Classes
Any
def chorusAPICaller: ChorusAPICaller

Definition Classes
SparkExecutionContextMock → ExecutionContext
def chorusUserInfo: ChorusUserInfo

Definition Classes
SparkExecutionContextMock → ExecutionContext
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
def config: CustomOperatorConfig

Definition Classes
SparkExecutionContextMock → ExecutionContext
def createPath(path: String, overwrite: Boolean): OutputStream

Create a HDFS path for writing.
Create a HDFS path for writing.
path
The HDFS path that we want to create and write to.
overwrite
Whether to overwrite the given path if it exists.
returns
OutputStream corresponding to the path.

Definition Classes
SparkExecutionContextMock → SparkExecutionContext
def deletePath(path: String, recursive: Boolean): Boolean

Delete the given HDFS path.
Delete the given HDFS path.
path
The HDFS path that we want to delete.
recursive
If it's a directory, whether we want to delete the directory recursively.
returns
true if successful, false otherwise.

Definition Classes
SparkExecutionContextMock → SparkExecutionContext
def doHdfsAction[T](fs: (FileSystem) ⇒ T): T

Definition Classes
SparkExecutionContextMock → SparkExecutionContext
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def exists(path: String): Boolean

Determine whether the given path exists in the HDFS or not.
Determine whether the given path exists in the HDFS or not.
path
The path that we want to check.
returns
true if it exists, false otherwise.

Definition Classes
SparkExecutionContextMock → SparkExecutionContext
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def getSparkAutoTunedParameters[I <: IOBase, JOB <: SparkIOTypedPluginJob[I, _]](jobClass: Class[JOB], input: I, params: OperatorParameters, sparkConf: SparkJobConfiguration, listener: OperatorListener): Nothing

Returns the map of Spark parameters after autoTuning algorithm is applied.
Returns the map of Spark parameters after autoTuning algorithm is applied. This is the final set of Spark properties ready to be passed with Spark job submission (except spark.job.name) It leverages: - user-defined Spark properties at the operator level (Spark Advanced Settings box), workflow level and data source level (in this order of precedence). - AutoTunerOptions set in the SparkJobConfiguration (@param sparkConf) - auto-tuned parameters that were not user-specified
I
Input type.
jobClass
IO typed job class.
input
Input to the job. This automatically gets serialized.
params
Parameters into the job.
sparkConf
Spark job configuration.
listener
Listener to pass to the job. The spark job should be able to communicate directly with Alpine as it's running.
returns
The map of relevant Spark properties after auto tuning algorithm was applied.

Definition Classes
SparkExecutionContextMock → SparkExecutionContext
def hashCode(): Int

Definition Classes
AnyRef → Any
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
def mkdir(path: String): Boolean

Create the directory path.
Create the directory path.
path
The directory path that we want to create.
returns
true if it succeeds, false otherwise.

Definition Classes
SparkExecutionContextMock → SparkExecutionContext
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
def openPath(path: String): InputStream

Open a HDFS path for reading.
Open a HDFS path for reading.
path
The HDFS path that we want to read from.
returns
InputStream corresponding to the path.

Definition Classes
SparkExecutionContextMock → SparkExecutionContext
def recommendedTempDir: File

Definition Classes
SparkExecutionContextMock → ExecutionContext
def submitJob[I <: IOBase, O <: IOBase, JOB <: SparkIOTypedPluginJob[I, O]](jobClass: Class[JOB], input: I, params: OperatorParameters, sparkConf: SparkJobConfiguration, listener: OperatorListener): SubmittedSparkJob[O]

This is to be the function to submit the IO typed job to Spark.
This is to be the function to submit the IO typed job to Spark. IO typed Spark jobs will automatically serialize/deserialize input/outputs. TODO: Not supported as of yet.
I
Input type.
O
Output type.
JOB
The job type.
jobClass
IO typed job class.
input
Input to the job. This automatically gets serialized.
params
Parameters into the job.
sparkConf
Spark job configuration.
listener
Listener to pass to the job. The spark job should be able to communicate directly with Alpine as it's running.
returns
A submitted job object.

Definition Classes
SparkExecutionContextMock → SparkExecutionContext
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def toString(): String

Definition Classes
AnyRef → Any
def visualModelHelper: HDFSVisualModelHelper

Definition Classes
SparkExecutionContextMock → SparkExecutionContext
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
def workflowInfo: WorkflowInfo

Definition Classes
SparkExecutionContextMock → ExecutionContext

Related Doc: package mock

class SparkExecutionContextMock extends SparkExecutionContext

Instance Constructors

new SparkExecutionContextMock(chorusAPICallerMock: ChorusAPICaller)

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

def appendPath(path: String): OutputStream

final def asInstanceOf[T0]: T0

def chorusAPICaller: ChorusAPICaller

def chorusUserInfo: ChorusUserInfo

def clone(): AnyRef

def config: CustomOperatorConfig

def createPath(path: String, overwrite: Boolean): OutputStream

def deletePath(path: String, recursive: Boolean): Boolean

def doHdfsAction[T](fs: (FileSystem) ⇒ T): T

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def exists(path: String): Boolean

def finalize(): Unit

final def getClass(): Class[_]

def getSparkAutoTunedParameters[I <: IOBase, JOB <: SparkIOTypedPluginJob[I, _]](jobClass: Class[JOB], input: I, params: OperatorParameters, sparkConf: SparkJobConfiguration, listener: OperatorListener): Nothing

def hashCode(): Int

final def isInstanceOf[T0]: Boolean

def mkdir(path: String): Boolean

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

def openPath(path: String): InputStream

def recommendedTempDir: File

def submitJob[I <: IOBase, O <: IOBase, JOB <: SparkIOTypedPluginJob[I, O]](jobClass: Class[JOB], input: I, params: OperatorParameters, sparkConf: SparkJobConfiguration, listener: OperatorListener): SubmittedSparkJob[O]

final def synchronized[T0](arg0: ⇒ T0): T0

def toString(): String

def visualModelHelper: HDFSVisualModelHelper

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

def workflowInfo: WorkflowInfo

Inherited from SparkExecutionContext

Inherited from ExecutionContext

Inherited from AnyRef

Inherited from Any

Ungrouped