SparkExecutionContext

Abstract Value Members

abstract def appendPath(path: String): OutputStream

Append contents to the given HDFS path.
Append contents to the given HDFS path.
path
The HDFS path that we want to append to.
returns
OutputStream corresponding to the path.
abstract def chorusAPICaller: ChorusAPICaller

Definition Classes
ExecutionContext
abstract def chorusUserInfo: ChorusUserInfo

Definition Classes
ExecutionContext
abstract def config: CustomOperatorConfig

Definition Classes
ExecutionContext
abstract def createPath(path: String, overwrite: Boolean): OutputStream

Create a HDFS path for writing.
Create a HDFS path for writing.
path
The HDFS path that we want to create and write to.
overwrite
Whether to overwrite the given path if it exists.
returns
OutputStream corresponding to the path.
abstract def deletePath(path: String, recursive: Boolean): Boolean

Delete the given HDFS path.
Delete the given HDFS path.
path
The HDFS path that we want to delete.
recursive
If it's a directory, whether we want to delete the directory recursively.
returns
true if successful, false otherwise.
abstract def doHdfsAction[T](fs: (FileSystem) ⇒ T): T
abstract def exists(path: String): Boolean

Determine whether the given path exists in the HDFS or not.
Determine whether the given path exists in the HDFS or not.
path
The path that we want to check.
returns
true if it exists, false otherwise.
abstract def getSparkAutoTunedParameters[I <: IOBase, JOB <: SparkIOTypedPluginJob[I, _]](jobClass: Class[JOB], input: I, params: OperatorParameters, sparkConf: SparkJobConfiguration, listener: OperatorListener): Map[String, String]

Returns the map of Spark parameters after autoTuning algorithm is applied.
Returns the map of Spark parameters after autoTuning algorithm is applied. This is the final set of Spark properties ready to be passed with Spark job submission (except spark.job.name) It leverages: - user-defined Spark properties at the operator level (Spark Advanced Settings box), workflow level and data source level (in this order of precedence). - AutoTunerOptions set in the SparkJobConfiguration (@param sparkConf) - auto-tuned parameters that were not user-specified
I
Input type.
jobClass
IO typed job class.
input
Input to the job. This automatically gets serialized.
params
Parameters into the job.
sparkConf
Spark job configuration.
listener
Listener to pass to the job. The spark job should be able to communicate directly with Alpine as it's running.
returns
The map of relevant Spark properties after auto tuning algorithm was applied.
abstract def mkdir(path: String): Boolean

Create the directory path.
Create the directory path.
path
The directory path that we want to create.
returns
true if it succeeds, false otherwise.
abstract def openPath(path: String): InputStream

Open a HDFS path for reading.
Open a HDFS path for reading.
path
The HDFS path that we want to read from.
returns
InputStream corresponding to the path.
abstract def recommendedTempDir: File

Definition Classes
ExecutionContext
abstract def submitJob[I <: IOBase, O <: IOBase, JOB <: SparkIOTypedPluginJob[I, O]](jobClass: Class[JOB], input: I, params: OperatorParameters, sparkConf: SparkJobConfiguration, listener: OperatorListener): SubmittedSparkJob[O]

This is to be the function to submit the IO typed job to Spark.
This is to be the function to submit the IO typed job to Spark. IO typed Spark jobs will automatically serialize/deserialize input/outputs. TODO: Not supported as of yet.
I
Input type.
O
Output type.
JOB
The job type.
jobClass
IO typed job class.
input
Input to the job. This automatically gets serialized.
params
Parameters into the job.
sparkConf
Spark job configuration.
listener
Listener to pass to the job. The spark job should be able to communicate directly with Alpine as it's running.
returns
A submitted job object.
abstract def visualModelHelper: HDFSVisualModelHelper
abstract def workflowInfo: WorkflowInfo

Definition Classes
ExecutionContext

Concrete Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def asInstanceOf[T0]: T0

Definition Classes
Any
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def hashCode(): Int

Definition Classes
AnyRef → Any
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def toString(): String

Definition Classes
AnyRef → Any
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )

Related Doc: package spark

trait SparkExecutionContext extends ExecutionContext

Abstract Value Members

abstract def appendPath(path: String): OutputStream

abstract def chorusAPICaller: ChorusAPICaller

abstract def chorusUserInfo: ChorusUserInfo

abstract def config: CustomOperatorConfig

abstract def createPath(path: String, overwrite: Boolean): OutputStream

abstract def deletePath(path: String, recursive: Boolean): Boolean

abstract def doHdfsAction[T](fs: (FileSystem) ⇒ T): T

abstract def exists(path: String): Boolean

abstract def getSparkAutoTunedParameters[I <: IOBase, JOB <: SparkIOTypedPluginJob[I, _]](jobClass: Class[JOB], input: I, params: OperatorParameters, sparkConf: SparkJobConfiguration, listener: OperatorListener): Map[String, String]

abstract def mkdir(path: String): Boolean

abstract def openPath(path: String): InputStream

abstract def recommendedTempDir: File

abstract def submitJob[I <: IOBase, O <: IOBase, JOB <: SparkIOTypedPluginJob[I, O]](jobClass: Class[JOB], input: I, params: OperatorParameters, sparkConf: SparkJobConfiguration, listener: OperatorListener): SubmittedSparkJob[O]

abstract def visualModelHelper: HDFSVisualModelHelper

abstract def workflowInfo: WorkflowInfo

Concrete Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

final def asInstanceOf[T0]: T0

def clone(): AnyRef

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def finalize(): Unit

final def getClass(): Class[_]

def hashCode(): Int

final def isInstanceOf[T0]: Boolean

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

final def synchronized[T0](arg0: ⇒ T0): T0

def toString(): String

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

Inherited from ExecutionContext

Inherited from AnyRef

Inherited from Any

Ungrouped