SparkExecutionContext

Abstract Value Members

abstract def appendPath(path: String): OutputStream

Append contents to the given HDFS path.
Append contents to the given HDFS path.
path
The HDFS path that we want to append to.
returns
OutputStream corresponding to the path.
abstract def createPath(path: String, overwrite: Boolean): OutputStream

Create a HDFS path for writing.
Create a HDFS path for writing.
path
The HDFS path that we want to create and write to.
overwrite
Whether to overwrite the given path if it exists.
returns
OutputStream corresponding to the path.
abstract def deletePath(path: String, recursive: Boolean): Boolean

Delete the given HDFS path.
Delete the given HDFS path.
path
The HDFS path that we want to delete.
recursive
If it's a directory, whether we want to delete the directory recursively.
returns
true if successful, false otherwise.
abstract def exists(path: String): Boolean

Determine whether the given path exists in the HDFS or not.
Determine whether the given path exists in the HDFS or not.
path
The path that we want to check.
returns
true if it exists, false otherwise.
abstract def mkdir(path: String): Boolean

Create the directory path.
Create the directory path.
path
The directory path that we want to create.
returns
true if it succeeds, false otherwise.
abstract def openPath(path: String): InputStream

Open a HDFS path for reading.
Open a HDFS path for reading.
path
The HDFS path that we want to read from.
returns
InputStream corresponding to the path.
abstract def submitJob[I <: IOBase, O <: IOBase, JOB <: SparkIOTypedPluginJob[I, O]](jobClass: Class[JOB], input: I, params: OperatorParameters, sparkConf: SparkJobConfiguration, listener: OperatorListener): SubmittedSparkJob[O]

This is to be the function to submit the IO typed job to Spark.
This is to be the function to submit the IO typed job to Spark. IO typed Spark jobs will automatically serialize/deserialize input/outputs. TODO: Not supported as of yet.
I
Input type.
O
Output type.
JOB
The job type.
jobClass
IO typed job class.
input
Input to the job. This automatically gets serialized.
params
Parameters into the job.
sparkConf
Spark job configuration.
listener
Listener to pass to the job. The spark job should be able to communicate directly with Alpine as it's running.
returns
A submitted job object.

Concrete Value Members

final def !=(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def !=(arg0: Any): Boolean

Definition Classes
Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def ==(arg0: Any): Boolean

Definition Classes
Any
final def asInstanceOf[T0]: T0

Definition Classes
Any
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def hashCode(): Int

Definition Classes
AnyRef → Any
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def toString(): String

Definition Classes
AnyRef → Any
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )

trait SparkExecutionContext extends ExecutionContext

Abstract Value Members

abstract def appendPath(path: String): OutputStream

abstract def createPath(path: String, overwrite: Boolean): OutputStream

abstract def deletePath(path: String, recursive: Boolean): Boolean

abstract def exists(path: String): Boolean

abstract def mkdir(path: String): Boolean

abstract def openPath(path: String): InputStream

abstract def submitJob[I <: IOBase, O <: IOBase, JOB <: SparkIOTypedPluginJob[I, O]](jobClass: Class[JOB], input: I, params: OperatorParameters, sparkConf: SparkJobConfiguration, listener: OperatorListener): SubmittedSparkJob[O]

Concrete Value Members

final def !=(arg0: AnyRef): Boolean

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: AnyRef): Boolean

final def ==(arg0: Any): Boolean

final def asInstanceOf[T0]: T0

def clone(): AnyRef

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def finalize(): Unit

final def getClass(): Class[_]

def hashCode(): Int

final def isInstanceOf[T0]: Boolean

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

final def synchronized[T0](arg0: ⇒ T0): T0

def toString(): String

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

Inherited from ExecutionContext

Inherited from AnyRef

Inherited from Any

Ungrouped