SparkParameterUtils

Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def addStandardSparkOptions(operatorDialog: OperatorDialog, additionalSparkParameters: List[SparkParameter]): Unit
def addStandardSparkOptionsWithStorageLevel(operatorDialog: OperatorDialog, defaultNumExecutors: Int, defaultExecutorMemoryMB: Int, defaultDriverMemoryMB: Int, defaultNumExecutorCores: Int, defaultStorageLevel: String, additionalSparkParameters: List[SparkParameter] = List.empty[SparkParameter]): Unit

A more advanced method for adding SparkP Parameters.
A more advanced method for adding SparkP Parameters. Will also add a "StorageLevel" Parameter which will indicate what level of persistence to use within a Spark job. NOTE: The storage level parameter cannot be set automatically during runtime. To have any effect the custom operator developer must implement RDD persistence with this value (retrievable with 'getStorageLevel' method) in the Spark Job class of their operator.
defaultStorageLevel
- default storage level e.g. NONE or "MEMORY_AND_DISK.
additionalSparkParameters
- a list of a additional Spark Parameters.
final def asInstanceOf[T0]: T0

Definition Classes
Any
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
val disableDynamicAllocationParamId: String
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def getRepartition(operatorParameters: OperatorParameters): Boolean

Return true if the repartition parameter was added AND the user checked it.
Return true if the repartition parameter was added AND the user checked it. Return false otherwise. Note that because this parameter is exposed as a check box by the alpine engine, the value of the parameter will be either "true" or "false" (string representation of java booleans).
def getStorageLevel(operatorParameters: OperatorParameters): Option[String]

Retrieve storage level param added via "makeStorageLevelParam" from advanced parameters box.
Retrieve storage level param added via "makeStorageLevelParam" from advanced parameters box. Return NONE if the parameter was not added. NOTE: this method does not validate the String, so if the users put in an invalid storage level parameter, calling StorageLevel.fromString(s) on the result of this method will fail.
def getUserSetNumPartitions(operatorParameters: OperatorParameters): Option[Int]

Retrieve the value of the number of partitions parameter added to the advanced spark box.
Retrieve the value of the number of partitions parameter added to the advanced spark box. However, the CO developer should be aware that Alpine Auto Tuning determines an optimal number of partitions to use and sets that value to spark.default.parallelism. So before repartitioning the developer should check that that is set if this method returns None.
def hashCode(): Int

Definition Classes
AnyRef → Any
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
def makeNumPartitionsParam: SparkParameter

Create a "Number of Partitions" parameter to let the user determine how many partitions should be used either when repartitioning the data (controlled by above param) or in when shuffling generally.
Create a "Number of Partitions" parameter to let the user determine how many partitions should be used either when repartitioning the data (controlled by above param) or in when shuffling generally. If this parameter is not set, a value will be selected by auto tuning. If a value is selected this value will be used to set the "spark.default.parallelism" (or "spark.sql.shuffle.partitions" for Spark SQL) parameters which controls the default number of parameter used in a wide transformation in Spark.
def makeRepartitionParam: SparkParameter

Create a "Repartition Data" checkbox to let the user determine whether the input data should be shuffled to increase the number of partitions.
def makeStorageLevelParam(defaultStorageLevel: String): SparkParameter

Add storage level param.
Add storage level param. Default must be the string representation of a Spark Storage Level e.g. "MEMORY_AND_DISK"
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
val numPartitionsId: String
val repartitionRDDId: String
val sparkDriverMBElementId: String
val sparkExecutorMBElementId: String
val sparkNumExecutorCoresElementId: String
val sparkNumExecutorsElementId: String
val storageLevelParamId: String
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def toString(): String

Definition Classes
AnyRef → Any
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )

Deprecated Value Members

def addStandardSparkOptions(operatorDialog: OperatorDialog, defaultNumExecutors: Int, defaultExecutorMemoryMB: Int, defaultDriverMemoryMB: Int, defaultNumExecutorCores: Int): Unit

Annotations
@deprecated
Deprecated
def addStandardSparkOptions(operatorDialog: OperatorDialog, defaultNumExecutors: Int, defaultExecutorMemoryMB: Int, defaultDriverMemoryMB: Int, defaultNumExecutorCores: Int, additionalSparkParameters: List[SparkParameter]): Unit

Annotations
@deprecated
Deprecated

Related Doc: package utils

object SparkParameterUtils

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

def addStandardSparkOptions(operatorDialog: OperatorDialog, additionalSparkParameters: List[SparkParameter]): Unit

final def asInstanceOf[T0]: T0

def clone(): AnyRef

val disableDynamicAllocationParamId: String

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def finalize(): Unit

final def getClass(): Class[_]

def getRepartition(operatorParameters: OperatorParameters): Boolean

def getStorageLevel(operatorParameters: OperatorParameters): Option[String]

def getUserSetNumPartitions(operatorParameters: OperatorParameters): Option[Int]

def hashCode(): Int

final def isInstanceOf[T0]: Boolean

def makeNumPartitionsParam: SparkParameter

def makeRepartitionParam: SparkParameter

def makeStorageLevelParam(defaultStorageLevel: String): SparkParameter

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

val numPartitionsId: String

val repartitionRDDId: String

val sparkDriverMBElementId: String

val sparkExecutorMBElementId: String

val sparkNumExecutorCoresElementId: String

val sparkNumExecutorsElementId: String

val storageLevelParamId: String

final def synchronized[T0](arg0: ⇒ T0): T0

def toString(): String

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

Deprecated Value Members

def addStandardSparkOptions(operatorDialog: OperatorDialog, defaultNumExecutors: Int, defaultExecutorMemoryMB: Int, defaultDriverMemoryMB: Int, defaultNumExecutorCores: Int): Unit

def addStandardSparkOptions(operatorDialog: OperatorDialog, defaultNumExecutors: Int, defaultExecutorMemoryMB: Int, defaultDriverMemoryMB: Int, defaultNumExecutorCores: Int, additionalSparkParameters: List[SparkParameter]): Unit

Inherited from AnyRef

Inherited from Any

Ungrouped