HdfsParameterUtils

Value Members

final def !=(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def !=(arg0: Any): Boolean

Definition Classes
Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def ==(arg0: Any): Boolean

Definition Classes
Any
val DEFAULT_NUMBER_ROWS: Int
def addDropIfExistsParameter(operatorDialog: OperatorDialog, defaultValue: Boolean = true): DialogElement

For use with database, not HDFS.
For use with database, not HDFS.
operatorDialog
From the 'onExecution' method's parameters. This method adds the parameters to this object.
defaultValue
boolean indicating default value of the parameter. True for "Yes", false for "No".
returns
The dialog element added.

Definition Classes
OutputParameterUtils
def addHdfsStorageFormatParameter(operatorDialog: OperatorDialog, defaultFormat: HdfsStorageFormatType): DialogElement

Adds a dropdown menu to select the storage format for a tabular dataset output.
Adds a dropdown menu to select the storage format for a tabular dataset output. I.e., it'll add a dropdown menu with available selections 'CSV', 'Parquet' and 'Avro'.
operatorDialog
The operator dialog where you are going to add the dropdown menu.
defaultFormat
The default format one wants to use.
returns
The dropdown dialog element.
def addNullDataReportParameter(operatorDialog: OperatorDialog, message: String = ...): DialogElement
def addOutputDirectorySelector(operatorDialog: OperatorDialog): DialogElement

Adds directory selector box to let the user select the location in HDFS where the results of the operator will be written
Adds directory selector box to let the user select the location in HDFS where the results of the operator will be written
operatorDialog
The dialog to which the parameter will be added.
returns
def addOutputNameParameter(operatorDialog: OperatorDialog, defaultOutputName: String): DialogElement

adds a string dialog box to let the user define the name of the file with the output.
adds a string dialog box to let the user define the name of the file with the output.
operatorDialog
The dialog to which the parameter will be added.
defaultOutputName
The default value to be used for the parameter.
returns
def addOverwriteParameter(operatorDialog: OperatorDialog, defaultValue: Boolean = true): DialogElement

Definition Classes
OutputParameterUtils
def addStandardHdfsOutputParameters(operatorDialog: OperatorDialog, defaultOutputName: String = operatorNameUUIDVariable): Seq[DialogElement]

Adds -- "outputDirectory": an HDFS directory selector for the location of the output -- "outputName": a StringBox parameter for the name of the output.
Adds -- "outputDirectory": an HDFS directory selector for the location of the output -- "outputName": a StringBox parameter for the name of the output. -- "overwrite": A Boolean parameter asking if the user wants to overwrite old output.
These are the standard parameters to be used when an operator outputs a HDFS dataset.
The default value of the output name will be @operator_name_uuid, which will be replaced at runtime with the actual operator name and uuid concatenated with a underscore, sanitized to make it a valid file name.
defaultOutputName
The default value for the output name parameter.
returns
A sequence of the dialog elements added.
final def asInstanceOf[T0]: T0

Definition Classes
Any
val badDataLocation: String
val badDataParameterOptions: Seq[String]
val badDataReportALL: String
val badDataReportNO: String
val badDataReportNO_COUNT: String
val badDataReportNROWS: String
val badDataReportParameterID: String
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
def countRowsRemovedDueToNullData(parameters: OperatorParameters): Boolean
val dropIfExists: String

Definition Classes
OutputParameterUtils
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
def getAmountOfBadDataToWrite(parameters: OperatorParameters): Option[Long]
def getBadDataPath(parameters: OperatorParameters): String
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def getDropIfExistsParameterValue(parameters: OperatorParameters): Boolean

Gets the value of the dropIfExists parameter, returning true if the String value is "Yes", false if it is a different value or missing.
Gets the value of the dropIfExists parameter, returning true if the String value is "Yes", false if it is a different value or missing.
parameters
OperatorParameters instance [containing the dropIfExists parameter].
returns
Boolean representation of the dropIfExists parameter.

Definition Classes
OutputParameterUtils
def getHdfsStorageFormatType(parameters: OperatorParameters): HdfsStorageFormatType

Get the Hdfs storage format from the parameters object.
Get the Hdfs storage format from the parameters object.
parameters
This must contain the format parameter. I.e., the user should've called addHdfsStorageFormatParameter before.
returns
The selected Hdfs storage format.
def getNullDataReportParameter(parameters: OperatorParameters): String
def getOutputPath(parameters: OperatorParameters): String

Concatenates the string values of parameters with keys "outputDirectory" and "outputName" with a file separator.
Concatenates the string values of parameters with keys "outputDirectory" and "outputName" with a file separator.
returns
The output path corresponding to parameters outputDirectory and outputName.
def getOverwriteParameterValue(parameters: OperatorParameters): Boolean

Gets the value of the overwrite parameter, returning true if the String value is "true", false if it is a different value or missing.
Gets the value of the overwrite parameter, returning true if the String value is "true", false if it is a different value or missing.
parameters
OperatorParameters instance [containing the overwrite parameter].
returns
Boolean representation of the overwrite parameter.

Definition Classes
OutputParameterUtils
def hashCode(): Int

Definition Classes
AnyRef → Any
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
val operatorNameUUIDVariable: String

Definition Classes
OutputParameterUtils
val outputDirectoryParameterID: String
val outputNameParameterID: String
val overwriteParameterID: String

Definition Classes
OutputParameterUtils
val storageFormatParameterID: String
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def toString(): String

Definition Classes
AnyRef → Any
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )

Deprecated Value Members

def addBadDataReportParameter(operatorDialog: OperatorDialog): DialogElement

Annotations
@deprecated
Deprecated
Use addNullDataReportParameter
def addHdfsStorageFormatParameter(operatorDialog: OperatorDialog, defaultFormat: HdfsStorageFormat.HdfsStorageFormat = HdfsStorageFormat.TSV): DialogElement

Annotations
@deprecated
Deprecated
Use method which takes HdfsStorageFormatType case class rather than HdfsStorageFormat enum
def getHdfsStorageFormat(parameters: OperatorParameters): HdfsStorageFormat.HdfsStorageFormat

Get the Hdfs storage format from the parameters object.
Get the Hdfs storage format from the parameters object.
parameters
This must contain the format parameter. I.e., the user should've called addHdfsStorageFormatParameter before.
returns
The selected Hdfs storage format.

Annotations
@deprecated
Deprecated
Use method which returns HdfsStorageFormatType case class rather than HdfsStorageFormat enum
def getTabularFormatAttributes(storageFormat: HdfsStorageFormatType): TabularFormatAttributes

Get default tabular format attributes to use (e.
Get default tabular format attributes to use (e.g., delimiter, quote information for CSV/CSV). This is useful if one wants to define output formats using default values.
storageFormat
The HdfsStorageFormatType.
returns
Tabular format attributes.

Annotations
@deprecated
Deprecated
Use the TabularSchema definition without TabularFormatAttributes, and then you will not need this method.
def getTabularFormatAttributes(storageFormat: HdfsStorageFormat.HdfsStorageFormat): TabularFormatAttributes

Get default tabular format attributes to use (e.
Get default tabular format attributes to use (e.g., delimiter, quote information for CSV/CSV). This is useful if one wants to define output formats using default values.
storageFormat
The HdfsStorageFormatType
returns
Tabular format attributes.

Annotations
@deprecated
Deprecated
Use method which takes HdfsStorageFormatType object rather than HdfsStorageFormat enum

object HdfsParameterUtils extends OutputParameterUtils

Value Members

final def !=(arg0: AnyRef): Boolean

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: AnyRef): Boolean

final def ==(arg0: Any): Boolean

val DEFAULT_NUMBER_ROWS: Int

def addDropIfExistsParameter(operatorDialog: OperatorDialog, defaultValue: Boolean = true): DialogElement

def addHdfsStorageFormatParameter(operatorDialog: OperatorDialog, defaultFormat: HdfsStorageFormatType): DialogElement

def addNullDataReportParameter(operatorDialog: OperatorDialog, message: String = ...): DialogElement

def addOutputDirectorySelector(operatorDialog: OperatorDialog): DialogElement

def addOutputNameParameter(operatorDialog: OperatorDialog, defaultOutputName: String): DialogElement

def addOverwriteParameter(operatorDialog: OperatorDialog, defaultValue: Boolean = true): DialogElement

def addStandardHdfsOutputParameters(operatorDialog: OperatorDialog, defaultOutputName: String = operatorNameUUIDVariable): Seq[DialogElement]

final def asInstanceOf[T0]: T0

val badDataLocation: String

val badDataParameterOptions: Seq[String]

val badDataReportALL: String

val badDataReportNO: String

val badDataReportNO_COUNT: String

val badDataReportNROWS: String

val badDataReportParameterID: String

def clone(): AnyRef

def countRowsRemovedDueToNullData(parameters: OperatorParameters): Boolean

val dropIfExists: String

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def finalize(): Unit

def getAmountOfBadDataToWrite(parameters: OperatorParameters): Option[Long]

def getBadDataPath(parameters: OperatorParameters): String

final def getClass(): Class[_]

def getDropIfExistsParameterValue(parameters: OperatorParameters): Boolean

def getHdfsStorageFormatType(parameters: OperatorParameters): HdfsStorageFormatType

def getNullDataReportParameter(parameters: OperatorParameters): String

def getOutputPath(parameters: OperatorParameters): String

def getOverwriteParameterValue(parameters: OperatorParameters): Boolean

def hashCode(): Int

final def isInstanceOf[T0]: Boolean

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

val operatorNameUUIDVariable: String

val outputDirectoryParameterID: String

val outputNameParameterID: String

val overwriteParameterID: String

val storageFormatParameterID: String

final def synchronized[T0](arg0: ⇒ T0): T0

def toString(): String

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

Deprecated Value Members

def addBadDataReportParameter(operatorDialog: OperatorDialog): DialogElement

def addHdfsStorageFormatParameter(operatorDialog: OperatorDialog, defaultFormat: HdfsStorageFormat.HdfsStorageFormat = HdfsStorageFormat.TSV): DialogElement

def getHdfsStorageFormat(parameters: OperatorParameters): HdfsStorageFormat.HdfsStorageFormat

def getTabularFormatAttributes(storageFormat: HdfsStorageFormatType): TabularFormatAttributes

def getTabularFormatAttributes(storageFormat: HdfsStorageFormat.HdfsStorageFormat): TabularFormatAttributes

Inherited from OutputParameterUtils

Inherited from AnyRef

Inherited from Any

Ungrouped