MLUtils

Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def appendBias(vector: Vector): Vector

Returns a new vector with 1.0 (bias) appended to the input vector.
Returns a new vector with 1.0 (bias) appended to the input vector.

Annotations
@Since( "1.0.0" )
final def asInstanceOf[T0]: T0

Definition Classes
Any
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
def convertMatrixColumnsFromML(dataset: Dataset[_], cols: String*): DataFrame

Converts matrix columns in an input Dataset to the org.apache.spark.mllib.linalg.Matrix type from the new org.apache.spark.ml.linalg.Matrix type under the spark.ml package.
Converts matrix columns in an input Dataset to the org.apache.spark.mllib.linalg.Matrix type from the new org.apache.spark.ml.linalg.Matrix type under the spark.ml package.
dataset
input dataset
cols
a list of matrix columns to be converted. Old matrix columns will be ignored. If unspecified, all new matrix columns will be converted except nested ones.
returns
the input DataFrame with new matrix columns converted to the old matrix type

Annotations
@Since( "2.0.0" ) @varargs()
def convertMatrixColumnsToML(dataset: Dataset[_], cols: String*): DataFrame

Converts Matrix columns in an input Dataset from the org.apache.spark.mllib.linalg.Matrix type to the new org.apache.spark.ml.linalg.Matrix type under the spark.ml package.
Converts Matrix columns in an input Dataset from the org.apache.spark.mllib.linalg.Matrix type to the new org.apache.spark.ml.linalg.Matrix type under the spark.ml package.
dataset
input dataset
cols
a list of matrix columns to be converted. New matrix columns will be ignored. If unspecified, all old matrix columns will be converted except nested ones.
returns
the input DataFrame with old matrix columns converted to the new matrix type

Annotations
@Since( "2.0.0" ) @varargs()
def convertVectorColumnsFromML(dataset: Dataset[_], cols: String*): DataFrame

Converts vector columns in an input Dataset to the org.apache.spark.mllib.linalg.Vector type from the new org.apache.spark.ml.linalg.Vector type under the spark.ml package.
Converts vector columns in an input Dataset to the org.apache.spark.mllib.linalg.Vector type from the new org.apache.spark.ml.linalg.Vector type under the spark.ml package.
dataset
input dataset
cols
a list of vector columns to be converted. Old vector columns will be ignored. If unspecified, all new vector columns will be converted except nested ones.
returns
the input DataFrame with new vector columns converted to the old vector type

Annotations
@Since( "2.0.0" ) @varargs()
def convertVectorColumnsToML(dataset: Dataset[_], cols: String*): DataFrame

Converts vector columns in an input Dataset from the org.apache.spark.mllib.linalg.Vector type to the new org.apache.spark.ml.linalg.Vector type under the spark.ml package.
Converts vector columns in an input Dataset from the org.apache.spark.mllib.linalg.Vector type to the new org.apache.spark.ml.linalg.Vector type under the spark.ml package.
dataset
input dataset
cols
a list of vector columns to be converted. New vector columns will be ignored. If unspecified, all old vector columns will be converted except nested ones.
returns
the input DataFrame with old vector columns converted to the new vector type

Annotations
@Since( "2.0.0" ) @varargs()
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def hashCode(): Int

Definition Classes
AnyRef → Any
def initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean = false): Boolean

Attributes
protected
Definition Classes
Logging
def initializeLogIfNecessary(isInterpreter: Boolean): Unit

Attributes
protected
Definition Classes
Logging
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
def isTraceEnabled(): Boolean

Attributes
protected
Definition Classes
Logging
def kFold[T](rdd: RDD[T], numFolds: Int, seed: Long)(implicit arg0: ClassTag[T]): Array[(RDD[T], RDD[T])]

Version of kFold() taking a Long seed.
Version of kFold() taking a Long seed.

Annotations
@Since( "2.0.0" )
def kFold[T](rdd: RDD[T], numFolds: Int, seed: Int)(implicit arg0: ClassTag[T]): Array[(RDD[T], RDD[T])]

Return a k element array of pairs of RDDs with the first element of each pair containing the training data, a complement of the validation data and the second element, the validation data, containing a unique 1/kth of the data.
Return a k element array of pairs of RDDs with the first element of each pair containing the training data, a complement of the validation data and the second element, the validation data, containing a unique 1/kth of the data. Where k=numFolds.

Annotations
@Since( "1.0.0" )
def loadLabeledPoints(sc: SparkContext, dir: String): RDD[LabeledPoint]

Loads labeled points saved using RDD[LabeledPoint].saveAsTextFile with the default number of partitions.
Loads labeled points saved using RDD[LabeledPoint].saveAsTextFile with the default number of partitions.

Annotations
@Since( "1.1.0" )
def loadLabeledPoints(sc: SparkContext, path: String, minPartitions: Int): RDD[LabeledPoint]

Loads labeled points saved using RDD[LabeledPoint].saveAsTextFile.
Loads labeled points saved using RDD[LabeledPoint].saveAsTextFile.
sc
Spark context
path
file or directory path in any Hadoop-supported file system URI
minPartitions
min number of partitions
returns
labeled points stored as an RDD[LabeledPoint]

Annotations
@Since( "1.1.0" )
def loadLibSVMFile(sc: SparkContext, path: String): RDD[LabeledPoint]

Loads binary labeled data in the LIBSVM format into an RDD[LabeledPoint], with number of features determined automatically and the default number of partitions.
Loads binary labeled data in the LIBSVM format into an RDD[LabeledPoint], with number of features determined automatically and the default number of partitions.

Annotations
@Since( "1.0.0" )
def loadLibSVMFile(sc: SparkContext, path: String, numFeatures: Int): RDD[LabeledPoint]

Loads labeled data in the LIBSVM format into an RDD[LabeledPoint], with the default number of partitions.
Loads labeled data in the LIBSVM format into an RDD[LabeledPoint], with the default number of partitions.

Annotations
@Since( "1.0.0" )
def loadLibSVMFile(sc: SparkContext, path: String, numFeatures: Int, minPartitions: Int): RDD[LabeledPoint]

Loads labeled data in the LIBSVM format into an RDD[LabeledPoint].
Loads labeled data in the LIBSVM format into an RDD[LabeledPoint]. The LIBSVM format is a text-based format used by LIBSVM and LIBLINEAR. Each line represents a labeled sparse feature vector using the following format:
```
label index1:value1 index2:value2 ...
```
where the indices are one-based and in ascending order. This method parses each line into a org.apache.spark.mllib.regression.LabeledPoint, where the feature indices are converted to zero-based.
sc
Spark context
path
file or directory path in any Hadoop-supported file system URI
numFeatures
number of features, which will be determined from the input data if a nonpositive value is given. This is useful when the dataset is already split into multiple files and you want to load them separately, because some features may not present in certain files, which leads to inconsistent feature dimensions.
minPartitions
min number of partitions
returns
labeled data stored as an RDD[LabeledPoint]

Annotations
@Since( "1.0.0" )
def loadVectors(sc: SparkContext, path: String): RDD[Vector]

Loads vectors saved using RDD[Vector].saveAsTextFile with the default number of partitions.
Loads vectors saved using RDD[Vector].saveAsTextFile with the default number of partitions.

Annotations
@Since( "1.1.0" )
def loadVectors(sc: SparkContext, path: String, minPartitions: Int): RDD[Vector]

Loads vectors saved using RDD[Vector].saveAsTextFile.
Loads vectors saved using RDD[Vector].saveAsTextFile.
sc
Spark context
path
file or directory path in any Hadoop-supported file system URI
minPartitions
min number of partitions
returns
vectors stored as an RDD[Vector]

Annotations
@Since( "1.1.0" )
def log: Logger

Attributes
protected
Definition Classes
Logging
def logDebug(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logDebug(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logError(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logError(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logInfo(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logInfo(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logName: String

Attributes
protected
Definition Classes
Logging
def logTrace(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logTrace(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
def logWarning(msg: ⇒ String, throwable: Throwable): Unit

Attributes
protected
Definition Classes
Logging
def logWarning(msg: ⇒ String): Unit

Attributes
protected
Definition Classes
Logging
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
def saveAsLibSVMFile(data: RDD[LabeledPoint], dir: String): Unit

Save labeled data in LIBSVM format.
Save labeled data in LIBSVM format.
data
an RDD of LabeledPoint to be saved
dir
directory to save the data

Annotations
@Since( "1.0.0" )
See also
org.apache.spark.mllib.util.MLUtils.loadLibSVMFile
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def toString(): String

Definition Classes
AnyRef → Any
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )

Related Doc: package util

object MLUtils extends Logging

Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

def appendBias(vector: Vector): Vector

final def asInstanceOf[T0]: T0

def clone(): AnyRef

def convertMatrixColumnsFromML(dataset: Dataset[_], cols: String*): DataFrame

def convertMatrixColumnsToML(dataset: Dataset[_], cols: String*): DataFrame

def convertVectorColumnsFromML(dataset: Dataset[_], cols: String*): DataFrame

def convertVectorColumnsToML(dataset: Dataset[_], cols: String*): DataFrame

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def finalize(): Unit

final def getClass(): Class[_]

def hashCode(): Int

def initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean = false): Boolean

def initializeLogIfNecessary(isInterpreter: Boolean): Unit

final def isInstanceOf[T0]: Boolean

def isTraceEnabled(): Boolean

def kFold[T](rdd: RDD[T], numFolds: Int, seed: Long)(implicit arg0: ClassTag[T]): Array[(RDD[T], RDD[T])]

def kFold[T](rdd: RDD[T], numFolds: Int, seed: Int)(implicit arg0: ClassTag[T]): Array[(RDD[T], RDD[T])]

def loadLabeledPoints(sc: SparkContext, dir: String): RDD[LabeledPoint]

def loadLabeledPoints(sc: SparkContext, path: String, minPartitions: Int): RDD[LabeledPoint]

def loadLibSVMFile(sc: SparkContext, path: String): RDD[LabeledPoint]

def loadLibSVMFile(sc: SparkContext, path: String, numFeatures: Int): RDD[LabeledPoint]

def loadLibSVMFile(sc: SparkContext, path: String, numFeatures: Int, minPartitions: Int): RDD[LabeledPoint]

def loadVectors(sc: SparkContext, path: String): RDD[Vector]

def loadVectors(sc: SparkContext, path: String, minPartitions: Int): RDD[Vector]

def log: Logger

def logDebug(msg: ⇒ String, throwable: Throwable): Unit

def logDebug(msg: ⇒ String): Unit

def logError(msg: ⇒ String, throwable: Throwable): Unit

def logError(msg: ⇒ String): Unit

def logInfo(msg: ⇒ String, throwable: Throwable): Unit

def logInfo(msg: ⇒ String): Unit

def logName: String

def logTrace(msg: ⇒ String, throwable: Throwable): Unit

def logTrace(msg: ⇒ String): Unit

def logWarning(msg: ⇒ String, throwable: Throwable): Unit

def logWarning(msg: ⇒ String): Unit

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

def saveAsLibSVMFile(data: RDD[LabeledPoint], dir: String): Unit

final def synchronized[T0](arg0: ⇒ T0): T0

def toString(): String

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

Inherited from Logging

Inherited from AnyRef

Inherited from Any

Ungrouped