JavaDStreamLike

Abstract Value Members

implicit abstract val classTag: ClassTag[T]
abstract def dstream: DStream[T]
abstract def wrapRDD(in: RDD[T]): R

Concrete Value Members

final def !=(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def ##(): Int

Definition Classes
AnyRef → Any
final def ==(arg0: Any): Boolean

Definition Classes
AnyRef → Any
final def asInstanceOf[T0]: T0

Definition Classes
Any
def checkpoint(interval: Duration): DStream[T]

Enable periodic checkpointing of RDDs of this DStream.
Enable periodic checkpointing of RDDs of this DStream.
interval
Time interval after which generated RDD will be checkpointed
def clone(): AnyRef

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( ... )
def context(): StreamingContext

Return the org.apache.spark.streaming.StreamingContext associated with this DStream
def count(): JavaDStream[Long]

Return a new DStream in which each RDD has a single element generated by counting each RDD of this DStream.
def countByValue(numPartitions: Int): JavaPairDStream[T, Long]

Return a new DStream in which each RDD contains the counts of each distinct value in each RDD of this DStream.
Return a new DStream in which each RDD contains the counts of each distinct value in each RDD of this DStream. Hash partitioning is used to generate the RDDs with numPartitions partitions.
numPartitions
number of partitions of each RDD in the new DStream.
def countByValue(): JavaPairDStream[T, Long]

Return a new DStream in which each RDD contains the counts of each distinct value in each RDD of this DStream.
Return a new DStream in which each RDD contains the counts of each distinct value in each RDD of this DStream. Hash partitioning is used to generate the RDDs with Spark's default number of partitions.
def countByValueAndWindow(windowDuration: Duration, slideDuration: Duration, numPartitions: Int): JavaPairDStream[T, Long]

Return a new DStream in which each RDD contains the count of distinct elements in RDDs in a sliding window over this DStream.
Return a new DStream in which each RDD contains the count of distinct elements in RDDs in a sliding window over this DStream. Hash partitioning is used to generate the RDDs with numPartitions partitions.
windowDuration
width of the window; must be a multiple of this DStream's batching interval
slideDuration
sliding interval of the window (i.e., the interval after which the new DStream will generate RDDs); must be a multiple of this DStream's batching interval
numPartitions
number of partitions of each RDD in the new DStream.
def countByValueAndWindow(windowDuration: Duration, slideDuration: Duration): JavaPairDStream[T, Long]

Return a new DStream in which each RDD contains the count of distinct elements in RDDs in a sliding window over this DStream.
Return a new DStream in which each RDD contains the count of distinct elements in RDDs in a sliding window over this DStream. Hash partitioning is used to generate the RDDs with Spark's default number of partitions.
windowDuration
width of the window; must be a multiple of this DStream's batching interval
slideDuration
sliding interval of the window (i.e., the interval after which the new DStream will generate RDDs); must be a multiple of this DStream's batching interval
def countByWindow(windowDuration: Duration, slideDuration: Duration): JavaDStream[Long]

Return a new DStream in which each RDD has a single element generated by counting the number of elements in a window over this DStream.
Return a new DStream in which each RDD has a single element generated by counting the number of elements in a window over this DStream. windowDuration and slideDuration are as defined in the window() operation. This is equivalent to window(windowDuration, slideDuration).count()
final def eq(arg0: AnyRef): Boolean

Definition Classes
AnyRef
def equals(arg0: Any): Boolean

Definition Classes
AnyRef → Any
def finalize(): Unit

Attributes
protected[java.lang]
Definition Classes
AnyRef
Annotations
@throws( classOf[java.lang.Throwable] )
def flatMap[U](f: FlatMapFunction[T, U]): JavaDStream[U]

Return a new DStream by applying a function to all elements of this DStream, and then flattening the results
def flatMapToPair[K2, V2](f: PairFlatMapFunction[T, K2, V2]): JavaPairDStream[K2, V2]

Return a new DStream by applying a function to all elements of this DStream, and then flattening the results
def foreachRDD(foreachFunc: VoidFunction2[R, Time]): Unit

Apply a function to each RDD in this DStream.
Apply a function to each RDD in this DStream. This is an output operator, so 'this' DStream will be registered as an output stream and therefore materialized.
def foreachRDD(foreachFunc: VoidFunction[R]): Unit

Apply a function to each RDD in this DStream.
Apply a function to each RDD in this DStream. This is an output operator, so 'this' DStream will be registered as an output stream and therefore materialized.
final def getClass(): Class[_]

Definition Classes
AnyRef → Any
def glom(): JavaDStream[List[T]]

Return a new DStream in which each RDD is generated by applying glom() to each RDD of this DStream.
Return a new DStream in which each RDD is generated by applying glom() to each RDD of this DStream. Applying glom() to an RDD coalesces all elements within each partition into an array.
def hashCode(): Int

Definition Classes
AnyRef → Any
final def isInstanceOf[T0]: Boolean

Definition Classes
Any
def map[U](f: Function[T, U]): JavaDStream[U]

Return a new DStream by applying a function to all elements of this DStream.
def mapPartitions[U](f: FlatMapFunction[Iterator[T], U]): JavaDStream[U]

Return a new DStream in which each RDD is generated by applying mapPartitions() to each RDDs of this DStream.
Return a new DStream in which each RDD is generated by applying mapPartitions() to each RDDs of this DStream. Applying mapPartitions() to an RDD applies a function to each partition of the RDD.
def mapPartitionsToPair[K2, V2](f: PairFlatMapFunction[Iterator[T], K2, V2]): JavaPairDStream[K2, V2]

Return a new DStream in which each RDD is generated by applying mapPartitions() to each RDDs of this DStream.
Return a new DStream in which each RDD is generated by applying mapPartitions() to each RDDs of this DStream. Applying mapPartitions() to an RDD applies a function to each partition of the RDD.
def mapToPair[K2, V2](f: PairFunction[T, K2, V2]): JavaPairDStream[K2, V2]

Return a new DStream by applying a function to all elements of this DStream.
final def ne(arg0: AnyRef): Boolean

Definition Classes
AnyRef
final def notify(): Unit

Definition Classes
AnyRef
final def notifyAll(): Unit

Definition Classes
AnyRef
def print(num: Int): Unit

Print the first num elements of each RDD generated in this DStream.
Print the first num elements of each RDD generated in this DStream. This is an output operator, so this DStream will be registered as an output stream and there materialized.
def print(): Unit

Print the first ten elements of each RDD generated in this DStream.
Print the first ten elements of each RDD generated in this DStream. This is an output operator, so this DStream will be registered as an output stream and there materialized.
def reduce(f: Function2[T, T, T]): JavaDStream[T]

Return a new DStream in which each RDD has a single element generated by reducing each RDD of this DStream.
def reduceByWindow(reduceFunc: Function2[T, T, T], invReduceFunc: Function2[T, T, T], windowDuration: Duration, slideDuration: Duration): JavaDStream[T]

Return a new DStream in which each RDD has a single element generated by reducing all elements in a sliding window over this DStream.
Return a new DStream in which each RDD has a single element generated by reducing all elements in a sliding window over this DStream. However, the reduction is done incrementally using the old window's reduced value :
1. reduce the new values that entered the window (e.g., adding new counts) 2. "inverse reduce" the old values that left the window (e.g., subtracting old counts) This is more efficient than reduceByWindow without "inverse reduce" function. However, it is applicable to only "invertible reduce functions".
reduceFunc
associative and commutative reduce function
invReduceFunc
inverse reduce function; such that for all y, invertible x: invReduceFunc(reduceFunc(x, y), x) = y
windowDuration
width of the window; must be a multiple of this DStream's batching interval
slideDuration
sliding interval of the window (i.e., the interval after which the new DStream will generate RDDs); must be a multiple of this DStream's batching interval
def reduceByWindow(reduceFunc: Function2[T, T, T], windowDuration: Duration, slideDuration: Duration): JavaDStream[T]

Return a new DStream in which each RDD has a single element generated by reducing all elements in a sliding window over this DStream.
Return a new DStream in which each RDD has a single element generated by reducing all elements in a sliding window over this DStream.
reduceFunc
associative and commutative reduce function
windowDuration
width of the window; must be a multiple of this DStream's batching interval
slideDuration
sliding interval of the window (i.e., the interval after which the new DStream will generate RDDs); must be a multiple of this DStream's batching interval
implicit def scalaIntToJavaLong(in: DStream[Long]): JavaDStream[Long]
def slice(fromTime: Time, toTime: Time): List[R]

Return all the RDDs between 'fromDuration' to 'toDuration' (both included)
final def synchronized[T0](arg0: ⇒ T0): T0

Definition Classes
AnyRef
def toString(): String

Definition Classes
AnyRef → Any
def transform[U](transformFunc: Function2[R, Time, JavaRDD[U]]): JavaDStream[U]

Return a new DStream in which each RDD is generated by applying a function on each RDD of 'this' DStream.
def transform[U](transformFunc: Function[R, JavaRDD[U]]): JavaDStream[U]

Return a new DStream in which each RDD is generated by applying a function on each RDD of 'this' DStream.
def transformToPair[K2, V2](transformFunc: Function2[R, Time, JavaPairRDD[K2, V2]]): JavaPairDStream[K2, V2]

Return a new DStream in which each RDD is generated by applying a function on each RDD of 'this' DStream.
def transformToPair[K2, V2](transformFunc: Function[R, JavaPairRDD[K2, V2]]): JavaPairDStream[K2, V2]

Return a new DStream in which each RDD is generated by applying a function on each RDD of 'this' DStream.
def transformWith[K2, V2, W](other: JavaPairDStream[K2, V2], transformFunc: Function3[R, JavaPairRDD[K2, V2], Time, JavaRDD[W]]): JavaDStream[W]

Return a new DStream in which each RDD is generated by applying a function on each RDD of 'this' DStream and 'other' DStream.
def transformWith[U, W](other: JavaDStream[U], transformFunc: Function3[R, JavaRDD[U], Time, JavaRDD[W]]): JavaDStream[W]

Return a new DStream in which each RDD is generated by applying a function on each RDD of 'this' DStream and 'other' DStream.
def transformWithToPair[K2, V2, K3, V3](other: JavaPairDStream[K2, V2], transformFunc: Function3[R, JavaPairRDD[K2, V2], Time, JavaPairRDD[K3, V3]]): JavaPairDStream[K3, V3]

Return a new DStream in which each RDD is generated by applying a function on each RDD of 'this' DStream and 'other' DStream.
def transformWithToPair[U, K2, V2](other: JavaDStream[U], transformFunc: Function3[R, JavaRDD[U], Time, JavaPairRDD[K2, V2]]): JavaPairDStream[K2, V2]

Return a new DStream in which each RDD is generated by applying a function on each RDD of 'this' DStream and 'other' DStream.
final def wait(): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long, arg1: Int): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )
final def wait(arg0: Long): Unit

Definition Classes
AnyRef
Annotations
@throws( ... )

Related Doc: package java

trait JavaDStreamLike[T, This <: JavaDStreamLike[T, This, R], R <: JavaRDDLike[T, R]] extends Serializable

Abstract Value Members

implicit abstract val classTag: ClassTag[T]

abstract def dstream: DStream[T]

abstract def wrapRDD(in: RDD[T]): R

Concrete Value Members

final def !=(arg0: Any): Boolean

final def ##(): Int

final def ==(arg0: Any): Boolean

final def asInstanceOf[T0]: T0

def checkpoint(interval: Duration): DStream[T]

def clone(): AnyRef

def context(): StreamingContext

def count(): JavaDStream[Long]

def countByValue(numPartitions: Int): JavaPairDStream[T, Long]

def countByValue(): JavaPairDStream[T, Long]

def countByValueAndWindow(windowDuration: Duration, slideDuration: Duration, numPartitions: Int): JavaPairDStream[T, Long]

def countByValueAndWindow(windowDuration: Duration, slideDuration: Duration): JavaPairDStream[T, Long]

def countByWindow(windowDuration: Duration, slideDuration: Duration): JavaDStream[Long]

final def eq(arg0: AnyRef): Boolean

def equals(arg0: Any): Boolean

def finalize(): Unit

def flatMap[U](f: FlatMapFunction[T, U]): JavaDStream[U]

def flatMapToPair[K2, V2](f: PairFlatMapFunction[T, K2, V2]): JavaPairDStream[K2, V2]

def foreachRDD(foreachFunc: VoidFunction2[R, Time]): Unit

def foreachRDD(foreachFunc: VoidFunction[R]): Unit

final def getClass(): Class[_]

def glom(): JavaDStream[List[T]]

def hashCode(): Int

final def isInstanceOf[T0]: Boolean

def map[U](f: Function[T, U]): JavaDStream[U]

def mapPartitions[U](f: FlatMapFunction[Iterator[T], U]): JavaDStream[U]

def mapPartitionsToPair[K2, V2](f: PairFlatMapFunction[Iterator[T], K2, V2]): JavaPairDStream[K2, V2]

def mapToPair[K2, V2](f: PairFunction[T, K2, V2]): JavaPairDStream[K2, V2]

final def ne(arg0: AnyRef): Boolean

final def notify(): Unit

final def notifyAll(): Unit

def print(num: Int): Unit

def print(): Unit

def reduce(f: Function2[T, T, T]): JavaDStream[T]

def reduceByWindow(reduceFunc: Function2[T, T, T], invReduceFunc: Function2[T, T, T], windowDuration: Duration, slideDuration: Duration): JavaDStream[T]

def reduceByWindow(reduceFunc: Function2[T, T, T], windowDuration: Duration, slideDuration: Duration): JavaDStream[T]

implicit def scalaIntToJavaLong(in: DStream[Long]): JavaDStream[Long]

def slice(fromTime: Time, toTime: Time): List[R]

final def synchronized[T0](arg0: ⇒ T0): T0

def toString(): String

def transform[U](transformFunc: Function2[R, Time, JavaRDD[U]]): JavaDStream[U]

def transform[U](transformFunc: Function[R, JavaRDD[U]]): JavaDStream[U]

def transformToPair[K2, V2](transformFunc: Function2[R, Time, JavaPairRDD[K2, V2]]): JavaPairDStream[K2, V2]

def transformToPair[K2, V2](transformFunc: Function[R, JavaPairRDD[K2, V2]]): JavaPairDStream[K2, V2]

def transformWith[K2, V2, W](other: JavaPairDStream[K2, V2], transformFunc: Function3[R, JavaPairRDD[K2, V2], Time, JavaRDD[W]]): JavaDStream[W]

def transformWith[U, W](other: JavaDStream[U], transformFunc: Function3[R, JavaRDD[U], Time, JavaRDD[W]]): JavaDStream[W]

def transformWithToPair[K2, V2, K3, V3](other: JavaPairDStream[K2, V2], transformFunc: Function3[R, JavaPairRDD[K2, V2], Time, JavaPairRDD[K3, V3]]): JavaPairDStream[K3, V3]

def transformWithToPair[U, K2, V2](other: JavaDStream[U], transformFunc: Function3[R, JavaRDD[U], Time, JavaPairRDD[K2, V2]]): JavaPairDStream[K2, V2]

final def wait(): Unit

final def wait(arg0: Long, arg1: Int): Unit

final def wait(arg0: Long): Unit

Inherited from Serializable

Inherited from Serializable

Inherited from AnyRef

Inherited from Any

Ungrouped