工具：Idea
Scala：版本2.10.6<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming_2.10</artifactId>
<version>1.6.0</version>
</dependency>
//以下不加否则本地启动会失败
pom中的provided指的是编译需要，发布不需要，当我们通过spark-submit提交时，spark会提供需要的streaming包，
而Intellij是通过java提交的，在运行时依然需要streaming的包，所以需要去掉．
<scope>provided</scope>
package SparkStreaming
import org.apache.spark.SparkConf
import org.apache.spark.streaming.{Seconds, StreamingContext}
import org.apache.spark.streaming.dstream.{DStream, ReceiverInputDStream}
/**
* receiver 模式（）旧
*/
object StreamingWC {
def main(args: Array[String]):Unit = {
val conf =new SparkConf().setAppName("wc").setMaster("local[2]")
val ssc =new StreamingContext(conf,Seconds(5))
val text: ReceiverInputDStream[String] = ssc.socketTextStream("192.168.23.101",6666)
val maps: DStream[String] = text.flatMap(_.split(" "))
val m = maps.map((_,1))
val reduce = m.reduceByKey(_+_)
//打印统计结果
reduce.print()
//开始
ssc.start()
// 等待停止线程
ssc.awaitTermination()
}
}
[root@node1 ~]# jps
6469 Jps
5687 Kafka
4797 DFSZKFailoverController
4495 NameNode
[root@node1 ~]# nc -lk 6666
123
123 123
123 123
1 1
1 2
-------------------------------------------
Time: 1551356090000 ms
-------------------------------------------
(2,1)
(1,1)