DStreamWordCount

sdfwe

import org.apache.spark.storage.StorageLevel
import org.apache.spark.streaming.dstream.ReceiverInputDStream
import org.apache.spark.streaming.{StreamingContext, Seconds}
import org.apache.spark.{SparkContext, SparkConf}

/**
* Created by Administrator on 2015/10/19.
*/
object DStreamWordCount {
   def main (args: Array[String]) {
     val conf=new SparkConf().setMaster("local[2]").setAppName("wordcount")
     val sc=new SparkContext(conf);
     val ssc=new StreamingContext(sc,Seconds(10));

     //val lines=ssc.socketTextStream("localhost",8888,StorageLevel.MEMORY_AND_DISK_SER);
     val line= ssc.receiverStream(new CustomReceiver("E:\\backup\\test"))
     val wordCount=line.map(x=>(x,1)).reduceByKey(_ + _)
     wordCount.print()
     ssc.start()
     ssc.awaitTermination()
  }

}





import java.io.File
import org.apache.spark.{ SparkConf, Logging }
import org.apache.spark.storage.StorageLevel
import org.apache.spark.streaming.{ Seconds, StreamingContext }
import org.apache.spark.streaming.receiver.Receiver

class CustomReceiver(dir: String)
  extends Receiver[String](StorageLevel.MEMORY_AND_DISK_2) with Logging {

  def onStart() {
    // Start the thread that receives data over a connection
    new Thread("File Receiver") {
      override def run() { receive() }
    }.start()
  }

  def onStop() {
    // There is nothing much to do as the thread calling receive()
    // is designed to stop by itself isStopped() returns false
  }

  def recursiveListFiles(f: File): Array[File] = {
    val these = f.listFiles
    these ++ these.filter(_.isDirectory).flatMap(recursiveListFiles)
  }

  private def receive() {

    for (f <- recursiveListFiles(new File(dir))) {

      val source = scala.io.Source.fromFile(f)
      val lines = source.getLines

      lines.foreach(str=>{
        str.split(",").foreach(strchar=>{
          store(strchar)
        })
      })

      source.close()
    }
  }
}