zhuzher日志log

 val log=sc.textFile("/data/logstash/data/*.log")
 val rowRDD2=log2.map(line=>(line.split("\"message\":\"").last.split(" ").head.trim(),line.split("account: ").last.split(", args:").head))
 val k2=rowRDD.filter({case(k,v) =>k.length==10 && !k.contains("TypeError:")}).filter({case(k,v)=>v.length==8})
 val k2=k.filter({case(k,v) =>v.length==7})
 val staff =sqlContext.jdbc("jdbc:mysql://rdssw603u1t68figaia7.mysql.rds.aliyuncs.com:3306/falcon?user=wy_app&password=V0tkEIve2","staff")
 val s=staff.toDF().registerTempTable("staff")
 val staffjob =sqlContext.jdbc("jdbc:mysql://rdssw603u1t68figaia7.mysql.rds.aliyuncs.com:3306/falcon?user=wy_app&password=V0tkEIve2","staff_and_job")
 val s2=staffjob.toDF().registerTempTable("staffjob")

 val job =sqlContext.jdbc("jdbc:mysql://rdssw603u1t68figaia7.mysql.rds.aliyuncs.com:3306/falcon?user=wy_app&password=V0tkEIve2","job")
 val s3=job.toDF().registerTempTable("job")
 val project=sqlContext.jdbc("jdbc:mysql://rdssw603u1t68figaia7.mysql.rds.aliyuncs.com:3306/falcon?user=wy_app&password=V0tkEIve2","project")
 val s4=project.toDF().registerTempTable("project")
 val  m=sqlContext.sql("select project.code,project.Name,staff.id from staff  left join staffjob   on staff.id=staffjob.staff_id  left join  job on staffjob.job_id=job.id left join project  on job.project_code=project.code")

val n=sqlContext.sql("select count(distinct m.No) from m join l on m.No=l.No where l.date >= '2016-04-25' and l.date<='2016-05-01' ")

n.repartition(1).rdd.saveAsTextFile("file:////data/projectq")