Scala【需求二：求各省市的各个指标】

需求处理步骤

原始数据->json->过滤->列裁剪

需求二：求各省市的各个指标

原始数据

文本pmt.json，每一行都是一个json字符串。里面包含ip等信息

{"sessionid":"4KT69Su8FavGfydclctzpUBQwYfRT0KW","advertisersid":19,"adorderid":188182,"adcreativeid":2123233,"adplatformproviderid":353466,"sdkversion":"Android 5.0","adplatformkey":"hUHtJfmzYgkKQmBgS1XnmIwT1lwWJZis","putinmodeltype":1,"requestmode":1,"adprice":4295.0,"adppprice":5153.0,"requestdate":"2018-10-06","ip":"106.82.41.165","appid":"XRX1000057","appname":"Face++","uuid":"PXUw6oNaBbOLgE4qHzy1eRR0AP6zl0LC","device":"BLACK BARRY","client":4,"osversion":"","density":"","pw":960,"ph":640,"lang":"","lat":"","provincename":"","cityname":"","ispid":46003,"ispname":"电信","networkmannerid":0,"networkmannername":"WIFI","iseffective":1,"isbilling":1,"adspacetype":3,"adspacetypename":"全屏","devicetype":1,"processnode":1,"apptype":0,"district":"district","paymode":1,"isbid":1,"bidprice":4884.0,"winprice":74754.0,"iswin":1,"cur":"rmb","rate":0.0,"cnywinprice":0.0,"imei":"","mac":"52:54:00:b4:e6:10","idfa":"JOQYVMIIPWAEKPHZRDZNCDLJIUZFSBLZ","openudid":"","androidid":"","rtbprovince":"","rtbcity":"","rtbdistrict":"","rtbstreet":"","storeurl":"","realip":"222.89.26.142","isqualityapp":0,"bidfloor":0.0,"aw":0,"ah":0,"imeimd5":"","macmd5":"","idfamd5":"","openudidmd5":"","androididmd5":"","imeisha1":"","macsha1":"","idfasha1":"","openudidsha1":"","androididsha1":"","uuidunknow":"","userid":"YRNo94gOpa3hCANOpFhUBUpQKWfkDblZ","reqdate":null,"reqhour":null,"iptype":1,"initbidprice":0.0,"adpayment":171671.0,"agentrate":0.0,"lomarkrate":0.0,"adxrate":0.0,"title":"非常经典的句子：没有改变不了的未来，只有不想改变的过去","keywords":"莲藕,甘蔗,辣椒,美文","tagid":"bCvpm912U8soUBaF6QxAIC0PXn4E0KD3","callbackdate":"2018-10-06","channelid":"123489","mediatype":1,"email":"2ki0i@hotmail.com","tel":"13404821298","age":"54","sex":"1"}
{"sessionid":"retXIU76Vpp8VZzc7uQvDtObLjRHLtRe","advertisersid":81,"adorderid":140687,"adcreativeid":2321312,"adplatformproviderid":1036820,"sdkversion":"IOS 11.2","adplatformkey":"U8oDbQfH66KCkAtU092evNM1OLvlIQcK","putinmodeltype":1,"requestmode":2,"adprice":7402.0,"adppprice":3552.0,"requestdate":"2018-10-06","ip":"123.234.117.194","appid":"XRX1000033","appname":"蝉大师","uuid":"xExEur14ellSeYq1wbsDzmw9aMTcW6BU","device":"IPHONE6","client":2,"osversion":"","density":"","pw":1334,"ph":750,"lang":"","lat":"","provincename":"","cityname":"","ispid":46000,"ispname":"移动","networkmannerid":3,"networkmannername":"2G","iseffective":1,"isbilling":1,"adspacetype":2,"adspacetypename":"插屏","devicetype":1,"processnode":3,"apptype":0,"district":"district","paymode":1,"isbid":1,"bidprice":9514.0,"winprice":48180.0,"iswin":1,"cur":"rmb","rate":0.0,"cnywinprice":0.0,"imei":"778207196118215","mac":"52:54:00:a0:6b:b1","idfa":"","openudid":"","androidid":"","rtbprovince":"","rtbcity":"","rtbdistrict":"","rtbstreet":"","storeurl":"","realip":"210.41.145.252","isqualityapp":0,"bidfloor":0.0,"aw":0,"ah":0,"imeimd5":"","macmd5":"","idfamd5":"","openudidmd5":"","androididmd5":"","imeisha1":"","macsha1":"","idfasha1":"","openudidsha1":"","androididsha1":"","uuidunknow":"","userid":"G2KTkDDjamgwbP5uFngqzZPplfesjRQ4","reqdate":null,"reqhour":null,"iptype":1,"initbidprice":0.0,"adpayment":69642.0,"agentrate":0.0,"lomarkrate":0.0,"adxrate":0.0,"title":"非常经典的句子：生活中的点点滴滴，都是因果关系","keywords":"美文","tagid":"lvcFc7R4YuaOzOPZ0W3QDBgClZVCIkWk","callbackdate":"2018-10-06","channelid":"123500","mediatype":2,"email":"lgd8554@hotmail.com","tel":"13704892122","age":"35","sex":"0"}

需求说明

期望数据

按照省市为单位，输出各个指标信息

1.导入依赖

    <dependencies>
        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>fastjson</artifactId>
            <version>1.2.62</version>
        </dependency>
        <dependency>
            <groupId>commons-httpclient</groupId>
            <artifactId>commons-httpclient</artifactId>
            <version>3.0.1</version>
        </dependency>
        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>4.12</version>
        </dependency>
    </dependencies>
    <build>
        <plugins>
            <plugin>
                <artifactId>maven-compiler-plugin</artifactId>
                <version>2.3.2</version>
                <configuration>
                    <source>1.8</source>
                    <target>1.8</target>
                </configuration>
            </plugin>
            <plugin>
                <artifactId>maven-assembly-plugin</artifactId>
                <configuration>
                    <descriptorRefs>
                        <descriptorRef>jar-with-dependencies</descriptorRef>
                    </descriptorRefs>
                </configuration>
                <executions>
                    <execution>
                        <id>make-assembly</id>
                        <phase>package</phase>
                        <goals>
                            <goal>single</goal>
                        </goals>
                    </execution>
                </executions>
            </plugin>
        </plugins>
    </build>

2.代码

package com.bigdata.scala.homework

import scala.io.Source
import com.alibaba.fastjson.{JSON, JSONObject}
import org.apache.commons.httpclient.HttpClient
import org.apache.commons.httpclient.methods.GetMethod

/**
 * @description: TODO
 * @author: HaoWu
 * @create: 2020年07月25日
 */
object HomeWork {
  def main(args: Array[String]): Unit = {
    //1.读文件转为list
    val source = Source.fromFile("C:\\Users\\HaoWu\\Desktop\\pmt.json", "utf-8").getLines().toList
    //2.转为json数据
    val data = source.map(jsonStr => {
      val json = JSON.parseObject(jsonStr)
      val ip = json.getString("ip")
      val provincename = json.getString("provincename")
      val cityname = json.getString("cityname")
      val adplatformproviderid = json.getLong("adplatformproviderid")
      val requestmode = json.getInteger("requestmode")
      val processnode = json.getInteger("processnode")
      val iseffective = json.getInteger("iseffective")
      val isbilling = json.getInteger("isbilling")
      val isbid = json.getInteger("isbid")
      val iswin = json.getInteger("iswin")
      val adorderid = json.getLong("adorderid")
      val adcreativeid = json.getLong("adplatformproviderid")
      val winprice = json.getDouble("winprice")
      val adpayment = json.getDouble("adpayment")
      (ip, provincename, cityname, adplatformproviderid, requestmode, processnode, iseffective, isbilling, isbid, iswin, adorderid, adcreativeid, winprice, adpayment)
    })
      //3.过滤出ip为空的数据
      .filter({
        case (ip, provincename, cityname, adplatformproviderid, requestmode, processnode, iseffective, isbilling, isbid, iswin, adorderid, adcreativeid, winprice, adpayment) => "" != ip && null != ip
      })
      //4.//发起http请求根据ip获取省、市,返回省、市...信息
      .map({
        case (ip, provincename, cityname, adplatformproviderid, requestmode, processnode, iseffective, isbilling, isbid, iswin, adorderid, adcreativeid, winprice, adpayment) => {
          val client = new HttpClient()
          val url = s"https://restapi.amap.com/v3/ip?ip=${ip}&key=f75418e64363b8a96d3565108638c5f1"
          val method = new GetMethod(url)
          val code = client.executeMethod(method)
          var provincename = ""
          var cityname = ""
          if (code == 200) {
            val responseBodyAsString = method.getResponseBodyAsString
            provincename = JSON.parseObject(responseBodyAsString).getString("province")
            cityname = JSON.parseObject(responseBodyAsString).getString("city")
          }
          //
          (provincename, cityname, adplatformproviderid, requestmode, processnode, iseffective, isbilling, isbid, iswin, adorderid, adcreativeid, winprice, adpayment)
        }
      }) //List(([],[],804821,1,2,1,1,0,0,31547,804821,26153.0,37318.0), (陕西省,西安市,215884,3,3,1,1,1,0,167967,215884,14094.0,46195.0), (上海市,上海市,405441,1,3,1,1,0,1,52433,405441,22976.0,103778.0),
      //5.//过滤出省、市不为空的数据
      .filter({
        case (provincename, cityname, adplatformproviderid, requestmode,
        processnode, iseffective, isbilling, isbid, iswin, adorderid, adcreativeid, winprice, adpayment) =>
          "" != provincename && null != provincename && "" != cityname && null != cityname
      }) //List((安徽省,合肥市,213685,2,3,1,1,1,0,41256,213685,7601.0,11174.0), (陕西省,西安市,580944,3,2,1,1,1,1,29928,580944,69692.0,91727.0),...)

    //6.按照省市分组
    val result = data.groupBy(x => (x._1, x._2)) //((安徽省,合肥市),List((安徽省,合肥市,213685,2,3,1,1,1,0,41256,213685,7601.0,11174.0),(安徽省,合肥市,213685,2,3,1,1,1,0,41256,213685,7601.0,11174.0),..))
      .map({
        y => {
          //省、市
          val province_city = y._1
          //原始请求数
          val requestAcount = y._2.filter({
            case (provincename, cityname, adplatformproviderid, requestmode,
            processnode, iseffective, isbilling, isbid, iswin, adorderid, adcreativeid, winprice, adpayment) => (requestmode == 1 && processnode >= 1)
            case _ => false
          }).size
          //广告消费
          val advertConsume = y._2.filter({
            case (provincename, cityname, adplatformproviderid, requestmode,
            processnode, iseffective, isbilling, isbid, iswin, adorderid, adcreativeid, winprice, adpayment) =>
              (adplatformproviderid >= 100000 && iseffective == 1 && isbilling == 1 && iswin == 1 && adorderid > 200000 && adcreativeid > 2000000)
          }).map(x => {
            x._13 / 1000
          }).sum
          (province_city, requestAcount, advertConsume)
        }
      })
    println(result) //List(((四川省,泸州市),1,0.0), ((宁夏回族自治区,吴忠市),0,0.0), ((上海市,上海市),54,0.0), ((吉林省,松原市),2,0.0)...)
  }
}