sparkML原始数据转换成label-features方法

数据1：kaggle-旧金山犯罪分类数据
格式如下：
Dates,Category,Descript,DayOfWeek,PdDistrict,Resolution,Address,X,Y
-- ::,WARRANTS,WARRANT ARREST,Wednesday,NORTHERN,"ARREST, BOOKED",OAK ST / LAGUNA ST,-122.425891675136,37.7745985956747
-- ::,OTHER OFFENSES,TRAFFIC VIOLATION ARREST,Wednesday,NORTHERN,"ARREST, BOOKED",OAK ST / LAGUNA ST,-122.425891675136,37.7745985956747
-- ::,OTHER OFFENSES,TRAFFIC VIOLATION ARREST,Wednesday,NORTHERN,"ARREST, BOOKED",VANNESS AV / GREENWICH ST,-122.42436302145,37.8004143219856
-- ::,LARCENY/THEFT,GRAND THEFT FROM LOCKED AUTO,Wednesday,NORTHERN,NONE, Block of LOMBARD ST,-122.42699532676599,37.80087263276921
-- ::,LARCENY/THEFT,GRAND THEFT FROM LOCKED AUTO,Wednesday,PARK,NONE, Block of BRODERICK ST,-122.438737622757,37.771541172057795
-- ::,LARCENY/THEFT,GRAND THEFT FROM UNLOCKED AUTO,Wednesday,INGLESIDE,NONE, Block of TEDDY AV,-122.40325236121201,37.713430704116
-- ::,VEHICLE THEFT,STOLEN AUTOMOBILE,Wednesday,INGLESIDE,NONE,AVALON AV / PERU AV,-122.423326976668,37.7251380403778
-- ::,VEHICLE THEFT,STOLEN AUTOMOBILE,Wednesday,BAYVIEW,NONE,KIRKWOOD AV / DONAHUE ST,-122.371274317441,37.7275640719518
-- ::,LARCENY/THEFT,GRAND THEFT FROM LOCKED AUTO,Wednesday,RICHMOND,NONE, Block of 47TH AV,-122.508194031117,37.776601260681204

测试代码：

    public static void main(String[] args) {

        SparkSession spark = SparkSession.builder().enableHiveSupport()
                .getOrCreate();
        Dataset<Row> dataset = spark
                .read()
                .format("org.apache.spark.sql.execution.datasources.csv.CSVFileFormat")
                .option("header", true)
                .option("inferSchema", true)
                .option("delimiter", ",")
                .load("file:///E:/git/bigdata_sparkIDE/spark-ide/workspace/test/SparkMLTest/SanFranciscoCrime/document/kaggle-旧金山犯罪分类/train-new.csv")
                .persist();

        DataPreProcess(dataset);

    }

   //此函数包含StringIndexer，OneHotEncoder，VectorAssembler,VectorIndexer数据转换方法
    public static Dataset<Row> DataPreProcess(Dataset<Row> data) {

        //Dataset<Row> df = data.selectExpr("cast(Dates as String) ,DayOfWeek,PdDistrict,Category".split(","));

        Dataset<Row> df = data.select(data.col("Dates").cast("String").alias("Dates"),data.col("DayOfWeek").alias("DayOfWeek"),data.col("PdDistrict"),data.col("Category"));
        df.printSchema();
        // 重新索引标签值

        SparkLog.info(data.select("Category").distinct().count());

        //将非数字类型标签转换成数字类型，按照标签去重的个数n,编号0~n,相同标签的多行记录转换后的数字标签编号相同
        //这个适合所有非数字且不连续的有限类别数据编号，不仅仅是只能编号标签
        StringIndexerModel labelIndexer = new StringIndexer()
                .setInputCol("Category").setOutputCol("label").fit(df);

        StringIndexerModel DateIndexer = new StringIndexer()
                .setInputCol("Dates").setOutputCol("DatesNum").fit(df);

        StringIndexerModel DayOfWeekIndexer = new StringIndexer()
                .setInputCol("DayOfWeek").setOutputCol("dfNum").fit(df);

        StringIndexerModel PdDistrictIndexer = new StringIndexer()
                .setInputCol("PdDistrict").setOutputCol("pdNum").fit(df);

        /*独热编码将类别特征（离散的，已经转换为数字编号形式（这个是必须的，否则会报错），
        映射成独热编码，生成的是一个稀疏向量
        比如字符串"abcab"的映射规则:去重后的特征个数n即为稀疏向量的维数，而数字编号代
        表该特征对应的向量中非0值的下标，最后生成0-1编码的向量
        a  1 0 0
        b  0 1 0
        c  0 0 1
        a  1 0 0
        b  0 1 0
        */

        //OneHotEncoder不需要fit
        OneHotEncoder encoder = new OneHotEncoder().setInputCol("dfNum")
                .setOutputCol("dfvec")
                .setDropLast(false);  // 设置最后一个是否包含

        OneHotEncoder encoder1 = new OneHotEncoder().setInputCol("pdNum")
                .setOutputCol("pdvec")
                .setDropLast(false);// 设置最后一个是否包含

        OneHotEncoder encoder2 = new OneHotEncoder().setInputCol("DatesNum")
                .setOutputCol("Datesvec")
                .setDropLast(false);// 设置最后一个是否包含

        //将多个列拼接成一个向量，列的类型可以是向量
        VectorAssembler assembler = new VectorAssembler().setInputCols(
                "Datesvec,dfvec,pdvec".split(",")).setOutputCol("features");

        // Dataset<Row> assembledFeatures = assembler.transform(df);

        Pipeline pipeline = new Pipeline().setStages(new PipelineStage[] {
                DateIndexer, DayOfWeekIndexer, PdDistrictIndexer, encoder,
                encoder1, encoder2, labelIndexer, assembler });

        // Train model. This also runs the indexers.
        PipelineModel model = pipeline.fit(df);

        // Make predictions.
        Dataset<Row> predictions = model.transform(df);
        predictions.describe("label").show();
        predictions.show(, false);

        return predictions;

    }

+-------------------+---------+----------+--------------+--------+-----+-----+-------------+--------------+-----------------------+-----+---------------------------------------------+
|Dates |DayOfWeek|PdDistrict|Category |DatesNum|dfNum|pdNum|dfvec |pdvec |Datesvec |label|features |
+-------------------+---------+----------+--------------+--------+-----+-----+-------------+--------------+-----------------------+-----+---------------------------------------------+
|-- ::|Wednesday|NORTHERN |WARRANTS |172231.0|1.0 |2.0 |(,[],[1.0])|(,[],[1.0])|(,[],[1.0])|7.0 |(,[,,],[1.0,1.0,1.0])|
|-- ::|Wednesday|NORTHERN |OTHER OFFENSES|172231.0|1.0 |2.0 |(,[],[1.0])|(,[],[1.0])|(,[],[1.0])|1.0 |(,[,,],[1.0,1.0,1.0])|
|-- ::|Wednesday|BAYVIEW |LARCENY/THEFT |330092.0|1.0 |3.0 |(,[],[1.0])|(,[],[1.0])|(,[],[1.0])|0.0 |(,[,,],[1.0,1.0,1.0])|
|-- ::|Wednesday|MISSION |OTHER OFFENSES|387792.0|1.0 |1.0 |(,[],[1.0])|(,[],[1.0])|(,[],[1.0])|1.0 |(,[,,],[1.0,1.0,1.0])|
|-- ::|Wednesday|SOUTHERN |BURGLARY |32607.0 |1.0 |0.0 |(,[],[1.0])|(,[],[1.0])|(,[],[1.0]) |8.0 |(,[,,],[1.0,1.0,1.0]) |
|-- ::|Wednesday|BAYVIEW |LARCENY/THEFT |32607.0 |1.0 |3.0 |(,[],[1.0])|(,[],[1.0])|(,[],[1.0]) |0.0 |(,[,,],[1.0,1.0,1.0]) |
|-- ::|Wednesday|PARK |LARCENY/THEFT |32607.0 |1.0 |8.0 |(,[],[1.0])|(,[],[1.0])|(,[],[1.0]) |0.0 |(,[,,],[1.0,1.0,1.0]) |
+-------------------+---------+----------+--------------+--------+-----+-----+-------------+--------------+-----------------------+-----+---------------------------------------------+
only showing top  rows

*******************************************************************************************************************

数据2：

id,name,age,sex,rate
,lyy,,F,0.6
,rdd,,M,0.4
,nyc,,M,0.55
,mzy,,M,0.21

 //Binarizer二值化： 将该列数据二值化，大于阈值的为1.0，否则为0.0  spark源码:udf { in: Double => if (in > td) 1.0 else 0.0 }

 Dataset<Row> result = new Binarizer()
                 .setInputCol("rate")
                 .setOutputCol("flag")
                 .setThreshold(0.5).transform(data);

                 result.show(, false);

+---+----+---+---+----+----+
|id |name|age|sex|rate|flag|
+---+----+---+---+----+----+
| |lyy | |F |0.6 |1.0 |
| |rdd | |M |0.4 |0.0 |
| |nyc | |M |0.55|1.0 |
| |mzy | |M |0.21|0.0 |
+---+----+---+---+----+----+

 //IndexToString将stringindexder转换的数据转回到原始的数据

  StringIndexer labelIndexer = new StringIndexer()
                  .setInputCol("sex")
                  .setOutputCol("label");

                  IndexToString IndexToSex = new  IndexToString()
                              .setInputCol("label")
                              .setOutputCol("orisex");

                  Pipeline pipeline = new Pipeline().setStages(new PipelineStage[] {    labelIndexer,    IndexToSex});
                  PipelineModel model = pipeline.fit(data);

                  // Make predictions.
                  Dataset<Row> result = model.transform(data);

                  result.show(, false);

                 //Bucketizer 分箱（分段处理）：将连续数值转换为离散类别
                 //比如特征是年龄，是一个连续数值，需要将其转换为离散类别(未成年人、青年人、中年人、老年人），就要用到Bucketizer了
                 //如age > 55 老年人
                 double[] splits={,,,,Double.POSITIVE_INFINITY};//[0,18),[18,35),[35,55),[55,正无穷)
                 Dataset<Row> result=new Bucketizer()
                  .setInputCol("age")
                  .setOutputCol("bucketCategory")
                  .setSplits(splits)//设置分段标准
                  .transform(data);

                 result.show(, false);