MongDB日志分析

Result文件数据说明：

Ip：106.39.41.166,（城市）

Date：10/Nov/2016:00:01:02 +0800,（日期）

Day：10,（天数）

Traffic: 54 ,（流量）

Type: video,（类型：视频video或文章article）

Id: 8701（视频或者文章的id）

测试要求：

1、 数据清洗：按照进行数据清洗，并将清洗后的数据导入MongDB数据库中。

两阶段数据清洗：

（1）第一阶段：把需要的信息从原始日志中提取出来

ip:    199.30.25.88

time:  10/Nov/2016:00:01:03 +0800

traffic:  62

文章： article/11325

视频： video/3235

结果实现：

package com.mongodb;

import com.mongodb.client.FindIterable;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.MongoCursor;
import com.mongodb.client.MongoDatabase;
import com.mongodb.client.model.Filters;
import org.bson.Document;

import java.util.ArrayList;
import java.util.List;

/**
 * @author June
 * @date 2021/11/4 14:04
 *                                      原始日志提取结果
 */
public class SelectNo1 {
    public static void main(String[] args) {

        ServerAddress serverAddress = new ServerAddress("192.168.154.129",27017);
        List<ServerAddress> addrs = new ArrayList<ServerAddress>();
        addrs.add(serverAddress);

        //MongoCredential.createScramSha1Credential()三个参数分别为 用户名 数据库名称 密码
        MongoCredential credential = MongoCredential.createScramSha1Credential("rootx", "admin", ".Xuan0613".toCharArray());
        List<MongoCredential> credentials = new ArrayList<MongoCredential>();
        credentials.add(credential);

        //通过连接认证获取MongoDB连接
        MongoClient mongoClient = new MongoClient(addrs,credentials);
        //获取操作数据库
        MongoDatabase mongoDatabase = mongoClient.getDatabase("chaoba");
        //获取集合。后面的操作，大部分都是基于集合操作
        MongoCollection<Document> contections=mongoDatabase.getCollection("result");
        MongoCollection<Document> contection=mongoDatabase.getCollection("result1");

        List<Document> documents=new ArrayList<>();
        MongoCursor result1= contections.find().projection(new BasicDBObject("Ip",1).append("Date", 1).append("Traffic",1).append("Type",1)).iterator();
        while(result1.hasNext()) {
            documents.add((Document)result1.next());
        }
        contection.insertMany(documents);

        //删除
        contections.deleteOne(Filters.eq("chaoba",1));

    }
}

（2）第二阶段：根据提取出来的信息做精细化操作

ip--->城市 city（IP）

date--> time:2016-11-10 00:01:03

day: 10

traffic:62

type:article/video

id:11325

实现代码：

package com.mongodb;

import com.mongodb.client.MongoCollection;
import com.mongodb.client.MongoCursor;
import com.mongodb.client.MongoDatabase;
import com.mongodb.client.model.Filters;
import com.mongodb.client.result.UpdateResult;
import org.bson.Document;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.TransformerException;
import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.List;

/**
 * @author June
 * @date 2021/11/4 14:26
 */
public class Timeclean {
    public static void main(String[] args) throws ParserConfigurationException, IOException, SAXException, TransformerException {

        ServerAddress serverAddress = new ServerAddress("192.168.154.129",27017);
        List<ServerAddress> addrs = new ArrayList<ServerAddress>();
        addrs.add(serverAddress);

        //MongoCredential.createScramSha1Credential()三个参数分别为 用户名 数据库名称 密码
        MongoCredential credential = MongoCredential.createScramSha1Credential("rootx", "admin", ".Xuan0613".toCharArray());
        List<MongoCredential> credentials = new ArrayList<MongoCredential>();
        credentials.add(credential);

        //通过连接认证获取MongoDB连接
        MongoClient mongoClient = new MongoClient(addrs,credentials);
        //获取操作数据库
        MongoDatabase mongoDatabase = mongoClient.getDatabase("chaoba");
        //获取集合。后面的操作，大部分都是基于集合操作
        MongoCollection<Document> contections=mongoDatabase.getCollection("result");
        MongoCollection<Document> contection=mongoDatabase.getCollection("result1");

        public void updatetime() {
            List<test> list = mongoTemplate.findAll(test.class);
            for (int i = 0; i < list.size(); i++) {
                Update update = null;
                try {
                    update = new Update().set("time", datatime.parseDate(list.get(i).getTime()));
                } catch (ParseException e) {
                    e.printStackTrace();
                }
                Query query = new Query(Criteria.where("_id").is(list.get(i).get_id()));
                mongoTemplate.updateFirst(query, update, test.class);
            }
        }

        //删除
        contections.deleteOne(Filters.eq("chaoba",1));

    }
}

（3）MongDB数据库表结构:

create table data(  ip string,  time string , day string, traffic bigint,

type string, id   string )

2、数据处理：

·统计最受欢迎的视频/文章的Top10访问次数 （video/article）

·按照地市统计最受欢迎的Top10课程 （ip）

·按照流量统计最受欢迎的Top10课程 （traffic）

package com.mongodb;

import com.mongodb.client.FindIterable;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.MongoCursor;
import com.mongodb.client.MongoDatabase;
import com.mongodb.client.model.Filters;
import org.bson.Document;

import java.util.ArrayList;
import java.util.List;

/**
 * @author June
 * @date 2021/11/4 15:23
 */
public class TypeNo {
    public static void main(String[] args) {

        ServerAddress serverAddress = new ServerAddress("192.168.154.129",27017);
        List<ServerAddress> addrs = new ArrayList<ServerAddress>();
        addrs.add(serverAddress);

        //MongoCredential.createScramSha1Credential()三个参数分别为 用户名 数据库名称 密码
        MongoCredential credential = MongoCredential.createScramSha1Credential("rootx", "admin", ".Xuan0613".toCharArray());
        List<MongoCredential> credentials = new ArrayList<MongoCredential>();
        credentials.add(credential);

        //通过连接认证获取MongoDB连接
        MongoClient mongoClient = new MongoClient(addrs,credentials);
        //获取操作数据库
        MongoDatabase mongoDatabase = mongoClient.getDatabase("chaoba");
        //获取集合。后面的操作，大部分都是基于集合操作
        MongoCollection<Document> contection=mongoDatabase.getCollection("result");
        MongoCollection<Document> contections=mongoDatabase.getCollection("result2");

        int i=0;
        List<Document> documents=new ArrayList<>();
        BasicDBObject dbObject = new BasicDBObject();
        dbObject.put("Traffic",-1);
        MongoCursor result1=contection.find().projection(new BasicDBObject("Ip",1).append("Traffic",1).append("Id",1)).sort(dbObject).iterator();
        while(result1.hasNext()) {
            //System.out.println(result1.next());
            documents.add((Document)result1.next());
            i++;
            if(i>=10)
                break;
        }
        contections.insertMany(documents);
        //删除
        contections.deleteOne(Filters.eq("chaoba",1));

    }

}