大数据实战手册-开发篇之spark实战案例：实时日志分析

2.6 spark实战案例：实时日志分析

• 2.6.1 交互流程图

• 2.6.2 客户端监听器（java）

@SuppressWarnings("static-access")
	private void handleSocket() {
		lock.lock();
		Writer writer = null;
		RandomAccessFile raf = null;

		try {
			File file = new File(filepath);
			raf = new RandomAccessFile(file, "r");
			raf.seek(pointer);
			writer = new OutputStreamWriter(socket.getOutputStream(), "UTF-8");

			String line = null;

			while ((line = raf.readLine()) != null) {
				if (Strings.isBlank(line)) {
					continue;
				}

				line = new String(line.getBytes("ISO-8859-1"), "UTF-8");
				writer.write(line.concat("\n"));
				writer.flush();
				logger.info("线程：{}----起始位置：{}----读取文件\n{} :",Thread.currentThread().getName(), pointer, line);

				pointer = raf.getFilePointer();
			}
			Thread.currentThread().sleep(2000);
		} catch (Exception e) {
			logger.error(e.getMessage());
			e.printStackTrace();
		} finally {
			lock.unlock();
			fclose(writer, raf);
		}

	}

• 2.6.3 sparkStream实时数据接收（python）

conf = SparkConf()
conf.setAppName("HIS实时日志分析")
conf.setMaster('yarn') # spark standalone
conf.set('spark.executor.instances', 8) # cluster on yarn
conf.set('spark.executor.memory', '1g')
conf.set('spark.executor.cores', '1')
# conf.set('spark.cores.max', '2')
# conf.set('spark.logConf', True)
conf.set('spark.streaming.blockInterval', 1000*4)  # restart receiver interval

sc = SparkContext(conf = conf)
sc.setLogLevel('ERROR')
sc.setCheckpointDir('hdfs://hadoop01:9000/hadoop/upload/checkpoint/')

ssc = StreamingContext(sc, 30)  # time interval at which splits streaming data into block

lines = ssc.socketTextStream(str(ip), int(port))
# lines.pprint()
lines.foreachRDD(requestLog)
lines.foreachRDD(errorLog)
ssc.start()
ssc.awaitTermination()

• 2.6.4 sparklSQL、RDD结算、结构化搜索、结构存储mongoDB（python）

 def getSparkSessionInstance(sparkConf):
    '''
    :@desc 多个RDD全局共享sparksession
     .config("spark.mongodb.input.uri", "mongodb://127.0.0.1/test.coll") \
     .config("spark.mongodb.output.uri", "mongodb://adxkj:123456@192.168.0.252:27017/") \
    :param sparkConf:
    :return:
    '''
    if ('sparkSessionSingletonInstance' not in globals()):
        globals()['sparkSessionSingletonInstance'] = SparkSession \
            .builder \
            .config(conf=sparkConf) \
            .getOrCreate()
    return globals()['sparkSessionSingletonInstance']

def timeFomate(x):
    '''
    :@desc 处理时间
    :param x:
    :return:
    '''
    if not isinstance(x, list):
        return None

    # filter microsenconds
    x.insert(0, ' '.join(x[0:2]))
    x.pop(1)
    x.pop(1)

    # filter '[]'
    rx = re.compile('([\[\]\',])')
    # text = rx.sub(r'\\\1', text)
    x = [rx.sub(r'', x[i]) for i in range(len(x))]

    # string to time
    x[0] = x[0][: x[0].find('.')]
    x[0] = ''.join(x[0])
    x[0] = datetime.strptime(x[0], '%Y-%m-%d %H:%M:%S')

    return x

def sqlMysql(sqlResult, table, url="jdbc:mysql://192.168.0.252:3306/hisLog", user='root', password=""):
    '''
    :@desc sql结果保存
    :param sqlResult:
    :param table:
    :param url:
    :param user:
    :param password:
    :return:
    '''
    try:
        sqlResult.write \
            .mode('append') \
            .format("jdbc") \
            .option("url", url) \
            .option("dbtable", table) \
            .option("user", user) \
            .option("password", password) \
            .save()
    except:
        excType, excValue, excTraceback = sys.exc_info()
        traceback.print_exception(excType, excValue, excTraceback, limit=3)
        # print(excValue)
        # traceback.print_tb(excTraceback)

def sqlMongodb(sqlResult, table):
    '''
    :@desc sql结果保存
    :param sqlResult:
    :param table:
    :param url:
    :param user:
    :param password:
    :return:
    '''
    try:
        sqlResult.\
            write.\
            format("com.mongodb.spark.sql.DefaultSource"). \
            options(uri="mongodb://adxkj:123456@192.168.0.252:27017/hislog",
                    database="hislog", collection=table, user="adxkj", password="123456").\
            mode("append").\
            save()
    except:
        excType, excValue, excTraceback = sys.exc_info()
        traceback.print_exception(excType, excValue, excTraceback, limit=3)
        # print(excValue)
        # traceback.print_tb(excTraceback)

def decodeStr(x) :
    '''
    :@desc base64解码
    :param x:
    :return:
    '''

    try:
        if x[9].strip() != '' :
            x[9] = base64.b64decode(x[9].encode("utf-8")).decode("utf-8")
            # x[9] = x[9][:5000] #mysql

        if x[11].strip() != '':
            x[11] = base64.b64decode(x[11].encode("utf-8")).decode("utf-8")
            # x[11] = x[11][:5000] #mysql

        if len(x) > 12 and x[12].strip() != '':
            x[12] = base64.b64decode(x[12].encode("utf-8")).decode("utf-8")

    except Exception as e:
        print("不能解码：", x, e)

    return x

def analyMod(x) :
    '''
    :@desc 通过uri匹配模块
    :param x:
    :return:
    '''
    if x[6].strip() == ' ':
        return None

    hasMatch = False

    for k, v in URI_MODULES.items() :
        if x[6].strip().startswith('/' + k) :
            hasMatch = True
            x.append(v)

    if not hasMatch:
        x.append('公共模块')

    return x

def requestLog(time, rdd):
    '''
    :@desc 请求日志分析
    :param time:
    :param rdd:
    :return:
    '''

    logging.info("+++++handle request log：length：%d，获取内容：++++++++++" % (rdd.count()))

    if rdd.isEmpty():
        return None

    logging.info("++++++++++++++++++++++处理requestLog+++++++++++++++++++++++++++++++")

    reqrdd = rdd.map(lambda x: x.split(' ')).\
        filter(lambda x: len(x) > 12 and x[4].find('http-nio-') > 0 and x[2].strip() == 'INFO').\
        filter(lambda x: x[8].strip().upper().startswith('POST') or x[8].strip().upper().startswith('GET')).\
        map(timeFomate).\
        map(decodeStr).\
        map(analyMod)

    reqrdd.cache()
    reqrdd.checkpoint()  # checkpoint先cache避免计算两次，以前的rdd销毁

    sqlRdd = reqrdd.map(lambda x: Row(time=x[0], level=x[1], clz=x[2], thread=x[3], user=x[4], depart=x[5],
                          uri=x[6], method=x[7], ip=x[8], request=x[9], oplen=x[10],
                          respone=x[11], mod=x[12]))

    # rdd持久化，降低内存消耗, cache onliy for StorageLevel.MEMORY_ONLY
    # reqrdd.persist(storageLevel=StorageLevel.MEMORY_AND_DISK_SER)

    if reqrdd.isEmpty():
        return None

    spark = getSparkSessionInstance(rdd.context.getConf())
    df = spark.createDataFrame(sqlRdd)
    df.createOrReplaceTempView(REQUEST_TABLE)

    # 结构化后再分析
    sqlresult = spark.sql("SELECT * FROM " + REQUEST_TABLE)
    sqlresult.show()

    # 保存
    sqlMongodb(sqlresult, REQUEST_TABLE)

def errorLog(time, rdd):
    '''
    :@desc 错误日志分析
    :param time:
    :param rdd:
    :return:
    '''

    logging.info("+++++handle error log：length：%d，获取内容：++++++++++" % (rdd.count()))

    if rdd.isEmpty():
        return None

    logging.info("++++++++++++++++++++++处理errorLog+++++++++++++++++++++++++++++++")

    errorrdd = rdd.map(lambda x: x.split(' ')). \
        filter(lambda x: len(x) > 13 and x[2].strip().upper().startswith('ERROR')). \
        map(timeFomate). \
        map(decodeStr). \
        map(analyMod). \
        map(lambda x: Row(time=x[0], level=x[1], clz=x[2], thread=x[3], user=x[4], depart=x[5],
                          uri=x[6], method=x[7], ip=x[8], request=x[9], oplen=x[10],
                          respone=x[11], stack=x[12], mod=x[13]))
    # rdd持久化，降低内存消耗
    errorrdd.persist(storageLevel=StorageLevel.MEMORY_AND_DISK_SER)

    if errorrdd.isEmpty():
        return None

    spark = getSparkSessionInstance(rdd.context.getConf())
    df = spark.createDataFrame(errorrdd)
    df.createOrReplaceTempView(ERROR_TABLE)

    # 结构化后再分析
    sqlresult = spark.sql("SELECT * FROM " + ERROR_TABLE)
    sqlresult.show()

    # 保存
    sqlMongodb(sqlresult, ERROR_TABLE)

备注：需要完整代码请联系作者@狼