SparkContext自定义扩展textFiles，支持从多个目录中输入文本文件

需求

 

SparkContext自定义扩展textFiles，支持从多个目录中输入文本文件

 

扩展

 

class SparkContext(pyspark.SparkContext):

    def __init__(self, master=None, appName=None, sparkHome=None, pyFiles=None, environment=None, batchSize=0, serializer=PickleSerializer(), conf=None, gateway=None, jsc=None):
        pyspark.SparkContext.__init__(self, master=master, appName=appName, sparkHome=sparkHome, pyFiles=pyFiles,
                                      environment=environment, batchSize=batchSize, serializer=serializer, conf=conf, gateway=gateway, jsc=jsc)

    def textFiles(self, dirs):
        hadoopConf = {"mapreduce.input.fileinputformat.inputdir": ",".join(
            dirs), "mapreduce.input.fileinputformat.input.dir.recursive": "true"}

        pair = self.hadoopRDD(inputFormatClass="org.apache.hadoop.mapred.TextInputFormat",
                              keyClass="org.apache.hadoop.io.LongWritable", valueClass="org.apache.hadoop.io.Text", conf=hadoopConf)

        text = pair.map(lambda pair: pair[1])

        return text

 

示例

 

from pyspark import SparkConf
from dip.spark import SparkContext

conf = SparkConf().setAppName("spark_textFiles_test")

sc = SparkContext(conf=conf)

dirs = ["hdfs://dip.cdh5.dev:8020/user/yurun/dir1",
        "hdfs://dip.cdh5.dev:8020/user/yurun/dir2"]

def printLines(lines):
    if lines:
        for line in lines:
            print line

lines = sc.textFiles(dirs).collect()

printLines(lines)

sc.stop()