简单的Http请求数据保存到Hdfs

使用okhttp工具集来开发：（如果文件已经存在会报错）

package com.etl;

import java.io.IOException;

import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

import okhttp3.Call;
import okhttp3.Callback;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.Response;

public class LinkHttp {

    private  static Configuration conf = null;
    private static String fsName = "fs.defaultFS";
    private static String fsURI = null;

    public static void main(String[] args) throws Exception {

        String name = args[0];
        String uri =  args[1];
        String url = args[2];
        String targetFile = args[3];  //文件全路径

        //初始化
        init(name, uri);
        OkHttpClient client = new OkHttpClient();
        final Request request = new Request.Builder().url(url).get().build();
        Call call = client.newCall(request);
        call.enqueue(new Callback() {

            @Override
            public void onFailure(Call call, IOException e) {
                System.out.println("Fail");
            }

            @Override
            public void onResponse(Call call, Response response) throws IOException {
                FileSystem fs = null;
                try {
                        Path dstPath = new Path(targetFile);
                        fs = FileSystem.get(conf);
                        FSDataOutputStream outputStream = fs.create(dstPath);
                        if(response.isSuccessful()) {
            //                    System.out.println(response.body().string());
                                outputStream.write(response.body().bytes());
                                outputStream.close();
                                System.out.println("create file " + targetFile + " success!");
                                //fs.close();
                        }
                }catch (Exception e){
                    e.printStackTrace();
                }finally {
                    fs.close();
                }
                System.out.println("run writeHdfs end");

                //关闭
                if(response.body()!=null) {
                    response.body().close();
                }
            }
        });

    }

    private static void init(String name, String uri) {
            if(StringUtils.isNotBlank(fsName)){
                fsName = name;
            }
             fsURI  = uri;
             conf = new Configuration();
             conf.set(fsName, fsURI);
    }

}

配置启动脚本如下：

#!/bin/sh
name=fs.defaultFS        #固定不变
uri=dwpro-name1:8020     #hdfs文件系统地址
url=http://www.cnblogs.com/30go/   #待保存的http地址
targetPath=/tmp/test/king.txt   # 目标的文件名
java -Djava.ext.dirs=lib com.etl.LinkHttp \
${name} ${uri} ${url} ${targetPath}  >> test.log >& &