Kettle Rest大文件上传(RestUploadFile.ktr) Rest文件下载(FileDownload.ktr)

1． Rest大文件上传(RestUploadFile.ktr)

需求描述

1. 上传文件大于10M小于500M
2. 上传文件进行分片（5M一片要比1M分片整体时间快）
3. 先使用java类进行功能模拟在迁移Ktr
4. 使用Kettle+Java片段代码开发
5. 启动步骤时可以自定义必须参数
6. 增加UserId（如：testXiaoYu目录）
7. 上传地址：http://**:8089/api/dlapiservice/v1/file/userdata
8. 下载地址：http://**:8089/api/dlapiservice/v1/file/userdata/
9. HDFS地址：http://**:50070/explorer.html#/testXiaoYu（需要查看需要92服务器远程到153服务器查看）

截图步骤说明

　　指定大文件上传

　　片段代码

　　运行成功结果

上传片段代码

 import java.io.*;
 import org.apache.http.HttpResponse;
 import org.apache.http.client.HttpClient;
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.entity.mime.HttpMultipartMode;
 import org.apache.http.entity.mime.MultipartEntityBuilder;
 import org.apache.http.impl.client.HttpClients;

 public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {

 Object[] r = getRow();

     if (r == null) {
         setOutputDone();
         return false;
     }

  r = createOutputRow(r, data.outputRowMeta.size());

  String urlString = get(Fields.In, "url").getString(r);
  String filename = get(Fields.In, "filename").getString(r);
  String filepath = get(Fields.In, "filepath").getString(r);
  // String action = get(Fields.In, "action").getString(r);
  String userId = get(Fields.In, "userid").getString(r);

         int partsize = 1024 * 1024 * 5;
         File file = new File(filename);
           HttpClient httpClient = HttpClients.createDefault();
          HttpPost post = new HttpPost(urlString);
          MultipartEntityBuilder entity = MultipartEntityBuilder.create();
         entity.setMode(HttpMultipartMode.BROWSER_COMPATIBLE);
         entity.addTextBody("filepath", filepath);
         entity.addTextBody("userId", userId);

  try {
             //Make HTTP Call
             if (file == null || file.getAbsolutePath() == null) {
                 get(Fields.Out, "http_status").setValue(r,"失败");
             } else {

                 long filelenght = file.length();
                 if (filelenght <= partsize) {
                     entity.addBinaryBody("file", file);
                     entity.addTextBody("action", "create");
                     post = new HttpPost(urlString);
                     post.setEntity(entity.build());
                     HttpResponse response = httpClient.execute(post);
                      //设置返回值
                     String httpStatusCode = String.valueOf(response.getStatusLine().getStatusCode());
                     // System.out.println("****上传完成*************:" + httpStatusCode + "------result:" + result);
                     if(httpStatusCode.equals("200")||httpStatusCode.equals("201"))
                     {
                         get(Fields.Out, "http_status").setValue(r,"成功");
                     }
                     else
                     {
                         get(Fields.Out, "http_status").setValue(r,"失败");
                     }
                 }else {
                     int endPosition = 0;//子文件结束位置
                     int count = (filelenght % partsize != 0) ? (int) (filelenght / partsize + 1) : (int) (filelenght / partsize);

                      try {
                           FileInputStream fileInputStream = new FileInputStream(file);
                          int byteslength = 0;
                         byte[] tempbytes = new byte[partsize];
                         byte[] array = null;
                         int i = 1;

                          while ((byteslength = fileInputStream.read(tempbytes)) != -1) {
                             endPosition += partsize;
                             endPosition = (endPosition > filelenght) ? (int) filelenght : endPosition;
                             array = new byte[byteslength];
                              //System.arraycopy(tempbytes, 0, array, 0, byteslength);
                              entity = MultipartEntityBuilder.create();
                             if (endPosition == partsize) {
                                 entity.addTextBody("action", "create");
                             } else {
                                 entity.addTextBody("action", "append");
                             }
                             entity.setMode(HttpMultipartMode.BROWSER_COMPATIBLE);
                             entity.addTextBody("filepath", filepath);
                             entity.addTextBody("userId", userId);
                             File tempfile = new File(String.valueOf(0));
                              FileOutputStream temfileStream = new FileOutputStream(tempfile);
                             temfileStream.write(array);
                             entity.addBinaryBody("file", tempfile);
                             post = new HttpPost(urlString);
                             post.setEntity(entity.build());
                              temfileStream.close();
                              httpClient = HttpClients.createDefault();
                             HttpResponse response = httpClient.execute(post);
                             //设置返回值
                             String httpStatusCode = String.valueOf(response.getStatusLine().getStatusCode());
                             get(Fields.Out, "http_statuscode").setValue(r, httpStatusCode);
                              if (httpStatusCode.equals("200") || httpStatusCode.equals("201")) {
                                 get(Fields.Out, "http_status").setValue(r,"成功");
                             } else {
                                 get(Fields.Out, "http_status").setValue(r,"失败");
                                 break;
                             }
                              i++;
                         }

                     }catch (Exception e) {
                          get(Fields.Out, "http_statuscode").setValue(r, -1);
                         get(Fields.Out, "http_status").setValue(r, "失败:"+e.getMessage());
                     }

                 }
             }

         } catch (Exception e) {
             //System.out.println("==================" + e.getMessage());
             // Set value of HTTP Status to -1 since HTTP Post caused exception
             get(Fields.Out, "http_statuscode").setValue(r, -1);
             get(Fields.Out, "http_status").setValue(r, "失败:"+e.getMessage());
         } finally {

         }

             // get(Fields.Out, "http_statuscode").setValue(r, -1);
             // get(Fields.Out, "http_status").setValue(r, "失败");

      // Send the row on to the next step.
     putRow(data.outputRowMeta, r);
       return true;
 }

2． Rest文件下载(FileDownload.ktr)

需求描述

1. 下载上传的文件
2. （5M一片要比1M分片整体时间快）
3. 下载地址：http://**:8089/api/dlapiservice/v1/file/userdata/
4. HDFS地址：http:// **:50070/explorer.html#/testXiaoYu（需要查看需要92服务器远程到153服务器查看）

下载片段代码

 import org.apache.http.*;
 import org.apache.http.client.ClientProtocolException;
 import org.apache.http.client.HttpClient;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.client.utils.URIBuilder;
 import org.apache.http.impl.client.HttpClients;

 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.URISyntaxException;

 public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {

 Object[] r = getRow();

     if (r == null) {
         setOutputDone();
         return false;
     }

  r = createOutputRow(r, data.outputRowMeta.size());

  String urlString = get(Fields.In, "url").getString(r);
  String filepath = get(Fields.In, "filepath").getString(r);
  String folder = get(Fields.In, "folder").getString(r);
  String filename ="";
  String userId = get(Fields.In, "userid").getString(r);
  HttpClient httpClient = HttpClients.createDefault();

         try {
             URIBuilder builder = new URIBuilder(urlString+"/" + userId);
             builder.addParameter("filepath", filepath);
             HttpGet httpGet = new HttpGet(builder.build());
             HttpResponse response = httpClient.execute(httpGet);
             HttpEntity entity = response.getEntity();
             InputStream is = entity.getContent();
             filename = getFileName(response);
             //System.out.println("-----filename--------:" + filename);
             File file = new File(folder + filename);
             file.getParentFile().mkdirs();
             FileOutputStream fileout = new FileOutputStream(file);
             byte[] buffer = new byte[1024 * 1024];
             int ch = 0;
             while ((ch = is.read(buffer)) != -1) {
                 fileout.write(buffer, 0, ch);
             }
             is.close();
             fileout.flush();
             fileout.close();
             get(Fields.Out, "filename").setValue(r,filename);
             get(Fields.Out, "http_status").setValue(r, "成功");
         } catch (URISyntaxException e) {
             //e.printStackTrace();
             get(Fields.Out, "exception").setValue(r, "失败:"+e.getMessage());

         } catch (ClientProtocolException e) {
             //e.printStackTrace();
             get(Fields.Out, "exception").setValue(r, "失败:"+e.getMessage());

         } catch (IOException e) {
             //e.printStackTrace();
             get(Fields.Out, "exception").setValue(r, "失败:"+e.getMessage());

         }

      // Send the row on to the next step.
     putRow(data.outputRowMeta, r);
       return true;
 }

 public static String getFileName(HttpResponse response) {
         Header contentHeader = response.getFirstHeader("Content-Disposition");
         String filename = null;
         if (contentHeader != null) {
             HeaderElement[] values = contentHeader.getElements();
             if (values.length == 1) {
                 NameValuePair param = values[0].getParameterByName("filename");
                 if (param != null) {
                     try {
                         filename = param.getValue();
                     } catch (Exception e) {
                         e.printStackTrace();
                     }
                 }
             }
         }
         return filename;
     }

　　

1. KettleDemo整体原型前提条件，Java片段代码需要引用Jar包下载路径： http://hc.apache.org/downloads.cgi

1. Jar包拷贝