springMVC + hadoop + httpclient 文件上传请求直接写入hdfs

1、首先是一个基于httpclient的java 应用程序，代码在这篇文章的开头：点击打开链接

2、我们首先写一个基于springMVC框架的简单接收请求上传的文件保存本地文件系统的demo，程序代码在这：点击打开链接，不过因为之后与hadoop所以来的jar包同时使用，加上有一些三方jar包可能会冲突，这里我们简化这个demo里面maven所依赖的选项，更新的pom文件如下：

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  <modelVersion>4.0.0</modelVersion>
  <groupId>springmvx</groupId>
  <artifactId>springmvx</artifactId>
  <version>0.0.1-SNAPSHOT</version>
  <packaging>war</packaging>
  <build>
    <sourceDirectory>src</sourceDirectory>
    <plugins>
      <plugin>
        <artifactId>maven-compiler-plugin</artifactId>
        <version>3.3</version>
        <configuration>
          <source>1.7</source>
          <target>1.7</target>
        </configuration>
      </plugin>
      <plugin>
        <artifactId>maven-war-plugin</artifactId>
        <version>2.6</version>
        <configuration>
          <warSourceDirectory>WebContent</warSourceDirectory>
          <failOnMissingWebXml>false</failOnMissingWebXml>
        </configuration>
      </plugin>
    </plugins>
  </build>
  <dependencies>
      <dependency>
        <groupId>jstl</groupId>
        <artifactId>jstl</artifactId>
        <version>1.2</version>
    </dependency>
    <dependency>
        <groupId>org.springframework</groupId>
        <artifactId>spring-webmvc</artifactId>
        <version>3.1.1.RELEASE</version>
    </dependency>
      <dependency>
        <groupId>org.springframework</groupId>
        <artifactId>spring-web</artifactId>
        <version>3.1.1.RELEASE</version>
    </dependency>
    <dependency>
        <groupId>commons-fileupload</groupId>
        <artifactId>commons-fileupload</artifactId>
        <version>1.3.1</version>
    </dependency>
    <dependency>
        <groupId>jdk.tools</groupId>
        <artifactId>jdk.tools</artifactId>
        <version>1.7</version>
        <scope>system</scope>
        <systemPath>${JAVA_HOME}/lib/tools.jar</systemPath>
    </dependency>
    <dependency>
        <groupId>org.apache.hadoop</groupId>
        <artifactId>hadoop-client</artifactId>
        <version>2.6.0</version>
    </dependency>
  </dependencies>
</project>

3、我们将springMVC控制器里的代码稍加修改，改为直接上传文件到hdfs，而不是本地文件系统，代码如下：

package net.spring.controller;

import java.io.File;
import java.io.IOException;
import java.util.Iterator;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

import org.apache.commons.fileupload.disk.DiskFileItem;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.multipart.MultipartFile;
import org.springframework.web.multipart.MultipartHttpServletRequest;
import org.springframework.web.multipart.commons.CommonsMultipartFile;
import org.springframework.web.multipart.commons.CommonsMultipartResolver;
import org.springframework.web.servlet.ModelAndView;

import com.busymonkey.hadoop.*;

@Controller
public class HelloWorldController {

    @RequestMapping("/hello")
    public ModelAndView hello() {
        String message = "Hello World, Spring 3.0!";
        System.out.println(message);
        return new ModelAndView("hello1", "message", message);
    }
    @RequestMapping("/upload")
    public String upLoad(HttpServletRequest request, HttpServletResponse response)
            throws IllegalStateException, IOException{
        //解析器解析request的上下文
        CommonsMultipartResolver multipartResolver = new CommonsMultipartResolver(request.getSession().getServletContext());
        //先判断request中是否包涵multipart类型的数据，
        if(multipartResolver.isMultipart(request)) {
            //再将request中的数据转化成multipart类型的数据
            MultipartHttpServletRequest multiRequest = (MultipartHttpServletRequest) request;
            Iterator<String> iter = multiRequest.getFileNames();
            while(iter.hasNext()) {
                MultipartFile file = multiRequest.getFile((String)iter.next());
                if(file != null) {
                    CommonsMultipartFile cf= (CommonsMultipartFile)file;
                    DiskFileItem fi = (DiskFileItem)cf.getFileItem();
                    File inputFile = fi.getStoreLocation();
                    HdfsFileSystem.createFile(inputFile, "hdfs://192.168.241.149:9000/upload/1.mp4");
                }
            }
        }
        return "/hello1";
    }
}

4、同时我们再新建一个hdfs的工具类，在上面的类中有引用，代码如下：

package com.busymonkey.hadoop;

import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;

public class HdfsFileSystem {
    /**
     * 按路径上传文件到hdfs
     * @param conf
     * @param local
     * @param remote
     * @throws IOException
     */
    public static void copyFile(Configuration conf , String uri , String local, String remote) throws IOException {
        FileSystem fs = FileSystem.get(URI.create(uri), conf);
        fs.copyFromLocalFile(new Path(local), new Path(remote));
        System.out.println("copy from: " + local + " to " + remote);
        fs.close();
    }  

    /**
     * 按路径下载hdfs上的文件
     * @param conf
     * @param uri
     * @param remote
     * @param local
     * @throws IOException
     */
    public static void download(Configuration conf , String uri ,String remote, String local) throws IOException {
           Path path = new Path(remote);
           FileSystem fs = FileSystem.get(URI.create(uri), conf);
           fs.copyToLocalFile(path, new Path(local));
           System.out.println("download: from" + remote + " to " + local);
           fs.close();
   } 

    /**
     * File对象上传到hdfs
     * @param conf
     * @param uri
     * @param remote
     * @param local
     * @throws IOException
     */
    public static void createFile(File localPath, String hdfsPath) throws IOException {
        InputStream in = null;
        try {
            Configuration conf = new Configuration();
            FileSystem fileSystem = FileSystem.get(URI.create(hdfsPath), conf);
            FSDataOutputStream out = fileSystem.create(new Path(hdfsPath));
            in = new BufferedInputStream(new FileInputStream(localPath));
            IOUtils.copyBytes(in, out, 4096, false);
            out.hsync();
            out.close();
            System.out.println("create file in hdfs:" + hdfsPath);
        } finally {
            IOUtils.closeStream(in);
        }
    }
}

 

其他代码跟本地上传文件的demo是一样的。结果如下：

5、jsp页面提交文件上传请求，不使用 java 应用 httpclient，将 index.jsp 修改如下：

<%@ page language="java" contentType="text/html; charset=ISO-8859-1"  pageEncoding="ISO-8859-1"%>
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
<html>
<head>
    <title>Spring 3.0 MVC demo</title>
</head>
<body>
    <a href="hello.html">Say Hello</a>
    <h3>upload :</h3>
    Please choose the file :<br />
    <form action="upload.html" method="post" enctype="multipart/form-data">
        <input type="file" name="file" size="50" />
        <br />
    <input type="submit" value="Upload" />
    </form>
</body>
</html>

 

6、将hello1.jsp 重命名为 result.jsp，代码如下：

<%@ page language="java" contentType="text/html; charset=ISO-8859-1"
    pageEncoding="ISO-8859-1"%>
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
<html>
    <head>
        <meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
        <title>result</title>
    </head>
    <body>
        <p>success!</p>
    </body>
</html>

原文：http://blog.csdn.net/dopamy_busymonkey/article/details/51036248