[b0014] HDFS 常用JAVA 操作实战

目的:

学习用java进行的常用hdfs操作

参考：

[b0002] Hadoop HDFS cmd常用命令练手

环境：

hadoop2.6.4

win7 下的eclipse环境调试已经配置好，参考前面的文章

代码：

1. 创建文件夹

 package hdfs;

 import java.io.IOException;

 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;

 /**
  *
  * @author Administrator
  *  创建文件夹,如果不存在
  */
 public class CreateFolder {

     public static void main(String[] args) throws IOException {
         Configuration conf =new Configuration();
         conf.set("fs.defaultFS", "hdfs://ssmaster:9000/");
         FileSystem fs =  FileSystem.get(conf) ;
         Path path = new Path("/output");

         if(! fs.exists(path)){
             fs.mkdirs(path);
         }
     }
 }

以流的方式下载文件

 package hdfs;

 import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.IOException;

 import org.apache.commons.compress.utils.IOUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;

 /**
  *  功能：      将 hdfs://ssmaster:9000/data/paper.txt下载到Windows下c:\paper.txt
  *  调用方式：windows下执行,eclipse中执行
  */

 public class Down_Load {

     public static void main(String[] args) {

     Configuration conf =new Configuration();
     conf.set("fs.defaultFS", "hdfs://ssmaster:9000/");

     FileSystem fs = null;
     Path src = null;
     FSDataInputStream in = null;
     FileOutputStream out = null;

     src = new Path("hdfs://ssmaster:9000/data/paper.txt" );

     try {

       fs = FileSystem.get(conf) ;
       in = fs.open(src);

        } catch (IOException e) {
         e.printStackTrace();
     }

     try {
         out = new FileOutputStream ("c:\\paper.txt"); //等效  c:/paper.txt
     } catch (FileNotFoundException e) {
         e.printStackTrace();
     }

     try {
         IOUtils.copy(in, out);
     } catch (IOException e) {
         e.printStackTrace();
     }

 }
 }

2 上传文件

 package hdfs;

 import java.io.IOException;

 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;

 /**
  *
  * @author Administrator
  *  上传本地文件
  */
 public class UploadFile {

     public static void main(String[] args) throws IOException {
         Configuration conf =new Configuration();
         conf.set("fs.defaultFS", "hdfs://ssmaster:9000/");
         FileSystem fs =  FileSystem.get(conf) ;
         Path path = new Path("/output");
         Path src  = new Path("c:/paper.txt");

         fs.copyFromLocalFile(false, true, src, path);

     }
 }

3 下载文件

 package hdfs;

 import java.io.IOException;

 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;

 /**
  *
  * @author Administrator
  *  上传本地文件
  */
 public class DownFile {

     public static void main(String[] args) throws IOException {
         Configuration conf =new Configuration();
         conf.set("fs.defaultFS", "hdfs://ssmaster:9000/");
         FileSystem fs =  FileSystem.get(conf) ;
         Path hdfs = new Path("/output/paper.txt");
         Path win7  = new Path("c:/paper_download.txt");

         fs.copyToLocalFile(hdfs, win7);

     }
 }

4 删除文件

 package hdfs;

 import java.io.IOException;

 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;

 /**
  *
  * @author Administrator
  *  删除hdfs文件,如何文件不存在，也运行正常
  */
 public class DeleteFile {

     public static void main(String[] args) throws IOException {
         Configuration conf =new Configuration();
         conf.set("fs.defaultFS", "hdfs://ssmaster:9000/");
         FileSystem fs =  FileSystem.get(conf) ;
         Path hdfs = new Path("/output/paper.txt");
         fs.delete(hdfs, true);

     }
 }

5 显示目录信息

 package hdfs;

 import java.io.IOException;

 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;

 /**
  *
  * @author Administrator
  *  显示某个目录下的文件
  */
 public class ListFiles {

     public static void main(String[] args) throws IOException {
         Configuration conf =new Configuration();
         conf.set("fs.defaultFS", "hdfs://ssmaster:9000/");
         FileSystem fs =  FileSystem.get(conf) ;
         Path hdfs = new Path("/");

         FileStatus [] files = fs.listStatus(hdfs);
         for (FileStatus file:files) {

            System.out.print(file.getPath().getName());
            System.out.print("\t"+ file.isDirectory());
            System.out.print("\t"+ file.getOwner());
            System.out.print("\n");
         }

     }
 }

总结：

HDFS JAVA API 调用初步学会使用。该篇章翻过去，后续用什么再学

后续：

有空将文件系统的常用操作实现，搜索、递归显示、查看文件内容

参考:

1 hadoop 2.x hdfs api