hdfs遍历文件方法

import org.apache.commons.lang.StringUtils;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.*;

 

import java.io.IOException;

import java.net.URI;

import java.net.URISyntaxException;

 

class HelloWord {

public static void main(String[] args){

 

 

Hdfs();

 

}

 

 

//方法1：

public static void Hdfs(){

try{

Configuration conf = new Configuration();

conf.set("fs.defaultFS","hdfs://10.8.6.126:8020");

FileSystem fs = null;

//fs = FileSystem.get(new URI("hdfs://10.8.6.126:8020"),conf); //这两种方式都可以配置hdfs ip

fs = FileSystem.get(conf);

 

RemoteIterator<LocatedFileStatus> lt = fs.listFiles(new Path("hdfs://10.8.6.126:8020/ada/lyy/App"), true);

while (lt.hasNext()) {

LocatedFileStatus file = lt.next();

if(file.isFile())

{

Path path = file.getPath();

System.out.println("文件:["+path.toString()+"]");

System.out.println("文件名:["+path.getName.toString()+"]"); //只是文件名，没有路径信息

}else{

Path path = file.getPath();

System.out.println("目录:["+path.toString()+"]");

}

}

 

}

catch( IOException e){

System.out.println(e.getStackTrace());

}

}

 

}

 

//方法2：

public static void Hdfs(){

try{

Configuration conf = new Configuration();

conf.set("fs.defaultFS","hdfs://10.8.6.126:8020");

FileSystem fs = null;

//fs = FileSystem.get(new URI("hdfs://10.8.6.126:8020"),conf); //这两种方式都可以配置hdfs ip

fs = FileSystem.get(conf);

 

Path path = new Path("/shell");

//通过fs的listStatus方法获取一个指定path的所有文件信息(status)，因此我们需要传入一个hdfs的路径，返回的是一个filStatus数组

FileStatus[] fileStatuses = fs.listStatus(path);

for (FileStatus fileStatus : fileStatuses) {

//判断当前迭代对象是否是目录

boolean isDir = fileStatus.isDirectory();

//获取当前文件的绝对路径

String fullPath = fileStatus.getPath().toString();

System.out.println("isDir:" + isDir + ",Path:" + fullPath);

}

}

 

 

 

同时读取本地和hdfs目录：

 

public class TestHdfs{

 

public static void main(String[] args) {

 

 

//读取配置文件

Configuration conf=new Configuration();

//String path = "hdfs://10.8.6.126:8020/ada/lyy/data/NaiveBayesModel.model";

String path = "file:///home/liyanyan/cluster/NaiveBayesModel.model";

 

 

 

String classfile = "";

 

//读取配置文件

FileStatus[] listFile = null;

FileSystem fs = null;

try {

if(path.startsWith("hdfs:")){

conf.set("fs.defaultFS",path.substring(0,path.indexOf('/', path.indexOf(':') + 3)));

fs = FileSystem.get(conf);

 

}else if(path.startsWith("file:")){

fs=FileSystem.getLocal(conf);

//获取文件目录

}

 

listFile =fs.listStatus(new Path(path+"/metadata"), new RegxRejectPathFilter("^[._]+\\w+.*$"));

 

if(listFile.length != 1){

}else{

classfile = listFile[0].getPath().toString();

}

 

}catch(IOException e){

e.printStackTrace();

}

 

System.out.println("classfile = "+classfile.replaceFirst("file:","file://"));

 

}

 

private static class RegxRejectPathFilter implements PathFilter {

private final String regex;

public RegxRejectPathFilter(String regex) {

this.regex=regex;

}

 

@Override

public boolean accept(Path path) {

// TODO 自动生成的方法存根

boolean flag=path.getName().toString().matches(regex);

return !flag;

}

}

}