Layui_Tree模块遍历HDFS

注：转载请署名

一、实体

package com.ebd.application.common.Base;

import java.util.List;

public class HDFSDir {

	private String id;      //自定id

	private String pid;		//父ID

	private String name;	//当前目录名称

	private String alias;	//目录别名,可不用

	private String dir;		//自"/"目录后的完整目录

	private boolean spread;  //是否展开(true,false)

	private List<HDFSDir> children; //子目录

	public String getId() {

		return id;

	}

	public void setId(String id) {

		this.id = id;

	}

	public String getPid() {

		return pid;

	}

	public void setPid(String pid) {

		this.pid = pid;

	}

	public String getName() {

		return name;

	}

	public void setName(String name) {

		this.name = name;

	}

	public String getAlias() {

		return alias;

	}

	public void setAlias(String alias) {

		this.alias = alias;

	}

	public String getDir() {

		return dir;

	}

	public void setDir(String dir) {

		this.dir = dir;

	}

	public boolean isSpread() {

		return spread;

	}

	public void setSpread(boolean spread) {

		this.spread = spread;

	}

	public List<HDFSDir> getChildren() {

		return children;

	}

	public void setChildren(List<HDFSDir> children) {

		this.children = children;

	}

}

二、工具类

package hdfstest;

import java.io.IOException;

import java.net.URI;

import java.util.ArrayList;

import java.util.List;

import org.apache.commons.lang3.StringUtils;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.FSDataInputStream;

import org.apache.hadoop.fs.FileStatus;

import org.apache.hadoop.fs.FileSystem;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.mapred.JobConf;

import com.ebd.application.common.Base.HDFSDir;

import com.ebd.application.common.utils.Identities;

import net.sf.json.JSONObject;

public class HdfsListTest {

	//HDFS访问地址

	private static final String HDFS = "hdfs://bigdata.hadoop.com:9000";

	public HdfsListTest(Configuration conf) {

		this(HDFS, conf);

	}

	public HdfsListTest(String hdfs, Configuration conf) {

		this.hdfsPath = hdfs;

		this.conf = conf;

	}

	//hdfs路径

	private String hdfsPath;

	//Hadoop系统配置

	private Configuration conf;

	//启动函数

	public static void main(String[] args) throws IOException {

		JobConf conf = config();

//		System.out.println(conf.get("hadoop.http.staticuser.user"));

//		System.out.println(System.getenv("HADOOP_HOME"));

		HdfsListTest hdfs = new HdfsListTest(conf);

//		hdfs.mkdirs("/testput");

//		hdfs.copyFile("C:\\Users\\Administrator\\Desktop\\testput", "/testput/testput2");

//		hdfs.catFile("/testput/testput");

//		hdfs.download("/testput/testput", "E:\\");

//		hdfs.ls("hdfs://bigdata.hadoop.com:9000/user");

//		hdfs.rmr("/testput");

//		List<String> fileList = hdfs.getTree("/","/","|-");

		List<HDFSDir> kk = new ArrayList<HDFSDir>();

		HDFSDir ds1 = new HDFSDir();

		HDFSDir ds2 = new HDFSDir();

		HDFSDir ds3 = new HDFSDir();

		ds1.setId(Identities.uuid());

		ds1.setDir("/testput");

		ds2.setId(Identities.uuid());

		ds2.setDir("/user");

		ds3.setId(Identities.uuid());

		ds3.setDir("/tmp");

//		kk.add(ds1);

//		kk.add(ds2);

//		kk.add(ds3);

		HDFSDir ds = new HDFSDir();

		ds.setId(Identities.uuid());

		ds.setDir("/");

		kk.add(ds);

//		List<HDFSDir> fileList = hdfs.getListTree("/","/user",0);

		HDFSDir hdfss = hdfs.getChildNode(ds);

		JSONObject object = JSONObject.fromObject(hdfss);

		System.out.println(dirJsonFunc(object.toString()));

	}      

	//加载Hadoop配置文件

	public static JobConf config(){

		JobConf conf = new JobConf(HdfsListTest.class);

		conf.setJobName("HdfsDAO");

		conf.addResource("hadoop/core-site.xml");

		conf.addResource("hadoop/hdfs-site.xml");

		conf.addResource("hadoop/mapred-site.xml");

		return conf;

	}

	//在根目录下创建文件夹

	public void mkdirs(String folder) throws IOException {

		Path path = new Path(folder);

		FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);

		if (!fs.exists(path)) {

			fs.mkdirs(path);

			System.out.println("Create: " + folder);

		}

		fs.close();

	}

	//某个文件夹的文件列表

	public FileStatus[] ls(String folder) throws IOException {

		Path path = new Path(folder);

		FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);

		FileStatus[] list = fs.listStatus(path);

		System.out.println("ls: " + folder);

		System.out.println("==========================================================");

		if(list != null)

			for (FileStatus f : list) {

				System.out.printf("name: %s, folder: %s, size: %d\n", f.getPath(), f.isDir(), f.getLen());

//				System.out.printf("%s, folder: %s, 大小: %dK\n", f.getPath().getName(), (f.isDir()?"目录":"文件"), f.getLen()/1024);

			}

		System.out.println("==========================================================");

		fs.close();

		return  list;

	}

	public void copyFile(String local, String remote) throws IOException {

		FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);

		//remote---/用户/用户下的文件或文件夹

		fs.copyFromLocalFile(new Path(local), new Path(remote));

		System.out.println("copy from: " + local + " to " + remote);

		fs.close();

	}

	public void catFile(String remote) throws IOException {

		FSDataInputStream instream = null;

		FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);

		Path path = new Path(remote);

		if(fs.isFile(path)){

			fs.open(path);

			instream = fs.open(path);

	        byte[] b = new byte[1024];

	        instream.read(b);

	        System.out.println(new String(b,"utf-8"));

			fs.close();

		}

	}

	List <String> treeList = new ArrayList<String>();

	public List<String> getTree(String top, String remote, String prefix) throws IOException {

		Path path = new Path(remote);

		FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);

		FileStatus[] list = fs.listStatus(path);

		if(list != null)

			for (FileStatus f : list) {

//				System.out.printf("name: %s, folder: %s, size: %d\n", f.getPath(), f.isDir(), f.getLen());

				System.out.println(prefix+ f.getPath().getName());

				top += f.getPath().getName();

				treeList.add(top);

				if(fs.isDirectory(f.getPath())){

					getTree(top,f.getPath().toString(),prefix+"-");

				}

			}

		return treeList;

	}

	int id = 0;

	static int pid = 0;

	List<HDFSDir> dirList = new ArrayList<HDFSDir>();

	HDFSDir hdfsDir = null;

	private List<HDFSDir> getListTree(String top, String remote, int pid) throws IOException {

		Path path = new Path(remote);

		FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);

		FileStatus[] list = fs.listStatus(path);

		if(list != null)

			for (FileStatus f : list) {

				if(f.isDirectory()){

					hdfsDir = new HDFSDir();

//					hdfsDir.setId(id++);

//					hdfsDir.setPid(pid);

					hdfsDir.setName(f.getPath().getName());

					hdfsDir.setAlias(f.getPath().getName());

					hdfsDir.setDir(f.getPath().toString().substring(HDFS.length()));

					hdfsDir.setSpread(false);

					System.out.println(f.getPath().getName()+"="+f.getPath().toString().substring(HDFS.length()));

					dirList.add(hdfsDir);

				}

//				System.out.printf("name: %s, folder: %s, size: %d\n", f.getPath(), f.isDir(), f.getLen());

//				System.out.println(prefix+ f.getPath().getName());

//				top += f.getPath().getName();

//				if(fs.isDirectory(f.getPath())){

//					getListTree(top,f.getPath().toString(),pid++);

//				}

			}

		return dirList;

	}

	List<HDFSDir> cDirList = null;

	public HDFSDir getChildNode(HDFSDir pDir) throws IOException{

		Path path = null;

		if(pDir.getChildren() != null && pDir.getChildren().size() >= 1){

			for(HDFSDir p : pDir.getChildren()){

				path = new Path(p.getDir());

				FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);

				FileStatus[] list = fs.listStatus(path);

				if(list != null){

					cDirList = new ArrayList<HDFSDir>();

					for (FileStatus f : list) {

						if(f.isDirectory()){

							hdfsDir = new HDFSDir();

							hdfsDir.setId(Identities.uuid());

							hdfsDir.setPid(p.getId());

							hdfsDir.setName(f.getPath().getName());

							hdfsDir.setAlias(f.getPath().getName());

							hdfsDir.setDir(f.getPath().toString().substring(HDFS.length()));

							hdfsDir.setSpread(false);

							cDirList.add(hdfsDir);

						}

					}

					p.setChildren(cDirList);

					for(HDFSDir pp : cDirList){

						getChildNode(pp);

					}

				}

			}

		}else{

			path = new Path(pDir.getDir());

			FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);

			FileStatus[] list = fs.listStatus(path);

			if(list != null){

				cDirList = new ArrayList<HDFSDir>();

				for (FileStatus f : list) {

					if(f.isDirectory()){

						hdfsDir = new HDFSDir();

						hdfsDir.setId(Identities.uuid());

						hdfsDir.setPid(pDir.getId());

						hdfsDir.setName(f.getPath().getName().equals("")?"/":f.getPath().getName());

						hdfsDir.setAlias(f.getPath().getName().equals("")?"/":f.getPath().getName());

						hdfsDir.setDir(f.getPath().toString().substring(HDFS.length()));

						hdfsDir.setSpread(false);

						cDirList.add(hdfsDir);

					}

				}

				pDir.setChildren(cDirList);

				for(HDFSDir pp : cDirList){

					getChildNode(pp);

				}

			}

		}

		return pDir;

	}

	public static String dirJsonFunc(String jsonStr) {

		 if (StringUtils.isNotBlank(jsonStr)) {

			 String[] reg_array = {"([\"])","(,['])","([']:)"};

			 String[] rpa_array = {"'",",",":"};

			 for(int i=0;i<reg_array.length;i++){

				 jsonStr = jsonStr.replaceAll(reg_array[i], rpa_array[i]);

			 }

			 jsonStr = jsonStr.replace("{'", "{");

			 jsonStr = jsonStr.replace("'}", "}");

		 }

		 return jsonStr;

	}

	//删除文件或文件夹

	public void rmr(String folder) throws IOException {

		Path path = new Path(folder);

		FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);

		fs.deleteOnExit(path);

		System.out.println("Delete: " + folder);

		fs.close();

	}

	//下载文件到本地系统

	public void download(String remote, String local) throws IOException {

		Path path = new Path(remote);

		FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);

		fs.copyToLocalFile(path, new Path(local));

		System.out.println("download: from" + remote + " to " + local);

		fs.close();

	}

}

转换工具类

package test;

import org.apache.commons.lang3.StringUtils;

import com.ebd.application.common.Base.HDFSDir;

import com.ebd.application.common.utils.Identities;

import net.sf.json.JSONArray;

public class TestObjectToJson {

	public static void main(String[] args) {

		HDFSDir ds = new HDFSDir();

		ds.setId(Identities.uuid());

		ds.setDir("/testput");

		JSONArray js = JSONArray.fromObject(ds);

//		System.out.println(js.toString());

		String jsonStr = js.toString();

//		String reg_1 = "([\"])"; //双引号转单引号

//		String reg_2 = "(,['])"; //去掉逗号后面的单引号

//		String reg_3 = "([']:)"; //去掉冒号前面的单引号

//		String reg_4 = "('{'['])"; //去掉开头大括号后面的单引号

//		Pattern pattern = Pattern.compile(regEx);

//		jsonStr =  jsonStr.replaceAll(reg_1, "'");

//		jsonStr =  jsonStr.replaceAll(reg_2, ",");

//		jsonStr =  jsonStr.replaceAll(reg_3, ":");

//		jsonStr =  jsonStr.replaceAll("{'", "{");

//        System.out.println(jsonStr);

	}

	public static String dirJsonFunc(String jsonStr) {

		 if (StringUtils.isNotBlank(jsonStr)) {

			 String[] reg_array = {"([\"])","(,['])","([']:)"};

			 String[] rpa_array = {"'",",",":"};

			 for(int i=0;i<reg_array.length;i++){

				 jsonStr = jsonStr.replaceAll(reg_array[i], rpa_array[i]);

			 }

			 jsonStr = jsonStr.replace("{'", "{");

			 jsonStr = jsonStr.replace("'}", "}");

		 }

		 return jsonStr;

	}

}

工具类

package hdfstest;

import java.io.File;

import java.io.FileOutputStream;

import java.io.IOException;

import java.io.OutputStream;

import java.net.URI;

import java.util.ArrayList;

import java.util.List;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.FSDataInputStream;

import org.apache.hadoop.fs.FileStatus;

import org.apache.hadoop.fs.FileSystem;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.IOUtils;

import org.apache.hadoop.mapred.JobConf;

import com.ebd.application.common.utils.CreateFileUtil;

public class HdfsDirConsoleTest {

	//HDFS访问地址

	private static final String HDFS = "hdfs://bigdata.hadoop.com:9000";

	public HdfsDirConsoleTest(Configuration conf) {

		this(HDFS, conf);

	}

	public HdfsDirConsoleTest(String hdfs, Configuration conf) {

		this.hdfsPath = hdfs;

		this.conf = conf;

	}

	//hdfs路径

	private String hdfsPath;

	//Hadoop系统配置

	private Configuration conf;

	//启动函数

	public static void main(String[] args) throws IOException {

		JobConf conf = config();

		System.out.println(conf.get("hadoop.http.staticuser.user"));

		System.out.println(System.getenv("HADOOP_HOME"));

		HdfsDirConsoleTest hdfs = new HdfsDirConsoleTest(conf);

//		hdfs.mkdirs("/testput");

//		hdfs.copyFile("C:\\Users\\Administrator\\Desktop\\testput", "/testput/testput2");

//		hdfs.catFile("/testput/testput");

		hdfs.download("/testput/testput", "D:/ss/ss",conf);

//		hdfs.ls("hdfs://bigdata.hadoop.com:9000/");

//		hdfs.rmr("/testput");

//		List<String> fileList = hdfs.getTree("/","/","|-");

//		for(int i=0;i<fileList.size();i++){

//			System.out.println(fileList.get(i));

//		}

		System.out.println("success!");

	}      

	//加载Hadoop配置文件

	public static JobConf config(){

		JobConf conf = new JobConf(HdfsDirConsoleTest.class);

		conf.setJobName("HdfsDAO");

		conf.addResource("hadoop/core-site.xml");

		conf.addResource("hadoop/hdfs-site.xml");

		conf.addResource("hadoop/mapred-site.xml");

		return conf;

	}

	//在根目录下创建文件夹

	public void mkdirs(String folder) throws IOException {

		Path path = new Path(folder);

		FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);

		if (!fs.exists(path)) {

			fs.mkdirs(path);

			System.out.println("Create: " + folder);

		}

		fs.close();

	}

	//某个文件夹的文件列表

	public FileStatus[] ls(String folder) throws IOException {

		Path path = new Path(folder);

		FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);

		FileStatus[] list = fs.listStatus(path);

		System.out.println("ls: " + folder);

		System.out.println("==========================================================");

		if(list != null)

			for (FileStatus f : list) {

				System.out.printf("name: %s, folder: %s, size: %d\n", f.getPath(), f.isDir(), f.getLen());

				System.out.println(f.getOwner()+"=="+f.getBlockSize()+"="+f.getModificationTime()+"--"+f.getPermission()+"="+f.getReplication());

//				System.out.printf("%s, folder: %s, 大小: %dK\n", f.getPath().getName(), (f.isDir()?"目录":"文件"), f.getLen()/1024);

			}

		System.out.println("==========================================================");

		fs.close();

		return  list;

	}

	public void copyFile(String local, String remote) throws IOException {

		FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);

		//remote---/用户/用户下的文件或文件夹

		fs.copyFromLocalFile(new Path(local), new Path(remote));

		System.out.println("copy from: " + local + " to " + remote);

		fs.close();

	}

	public void catFile(String remote) throws IOException {

		FSDataInputStream instream = null;

		FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);

		Path path = new Path(remote);

		if(fs.isFile(path)){

			fs.open(path);

			instream = fs.open(path);

	        byte[] b = new byte[1024];

	        instream.read(b);

	        System.out.println(new String(b,"utf-8"));

			fs.close();

		}

	}

	List <String> treeList = new ArrayList<String>();

	public List<String> getTree(String top, String remote, String prefix) throws IOException {

		Path path = new Path(remote);

		FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);

		FileStatus[] list = fs.listStatus(path);

		if(list != null)

			for (FileStatus f : list) {

//				System.out.printf("name: %s, folder: %s, size: %d\n", f.getPath(), f.isDir(), f.getLen());

				System.out.println(prefix+ f.getPath().getName());

				top += f.getPath().getName();

				treeList.add(top);

				if(fs.isDirectory(f.getPath())){

					getTree(top,f.getPath().toString(),prefix+"-");

				}

			}

		return treeList;

	}

	//删除文件或文件夹

	public void rmr(String folder) throws IOException {

		Path path = new Path(folder);

		FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);

		fs.deleteOnExit(path);

		System.out.println("Delete: " + folder);

		fs.close();

	}

	//下载文件到本地系统

	public void download(String remote, String local,JobConf conf) throws IOException {

//		Path path = new Path(remote);

//		FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);

//		fs.copyToLocalFile(path, new Path(local));

//		System.out.println("download: from" + remote + " to " + local);

//		fs.close();

		 FileSystem fs = FileSystem.get(URI.create(remote),conf);

		 FSDataInputStream fsdi = fs.open(new Path(remote));

		 if(CreateFileUtil.createDir(local)){

			 OutputStream output = new FileOutputStream(local+remote.substring(remote.lastIndexOf("/")));

			 IOUtils.copyBytes(fsdi,output,4096,true);

		 }

	}

	public static void makdir(String path) {

		String strPath = "E:/a/aa/";

		File file = new File(strPath);

		File fileParent = file.getParentFile();

		if(!fileParent.exists()){

		 fileParent.mkdirs();

		}

	}

}

Layui_Tree模块遍历HDFS的更多相关文章

【hadoop】python通过hdfs模块读hdfs数据
hdfs官网:http://hdfscli.readthedocs.io/en/latest/api.html 一个非常好的博客:http://blog.csdn.net/gamer_gyt/arti ...
Windows编程之模块遍历(C++实现)
Windows编程之模块遍历 PS: 主要扣代码使用,直接滑动到最下面使用. 遍历模块需要几个API,和一个结构体 1.创建进程快照 2.遍历首次模块 3.继续下次遍历 4.模块信息结构体 API 分 ...
[Windows编程]模块遍历
模块遍历整体思路 1.创建进程快照 2.遍历首次模块 3.继续下次遍历 4.模块信息结构体相关API的调用创建进程快照API HANDLE WINAPI CreateToolhelp32Snap ...
Python 使用 os 模块遍历目录/获取当前文件的路径
1.列出指定目录下所包含的目录 item = os.listdir("/Users/jinchengxie/go") 返回的是一个列表, 里面包含了指定目录下所包含的所有的目录 2 ...
python os模块遍历目录
#os #os ->tree命令 import os #递归 #目录 ->文件,文件夹 -> 文件文件夹 dirpath = input('请输入你要遍历的目录\n') def ge ...
[java开发篇][dom4j模块] 遍历xml文件
http://blog.csdn.net/chenleixing/article/details/44353491 在android studio 导入dom4j库(build-gradle(Moud ...
[java开发篇][dom4j模块]遍历，解析xml
package com.softwinner.performance.benchmark; /** * Created by Administrator on 2017/7/21. */ import ...
[java开发篇][dom模块] 遍历解析xml
http://blog.csdn.net/andie_guo/article/details/24844351 XML DOM节点树 XML DOM将XML文档作为树结构,树结构称为一个节点树.所有的 ...
hadoop(一HDFS)
hadoop(一HDFS) 介绍狭义上来说: hadoop指的是以下的三大系统: HDFS :分布式文件系统(高吞吐,没有延时要求,容错性,扩展能力) MapReduce : 分布式计算系统 Yar ...

随机推荐

node中npm安装模块的网络问题
最近使用node开发时,发现所有的依赖模块都安装不了啦,一直报错如下 rollbackFailedOptional: verb npm-session 5a4a66a1b8d06dc3 后来才发现是由 ...
ubuntu 14.04 sudo apt-get 简单安装 ffmpeg
一种方法是这样的sudo add-apt-repository ppa:kirillshkrogalev/ffmpeg-next sudo apt-get update sudo apt-get in ...
python简介及安装配置
概述 python是解释型语言,相对编译型语言,执行效率较低.python是通过c语言编写,官方解释器也是c语言编写cpython,也有其他的如用java编写的jpython.目前有2.0和3.0版本 ...
VD: error VERR_FILE_NOT_FOUND
virtualbox制作的镜像文件如果移动了位置,比如从C盘移到D盘,那么再次打开时会提示找不到文件. 解决办法: 打开virtualbox,在“管理”菜单中打开“虚拟介质管理”,在“虚拟硬盘”选项卡 ...
odooERP系统（框架）总结
1:Odoo 是一个现代化的商业应用套件,使用 AGPL 许可证,并具有客户关系管理(CRM),人力资源,销售,采购,会计,制造,仓库管理,项目管理,以及众多社区模块. 2:它是基于一个模块化,可扩展 ...
min-max 容斥
$\min - \max$ 容斥 Part 1 对于简单的$\min - \max$容斥有一般形式,表达为:$\max(S)=\sum\limits_{T\subseteq S}(-1)^{|T|-1 ...
UOJ224 NOI2016 旷野大计算构造、造计算机
传送门——UOJ 传送门——Luogu 这段时间请不要找Itst聊天,Itst已经做疯了事实证明大模拟题不可做 query 1 送分,加起来一起乘即可 I I + < - O query 2 ...
BootStrap学习(3)_导航菜单
一.导航元素 1.表格导航或标签以一个带有 class .nav 的无序列表开始. 添加 class .nav-tabs. <!DOCTYPE html> <html xmlns= ...
【CV】ICCV2015_Describing Videos by Exploiting Temporal Structure
Describing Videos by Exploiting Temporal Structure Note here: it's a learning note on the topic of v ...
PHP使用echo输出标签设置CSS样式问题
使用php是可以输出HTML标签的,这为页面设计带来很大方便. 在此记录php输出标签设置CSS样式的问题: echo可使用''.""或你不用引号,如果想要输出带CSS样式的HTM ...

Layui_Tree模块遍历HDFS

Layui_Tree模块遍历HDFS的更多相关文章

随机推荐

热门专题