Layui_Tree模块遍历HDFS

注：转载请署名

一、实体

package com.ebd.application.common.Base;

import java.util.List;

public class HDFSDir {

	private String id;      //自定id

	private String pid;		//父ID

	private String name;	//当前目录名称

	private String alias;	//目录别名,可不用

	private String dir;		//自"/"目录后的完整目录

	private boolean spread;  //是否展开(true,false)

	private List<HDFSDir> children; //子目录

	public String getId() {

		return id;

	}

	public void setId(String id) {

		this.id = id;

	}

	public String getPid() {

		return pid;

	}

	public void setPid(String pid) {

		this.pid = pid;

	}

	public String getName() {

		return name;

	}

	public void setName(String name) {

		this.name = name;

	}

	public String getAlias() {

		return alias;

	}

	public void setAlias(String alias) {

		this.alias = alias;

	}

	public String getDir() {

		return dir;

	}

	public void setDir(String dir) {

		this.dir = dir;

	}

	public boolean isSpread() {

		return spread;

	}

	public void setSpread(boolean spread) {

		this.spread = spread;

	}

	public List<HDFSDir> getChildren() {

		return children;

	}

	public void setChildren(List<HDFSDir> children) {

		this.children = children;

	}

}

二、工具类

package hdfstest;

import java.io.IOException;

import java.net.URI;

import java.util.ArrayList;

import java.util.List;

import org.apache.commons.lang3.StringUtils;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.FSDataInputStream;

import org.apache.hadoop.fs.FileStatus;

import org.apache.hadoop.fs.FileSystem;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.mapred.JobConf;

import com.ebd.application.common.Base.HDFSDir;

import com.ebd.application.common.utils.Identities;

import net.sf.json.JSONObject;

public class HdfsListTest {

	//HDFS访问地址

	private static final String HDFS = "hdfs://bigdata.hadoop.com:9000";

	public HdfsListTest(Configuration conf) {

		this(HDFS, conf);

	}

	public HdfsListTest(String hdfs, Configuration conf) {

		this.hdfsPath = hdfs;

		this.conf = conf;

	}

	//hdfs路径

	private String hdfsPath;

	//Hadoop系统配置

	private Configuration conf;

	//启动函数

	public static void main(String[] args) throws IOException {

		JobConf conf = config();

//		System.out.println(conf.get("hadoop.http.staticuser.user"));

//		System.out.println(System.getenv("HADOOP_HOME"));

		HdfsListTest hdfs = new HdfsListTest(conf);

//		hdfs.mkdirs("/testput");

//		hdfs.copyFile("C:\\Users\\Administrator\\Desktop\\testput", "/testput/testput2");

//		hdfs.catFile("/testput/testput");

//		hdfs.download("/testput/testput", "E:\\");

//		hdfs.ls("hdfs://bigdata.hadoop.com:9000/user");

//		hdfs.rmr("/testput");

//		List<String> fileList = hdfs.getTree("/","/","|-");

		List<HDFSDir> kk = new ArrayList<HDFSDir>();

		HDFSDir ds1 = new HDFSDir();

		HDFSDir ds2 = new HDFSDir();

		HDFSDir ds3 = new HDFSDir();

		ds1.setId(Identities.uuid());

		ds1.setDir("/testput");

		ds2.setId(Identities.uuid());

		ds2.setDir("/user");

		ds3.setId(Identities.uuid());

		ds3.setDir("/tmp");

//		kk.add(ds1);

//		kk.add(ds2);

//		kk.add(ds3);

		HDFSDir ds = new HDFSDir();

		ds.setId(Identities.uuid());

		ds.setDir("/");

		kk.add(ds);

//		List<HDFSDir> fileList = hdfs.getListTree("/","/user",0);

		HDFSDir hdfss = hdfs.getChildNode(ds);

		JSONObject object = JSONObject.fromObject(hdfss);

		System.out.println(dirJsonFunc(object.toString()));

	}      

	//加载Hadoop配置文件

	public static JobConf config(){

		JobConf conf = new JobConf(HdfsListTest.class);

		conf.setJobName("HdfsDAO");

		conf.addResource("hadoop/core-site.xml");

		conf.addResource("hadoop/hdfs-site.xml");

		conf.addResource("hadoop/mapred-site.xml");

		return conf;

	}

	//在根目录下创建文件夹

	public void mkdirs(String folder) throws IOException {

		Path path = new Path(folder);

		FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);

		if (!fs.exists(path)) {

			fs.mkdirs(path);

			System.out.println("Create: " + folder);

		}

		fs.close();

	}

	//某个文件夹的文件列表

	public FileStatus[] ls(String folder) throws IOException {

		Path path = new Path(folder);

		FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);

		FileStatus[] list = fs.listStatus(path);

		System.out.println("ls: " + folder);

		System.out.println("==========================================================");

		if(list != null)

			for (FileStatus f : list) {

				System.out.printf("name: %s, folder: %s, size: %d\n", f.getPath(), f.isDir(), f.getLen());

//				System.out.printf("%s, folder: %s, 大小: %dK\n", f.getPath().getName(), (f.isDir()?"目录":"文件"), f.getLen()/1024);

			}

		System.out.println("==========================================================");

		fs.close();

		return  list;

	}

	public void copyFile(String local, String remote) throws IOException {

		FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);

		//remote---/用户/用户下的文件或文件夹

		fs.copyFromLocalFile(new Path(local), new Path(remote));

		System.out.println("copy from: " + local + " to " + remote);

		fs.close();

	}

	public void catFile(String remote) throws IOException {

		FSDataInputStream instream = null;

		FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);

		Path path = new Path(remote);

		if(fs.isFile(path)){

			fs.open(path);

			instream = fs.open(path);

	        byte[] b = new byte[1024];

	        instream.read(b);

	        System.out.println(new String(b,"utf-8"));

			fs.close();

		}

	}

	List <String> treeList = new ArrayList<String>();

	public List<String> getTree(String top, String remote, String prefix) throws IOException {

		Path path = new Path(remote);

		FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);

		FileStatus[] list = fs.listStatus(path);

		if(list != null)

			for (FileStatus f : list) {

//				System.out.printf("name: %s, folder: %s, size: %d\n", f.getPath(), f.isDir(), f.getLen());

				System.out.println(prefix+ f.getPath().getName());

				top += f.getPath().getName();

				treeList.add(top);

				if(fs.isDirectory(f.getPath())){

					getTree(top,f.getPath().toString(),prefix+"-");

				}

			}

		return treeList;

	}

	int id = 0;

	static int pid = 0;

	List<HDFSDir> dirList = new ArrayList<HDFSDir>();

	HDFSDir hdfsDir = null;

	private List<HDFSDir> getListTree(String top, String remote, int pid) throws IOException {

		Path path = new Path(remote);

		FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);

		FileStatus[] list = fs.listStatus(path);

		if(list != null)

			for (FileStatus f : list) {

				if(f.isDirectory()){

					hdfsDir = new HDFSDir();

//					hdfsDir.setId(id++);

//					hdfsDir.setPid(pid);

					hdfsDir.setName(f.getPath().getName());

					hdfsDir.setAlias(f.getPath().getName());

					hdfsDir.setDir(f.getPath().toString().substring(HDFS.length()));

					hdfsDir.setSpread(false);

					System.out.println(f.getPath().getName()+"="+f.getPath().toString().substring(HDFS.length()));

					dirList.add(hdfsDir);

				}

//				System.out.printf("name: %s, folder: %s, size: %d\n", f.getPath(), f.isDir(), f.getLen());

//				System.out.println(prefix+ f.getPath().getName());

//				top += f.getPath().getName();

//				if(fs.isDirectory(f.getPath())){

//					getListTree(top,f.getPath().toString(),pid++);

//				}

			}

		return dirList;

	}

	List<HDFSDir> cDirList = null;

	public HDFSDir getChildNode(HDFSDir pDir) throws IOException{

		Path path = null;

		if(pDir.getChildren() != null && pDir.getChildren().size() >= 1){

			for(HDFSDir p : pDir.getChildren()){

				path = new Path(p.getDir());

				FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);

				FileStatus[] list = fs.listStatus(path);

				if(list != null){

					cDirList = new ArrayList<HDFSDir>();

					for (FileStatus f : list) {

						if(f.isDirectory()){

							hdfsDir = new HDFSDir();

							hdfsDir.setId(Identities.uuid());

							hdfsDir.setPid(p.getId());

							hdfsDir.setName(f.getPath().getName());

							hdfsDir.setAlias(f.getPath().getName());

							hdfsDir.setDir(f.getPath().toString().substring(HDFS.length()));

							hdfsDir.setSpread(false);

							cDirList.add(hdfsDir);

						}

					}

					p.setChildren(cDirList);

					for(HDFSDir pp : cDirList){

						getChildNode(pp);

					}

				}

			}

		}else{

			path = new Path(pDir.getDir());

			FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);

			FileStatus[] list = fs.listStatus(path);

			if(list != null){

				cDirList = new ArrayList<HDFSDir>();

				for (FileStatus f : list) {

					if(f.isDirectory()){

						hdfsDir = new HDFSDir();

						hdfsDir.setId(Identities.uuid());

						hdfsDir.setPid(pDir.getId());

						hdfsDir.setName(f.getPath().getName().equals("")?"/":f.getPath().getName());

						hdfsDir.setAlias(f.getPath().getName().equals("")?"/":f.getPath().getName());

						hdfsDir.setDir(f.getPath().toString().substring(HDFS.length()));

						hdfsDir.setSpread(false);

						cDirList.add(hdfsDir);

					}

				}

				pDir.setChildren(cDirList);

				for(HDFSDir pp : cDirList){

					getChildNode(pp);

				}

			}

		}

		return pDir;

	}

	public static String dirJsonFunc(String jsonStr) {

		 if (StringUtils.isNotBlank(jsonStr)) {

			 String[] reg_array = {"([\"])","(,['])","([']:)"};

			 String[] rpa_array = {"'",",",":"};

			 for(int i=0;i<reg_array.length;i++){

				 jsonStr = jsonStr.replaceAll(reg_array[i], rpa_array[i]);

			 }

			 jsonStr = jsonStr.replace("{'", "{");

			 jsonStr = jsonStr.replace("'}", "}");

		 }

		 return jsonStr;

	}

	//删除文件或文件夹

	public void rmr(String folder) throws IOException {

		Path path = new Path(folder);

		FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);

		fs.deleteOnExit(path);

		System.out.println("Delete: " + folder);

		fs.close();

	}

	//下载文件到本地系统

	public void download(String remote, String local) throws IOException {

		Path path = new Path(remote);

		FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);

		fs.copyToLocalFile(path, new Path(local));

		System.out.println("download: from" + remote + " to " + local);

		fs.close();

	}

}

转换工具类

package test;

import org.apache.commons.lang3.StringUtils;

import com.ebd.application.common.Base.HDFSDir;

import com.ebd.application.common.utils.Identities;

import net.sf.json.JSONArray;

public class TestObjectToJson {

	public static void main(String[] args) {

		HDFSDir ds = new HDFSDir();

		ds.setId(Identities.uuid());

		ds.setDir("/testput");

		JSONArray js = JSONArray.fromObject(ds);

//		System.out.println(js.toString());

		String jsonStr = js.toString();

//		String reg_1 = "([\"])"; //双引号转单引号

//		String reg_2 = "(,['])"; //去掉逗号后面的单引号

//		String reg_3 = "([']:)"; //去掉冒号前面的单引号

//		String reg_4 = "('{'['])"; //去掉开头大括号后面的单引号

//		Pattern pattern = Pattern.compile(regEx);

//		jsonStr =  jsonStr.replaceAll(reg_1, "'");

//		jsonStr =  jsonStr.replaceAll(reg_2, ",");

//		jsonStr =  jsonStr.replaceAll(reg_3, ":");

//		jsonStr =  jsonStr.replaceAll("{'", "{");

//        System.out.println(jsonStr);

	}

	public static String dirJsonFunc(String jsonStr) {

		 if (StringUtils.isNotBlank(jsonStr)) {

			 String[] reg_array = {"([\"])","(,['])","([']:)"};

			 String[] rpa_array = {"'",",",":"};

			 for(int i=0;i<reg_array.length;i++){

				 jsonStr = jsonStr.replaceAll(reg_array[i], rpa_array[i]);

			 }

			 jsonStr = jsonStr.replace("{'", "{");

			 jsonStr = jsonStr.replace("'}", "}");

		 }

		 return jsonStr;

	}

}

工具类

package hdfstest;

import java.io.File;

import java.io.FileOutputStream;

import java.io.IOException;

import java.io.OutputStream;

import java.net.URI;

import java.util.ArrayList;

import java.util.List;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.FSDataInputStream;

import org.apache.hadoop.fs.FileStatus;

import org.apache.hadoop.fs.FileSystem;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.IOUtils;

import org.apache.hadoop.mapred.JobConf;

import com.ebd.application.common.utils.CreateFileUtil;

public class HdfsDirConsoleTest {

	//HDFS访问地址

	private static final String HDFS = "hdfs://bigdata.hadoop.com:9000";

	public HdfsDirConsoleTest(Configuration conf) {

		this(HDFS, conf);

	}

	public HdfsDirConsoleTest(String hdfs, Configuration conf) {

		this.hdfsPath = hdfs;

		this.conf = conf;

	}

	//hdfs路径

	private String hdfsPath;

	//Hadoop系统配置

	private Configuration conf;

	//启动函数

	public static void main(String[] args) throws IOException {

		JobConf conf = config();

		System.out.println(conf.get("hadoop.http.staticuser.user"));

		System.out.println(System.getenv("HADOOP_HOME"));

		HdfsDirConsoleTest hdfs = new HdfsDirConsoleTest(conf);

//		hdfs.mkdirs("/testput");

//		hdfs.copyFile("C:\\Users\\Administrator\\Desktop\\testput", "/testput/testput2");

//		hdfs.catFile("/testput/testput");

		hdfs.download("/testput/testput", "D:/ss/ss",conf);

//		hdfs.ls("hdfs://bigdata.hadoop.com:9000/");

//		hdfs.rmr("/testput");

//		List<String> fileList = hdfs.getTree("/","/","|-");

//		for(int i=0;i<fileList.size();i++){

//			System.out.println(fileList.get(i));

//		}

		System.out.println("success!");

	}      

	//加载Hadoop配置文件

	public static JobConf config(){

		JobConf conf = new JobConf(HdfsDirConsoleTest.class);

		conf.setJobName("HdfsDAO");

		conf.addResource("hadoop/core-site.xml");

		conf.addResource("hadoop/hdfs-site.xml");

		conf.addResource("hadoop/mapred-site.xml");

		return conf;

	}

	//在根目录下创建文件夹

	public void mkdirs(String folder) throws IOException {

		Path path = new Path(folder);

		FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);

		if (!fs.exists(path)) {

			fs.mkdirs(path);

			System.out.println("Create: " + folder);

		}

		fs.close();

	}

	//某个文件夹的文件列表

	public FileStatus[] ls(String folder) throws IOException {

		Path path = new Path(folder);

		FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);

		FileStatus[] list = fs.listStatus(path);

		System.out.println("ls: " + folder);

		System.out.println("==========================================================");

		if(list != null)

			for (FileStatus f : list) {

				System.out.printf("name: %s, folder: %s, size: %d\n", f.getPath(), f.isDir(), f.getLen());

				System.out.println(f.getOwner()+"=="+f.getBlockSize()+"="+f.getModificationTime()+"--"+f.getPermission()+"="+f.getReplication());

//				System.out.printf("%s, folder: %s, 大小: %dK\n", f.getPath().getName(), (f.isDir()?"目录":"文件"), f.getLen()/1024);

			}

		System.out.println("==========================================================");

		fs.close();

		return  list;

	}

	public void copyFile(String local, String remote) throws IOException {

		FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);

		//remote---/用户/用户下的文件或文件夹

		fs.copyFromLocalFile(new Path(local), new Path(remote));

		System.out.println("copy from: " + local + " to " + remote);

		fs.close();

	}

	public void catFile(String remote) throws IOException {

		FSDataInputStream instream = null;

		FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);

		Path path = new Path(remote);

		if(fs.isFile(path)){

			fs.open(path);

			instream = fs.open(path);

	        byte[] b = new byte[1024];

	        instream.read(b);

	        System.out.println(new String(b,"utf-8"));

			fs.close();

		}

	}

	List <String> treeList = new ArrayList<String>();

	public List<String> getTree(String top, String remote, String prefix) throws IOException {

		Path path = new Path(remote);

		FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);

		FileStatus[] list = fs.listStatus(path);

		if(list != null)

			for (FileStatus f : list) {

//				System.out.printf("name: %s, folder: %s, size: %d\n", f.getPath(), f.isDir(), f.getLen());

				System.out.println(prefix+ f.getPath().getName());

				top += f.getPath().getName();

				treeList.add(top);

				if(fs.isDirectory(f.getPath())){

					getTree(top,f.getPath().toString(),prefix+"-");

				}

			}

		return treeList;

	}

	//删除文件或文件夹

	public void rmr(String folder) throws IOException {

		Path path = new Path(folder);

		FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);

		fs.deleteOnExit(path);

		System.out.println("Delete: " + folder);

		fs.close();

	}

	//下载文件到本地系统

	public void download(String remote, String local,JobConf conf) throws IOException {

//		Path path = new Path(remote);

//		FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);

//		fs.copyToLocalFile(path, new Path(local));

//		System.out.println("download: from" + remote + " to " + local);

//		fs.close();

		 FileSystem fs = FileSystem.get(URI.create(remote),conf);

		 FSDataInputStream fsdi = fs.open(new Path(remote));

		 if(CreateFileUtil.createDir(local)){

			 OutputStream output = new FileOutputStream(local+remote.substring(remote.lastIndexOf("/")));

			 IOUtils.copyBytes(fsdi,output,4096,true);

		 }

	}

	public static void makdir(String path) {

		String strPath = "E:/a/aa/";

		File file = new File(strPath);

		File fileParent = file.getParentFile();

		if(!fileParent.exists()){

		 fileParent.mkdirs();

		}

	}

}

Layui_Tree模块遍历HDFS的更多相关文章

【hadoop】python通过hdfs模块读hdfs数据
hdfs官网:http://hdfscli.readthedocs.io/en/latest/api.html 一个非常好的博客:http://blog.csdn.net/gamer_gyt/arti ...
Windows编程之模块遍历(C++实现)
Windows编程之模块遍历 PS: 主要扣代码使用,直接滑动到最下面使用. 遍历模块需要几个API,和一个结构体 1.创建进程快照 2.遍历首次模块 3.继续下次遍历 4.模块信息结构体 API 分 ...
[Windows编程]模块遍历
模块遍历整体思路 1.创建进程快照 2.遍历首次模块 3.继续下次遍历 4.模块信息结构体相关API的调用创建进程快照API HANDLE WINAPI CreateToolhelp32Snap ...
Python 使用 os 模块遍历目录/获取当前文件的路径
1.列出指定目录下所包含的目录 item = os.listdir("/Users/jinchengxie/go") 返回的是一个列表, 里面包含了指定目录下所包含的所有的目录 2 ...
python os模块遍历目录
#os #os ->tree命令 import os #递归 #目录 ->文件,文件夹 -> 文件文件夹 dirpath = input('请输入你要遍历的目录\n') def ge ...
[java开发篇][dom4j模块] 遍历xml文件
http://blog.csdn.net/chenleixing/article/details/44353491 在android studio 导入dom4j库(build-gradle(Moud ...
[java开发篇][dom4j模块]遍历，解析xml
package com.softwinner.performance.benchmark; /** * Created by Administrator on 2017/7/21. */ import ...
[java开发篇][dom模块] 遍历解析xml
http://blog.csdn.net/andie_guo/article/details/24844351 XML DOM节点树 XML DOM将XML文档作为树结构,树结构称为一个节点树.所有的 ...
hadoop(一HDFS)
hadoop(一HDFS) 介绍狭义上来说: hadoop指的是以下的三大系统: HDFS :分布式文件系统(高吞吐,没有延时要求,容错性,扩展能力) MapReduce : 分布式计算系统 Yar ...

随机推荐

Android学习之基础知识十五 — 最佳UI体验（Material Design实战）
一.前言长久以来,大多数人都认为Android系统的UI并不美观,至少没有iOS系统的美观.以至于很多IT公司在进行应用界面设计的时候,为了保证双平台的统一性,强制要求Android端的界面风格必须 ...
ELF格式文件分析以及运用
基于本文的一个实践<使用Python分析ELF文件优化Flash和Sram空间的案例>. 1.背景 ELF是Executable and Linkable Format缩写,其官方规范在& ...
Topographic ICA as a Model of Natural Image Statistics(作为自然图像统计模型的拓扑独立成分分析)
其实topographic independent component analysis 早在1999年由ICA的发明人等人就提出了,所以不算是个新技术,ICA是在1982年首先在一个神经生理学的背景 ...
使用yield返回IEnumber<T>集合
yield是对一种复杂行为的简化,就是将一段代码简化为一种简单的形式. 先看一下常规的写法,下面例子中,把找出字符串阵列中,某些元素包含有某些字符的元素. class Bi { public stri ...
odoo 订餐系统之消息提醒
打算入手odoo开发新的系统,先研究下开发的过程是如何的.案例模仿自带的订餐系统,此系统模块不多,但很典型,可以达到联系的目的.先记录下订餐系统消息提醒的开发过程. 1.添加自己的addons目录my ...
JDK8漫谈——增强接口
解决什么问题向下兼容.添加方法,所有的实现类必须实现此方法,否则会编译报错.这意味着每一次的接口升级都会伤筋动骨.但是这是一把双刃剑一定要把握好场景,不要滥用. 类爆炸.使用时,需要辅助类.即要记忆 ...
Sql_索引分析
「索引就像书的目录, 通过书的目录就准确的定位到了书籍具体的内容」,这句话描述的非常正确, 但就像脱了裤子放屁,说了跟没说一样,通过目录查找书的内容自然是要比一页一页的翻书找来的快,同样使用的索引的人 ...
LVM : 缩减文件系统的容量
有扩展就有缩减,我们在前文<LVM : 扩展文件系统的容量>中介绍了通过 LVM 扩展文件系统的方法,本文我们接着前文的 demo 介绍通过 LVM 缩减文件系统的方法.说明:本文的演示环 ...
MariaDB 安装与启动过程记录
1. 安装之前的准备工作 rpm -qa |grep mysql rpm -qa |grep mariadb 按照查出来的软件包使用 yum remove 全部卸载,当然也可以 yum remov ...
gitblit 配置图文详解
Windows平台下Git服务器搭建前提是确保存在JDK环境. 第一步:下载Gitblit.下载地址:http://www.gitblit.com/ 第二步:解压缩下载的压缩包即可,无需安装. 第三 ...

Layui_Tree模块遍历HDFS

Layui_Tree模块遍历HDFS的更多相关文章

随机推荐

热门专题