thrift实现HDFS文件操作
thrift 文件如下
namespace java com.pera.file.transform
struct File{
1:string path ,
2:string content,
}
service FileTransform {
bool exists(1:string path),
void mkdir(1:string path ),
void store(1:File file),
set<string> ls(1:string path),
bool isDirectory(1:string path),
string getParentFile(1:string path),
void createNewFile(1:string path),
void deleteFile(1:string path),
void deleteOnExit(1:string path),
bool isFile(1:string path),
void rename(1:string srcPath ,2:string destPath),
string getName(1:string path),
i64 totalSpace(1:string path),
i64 usedSpace(1:string path),
string read(1:string path)
}
客户端：
public class FileTransFormPair {
private TTransport transport ;
private Client client ;
public FileTransFormPair(){}
public FileTransFormPair( TTransport transport ,Client client ){
this.transport=transport ;
this.client =client;
}
public TTransport getTransport() {
return transport;
}
public void setTransport(TTransport transport) {
this.transport = transport;
}
public Client getClient() {
return client;
}
public void setClient(Client client) {
this.client = client;
}
}
public class FileTransFormClient implements Iface {
public static String SERVER_IP = "localhost";
public static int SERVER_PORT = 8090;
public static int TIMEOUT = 30000;
public FileTransFormClient(String host,int port ,int timeout){
SERVER_IP=host;
SERVER_PORT=port;
TIMEOUT=timeout;
}
private static FileTransFormPair getClient() throws TTransportException{
TTransport transport = new TSocket(SERVER_IP, SERVER_PORT, TIMEOUT);
TProtocol protocol = new TBinaryProtocol(transport);
Client client =new Client(protocol);
transport.open();
return new FileTransFormPair(transport, client);
}
@Override
public boolean exists(String path) throws TException {
boolean result =false;
FileTransFormPair parameter =getClient();
result =parameter.getClient().exists(path);
parameter.getTransport().close();
return result;
}
@Override
public void mkdir(String path) throws TException {
FileTransFormPair parameter =getClient();
parameter.getClient().mkdir(path);
parameter.getTransport().close();
}
@Override
public void store(File file) throws TException {
FileTransFormPair parameter =getClient();
parameter.getClient().store(file);
parameter.getTransport().close();
}
@Override
public Set<String> ls(String path) throws TException {
FileTransFormPair parameter =getClient();
Set<String> result =parameter.getClient().ls(path);
parameter.getTransport().close();
return result;
}
@Override
public boolean isDirectory(String path) throws TException {
boolean result =false ;
FileTransFormPair parameter =getClient();
result =parameter.getClient().isDirectory(path);
parameter.getTransport().close();
return result;
}
@Override
public String getParentFile(String path) throws TException {
FileTransFormPair parameter =getClient();
String result =parameter.getClient().getParentFile(path);
parameter.getTransport().close();
return result;
}
@Override
public void createNewFile(String path) throws TException {
FileTransFormPair parameter =getClient();
parameter.getClient().createNewFile(path);
parameter.getTransport().close();
}
@Override
public void deleteFile(String path) throws TException {
FileTransFormPair parameter =getClient();
parameter.getClient().deleteFile(path);
parameter.getTransport().close();
}
@Override
public void deleteOnExit(String path) throws TException {
FileTransFormPair parameter =getClient();
parameter.getClient().deleteOnExit(path);
parameter.getTransport().close();
}
@Override
public boolean isFile(String path) throws TException {
FileTransFormPair parameter =getClient();
boolean result =parameter.getClient().isFile(path);
parameter.getTransport().close();
return result ;
}
@Override
public void rename(String srcPath, String destPath) throws TException {
FileTransFormPair parameter =getClient();
parameter.getClient().rename(srcPath, destPath);
parameter.getTransport().close();
}
@Override
public String getName(String path) throws TException {
FileTransFormPair parameter =getClient();
String result =parameter.getClient().getName(path);
parameter.getTransport().close();
return result;
}
@Override
public long totalSpace(String path) throws TException {
FileTransFormPair parameter =getClient();
long result =parameter.getClient().totalSpace(path);
parameter.getTransport().close();
return result;
}
@Override
public long usedSpace(String path) throws TException {
FileTransFormPair parameter =getClient();
long result =parameter.getClient().usedSpace(path);
parameter.getTransport().close();
return result;
}
@Override
public String read(String path) throws TException {
FileTransFormPair parameter =getClient();
String result =parameter.getClient().read(path);
parameter.getTransport().close();
return result;
}
}
服务端：
public class ThriftServer {
public static int SERVER_PORT ;
private static TServer server =null;
private static Log LOG =LogFactory.getLog(ThriftServer.class);
public void init(){
SERVER_PORT = 9090;
LOG.info("init the thriftServer");
}
public void start() throws TTransportException {
TProcessor tprocessor = new FileTransform.Processor<FileTransform.Iface>(
new FileStoreService());
TServerSocket serverTransport = new TServerSocket(SERVER_PORT);
TThreadPoolServer.Args ttpsArgs = new TThreadPoolServer.Args(
serverTransport);
ttpsArgs.processor(tprocessor);
ttpsArgs.protocolFactory(new TBinaryProtocol.Factory());
server = new TThreadPoolServer(ttpsArgs);
server.serve();
LOG.info("start the thriftServer");
}
public void stop() {
if(null!=server){
server.stop();
LOG.info("stop the thriftServer");
}
}
public static void main(String[] args) {
ThriftServer server =new ThriftServer();
server.init();
try {
server.start();
} catch (TTransportException e) {
e.printStackTrace();
}
}
}
public class FileStoreService implements Iface{
private Configuration getConfiguration(){
Configuration conf =new Configuration();
Properties pro = new Properties();
try {
pro.load(new FileInputStream(new File("conf/filetransform-site.properties")));
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
// conf.set("dfs.default.name", pro.getProperty("dfs.default.name"));
return conf;
}
@Override
public boolean exists(String path) throws TException {
Configuration conf =getConfiguration();
FileSystem fs=null;
try {
fs = FileSystem.get(conf);
} catch (IOException e) {
throw new TException(e.getMessage());
}
boolean exists =false ;
try {
exists = fs.exists(new Path(path));
} catch (IOException e) {
throw new TException(e.getMessage());
}finally{
if(null!=fs){
try {
fs.close();
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
return exists;
}
@Override
public void mkdir(String path) throws TException {
Configuration conf =getConfiguration();
FileSystem fs=null;
try {
fs = FileSystem.get(conf);
fs.mkdirs(new Path(path));
} catch (IOException e) {
throw new TException(e.getMessage());
}finally{
if(null!=fs){
try {
fs.close();
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
}
@Override
public void store(com.pera.file.transform.gen.File file) throws TException {
String path =file.getPath();
String content =file.getContent();
Configuration conf =getConfiguration();
FileSystem fs =null;
FSDataOutputStream out=null;
try {
Path p =new Path(path);
fs =FileSystem.get(conf);
if(!fs.exists(p.getParent())){
throw new TException( path+" is not exists");
}
out =fs.create(p);
if(null!=content){
byte [] tmp =content.getBytes();
out.write(tmp);
out.flush();
out.close();
}
} catch (Exception e) {
throw new TException(e.getMessage());
}finally{
try {
if(null!=out){
out.close();
}
} catch (IOException e) {
throw new TException(e.getMessage());
}
try {
if(null!=fs){
fs.close();
}
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
@Override
public Set<String> ls(String path) throws TException {
Configuration conf =getConfiguration();
FileSystem fs =null;
Set<String> result =new HashSet<String>();
try {
fs=FileSystem.get(conf);
FileStatus [] fstatus=fs.listStatus(new Path(path));
int length =fstatus==null?0:fstatus.length;
for (int i = 0; i < length; i++) {
String url =fstatus[i].getPath().toString();
result.add(url);
}
} catch (IOException e) {
throw new TException(e.getMessage());
}finally{
if(null!=fs){
try {
fs.close();
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
return result;
}
@Override
@SuppressWarnings("deprecation")
public boolean isDirectory(String path) throws TException {
Configuration conf =getConfiguration();
FileSystem fs =null;
boolean result =false ;
try {
fs =FileSystem.get(conf);
result =fs.isDirectory(new Path(path));
} catch (IOException e) {
throw new TException(e.getMessage());
}finally{
if(null!=fs){
try {
fs.close();
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
return result;
}
@Override
public String getParentFile(String path) throws TException {
Path parentPath =new Path(path).getParent();
return parentPath.toString();
}
@Override
public void createNewFile(String path) throws TException {
Configuration conf =getConfiguration();
FileSystem fs =null;
try {
fs =FileSystem.get(conf);
fs.createNewFile(new Path(path));
} catch (IOException e) {
throw new TException(e.getMessage());
}finally{
if(null!=fs){
try {
fs.close();
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
}
@Override
public void deleteFile(String path) throws TException {
Configuration conf =getConfiguration();
FileSystem fs =null;
try {
fs =FileSystem.get(conf);
fs.deleteOnExit(new Path(path));
} catch (IOException e) {
throw new TException(e.getMessage());
}finally{
if(null!=fs){
try {
fs.close();
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
}
@Override
public void deleteOnExit(String path) throws TException {
Configuration conf =getConfiguration();
FileSystem fs =null;
try {
fs =FileSystem.get(conf);
fs.deleteOnExit(new Path(path));
} catch (IOException e) {
throw new TException(e.getMessage());
}finally{
if(null!=fs){
try {
fs.close();
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
}
@Override
public boolean isFile(String path) throws TException {
boolean isFile =false ;
Configuration conf =getConfiguration();
FileSystem fs =null;
try {
fs =FileSystem.get(conf);
isFile =fs.isFile(new Path(path));
} catch (IOException e) {
throw new TException(e.getMessage());
}finally{
if(null!=fs){
try {
fs.close();
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
return isFile;
}
@Override
public void rename(String srcPath, String destPath) throws TException {
Configuration conf =getConfiguration();
FileSystem fs =null;
try {
fs =FileSystem.get(conf);
fs.rename(new Path(srcPath), new Path(destPath));
} catch (IOException e) {
throw new TException(e.getMessage());
}finally{
if(null!=fs){
try {
fs.close();
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
}
@Override
public String getName(String path) throws TException {
String name =new Path(path).getName();
return name;
}
@Override
public long totalSpace(String path) throws TException {
Configuration conf =getConfiguration();
FileSystem fs =null;
long length =0 ;
try {
fs =FileSystem.get(conf);
length =fs.getFileStatus(new Path(path)).getLen();
} catch (IOException e) {
throw new TException(e.getMessage());
}finally{
if(null!=fs){
try {
fs.close();
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
return length;
}
@Override
public long usedSpace(String path) throws TException {
Configuration conf =getConfiguration();
FileSystem fs =null;
long used =0 ;
try {
fs =FileSystem.get(conf);
used =fs.getUsed();
} catch (IOException e) {
throw new TException(e.getMessage());
}finally{
if(null!=fs){
try {
fs.close();
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
return used;
}
@Override
public String read(String path) throws TException {
Configuration conf =getConfiguration();
FileSystem fs =null;
String result ="" ;
try {
fs =FileSystem.get(conf);
FSDataInputStream in =fs.open(new Path(path));
int length =in.available();
byte [] tmp = new byte [length] ;
in.readFully(tmp);
result =new String(tmp);
} catch (IOException e) {
throw new TException(e.getMessage());
}finally{
if(null!=fs){
try {
fs.close();
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
return result;
}
}
thrift实现HDFS文件操作的更多相关文章
- Hadoop之HDFS文件操作常有两种方式（转载）
摘要:Hadoop之HDFS文件操作常有两种方式,命令行方式和JavaAPI方式.本文介绍如何利用这两种方式对HDFS文件进行操作. 关键词:HDFS文件 命令行 Java API HD ...
- hadoop的hdfs文件操作实现上传文件到hdfs
这篇文章主要介绍了使用hadoop的API对HDFS上的文件访问,其中包括上传文件到HDFS上.从HDFS上下载文件和删除HDFS上的文件,需要的朋友可以参考下hdfs文件操作操作示例,包括上传文件到 ...
- HDFS文件操作(命令行)
HDFS是一种分布式文件系统,为MapReduce这种框架下的海量数据分布式处理而设计. Hadoop之HDFS文件操作常有两种方式,一种是命令行方式,即Hadoop提供了一套与Linux文件命令类似 ...
- Hadoop之HDFS文件操作
摘要:Hadoop之HDFS文件操作常有两种方式.命令行方式和JavaAPI方式.本文介绍怎样利用这两种方式对HDFS文件进行操作. 关键词:HDFS文件 命令行 Java API HD ...
- JAVA API 实现hdfs文件操作
java api 实现hdfs 文件操作会出现错误提示: Permission denied: user=hp, access=WRITE, inode="/":hdfs:supe ...
- HDFS文件操作
hadoop装好后,文件系统中没有任何目录与文件 1. 创建文件夹 hadoop fs -mkdir -p /hkx/learn 参数-p表示递归创建文件夹 2. 浏览文件 hadoop fs -ls ...
- eclipse 对 hadoop1.2.1 hdfs 文件操作
package com.hdfs; import java.io.FileInputStream;import java.io.FileNotFoundException;import java.io ...
- Hadoop HDFS文件操作
1.创建目录 import java.io.IOException; import org.apache.hadoop.conf.Configuration; import org.apache.ha ...
- [bigdata] 使用Flume hdfs sink， hdfs文件未关闭的问题
现象: 执行mapreduce任务时失败 通过hadoop fsck -openforwrite命令查看发现有文件没有关闭. [root@com ~]# hadoop fsck -openforwri ...
随机推荐
- 学习Android路上的一些感慨和总结，慢慢来，比较快！
学习Android路上的一些感慨和总结,慢慢来,比较快! 一直想对自己的学习路程做一个总结,来告别某一个阶段的过去,迎接某一个阶段的来临,一直抽不出时间来,于是零零散散的写了点-,到现在,也已经积攒了 ...
- Effective C++ ——设计与声明
条款18:让接口更容易的被使用,不易误用 接口设计主要是给应用接口的人使用的,他们可能不是接口的设计者,这样作为接口的设计者就要对接口的定义更加易懂,让使用者不宜发生误用,例如对于一个时间类: cla ...
- Android系统开机启动流程及init进程浅析
Android系统启动概述 Android系统开机流程基于Linux系统,总体可分为三个阶段: Boot Loader引导程序启动Linux内核启动Android系统启动,Launcher/app启动 ...
- SpringMVC系列之（二） springMVC和Struts异同
Spring MVC属于SpringFrameWork的后续产品,已经融合在Spring Web Flow里面.Spring 框架提供了构建 Web 应用程序的全功能 MVC 模块.使用 Spring ...
- Java进阶(一)Java内存解析
栈.堆.常量池等虽同属Java内存分配时操作的区域,但其适用范围和功用却大不相同.本文将深入Java核心,简单讲解Java内存分配方面的知识. 首先我们先来讲解一下内存中的各个区域. stack(栈) ...
- MyBatis主键生成器Jdbc3KeyGenerator（二）
上一篇博客MyBatis主键生成器KeyGenerator(一)中我们大体介绍了主键生成器的接口及配置等,接下来我们介绍一下KeyGenerator的实现类Jdbc3KeyGenerator Jdbc ...
- 【一天一道LeetCode】#85. Maximal Rectangle
一天一道LeetCode 本系列文章已全部上传至我的github,地址:ZeeCoder's Github 欢迎大家关注我的新浪微博,我的新浪微博 欢迎转载,转载请注明出处 (一)题目 Given a ...
- UNIX环境高级编程——文件和目录
一.获取文件/目录的属性信息 int stat(const char *path, struct stat *buf); int fstat(int fd, struct stat *buf); in ...
- 小强的HTML5移动开发之路（16）——神奇的拖放功能
来自:http://blog.csdn.net/dawanganban/article/details/18181273 在智能手机发展飞速的现在拖放功能已经成为一种时尚,但是在我们的浏览器上是不是还 ...
- 强力推荐各位攻城狮查看，收藏IT职业技能图谱（全套13张）
汇集整理泛 IT 技术领域(云计算,大数据,运维,安全,开发语言,智能硬件等)学习技能图谱,帮助程序员梳理知识框架结构,并尝试提供路径指导和精华资源,方便技术人学习成长. 运维工程师必备技能 程序开发 ...