thrift实现HDFS文件操作
thrift 文件如下
namespace java com.pera.file.transform
struct File{
1:string path ,
2:string content,
}
service FileTransform {
bool exists(1:string path),
void mkdir(1:string path ),
void store(1:File file),
set<string> ls(1:string path),
bool isDirectory(1:string path),
string getParentFile(1:string path),
void createNewFile(1:string path),
void deleteFile(1:string path),
void deleteOnExit(1:string path),
bool isFile(1:string path),
void rename(1:string srcPath ,2:string destPath),
string getName(1:string path),
i64 totalSpace(1:string path),
i64 usedSpace(1:string path),
string read(1:string path)
}
客户端：
public class FileTransFormPair {
private TTransport transport ;
private Client client ;
public FileTransFormPair(){}
public FileTransFormPair( TTransport transport ,Client client ){
this.transport=transport ;
this.client =client;
}
public TTransport getTransport() {
return transport;
}
public void setTransport(TTransport transport) {
this.transport = transport;
}
public Client getClient() {
return client;
}
public void setClient(Client client) {
this.client = client;
}
}
public class FileTransFormClient implements Iface {
public static String SERVER_IP = "localhost";
public static int SERVER_PORT = 8090;
public static int TIMEOUT = 30000;
public FileTransFormClient(String host,int port ,int timeout){
SERVER_IP=host;
SERVER_PORT=port;
TIMEOUT=timeout;
}
private static FileTransFormPair getClient() throws TTransportException{
TTransport transport = new TSocket(SERVER_IP, SERVER_PORT, TIMEOUT);
TProtocol protocol = new TBinaryProtocol(transport);
Client client =new Client(protocol);
transport.open();
return new FileTransFormPair(transport, client);
}
@Override
public boolean exists(String path) throws TException {
boolean result =false;
FileTransFormPair parameter =getClient();
result =parameter.getClient().exists(path);
parameter.getTransport().close();
return result;
}
@Override
public void mkdir(String path) throws TException {
FileTransFormPair parameter =getClient();
parameter.getClient().mkdir(path);
parameter.getTransport().close();
}
@Override
public void store(File file) throws TException {
FileTransFormPair parameter =getClient();
parameter.getClient().store(file);
parameter.getTransport().close();
}
@Override
public Set<String> ls(String path) throws TException {
FileTransFormPair parameter =getClient();
Set<String> result =parameter.getClient().ls(path);
parameter.getTransport().close();
return result;
}
@Override
public boolean isDirectory(String path) throws TException {
boolean result =false ;
FileTransFormPair parameter =getClient();
result =parameter.getClient().isDirectory(path);
parameter.getTransport().close();
return result;
}
@Override
public String getParentFile(String path) throws TException {
FileTransFormPair parameter =getClient();
String result =parameter.getClient().getParentFile(path);
parameter.getTransport().close();
return result;
}
@Override
public void createNewFile(String path) throws TException {
FileTransFormPair parameter =getClient();
parameter.getClient().createNewFile(path);
parameter.getTransport().close();
}
@Override
public void deleteFile(String path) throws TException {
FileTransFormPair parameter =getClient();
parameter.getClient().deleteFile(path);
parameter.getTransport().close();
}
@Override
public void deleteOnExit(String path) throws TException {
FileTransFormPair parameter =getClient();
parameter.getClient().deleteOnExit(path);
parameter.getTransport().close();
}
@Override
public boolean isFile(String path) throws TException {
FileTransFormPair parameter =getClient();
boolean result =parameter.getClient().isFile(path);
parameter.getTransport().close();
return result ;
}
@Override
public void rename(String srcPath, String destPath) throws TException {
FileTransFormPair parameter =getClient();
parameter.getClient().rename(srcPath, destPath);
parameter.getTransport().close();
}
@Override
public String getName(String path) throws TException {
FileTransFormPair parameter =getClient();
String result =parameter.getClient().getName(path);
parameter.getTransport().close();
return result;
}
@Override
public long totalSpace(String path) throws TException {
FileTransFormPair parameter =getClient();
long result =parameter.getClient().totalSpace(path);
parameter.getTransport().close();
return result;
}
@Override
public long usedSpace(String path) throws TException {
FileTransFormPair parameter =getClient();
long result =parameter.getClient().usedSpace(path);
parameter.getTransport().close();
return result;
}
@Override
public String read(String path) throws TException {
FileTransFormPair parameter =getClient();
String result =parameter.getClient().read(path);
parameter.getTransport().close();
return result;
}
}
服务端：
public class ThriftServer {
public static int SERVER_PORT ;
private static TServer server =null;
private static Log LOG =LogFactory.getLog(ThriftServer.class);
public void init(){
SERVER_PORT = 9090;
LOG.info("init the thriftServer");
}
public void start() throws TTransportException {
TProcessor tprocessor = new FileTransform.Processor<FileTransform.Iface>(
new FileStoreService());
TServerSocket serverTransport = new TServerSocket(SERVER_PORT);
TThreadPoolServer.Args ttpsArgs = new TThreadPoolServer.Args(
serverTransport);
ttpsArgs.processor(tprocessor);
ttpsArgs.protocolFactory(new TBinaryProtocol.Factory());
server = new TThreadPoolServer(ttpsArgs);
server.serve();
LOG.info("start the thriftServer");
}
public void stop() {
if(null!=server){
server.stop();
LOG.info("stop the thriftServer");
}
}
public static void main(String[] args) {
ThriftServer server =new ThriftServer();
server.init();
try {
server.start();
} catch (TTransportException e) {
e.printStackTrace();
}
}
}
public class FileStoreService implements Iface{
private Configuration getConfiguration(){
Configuration conf =new Configuration();
Properties pro = new Properties();
try {
pro.load(new FileInputStream(new File("conf/filetransform-site.properties")));
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
// conf.set("dfs.default.name", pro.getProperty("dfs.default.name"));
return conf;
}
@Override
public boolean exists(String path) throws TException {
Configuration conf =getConfiguration();
FileSystem fs=null;
try {
fs = FileSystem.get(conf);
} catch (IOException e) {
throw new TException(e.getMessage());
}
boolean exists =false ;
try {
exists = fs.exists(new Path(path));
} catch (IOException e) {
throw new TException(e.getMessage());
}finally{
if(null!=fs){
try {
fs.close();
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
return exists;
}
@Override
public void mkdir(String path) throws TException {
Configuration conf =getConfiguration();
FileSystem fs=null;
try {
fs = FileSystem.get(conf);
fs.mkdirs(new Path(path));
} catch (IOException e) {
throw new TException(e.getMessage());
}finally{
if(null!=fs){
try {
fs.close();
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
}
@Override
public void store(com.pera.file.transform.gen.File file) throws TException {
String path =file.getPath();
String content =file.getContent();
Configuration conf =getConfiguration();
FileSystem fs =null;
FSDataOutputStream out=null;
try {
Path p =new Path(path);
fs =FileSystem.get(conf);
if(!fs.exists(p.getParent())){
throw new TException( path+" is not exists");
}
out =fs.create(p);
if(null!=content){
byte [] tmp =content.getBytes();
out.write(tmp);
out.flush();
out.close();
}
} catch (Exception e) {
throw new TException(e.getMessage());
}finally{
try {
if(null!=out){
out.close();
}
} catch (IOException e) {
throw new TException(e.getMessage());
}
try {
if(null!=fs){
fs.close();
}
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
@Override
public Set<String> ls(String path) throws TException {
Configuration conf =getConfiguration();
FileSystem fs =null;
Set<String> result =new HashSet<String>();
try {
fs=FileSystem.get(conf);
FileStatus [] fstatus=fs.listStatus(new Path(path));
int length =fstatus==null?0:fstatus.length;
for (int i = 0; i < length; i++) {
String url =fstatus[i].getPath().toString();
result.add(url);
}
} catch (IOException e) {
throw new TException(e.getMessage());
}finally{
if(null!=fs){
try {
fs.close();
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
return result;
}
@Override
@SuppressWarnings("deprecation")
public boolean isDirectory(String path) throws TException {
Configuration conf =getConfiguration();
FileSystem fs =null;
boolean result =false ;
try {
fs =FileSystem.get(conf);
result =fs.isDirectory(new Path(path));
} catch (IOException e) {
throw new TException(e.getMessage());
}finally{
if(null!=fs){
try {
fs.close();
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
return result;
}
@Override
public String getParentFile(String path) throws TException {
Path parentPath =new Path(path).getParent();
return parentPath.toString();
}
@Override
public void createNewFile(String path) throws TException {
Configuration conf =getConfiguration();
FileSystem fs =null;
try {
fs =FileSystem.get(conf);
fs.createNewFile(new Path(path));
} catch (IOException e) {
throw new TException(e.getMessage());
}finally{
if(null!=fs){
try {
fs.close();
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
}
@Override
public void deleteFile(String path) throws TException {
Configuration conf =getConfiguration();
FileSystem fs =null;
try {
fs =FileSystem.get(conf);
fs.deleteOnExit(new Path(path));
} catch (IOException e) {
throw new TException(e.getMessage());
}finally{
if(null!=fs){
try {
fs.close();
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
}
@Override
public void deleteOnExit(String path) throws TException {
Configuration conf =getConfiguration();
FileSystem fs =null;
try {
fs =FileSystem.get(conf);
fs.deleteOnExit(new Path(path));
} catch (IOException e) {
throw new TException(e.getMessage());
}finally{
if(null!=fs){
try {
fs.close();
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
}
@Override
public boolean isFile(String path) throws TException {
boolean isFile =false ;
Configuration conf =getConfiguration();
FileSystem fs =null;
try {
fs =FileSystem.get(conf);
isFile =fs.isFile(new Path(path));
} catch (IOException e) {
throw new TException(e.getMessage());
}finally{
if(null!=fs){
try {
fs.close();
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
return isFile;
}
@Override
public void rename(String srcPath, String destPath) throws TException {
Configuration conf =getConfiguration();
FileSystem fs =null;
try {
fs =FileSystem.get(conf);
fs.rename(new Path(srcPath), new Path(destPath));
} catch (IOException e) {
throw new TException(e.getMessage());
}finally{
if(null!=fs){
try {
fs.close();
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
}
@Override
public String getName(String path) throws TException {
String name =new Path(path).getName();
return name;
}
@Override
public long totalSpace(String path) throws TException {
Configuration conf =getConfiguration();
FileSystem fs =null;
long length =0 ;
try {
fs =FileSystem.get(conf);
length =fs.getFileStatus(new Path(path)).getLen();
} catch (IOException e) {
throw new TException(e.getMessage());
}finally{
if(null!=fs){
try {
fs.close();
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
return length;
}
@Override
public long usedSpace(String path) throws TException {
Configuration conf =getConfiguration();
FileSystem fs =null;
long used =0 ;
try {
fs =FileSystem.get(conf);
used =fs.getUsed();
} catch (IOException e) {
throw new TException(e.getMessage());
}finally{
if(null!=fs){
try {
fs.close();
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
return used;
}
@Override
public String read(String path) throws TException {
Configuration conf =getConfiguration();
FileSystem fs =null;
String result ="" ;
try {
fs =FileSystem.get(conf);
FSDataInputStream in =fs.open(new Path(path));
int length =in.available();
byte [] tmp = new byte [length] ;
in.readFully(tmp);
result =new String(tmp);
} catch (IOException e) {
throw new TException(e.getMessage());
}finally{
if(null!=fs){
try {
fs.close();
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
return result;
}
}
thrift实现HDFS文件操作的更多相关文章
- Hadoop之HDFS文件操作常有两种方式（转载）
摘要:Hadoop之HDFS文件操作常有两种方式,命令行方式和JavaAPI方式.本文介绍如何利用这两种方式对HDFS文件进行操作. 关键词:HDFS文件 命令行 Java API HD ...
- hadoop的hdfs文件操作实现上传文件到hdfs
这篇文章主要介绍了使用hadoop的API对HDFS上的文件访问,其中包括上传文件到HDFS上.从HDFS上下载文件和删除HDFS上的文件,需要的朋友可以参考下hdfs文件操作操作示例,包括上传文件到 ...
- HDFS文件操作(命令行)
HDFS是一种分布式文件系统,为MapReduce这种框架下的海量数据分布式处理而设计. Hadoop之HDFS文件操作常有两种方式,一种是命令行方式,即Hadoop提供了一套与Linux文件命令类似 ...
- Hadoop之HDFS文件操作
摘要:Hadoop之HDFS文件操作常有两种方式.命令行方式和JavaAPI方式.本文介绍怎样利用这两种方式对HDFS文件进行操作. 关键词:HDFS文件 命令行 Java API HD ...
- JAVA API 实现hdfs文件操作
java api 实现hdfs 文件操作会出现错误提示: Permission denied: user=hp, access=WRITE, inode="/":hdfs:supe ...
- HDFS文件操作
hadoop装好后,文件系统中没有任何目录与文件 1. 创建文件夹 hadoop fs -mkdir -p /hkx/learn 参数-p表示递归创建文件夹 2. 浏览文件 hadoop fs -ls ...
- eclipse 对 hadoop1.2.1 hdfs 文件操作
package com.hdfs; import java.io.FileInputStream;import java.io.FileNotFoundException;import java.io ...
- Hadoop HDFS文件操作
1.创建目录 import java.io.IOException; import org.apache.hadoop.conf.Configuration; import org.apache.ha ...
- [bigdata] 使用Flume hdfs sink， hdfs文件未关闭的问题
现象: 执行mapreduce任务时失败 通过hadoop fsck -openforwrite命令查看发现有文件没有关闭. [root@com ~]# hadoop fsck -openforwri ...
随机推荐
- 使用QGIS将文本坐标转换为矢量文件
本文主要是说明如果使用QGIS将文本格式的点坐标转换为矢量文件(如shapefile格式). 所需工具:QGIS 所需数据:文本格式的点文件 所需要处理的点坐标文件如下所示, 114.2 22.15 ...
- 乐观的并发策略——基于CAS的自旋
悲观者与乐观者的做事方式完全不一样,悲观者的人生观是一件事情我必须要百分之百完全控制才会去做,否则就认为这件事情一定会出问题:而乐观者的人生观则相反,凡事不管最终结果如何,他都会先尝试去做,大不了最后 ...
- （一〇四）使用Xcode6创建framework动态静态库
在Xcode6以前,创建framework可以使用iOS-Universal-Framework模板来创建framework,现在苹果已经提供了模板,如下图选择: 使用此模版创建的默认是动态库,方法和 ...
- 北大青鸟Asp.net之颗粒归仓
自从小编走进编程的世界以来,学习的编程知识都是和C/S这个小伙伴握手,直到做完牛腩老师的新闻发布系统,才开始了小编的B/S学习生涯,和B/S初次谋面,小宇宙瞬间爆发了,看着自己的第一个B/S系统,牛腩 ...
- ORACLE ORA-01653: unable to extend table 的错误
ORACLE ORA-01653: unable to extend table 的错误 今天用PL SQL Developer往oracle数据库中导入数据时,突然报错,只能终止,错误的具体内容如下 ...
- 百度地图API的学习
我们可以进入百度API的网站学习百度地图API:http://dev.baidu.com/wiki/map/index.php,看完这些你应该基本上会掌握了,还有一些显示地图中一些很神奇的效果,需要一 ...
- 【Android 应用开发】 Application 使用分析
博客地址 : http://blog.csdn.net/shulianghan/article/details/40737419 代码下载 : Android 应用 Application 经典用法; ...
- Mybatis插件原理分析（二）
在上一篇中Mybatis插件原理分析(一)中我们主要介绍了一下Mybatis插件相关的几个类的源码,并对源码进行了一些解释,接下来我们通过一个简单的插件实现来对Mybatis插件的运行流程进行分析. ...
- 小试ImageMagik——开发篇
===================================================== ImageMagick的使用和开发的文章: 小试ImageMagik--使用篇 小试Imag ...
- python 操作mysql数据库demo
sudo apt-get install python-mysqldb #!/usr/bin/env python #encoding=utf-8 import sys import MySQLdb ...