thrift实现HDFS文件操作
thrift 文件如下
namespace java com.pera.file.transform
struct File{
1:string path ,
2:string content,
}
service FileTransform {
bool exists(1:string path),
void mkdir(1:string path ),
void store(1:File file),
set<string> ls(1:string path),
bool isDirectory(1:string path),
string getParentFile(1:string path),
void createNewFile(1:string path),
void deleteFile(1:string path),
void deleteOnExit(1:string path),
bool isFile(1:string path),
void rename(1:string srcPath ,2:string destPath),
string getName(1:string path),
i64 totalSpace(1:string path),
i64 usedSpace(1:string path),
string read(1:string path)
}
客户端：
public class FileTransFormPair {
private TTransport transport ;
private Client client ;
public FileTransFormPair(){}
public FileTransFormPair( TTransport transport ,Client client ){
this.transport=transport ;
this.client =client;
}
public TTransport getTransport() {
return transport;
}
public void setTransport(TTransport transport) {
this.transport = transport;
}
public Client getClient() {
return client;
}
public void setClient(Client client) {
this.client = client;
}
}
public class FileTransFormClient implements Iface {
public static String SERVER_IP = "localhost";
public static int SERVER_PORT = 8090;
public static int TIMEOUT = 30000;
public FileTransFormClient(String host,int port ,int timeout){
SERVER_IP=host;
SERVER_PORT=port;
TIMEOUT=timeout;
}
private static FileTransFormPair getClient() throws TTransportException{
TTransport transport = new TSocket(SERVER_IP, SERVER_PORT, TIMEOUT);
TProtocol protocol = new TBinaryProtocol(transport);
Client client =new Client(protocol);
transport.open();
return new FileTransFormPair(transport, client);
}
@Override
public boolean exists(String path) throws TException {
boolean result =false;
FileTransFormPair parameter =getClient();
result =parameter.getClient().exists(path);
parameter.getTransport().close();
return result;
}
@Override
public void mkdir(String path) throws TException {
FileTransFormPair parameter =getClient();
parameter.getClient().mkdir(path);
parameter.getTransport().close();
}
@Override
public void store(File file) throws TException {
FileTransFormPair parameter =getClient();
parameter.getClient().store(file);
parameter.getTransport().close();
}
@Override
public Set<String> ls(String path) throws TException {
FileTransFormPair parameter =getClient();
Set<String> result =parameter.getClient().ls(path);
parameter.getTransport().close();
return result;
}
@Override
public boolean isDirectory(String path) throws TException {
boolean result =false ;
FileTransFormPair parameter =getClient();
result =parameter.getClient().isDirectory(path);
parameter.getTransport().close();
return result;
}
@Override
public String getParentFile(String path) throws TException {
FileTransFormPair parameter =getClient();
String result =parameter.getClient().getParentFile(path);
parameter.getTransport().close();
return result;
}
@Override
public void createNewFile(String path) throws TException {
FileTransFormPair parameter =getClient();
parameter.getClient().createNewFile(path);
parameter.getTransport().close();
}
@Override
public void deleteFile(String path) throws TException {
FileTransFormPair parameter =getClient();
parameter.getClient().deleteFile(path);
parameter.getTransport().close();
}
@Override
public void deleteOnExit(String path) throws TException {
FileTransFormPair parameter =getClient();
parameter.getClient().deleteOnExit(path);
parameter.getTransport().close();
}
@Override
public boolean isFile(String path) throws TException {
FileTransFormPair parameter =getClient();
boolean result =parameter.getClient().isFile(path);
parameter.getTransport().close();
return result ;
}
@Override
public void rename(String srcPath, String destPath) throws TException {
FileTransFormPair parameter =getClient();
parameter.getClient().rename(srcPath, destPath);
parameter.getTransport().close();
}
@Override
public String getName(String path) throws TException {
FileTransFormPair parameter =getClient();
String result =parameter.getClient().getName(path);
parameter.getTransport().close();
return result;
}
@Override
public long totalSpace(String path) throws TException {
FileTransFormPair parameter =getClient();
long result =parameter.getClient().totalSpace(path);
parameter.getTransport().close();
return result;
}
@Override
public long usedSpace(String path) throws TException {
FileTransFormPair parameter =getClient();
long result =parameter.getClient().usedSpace(path);
parameter.getTransport().close();
return result;
}
@Override
public String read(String path) throws TException {
FileTransFormPair parameter =getClient();
String result =parameter.getClient().read(path);
parameter.getTransport().close();
return result;
}
}
服务端：
public class ThriftServer {
public static int SERVER_PORT ;
private static TServer server =null;
private static Log LOG =LogFactory.getLog(ThriftServer.class);
public void init(){
SERVER_PORT = 9090;
LOG.info("init the thriftServer");
}
public void start() throws TTransportException {
TProcessor tprocessor = new FileTransform.Processor<FileTransform.Iface>(
new FileStoreService());
TServerSocket serverTransport = new TServerSocket(SERVER_PORT);
TThreadPoolServer.Args ttpsArgs = new TThreadPoolServer.Args(
serverTransport);
ttpsArgs.processor(tprocessor);
ttpsArgs.protocolFactory(new TBinaryProtocol.Factory());
server = new TThreadPoolServer(ttpsArgs);
server.serve();
LOG.info("start the thriftServer");
}
public void stop() {
if(null!=server){
server.stop();
LOG.info("stop the thriftServer");
}
}
public static void main(String[] args) {
ThriftServer server =new ThriftServer();
server.init();
try {
server.start();
} catch (TTransportException e) {
e.printStackTrace();
}
}
}
public class FileStoreService implements Iface{
private Configuration getConfiguration(){
Configuration conf =new Configuration();
Properties pro = new Properties();
try {
pro.load(new FileInputStream(new File("conf/filetransform-site.properties")));
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
// conf.set("dfs.default.name", pro.getProperty("dfs.default.name"));
return conf;
}
@Override
public boolean exists(String path) throws TException {
Configuration conf =getConfiguration();
FileSystem fs=null;
try {
fs = FileSystem.get(conf);
} catch (IOException e) {
throw new TException(e.getMessage());
}
boolean exists =false ;
try {
exists = fs.exists(new Path(path));
} catch (IOException e) {
throw new TException(e.getMessage());
}finally{
if(null!=fs){
try {
fs.close();
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
return exists;
}
@Override
public void mkdir(String path) throws TException {
Configuration conf =getConfiguration();
FileSystem fs=null;
try {
fs = FileSystem.get(conf);
fs.mkdirs(new Path(path));
} catch (IOException e) {
throw new TException(e.getMessage());
}finally{
if(null!=fs){
try {
fs.close();
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
}
@Override
public void store(com.pera.file.transform.gen.File file) throws TException {
String path =file.getPath();
String content =file.getContent();
Configuration conf =getConfiguration();
FileSystem fs =null;
FSDataOutputStream out=null;
try {
Path p =new Path(path);
fs =FileSystem.get(conf);
if(!fs.exists(p.getParent())){
throw new TException( path+" is not exists");
}
out =fs.create(p);
if(null!=content){
byte [] tmp =content.getBytes();
out.write(tmp);
out.flush();
out.close();
}
} catch (Exception e) {
throw new TException(e.getMessage());
}finally{
try {
if(null!=out){
out.close();
}
} catch (IOException e) {
throw new TException(e.getMessage());
}
try {
if(null!=fs){
fs.close();
}
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
@Override
public Set<String> ls(String path) throws TException {
Configuration conf =getConfiguration();
FileSystem fs =null;
Set<String> result =new HashSet<String>();
try {
fs=FileSystem.get(conf);
FileStatus [] fstatus=fs.listStatus(new Path(path));
int length =fstatus==null?0:fstatus.length;
for (int i = 0; i < length; i++) {
String url =fstatus[i].getPath().toString();
result.add(url);
}
} catch (IOException e) {
throw new TException(e.getMessage());
}finally{
if(null!=fs){
try {
fs.close();
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
return result;
}
@Override
@SuppressWarnings("deprecation")
public boolean isDirectory(String path) throws TException {
Configuration conf =getConfiguration();
FileSystem fs =null;
boolean result =false ;
try {
fs =FileSystem.get(conf);
result =fs.isDirectory(new Path(path));
} catch (IOException e) {
throw new TException(e.getMessage());
}finally{
if(null!=fs){
try {
fs.close();
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
return result;
}
@Override
public String getParentFile(String path) throws TException {
Path parentPath =new Path(path).getParent();
return parentPath.toString();
}
@Override
public void createNewFile(String path) throws TException {
Configuration conf =getConfiguration();
FileSystem fs =null;
try {
fs =FileSystem.get(conf);
fs.createNewFile(new Path(path));
} catch (IOException e) {
throw new TException(e.getMessage());
}finally{
if(null!=fs){
try {
fs.close();
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
}
@Override
public void deleteFile(String path) throws TException {
Configuration conf =getConfiguration();
FileSystem fs =null;
try {
fs =FileSystem.get(conf);
fs.deleteOnExit(new Path(path));
} catch (IOException e) {
throw new TException(e.getMessage());
}finally{
if(null!=fs){
try {
fs.close();
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
}
@Override
public void deleteOnExit(String path) throws TException {
Configuration conf =getConfiguration();
FileSystem fs =null;
try {
fs =FileSystem.get(conf);
fs.deleteOnExit(new Path(path));
} catch (IOException e) {
throw new TException(e.getMessage());
}finally{
if(null!=fs){
try {
fs.close();
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
}
@Override
public boolean isFile(String path) throws TException {
boolean isFile =false ;
Configuration conf =getConfiguration();
FileSystem fs =null;
try {
fs =FileSystem.get(conf);
isFile =fs.isFile(new Path(path));
} catch (IOException e) {
throw new TException(e.getMessage());
}finally{
if(null!=fs){
try {
fs.close();
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
return isFile;
}
@Override
public void rename(String srcPath, String destPath) throws TException {
Configuration conf =getConfiguration();
FileSystem fs =null;
try {
fs =FileSystem.get(conf);
fs.rename(new Path(srcPath), new Path(destPath));
} catch (IOException e) {
throw new TException(e.getMessage());
}finally{
if(null!=fs){
try {
fs.close();
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
}
@Override
public String getName(String path) throws TException {
String name =new Path(path).getName();
return name;
}
@Override
public long totalSpace(String path) throws TException {
Configuration conf =getConfiguration();
FileSystem fs =null;
long length =0 ;
try {
fs =FileSystem.get(conf);
length =fs.getFileStatus(new Path(path)).getLen();
} catch (IOException e) {
throw new TException(e.getMessage());
}finally{
if(null!=fs){
try {
fs.close();
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
return length;
}
@Override
public long usedSpace(String path) throws TException {
Configuration conf =getConfiguration();
FileSystem fs =null;
long used =0 ;
try {
fs =FileSystem.get(conf);
used =fs.getUsed();
} catch (IOException e) {
throw new TException(e.getMessage());
}finally{
if(null!=fs){
try {
fs.close();
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
return used;
}
@Override
public String read(String path) throws TException {
Configuration conf =getConfiguration();
FileSystem fs =null;
String result ="" ;
try {
fs =FileSystem.get(conf);
FSDataInputStream in =fs.open(new Path(path));
int length =in.available();
byte [] tmp = new byte [length] ;
in.readFully(tmp);
result =new String(tmp);
} catch (IOException e) {
throw new TException(e.getMessage());
}finally{
if(null!=fs){
try {
fs.close();
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
return result;
}
}
thrift实现HDFS文件操作的更多相关文章
- Hadoop之HDFS文件操作常有两种方式（转载）
摘要:Hadoop之HDFS文件操作常有两种方式,命令行方式和JavaAPI方式.本文介绍如何利用这两种方式对HDFS文件进行操作. 关键词:HDFS文件 命令行 Java API HD ...
- hadoop的hdfs文件操作实现上传文件到hdfs
这篇文章主要介绍了使用hadoop的API对HDFS上的文件访问,其中包括上传文件到HDFS上.从HDFS上下载文件和删除HDFS上的文件,需要的朋友可以参考下hdfs文件操作操作示例,包括上传文件到 ...
- HDFS文件操作(命令行)
HDFS是一种分布式文件系统,为MapReduce这种框架下的海量数据分布式处理而设计. Hadoop之HDFS文件操作常有两种方式,一种是命令行方式,即Hadoop提供了一套与Linux文件命令类似 ...
- Hadoop之HDFS文件操作
摘要:Hadoop之HDFS文件操作常有两种方式.命令行方式和JavaAPI方式.本文介绍怎样利用这两种方式对HDFS文件进行操作. 关键词:HDFS文件 命令行 Java API HD ...
- JAVA API 实现hdfs文件操作
java api 实现hdfs 文件操作会出现错误提示: Permission denied: user=hp, access=WRITE, inode="/":hdfs:supe ...
- HDFS文件操作
hadoop装好后,文件系统中没有任何目录与文件 1. 创建文件夹 hadoop fs -mkdir -p /hkx/learn 参数-p表示递归创建文件夹 2. 浏览文件 hadoop fs -ls ...
- eclipse 对 hadoop1.2.1 hdfs 文件操作
package com.hdfs; import java.io.FileInputStream;import java.io.FileNotFoundException;import java.io ...
- Hadoop HDFS文件操作
1.创建目录 import java.io.IOException; import org.apache.hadoop.conf.Configuration; import org.apache.ha ...
- [bigdata] 使用Flume hdfs sink， hdfs文件未关闭的问题
现象: 执行mapreduce任务时失败 通过hadoop fsck -openforwrite命令查看发现有文件没有关闭. [root@com ~]# hadoop fsck -openforwri ...
随机推荐
- Spring的DataSource配置、将Hibernate配置全部写到Spring配置
DataSource可以集中管理数据库连接,减少维护工作量,使部署更简单: Spring的DataSource配置:(Spring数据源配置)这里使用dbcp,还有很多其他的如c3p0,jdbc,jn ...
- Dynamics CRM2016 站点地图Bug之KnowledgeArticle不显示
在CRM2016中对知识库功能进行了加强,新加入了KnowledgeArticle的实体,但在学习的过程中发现了微软的一个bug 下图是我在最新的2016online的版本上看到的service下的s ...
- Android核心安全机制(一)
Android六种核心安全机制-加密.密钥.签名与证书 对于移动开发,程序猿很容易会忘记一些安全问题,如一个MD5的加密,大部分人都知道怎么去使用,但是其中的一些加密原理,加密方式却只有少部分会去了解 ...
- Android平移动画
Android平移动画 核心方法 public void startAnimation(Animation animation) 执行动画,参数可以是各种动画的对象,Animation的多态,也可以是 ...
- 关于AndroidSDK配置时的tools目录下找不到adb.exe的错误
欢迎关注公众号,每天推送Android技术文章,二维码如下:(可扫描) 在配置android SDK的时候,有时会发现在android-sdk-windows\tools目录下并没有adb.exe,这 ...
- Android开发学习之路--UI之简单聊天界面
学了很多的ui的知识,这里就来实现个聊天的界面,首先来实现个layout的xml,代码如下: <?xml version="1.0" encoding="utf-8 ...
- Objc中触摸处理阻塞时消息派送的问题
在游戏场景中添加了touchBegan的处理: -(void)touchBegan:(CCTouch *)touch withEvent:(CCTouchEvent *)event{ if ([sel ...
- SimpleAdapter和Baseadapter填充listActivity-android学习之旅（）
简介 SimpleAdapter的功能是能够为AbsListView提供复杂的数据,需要构造ListView 代码示例 package peng.liu.testview; import androi ...
- 【一天一道LeetCode】#160. Intersection of Two Linked Lists
一天一道LeetCode 本系列文章已全部上传至我的github,地址:ZeeCoder's Github 欢迎大家关注我的新浪微博,我的新浪微博 欢迎转载,转载请注明出处 (一)题目 Write a ...
- Hessian源码分析--HessianServlet
Hessian可以通过Servlet来对外暴露服务,HessianServlet继承于HttpServlet,但这仅仅是一个外壳,使用web服务器来提供对外的Http请求,在web.xml中我们会进行 ...