thrift实现HDFS文件操作
thrift 文件如下
namespace java com.pera.file.transform
struct File{
1:string path ,
2:string content,
}
service FileTransform {
bool exists(1:string path),
void mkdir(1:string path ),
void store(1:File file),
set<string> ls(1:string path),
bool isDirectory(1:string path),
string getParentFile(1:string path),
void createNewFile(1:string path),
void deleteFile(1:string path),
void deleteOnExit(1:string path),
bool isFile(1:string path),
void rename(1:string srcPath ,2:string destPath),
string getName(1:string path),
i64 totalSpace(1:string path),
i64 usedSpace(1:string path),
string read(1:string path)
}
客户端：
public class FileTransFormPair {
private TTransport transport ;
private Client client ;
public FileTransFormPair(){}
public FileTransFormPair( TTransport transport ,Client client ){
this.transport=transport ;
this.client =client;
}
public TTransport getTransport() {
return transport;
}
public void setTransport(TTransport transport) {
this.transport = transport;
}
public Client getClient() {
return client;
}
public void setClient(Client client) {
this.client = client;
}
}
public class FileTransFormClient implements Iface {
public static String SERVER_IP = "localhost";
public static int SERVER_PORT = 8090;
public static int TIMEOUT = 30000;
public FileTransFormClient(String host,int port ,int timeout){
SERVER_IP=host;
SERVER_PORT=port;
TIMEOUT=timeout;
}
private static FileTransFormPair getClient() throws TTransportException{
TTransport transport = new TSocket(SERVER_IP, SERVER_PORT, TIMEOUT);
TProtocol protocol = new TBinaryProtocol(transport);
Client client =new Client(protocol);
transport.open();
return new FileTransFormPair(transport, client);
}
@Override
public boolean exists(String path) throws TException {
boolean result =false;
FileTransFormPair parameter =getClient();
result =parameter.getClient().exists(path);
parameter.getTransport().close();
return result;
}
@Override
public void mkdir(String path) throws TException {
FileTransFormPair parameter =getClient();
parameter.getClient().mkdir(path);
parameter.getTransport().close();
}
@Override
public void store(File file) throws TException {
FileTransFormPair parameter =getClient();
parameter.getClient().store(file);
parameter.getTransport().close();
}
@Override
public Set<String> ls(String path) throws TException {
FileTransFormPair parameter =getClient();
Set<String> result =parameter.getClient().ls(path);
parameter.getTransport().close();
return result;
}
@Override
public boolean isDirectory(String path) throws TException {
boolean result =false ;
FileTransFormPair parameter =getClient();
result =parameter.getClient().isDirectory(path);
parameter.getTransport().close();
return result;
}
@Override
public String getParentFile(String path) throws TException {
FileTransFormPair parameter =getClient();
String result =parameter.getClient().getParentFile(path);
parameter.getTransport().close();
return result;
}
@Override
public void createNewFile(String path) throws TException {
FileTransFormPair parameter =getClient();
parameter.getClient().createNewFile(path);
parameter.getTransport().close();
}
@Override
public void deleteFile(String path) throws TException {
FileTransFormPair parameter =getClient();
parameter.getClient().deleteFile(path);
parameter.getTransport().close();
}
@Override
public void deleteOnExit(String path) throws TException {
FileTransFormPair parameter =getClient();
parameter.getClient().deleteOnExit(path);
parameter.getTransport().close();
}
@Override
public boolean isFile(String path) throws TException {
FileTransFormPair parameter =getClient();
boolean result =parameter.getClient().isFile(path);
parameter.getTransport().close();
return result ;
}
@Override
public void rename(String srcPath, String destPath) throws TException {
FileTransFormPair parameter =getClient();
parameter.getClient().rename(srcPath, destPath);
parameter.getTransport().close();
}
@Override
public String getName(String path) throws TException {
FileTransFormPair parameter =getClient();
String result =parameter.getClient().getName(path);
parameter.getTransport().close();
return result;
}
@Override
public long totalSpace(String path) throws TException {
FileTransFormPair parameter =getClient();
long result =parameter.getClient().totalSpace(path);
parameter.getTransport().close();
return result;
}
@Override
public long usedSpace(String path) throws TException {
FileTransFormPair parameter =getClient();
long result =parameter.getClient().usedSpace(path);
parameter.getTransport().close();
return result;
}
@Override
public String read(String path) throws TException {
FileTransFormPair parameter =getClient();
String result =parameter.getClient().read(path);
parameter.getTransport().close();
return result;
}
}
服务端：
public class ThriftServer {
public static int SERVER_PORT ;
private static TServer server =null;
private static Log LOG =LogFactory.getLog(ThriftServer.class);
public void init(){
SERVER_PORT = 9090;
LOG.info("init the thriftServer");
}
public void start() throws TTransportException {
TProcessor tprocessor = new FileTransform.Processor<FileTransform.Iface>(
new FileStoreService());
TServerSocket serverTransport = new TServerSocket(SERVER_PORT);
TThreadPoolServer.Args ttpsArgs = new TThreadPoolServer.Args(
serverTransport);
ttpsArgs.processor(tprocessor);
ttpsArgs.protocolFactory(new TBinaryProtocol.Factory());
server = new TThreadPoolServer(ttpsArgs);
server.serve();
LOG.info("start the thriftServer");
}
public void stop() {
if(null!=server){
server.stop();
LOG.info("stop the thriftServer");
}
}
public static void main(String[] args) {
ThriftServer server =new ThriftServer();
server.init();
try {
server.start();
} catch (TTransportException e) {
e.printStackTrace();
}
}
}
public class FileStoreService implements Iface{
private Configuration getConfiguration(){
Configuration conf =new Configuration();
Properties pro = new Properties();
try {
pro.load(new FileInputStream(new File("conf/filetransform-site.properties")));
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
// conf.set("dfs.default.name", pro.getProperty("dfs.default.name"));
return conf;
}
@Override
public boolean exists(String path) throws TException {
Configuration conf =getConfiguration();
FileSystem fs=null;
try {
fs = FileSystem.get(conf);
} catch (IOException e) {
throw new TException(e.getMessage());
}
boolean exists =false ;
try {
exists = fs.exists(new Path(path));
} catch (IOException e) {
throw new TException(e.getMessage());
}finally{
if(null!=fs){
try {
fs.close();
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
return exists;
}
@Override
public void mkdir(String path) throws TException {
Configuration conf =getConfiguration();
FileSystem fs=null;
try {
fs = FileSystem.get(conf);
fs.mkdirs(new Path(path));
} catch (IOException e) {
throw new TException(e.getMessage());
}finally{
if(null!=fs){
try {
fs.close();
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
}
@Override
public void store(com.pera.file.transform.gen.File file) throws TException {
String path =file.getPath();
String content =file.getContent();
Configuration conf =getConfiguration();
FileSystem fs =null;
FSDataOutputStream out=null;
try {
Path p =new Path(path);
fs =FileSystem.get(conf);
if(!fs.exists(p.getParent())){
throw new TException( path+" is not exists");
}
out =fs.create(p);
if(null!=content){
byte [] tmp =content.getBytes();
out.write(tmp);
out.flush();
out.close();
}
} catch (Exception e) {
throw new TException(e.getMessage());
}finally{
try {
if(null!=out){
out.close();
}
} catch (IOException e) {
throw new TException(e.getMessage());
}
try {
if(null!=fs){
fs.close();
}
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
@Override
public Set<String> ls(String path) throws TException {
Configuration conf =getConfiguration();
FileSystem fs =null;
Set<String> result =new HashSet<String>();
try {
fs=FileSystem.get(conf);
FileStatus [] fstatus=fs.listStatus(new Path(path));
int length =fstatus==null?0:fstatus.length;
for (int i = 0; i < length; i++) {
String url =fstatus[i].getPath().toString();
result.add(url);
}
} catch (IOException e) {
throw new TException(e.getMessage());
}finally{
if(null!=fs){
try {
fs.close();
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
return result;
}
@Override
@SuppressWarnings("deprecation")
public boolean isDirectory(String path) throws TException {
Configuration conf =getConfiguration();
FileSystem fs =null;
boolean result =false ;
try {
fs =FileSystem.get(conf);
result =fs.isDirectory(new Path(path));
} catch (IOException e) {
throw new TException(e.getMessage());
}finally{
if(null!=fs){
try {
fs.close();
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
return result;
}
@Override
public String getParentFile(String path) throws TException {
Path parentPath =new Path(path).getParent();
return parentPath.toString();
}
@Override
public void createNewFile(String path) throws TException {
Configuration conf =getConfiguration();
FileSystem fs =null;
try {
fs =FileSystem.get(conf);
fs.createNewFile(new Path(path));
} catch (IOException e) {
throw new TException(e.getMessage());
}finally{
if(null!=fs){
try {
fs.close();
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
}
@Override
public void deleteFile(String path) throws TException {
Configuration conf =getConfiguration();
FileSystem fs =null;
try {
fs =FileSystem.get(conf);
fs.deleteOnExit(new Path(path));
} catch (IOException e) {
throw new TException(e.getMessage());
}finally{
if(null!=fs){
try {
fs.close();
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
}
@Override
public void deleteOnExit(String path) throws TException {
Configuration conf =getConfiguration();
FileSystem fs =null;
try {
fs =FileSystem.get(conf);
fs.deleteOnExit(new Path(path));
} catch (IOException e) {
throw new TException(e.getMessage());
}finally{
if(null!=fs){
try {
fs.close();
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
}
@Override
public boolean isFile(String path) throws TException {
boolean isFile =false ;
Configuration conf =getConfiguration();
FileSystem fs =null;
try {
fs =FileSystem.get(conf);
isFile =fs.isFile(new Path(path));
} catch (IOException e) {
throw new TException(e.getMessage());
}finally{
if(null!=fs){
try {
fs.close();
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
return isFile;
}
@Override
public void rename(String srcPath, String destPath) throws TException {
Configuration conf =getConfiguration();
FileSystem fs =null;
try {
fs =FileSystem.get(conf);
fs.rename(new Path(srcPath), new Path(destPath));
} catch (IOException e) {
throw new TException(e.getMessage());
}finally{
if(null!=fs){
try {
fs.close();
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
}
@Override
public String getName(String path) throws TException {
String name =new Path(path).getName();
return name;
}
@Override
public long totalSpace(String path) throws TException {
Configuration conf =getConfiguration();
FileSystem fs =null;
long length =0 ;
try {
fs =FileSystem.get(conf);
length =fs.getFileStatus(new Path(path)).getLen();
} catch (IOException e) {
throw new TException(e.getMessage());
}finally{
if(null!=fs){
try {
fs.close();
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
return length;
}
@Override
public long usedSpace(String path) throws TException {
Configuration conf =getConfiguration();
FileSystem fs =null;
long used =0 ;
try {
fs =FileSystem.get(conf);
used =fs.getUsed();
} catch (IOException e) {
throw new TException(e.getMessage());
}finally{
if(null!=fs){
try {
fs.close();
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
return used;
}
@Override
public String read(String path) throws TException {
Configuration conf =getConfiguration();
FileSystem fs =null;
String result ="" ;
try {
fs =FileSystem.get(conf);
FSDataInputStream in =fs.open(new Path(path));
int length =in.available();
byte [] tmp = new byte [length] ;
in.readFully(tmp);
result =new String(tmp);
} catch (IOException e) {
throw new TException(e.getMessage());
}finally{
if(null!=fs){
try {
fs.close();
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
return result;
}
}
thrift实现HDFS文件操作的更多相关文章
- Hadoop之HDFS文件操作常有两种方式（转载）
摘要:Hadoop之HDFS文件操作常有两种方式,命令行方式和JavaAPI方式.本文介绍如何利用这两种方式对HDFS文件进行操作. 关键词:HDFS文件 命令行 Java API HD ...
- hadoop的hdfs文件操作实现上传文件到hdfs
这篇文章主要介绍了使用hadoop的API对HDFS上的文件访问,其中包括上传文件到HDFS上.从HDFS上下载文件和删除HDFS上的文件,需要的朋友可以参考下hdfs文件操作操作示例,包括上传文件到 ...
- HDFS文件操作(命令行)
HDFS是一种分布式文件系统,为MapReduce这种框架下的海量数据分布式处理而设计. Hadoop之HDFS文件操作常有两种方式,一种是命令行方式,即Hadoop提供了一套与Linux文件命令类似 ...
- Hadoop之HDFS文件操作
摘要:Hadoop之HDFS文件操作常有两种方式.命令行方式和JavaAPI方式.本文介绍怎样利用这两种方式对HDFS文件进行操作. 关键词:HDFS文件 命令行 Java API HD ...
- JAVA API 实现hdfs文件操作
java api 实现hdfs 文件操作会出现错误提示: Permission denied: user=hp, access=WRITE, inode="/":hdfs:supe ...
- HDFS文件操作
hadoop装好后,文件系统中没有任何目录与文件 1. 创建文件夹 hadoop fs -mkdir -p /hkx/learn 参数-p表示递归创建文件夹 2. 浏览文件 hadoop fs -ls ...
- eclipse 对 hadoop1.2.1 hdfs 文件操作
package com.hdfs; import java.io.FileInputStream;import java.io.FileNotFoundException;import java.io ...
- Hadoop HDFS文件操作
1.创建目录 import java.io.IOException; import org.apache.hadoop.conf.Configuration; import org.apache.ha ...
- [bigdata] 使用Flume hdfs sink， hdfs文件未关闭的问题
现象: 执行mapreduce任务时失败 通过hadoop fsck -openforwrite命令查看发现有文件没有关闭. [root@com ~]# hadoop fsck -openforwri ...
随机推荐
- CSDN发表文章后老是待审核的原因
最近开始在csdn上写文章,发现老是文章被 待审核 ,于是便在网上搜集了下网友们的反馈,最后做出以下的整理. 1.CSDN检测到文章中的链接大于5,就会将文章列为"待审核",这个其 ...
- EBS各个应用简称
模块全称 Banking Center 模块简称 FPT 服务器目录 FPT_TOP Billing Connect CUE CUE_TOP CADView-3D DDD DDD_TOP CPG ...
- ormlite介绍一
概述 ORMlite是类似hibernate的对象映射框架,主要面向java语言,同时,是时下最流行的android面向数据库的的编程工具. 官方网站:http://ormlite.com ...
- 通过一个color创建一个image
使用的地方: [_addButton setBackgroundImage:[UIImage imageWithColor:[[UIColor whiteColor] colorWithAlphaCo ...
- 对N各集合中的任意元素进行排列组合问题
小李去市场买菜,有蔬菜(茄子.黄瓜.大白菜...等k中素菜),和肉类(牛肉,羊肉,鸡肉...等m种荤菜),及点心(麻饼,桃酥,枣花...等n中点心),现在老婆要求每天一荤一素一点心 并且每天的样式要尽 ...
- 学习TensorFlow，线性回归模型
学习TensorFlow,在MNIST数据集上建立softmax回归模型并测试 一.代码 <span style="font-size:18px;">from tens ...
- Swift基础之显示波纹样式
最近项目用到了蓝牙连接,搜索设备的内容,其中需要搜索过程中出现波纹的动画效果,在这里将项目中OC语言编写的这种动画效果,转换成Swift编写,下面简单介绍说明一下代码. 这里用到了两种方法实现波纹效果 ...
- Request中Attribute 和 Parameter 的区别
Attribute 和 Parameter 的区别 (1)HttpServletRequest类有setAttribute()方法,而没有setParameter()方法 (2)当两个Web组件之间为 ...
- Android进阶(二十二)设置TextView文字水平垂直居中
设置TextView文字水平垂直居中 有2种方法可以设置TextView文字居中: 一:在xml文件设置:android:gravity="center" 二:在程序中设置:m_T ...
- nginx root、alias、location指令使用方法
一．nginx root指令 1. Nginx配置 相关配置如下图: 通过配置root目录到"/wwwroot/html/"位置 在用虚拟主机方法,主机名称是test,需要大家配置 ...