转眼间已经接触了hadoop两周了，从之前的极力排斥到如今的有点喜欢,刚开始被搭建hadoop开发环境搞得几乎要放弃，如今学会了编写小程序,每天都在成长一点挺好的，好好努力,为自己的装备库再填一件武器挺好的，学习在于坚持不懈,加油！！！

马上就要过年了，在最后一天的上班时间内完成了hadoop如何去连接mysql数据库,自己感到很满足，下面就把自己编写的源码贡献给大家,希望能够帮到你们，如存在优化的地方还请大牛们指出，也希望有hadoop的大牛能够给点学习建议，一个来个HA初学者的心声。第一次发布竟然被退回，也不知道为什么，瞬间心情都不好了,但我还是坚持写自己的博客...

StudentRecord类:

package com.simope.mr.db;

import java.io.DataInput;

import java.io.DataOutput;

import java.io.IOException;

import java.sql.PreparedStatement;

import java.sql.ResultSet;

import java.sql.SQLException;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.io.Writable;

import org.apache.hadoop.mapred.lib.db.DBWritable;

public class StudentRecord implements Writable, DBWritable{

    int id;

    String name;

    int age;

    int departmentID;

    @Override

    public void readFields(DataInput in) throws IOException {

        this.id = in.readInt();

        this.name = Text.readString(in);

        this.age = in.readInt();

        this.departmentID = in.readInt();

    }

    @Override

    public void write(DataOutput out) throws IOException {

        out.write(this.id);

        Text.writeString(out, this.name);

        out.write(this.age);

        out.write(this.departmentID);

    }

    public void readFields(ResultSet rs) throws SQLException {

        this.id = rs.getInt(1);

        this.name = rs.getString(2);

        this.age = rs.getInt(3);

        this.departmentID = rs.getInt(4);

    }

    public void write(PreparedStatement ps) throws SQLException {

        ps.setInt(1, this.id);

        ps.setString(2, this.name);

        ps.setInt(3, this.age);

        ps.setInt(4, this.departmentID);

    }

    @Override

    public String toString() {

        return new String(this.name + "\t" + this.age + "\t" + this.departmentID);

    }

}

TeacherRecord类:

package com.simope.mr.db;

import java.io.DataInput;

import java.io.DataOutput;

import java.io.IOException;

import java.sql.PreparedStatement;

import java.sql.ResultSet;

import java.sql.SQLException;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.io.Writable;

import org.apache.hadoop.mapred.lib.db.DBWritable;

public class TeacherRecord implements Writable, DBWritable{

    int id;

    String name;

    int age;

    int departmentID;

    @Override

    public void readFields(DataInput in) throws IOException {

        this.id = in.readInt();

        this.name = Text.readString(in);

        this.age = in.readInt();

        this.departmentID = in.readInt();

    }

    @Override

    public void write(DataOutput out) throws IOException {

        out.write(this.id);

        Text.writeString(out, this.name);

        out.write(this.age);

        out.write(this.departmentID);

    }

    public void readFields(ResultSet rs) throws SQLException {

        this.id = rs.getInt(1);

        this.name = rs.getString(2);

        this.age = rs.getInt(3);

        this.departmentID = rs.getInt(4);

    }

    public void write(PreparedStatement ps) throws SQLException {

        ps.setInt(1, this.id);

        ps.setString(2, this.name);

        ps.setInt(3, this.age);

        ps.setInt(4, this.departmentID);

    }

    @Override

    public String toString() {

        return new String(this.name + "\t" + this.age + "\t" + this.departmentID);

    }

}

DBMapper类:

package com.simope.mr.db;

import java.io.IOException;

import org.apache.hadoop.io.LongWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapred.MapReduceBase;

import org.apache.hadoop.mapred.Mapper;

import org.apache.hadoop.mapred.OutputCollector;

import org.apache.hadoop.mapred.Reporter;

public class DBMapper extends MapReduceBase implements

        Mapper<LongWritable, TeacherRecord, LongWritable, Text> {

    public void map(LongWritable key, TeacherRecord value,

            OutputCollector<LongWritable, Text> collector, Reporter reporter)

            throws IOException {

        collector.collect(new LongWritable(value.id),

                new Text(value.toString()));

    }

}

DBReducer类：

package com.simope.mr.db;

import java.io.IOException;

import java.util.Iterator;

import org.apache.hadoop.io.LongWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapred.MapReduceBase;

import org.apache.hadoop.mapred.OutputCollector;

import org.apache.hadoop.mapred.Reducer;

import org.apache.hadoop.mapred.Reporter;

public class DBReducer extends MapReduceBase implements Reducer<LongWritable, Text, StudentRecord, Text>{

    @Override

    public void reduce(LongWritable key, Iterator<Text> values,

            OutputCollector<StudentRecord, Text> output, Reporter reporter)

            throws IOException {

         String[] InfoArr = values.next().toString().split("\t");

         StudentRecord s = new StudentRecord();

//         t.id = Integer.parseInt(InfoArr[0]);  //id是自增长

         s.name = InfoArr[0];

         s.age = Integer.parseInt(InfoArr[1]);

         s.departmentID = Integer.parseInt(InfoArr[2]);

         output.collect(s, new Text(s.name));

    }

}

DBJob类：(读取数据库表内容,并将数据写入hdfs文件中)数据库表-hdfs文件

package com.simope.mr.db;

import java.io.IOException;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.LongWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapred.FileOutputFormat;

import org.apache.hadoop.mapred.JobClient;

import org.apache.hadoop.mapred.JobConf;

import org.apache.hadoop.mapred.lib.IdentityReducer;

import org.apache.hadoop.mapred.lib.db.DBConfiguration;

import org.apache.hadoop.mapred.lib.db.DBInputFormat;

/**

 * @deprecated 读取数据库录入文件

 * @author JimLy

 * @see 20160202

 * */

public class DBJob {

    public static void main(String[] args) throws IOException{

        JobConf jobConf = new JobConf(DBJob.class);

        jobConf.setOutputKeyClass(LongWritable.class);

        jobConf.setOutputValueClass(Text.class);

        jobConf.setInputFormat(DBInputFormat.class);

        FileOutputFormat.setOutputPath(jobConf, new Path("/usr/output/db"));

        DBConfiguration.configureDB(jobConf, "com.mysql.jdbc.Driver", "jdbc:mysql://10.10.1.1:3306/my_hd", "root", "root");

        String[] fields = {"id", "name", "age", "departmentID"};

        //从my_hd数据库的teacher表查询数据

        DBInputFormat.setInput(jobConf, TeacherRecord.class, "teacher", null, "id", fields);

        jobConf.setMapperClass(DBMapper.class);

        jobConf.setReducerClass(IdentityReducer.class);

        JobClient.runJob(jobConf);

    }

}

DB2Job类:(读取数据库表内容,并将数据写入hdfs文件中)数据库表-hdfs文件

package com.simope.mr.db;

import java.io.IOException;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.LongWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapred.FileOutputFormat;

import org.apache.hadoop.mapred.JobClient;

import org.apache.hadoop.mapred.JobConf;

import org.apache.hadoop.mapred.lib.IdentityReducer;

import org.apache.hadoop.mapred.lib.db.DBConfiguration;

import org.apache.hadoop.mapred.lib.db.DBInputFormat;

/**

 * @deprecated 读取数据库录入文件

 * @author JimLy

 * @see 20160202

 * */

public class DB2Job {

    public static void main(String[] args) throws IOException{

        JobConf jobConf = new JobConf(DB2Job.class);

        jobConf.setOutputKeyClass(LongWritable.class);

        jobConf.setOutputValueClass(Text.class);

        jobConf.setInputFormat(DBInputFormat.class);

        FileOutputFormat.setOutputPath(jobConf, new Path("/usr/output/db"));

        DBConfiguration.configureDB(jobConf, "com.mysql.jdbc.Driver", "jdbc:mysql://10.10.1.1:3306/my_hd", "root", "root");

//        String[] fields = {"id", "name", "age", "departmentID"};

        String inputQuery = "SELECT * FROM teacher where id != 4";

        String inputCountQuery = "SELECT COUNT(1) FROM teacher where id != 4";

        //从my_hd数据库的teacher表查询数据

        DBInputFormat.setInput(jobConf, TeacherRecord.class, inputQuery, inputCountQuery);

        jobConf.setMapperClass(DBMapper.class);

        jobConf.setReducerClass(IdentityReducer.class);

        JobClient.runJob(jobConf);

    }

}

DB3Job类:(读取hdfs文件中的内容,并将数据写入指定的数据库表中)=>hdfs文件-数据库表

package com.simope.mr.db;

import java.io.IOException;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.mapred.FileInputFormat;

import org.apache.hadoop.mapred.JobClient;

import org.apache.hadoop.mapred.JobConf;

import org.apache.hadoop.mapred.TextInputFormat;

import org.apache.hadoop.mapred.lib.IdentityMapper;

import org.apache.hadoop.mapred.lib.db.DBConfiguration;

import org.apache.hadoop.mapred.lib.db.DBOutputFormat;

/**

 * @deprecated 读取文件录入数据库

 * @author JimLy

 * @see 20160202

 * */

public class DB3Job {

    public static void main(String[] args) throws IOException{

        JobConf jobConf = new JobConf(DB3Job.class);

        jobConf.setInputFormat(TextInputFormat.class);

        jobConf.setOutputFormat(DBOutputFormat.class);

        FileInputFormat.addInputPath(jobConf, new Path("/usr/input/db"));

        DBConfiguration.configureDB(jobConf, "com.mysql.jdbc.Driver", "jdbc:mysql://10.10.1.1:3306/my_hd", "root", "root");

        String[] fields = {"id", "name", "age", "departmentID"};

        DBOutputFormat.setOutput(jobConf, "teacher", fields);

        jobConf.setMapperClass(IdentityMapper.class);

        jobConf.setReducerClass(DBReducer.class);

        JobClient.runJob(jobConf);

    }

}

DB4Job类:(读取指定的数据库表信息,并将数据写入其他指定表中)=>数据库表-表

package com.simope.mr.db;

import java.io.IOException;

import org.apache.hadoop.io.LongWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapred.JobClient;

import org.apache.hadoop.mapred.JobConf;

import org.apache.hadoop.mapred.lib.db.DBConfiguration;

import org.apache.hadoop.mapred.lib.db.DBInputFormat;

import org.apache.hadoop.mapred.lib.db.DBOutputFormat;

/**

 * @deprecated 读取数据库表录入其他表

 * @author JimLy

 * @see 20160202

 * */

public class DB4Job {

    public static void main(String[] args) throws IOException{

        JobConf jobConf = new JobConf(DB4Job.class);

        jobConf.setOutputKeyClass(LongWritable.class);

        jobConf.setOutputValueClass(Text.class);

        jobConf.setInputFormat(DBInputFormat.class);

        jobConf.setOutputFormat(DBOutputFormat.class);

        DBConfiguration.configureDB(jobConf, "com.mysql.jdbc.Driver", "jdbc:mysql://10.10.1.1:3306/my_hd", "root", "root");

        String inputQuery = "SELECT * FROM teacher";

        String inputCountQuery = "SELECT COUNT(1) FROM teacher";

        //从my_hd数据库的teacher表查询数据

        DBInputFormat.setInput(jobConf, TeacherRecord.class, inputQuery, inputCountQuery);

        String[] fields = {"id", "name", "age", "departmentID"};

        DBOutputFormat.setOutput(jobConf, "student", fields);

        jobConf.setMapperClass(DBMapper.class);

        jobConf.setReducerClass(DBReducer.class);

        JobClient.runJob(jobConf);

    }

}

如果你觉得写的不错的,请点个推荐，你的推荐是我继续坚持写博客的动力。。。

如需转载的请注明出处：http://www.cnblogs.com/JimLy-BUG/

hadoop与mysql数据库的那点事的更多相关文章

  1. 本地通过Eclipse链接Hadoop操作Mysql数据库问题小结

    前一段时间,在上一篇博文中描述了自己抽时间在构建的完全分布式Hadoop环境过程中遇到的一些问题以及构建成功后,通过Eclipse操作HDFS的时候遇到的一些问题,最近又想进一步学习学习Hadoop操 ...

  2. 一步一步跟我学习hadoop(7)----hadoop连接mysql数据库运行数据读写数据库操作

        为了方便 MapReduce 直接訪问关系型数据库(Mysql,Oracle).Hadoop提供了DBInputFormat和DBOutputFormat两个类.通过DBInputFormat ...

  3. MapReduce 程序mysql JDBC驱动类找不到原因及学习hadoop写入数据到Mysql数据库的方法

    报错 :ClassNotFoundException: com.mysql.jdbc.Driver 需求描述: hadoop需要动态加载个三方jar包(比如mysql JDBC 驱动包),是在MR结束 ...

  4. 关于MySQL数据库中null的那些事

    在mysql数据库中,null是一个经常出现的情况,关于mysql中的null,有哪些注意事项呢?下面简单总结归纳下,后续会不断补充. 1. is null 首先判断数据库中某一列的值是否为null, ...

  5. 吴裕雄--天生自然 HADOOP大数据分布式处理：安装配置MYSQL数据库

    安装之前先安装基本环境:yum install -y perl perl-Module-Build net-tools autoconf libaio numactl-libs # 下载mysql源安 ...

  6. （转）运维角度浅谈MySQL数据库优化

    转自:http://lizhenliang.blog.51cto.com/7876557/1657465 一个成熟的数据库架构并不是一开始设计就具备高可用.高伸缩等特性的,它是随着用户量的增加,基础架 ...

  7. 运维角度浅谈MySQL数据库优化(转)

    一个成熟的数据库架构并不是一开始设计就具备高可用.高伸缩等特性的,它是随着用户量的增加,基础架构才逐渐完善.这篇博文主要谈MySQL数据库发展周期中所面临的问题及优化方案,暂且抛开前端应用不说,大致分 ...

  8. 从运维角度浅谈 MySQL 数据库优化

    一个成熟的数据库架构并不是一开始设计就具备高可用.高伸缩等特性的,它是随着用户量的增加,基础架构才逐渐完善.这篇博文主要谈MySQL数据库发展周期中所面临的问题及优化方案,暂且抛开前端应用不说,大致分 ...

  9. [转载] 运维角度浅谈：MySQL数据库优化

    一个成熟的数据库架构并不是一开始设计就具备高可用.高伸缩等特性的,它是随着用户量的增加,基础架构才逐渐完善. 作者:zhenliang8,本文转自51CTO博客,http://lizhenliang. ...

随机推荐

  1. win8+VS2012搭建OpenGL超级宝典的环境

    版权声明:本文为博主原创文章,未经博主同意不得转载. https://blog.csdn.net/booirror/article/details/36957799 自从公司搬到腾讯附近,每天上班都迟 ...

  2. 【[SDOI2014]旅行】

    听说这是动态开点主席树的板子题,但是发现我还不会,于是就来写一写 其实跟主席树一个样子的 这里就是存个板子吧 #include<cstdio> #include<cstring> ...

  3. Softmax vs. Softmax-Loss: Numerical Stability

    http://freemind.pluskid.org/machine-learning/softmax-vs-softmax-loss-numerical-stability/ softmax 在 ...

  4. Ajax去除缓存

    1.在Ajax发送请求钱加上anyAjaxObj.setRequestHeader("If-Modeified-Since","0").2.在Ajax发送请求钱 ...

  5. 【luogu P2764 最小路径覆盖问题】 模板

    题目链接:https://www.luogu.org/problemnew/show/P2764 把每个点在左边建一遍右边建一遍,再加上源点汇点,跑最大流,n-最大流就是答案. #include &l ...

  6. <head> 中的 JavaScript 函数

    <!DOCTYPE html><html><head><meta http-equiv="Content-Type" content=&q ...

  7. Python基础—11-面向对象（01）

    面向对象 面向对象 与面向过程对比: 面向过程:数学逻辑的映射,学会做个好员工 面向对象:生活逻辑的映射,学会做个好领导 生活实例: 类: 人 手机 电脑 对象: 我的手机.女朋友的手机 你的那部T4 ...

  8. ios学习路线—Objective-C(堆(heap)和栈(stack))

    Objective-C的对象在内存中是以堆的方式分配空间的,并且堆内存是由你释放的,即release 栈由编译器管理自动释放的,在方法中(函数体)定义的变量通常是在栈内,因此如果你的变量要跨函数的话就 ...

  9. c#项目总结

    写了将近10年代码了,最后休息,回想了下,感觉什么都没有. 所以打算写一些总结性的文章,先写几个项目,用于c#各个方向的封装使用 最后汇总成一个完善的解决方案.所有项目都在一个解决方案FastAIFr ...

  10. DLL DEF文件编写方法 VC++ 调用、调试DLL的方法 显式（静态）调用、隐式（动态）调用

    DLL 文件编写方法: 1.建立DLL工程 2.声明.定义要导出的函数 BOOL WINAPI InitDlg( HWND hTabctrl,TShareMem* pTshare,CRect* prc ...