配置RHadoop与运行WordCount例子

1、安装R语言环境

su -c 'rpm -Uvh http://download.fedoraproject.org/pub/epel/6/i386/epel-release-6-8.noarch.rpm'

su -c 'yum install foo'

yum list R-\*

yum install R

2、安装RStudio Desktop和Server

Desktop是rpm包，双击执行

Server安装命令：

yum install openssl098e # Required only for RedHat/CentOS 6 and 7

wget http://download2.rstudio.org/rstudio-server-0.98.1091-x86_64.rpm

yum install --nogpgcheck rstudio-server-0.98.1091-x86_64.rpm

添加r-user用户

3、安装gcc、git、pkg-config

yum install gcc git pkg-config

4、安装thrift0.9.0

yum install automake libtool flex bison pkgconfig gcc-c++ boost-devel libevent-devel zlib-devel python-devel ruby-devel

编译安装步骤：

Update the System

yum -y update

Install the Platform Development Tools

yum -y groupinstall "Development Tools"

Upgrade autoconf/automake/bison

yum install -y wget

Upgrade autoconf

wget http://ftp.gnu.org/gnu/autoconf/autoconf-2.69.tar.gz

tar xvf autoconf-2.69.tar.gz

cd autoconf-2.69

./configure --prefix=/usr

make

make install

Upgrade automake

wget http://ftp.gnu.org/gnu/automake/automake-1.14.tar.gz

tar xvf automake-1.14.tar.gz

cd automake-1.14

./configure --prefix=/usr

make

make install

Upgrade bison

wget http://ftp.gnu.org/gnu/bison/bison-2.5.1.tar.gz

tar xvf bison-2.5.1.tar.gz

cd bison-2.5.1

./configure --prefix=/usr

make

make install

Install C++ Lib Dependencies

yum -y install libevent-devel zlib-devel openssl-devel

Upgrade Boost

wget http://sourceforge.net/projects/boost/files/boost/1.55.0/boost_1_55_0.tar.gz

tar xvf boost_1_55_0.tar.gz

cd boost_1_55_0

./bootstrap.sh

./b2 install

Build and Install the Apache Thrift IDL Compiler

git clone https://git-wip-us.apache.org/repos/asf/thrift.git

cd thrift

./bootstrap.sh

./configure --with-lua=no

修改/thrift-0.9.1/lib/cpp/thrift.pc的includedir=${prefix}/include/thrift

make

make install

Update PKG_CONFIG_PATH:

export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:/usr/local/lib/pkgconfig/

Verifiy pkg-config path is correct:

pkg-config --cflags thrift

returns:

-I /usr/local/include/thrift

拷贝文件到lib文件夹

cp /usr/local/lib/libthrift-1.0.0-dev.so /usr/lib/

5、设置Linux环境变量

export HADOOP_PREFIX=/usr/lib/hadoop

export HADOOP_CMD=/usr/lib/hadoop/bin/hadoop

export HADOOP_STREAMING=/usr/lib/hadoop-mapreduce/hadoop-streaming.jar

6、root用户下开启R环境安装依赖包

install.packages(c("rJava", "Rcpp", "RJSONIO", "bitops", "digest",

"functional", "stringr", "plyr", "reshape2", "dplyr",

"R.methodsS3", "caTools", "Hmisc", "data.table", "memoise"))

7、root用户下开启R环境安装RHadoop包

install.packages("/root/RHadoop/rhdfs_1.0.8.tar.gz", repos=NULL, type="source")

install.packages("/root/RHadoop/rmr2_3.3.0.tar.gz", repos=NULL, type="source")

install.packages("/root/RHadoop/plyrmr_0.5.0.tar.gz", repos=NULL, type="source")

install.packages("/root/RHadoop/rhbase_1.2.1.tar.gz", repos=NULL, type="source")

8、配置ant 和 maven

export MAVEN_HOME=/root/apache-maven-3.2.5

export PATH=/root/apache-maven-3.2.5/bin:$PATH

export ANT_HOME=/root/apache-ant-1.9.4

export PATH=$ANT_HOME/bin:$PATH

9、测试RHadoop

Sys.setenv("HADOOP_PREFIX"="/usr/lib/hadoop")

Sys.setenv("HADOOP_CMD"="/usr/lib/hadoop/bin/hadoop")

Sys.setenv("HADOOP_STREAMING"="/usr/lib/hadoop-mapreduce/hadoop-streaming.jar")

library(rmr2)

bp = rmr.options("backend.parameters")

trans <- list(D="mapreduce.map.java.opts=-Xmx400M",

D="mapreduce.reduce.java.opts=-Xmx400M",

D="mapreduce.map.memory.mb=4096",

D="mapreduce.reduce.memory.mb=4096",

D="mapreduce.task.io.sort.mb=100")

bp <- list(hadoop=trans)

#### 没有使用的代码 开始 #######################

bp$hadoop[1]="mapreduce.map.java.opts=-Xmx400M"

bp$hadoop[2]="mapreduce.reduce.java.opts=-Xmx400M"

bp$hadoop[3]="mapreduce.map.memory.mb=1024"

bp$hadoop[4]="mapreduce.reduce.memory.mb=2048"

bp$hadoop[5]="mapreduce.task.io.sort.mb=100"

#### 没有使用的代码 结束 #######################

rmr.options(backend.parameters = bp)

rmr.options("backend.parameters")

## map function

map <- function(k,lines) {

words.list <- strsplit(lines, '\\s')

words <- unlist(words.list)

return( keyval(words, 1) )

}

## reduce function

reduce <- function(word, counts) {

keyval(word, sum(counts))

}

wordcount <- function (input, output=NULL) {

mapreduce(input=input, output=output, input.format="text",

map=map, reduce=reduce)

}

## delete previous result if any

system("/usr/lib/hadoop/bin/hadoop fs -rm -r /tmp/zhengcong/out")

## Submit job

hdfs.root <- '/tmp/zhengcong'

hdfs.data <- file.path(hdfs.root, 'hp')

hdfs.out <- file.path(hdfs.root, 'out')

out <- wordcount(hdfs.data, hdfs.out)

## Fetch results from HDFS

results <- from.dfs(out)

## check top 30 frequent words

results.df <- as.data.frame(results, stringsAsFactors=F)

colnames(results.df) <- c('word', 'count')

head(results.df[order(results.df$count, decreasing=T), ], 30)

10、错误解决

rJava无法加载，root用户下运行 R CMD javareconf -e

添加 export LD_LIBRARY_PATH=$JAVA_HOME/lib/amd64:$JAVA_HOME/jre/lib/amd64/server