<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"

         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">

    <modelVersion>4.0.0</modelVersion>

    <groupId>com.jd.ads</groupId>

    <modelVersion>4.0.0</modelVersion>
 
    <groupId>com.jd.ads</groupId>
    <artifactId>fun2spark</artifactId>
    <version>1.0.0</version>
    <packaging>jar</packaging>
    <properties>
        <spark.version>2.0.1</spark.version>
        <proto.path>${project.basedir}/src/main/proto</proto.path>
        <proto.file>*.proto</proto.file>
        <proto.gene>${project.basedir}/src/main/java/gen-java</proto.gene>
    </properties>
    <dependencies>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>2.2.0</version>
        </dependency>
        <dependency>
            <groupId>commons-codec</groupId>
            <artifactId>commons-codec</artifactId>
            <version>1.9</version>
        </dependency>
        <dependency>
            <groupId>org.scala-lang.modules</groupId>
            <artifactId>scala-xml_2.11</artifactId>
            <version>1.0.2</version>
        </dependency>
        <dependency>
            <groupId>org.scala-lang</groupId>
            <artifactId>scala-library</artifactId>
            <version>2.11.2</version>
        </dependency>
 
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-core_2.11</artifactId>
            <version>${spark.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-sql_2.11</artifactId>
            <version>${spark.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-hive_2.11</artifactId>
            <version>${spark.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-streaming_2.11</artifactId>
            <version>${spark.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-streaming-kafka-0-8_2.11</artifactId>
            <version>${spark.version}</version>
        </dependency>
 
        <dependency>
            <groupId>com.github.scopt</groupId>
            <artifactId>scopt_2.10</artifactId>
            <version>3.2.0</version>
        </dependency>
    </dependencies>
 
    <build>
        <resources>
            <resource>
                <directory>src/main/java</directory>
            </resource>
            <resource>
                <directory>src/main/scala</directory>
            </resource>
            <resource>
                <directory>src/main/resources</directory>
            </resource>
        </resources>
        <plugins>
 
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-clean-plugin</artifactId>
                <version>2.4.1</version>
                <configuration>
                    <verbose>true</verbose>
                    <filesets>
                        <fileset>
                            <directory>${proto.gene}</directory>
                        </fileset>
                    </filesets>
                </configuration>
            </plugin>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-compiler-plugin</artifactId>
                <version>2.3.2</version>
                <configuration>
                    <source>1.6</source>
                    <target>1.6</target>
                    <encoding>UTF-8</encoding>
                </configuration>
            </plugin>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-resources-plugin</artifactId>
                <version>2.4.3</version>
                <configuration>
                    <encoding>UTF-8</encoding>
                </configuration>
            </plugin>
            <plugin>
                <groupId>org.scala-tools</groupId>
                <artifactId>maven-scala-plugin</artifactId>
                <version>2.15.2</version>
                <executions>
                    <execution>
                        <goals>
                            <goal>compile</goal>
                            <goal>testCompile</goal>
                        </goals>
                    </execution>
                </executions>
            </plugin>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-jar-plugin</artifactId>
                <version>2.3.1</version>
            </plugin>
        </plugins>
    </build>
    <pluginRepositories>
        <pluginRepository>
            <id>scala</id>
            <name>Scala Tools</name>
            <url>http://scala-tools.org/repo-releases/</url>
            <releases>
                <enabled>true</enabled>
            </releases>
            <snapshots>
                <enabled>false</enabled>
            </snapshots>
        </pluginRepository>
    </pluginRepositories>
</project>

    <artifactId>fun2spark</artifactId>

    <version>1.0.0</version>

    <packaging>jar</packaging>

    <properties>

        <spark.version>2.0.1</spark.version>

        <proto.path>${project.basedir}/src/main/proto</proto.path>

        <proto.file>*.proto</proto.file>

        <proto.gene>${project.basedir}/src/main/java/gen-java</proto.gene>

    </properties>

    <dependencies>

        <dependency>

            <groupId>org.apache.hadoop</groupId>

            <artifactId>hadoop-client</artifactId>

            <version>2.2.0</version>

        </dependency>

        <dependency>

            <groupId>commons-codec</groupId>

            <artifactId>commons-codec</artifactId>

            <version>1.9</version>

        </dependency>

        <dependency>

            <groupId>org.scala-lang.modules</groupId>

            <artifactId>scala-xml_2.11</artifactId>

            <version>1.0.2</version>

        </dependency>

        <dependency>

            <groupId>org.scala-lang</groupId>

            <artifactId>scala-library</artifactId>

            <version>2.11.2</version>

        </dependency>

        <dependency>

            <groupId>org.apache.spark</groupId>

            <artifactId>spark-core_2.11</artifactId>

            <version>${spark.version}</version>

        </dependency>

        <dependency>

            <groupId>org.apache.spark</groupId>

            <artifactId>spark-sql_2.11</artifactId>

            <version>${spark.version}</version>

        </dependency>

        <dependency>

            <groupId>org.apache.spark</groupId>

            <artifactId>spark-hive_2.11</artifactId>

            <version>${spark.version}</version>

        </dependency>

        <dependency>

            <groupId>org.apache.spark</groupId>

            <artifactId>spark-streaming_2.11</artifactId>

            <version>${spark.version}</version>

        </dependency>

        <dependency>

            <groupId>org.apache.spark</groupId>

            <artifactId>spark-streaming-kafka-0-8_2.11</artifactId>

            <version>${spark.version}</version>

        </dependency>

        <dependency>

            <groupId>com.github.scopt</groupId>

            <artifactId>scopt_2.10</artifactId>

            <version>3.2.0</version>

        </dependency>

    </dependencies>

    <build>

        <resources>

            <resource>

                <directory>src/main/java</directory>

            </resource>

            <resource>

                <directory>src/main/scala</directory>

            </resource>

            <resource>

                <directory>src/main/resources</directory>

            </resource>

        </resources>

        <plugins>

            <plugin>

                <groupId>org.apache.maven.plugins</groupId>

                <artifactId>maven-clean-plugin</artifactId>

                <version>2.4.1</version>

                <configuration>

                    <verbose>true</verbose>

                    <filesets>

                        <fileset>

                            <directory>${proto.gene}</directory>

                        </fileset>

                    </filesets>

                </configuration>

            </plugin>

            <plugin>

                <groupId>org.apache.maven.plugins</groupId>

                <artifactId>maven-compiler-plugin</artifactId>

                <version>2.3.2</version>

                <configuration>

                    <source>1.6</source>

                    <target>1.6</target>

                    <encoding>UTF-8</encoding>

                </configuration>

            </plugin>

            <plugin>

                <groupId>org.apache.maven.plugins</groupId>

                <artifactId>maven-resources-plugin</artifactId>

                <version>2.4.3</version>

                <configuration>

                    <encoding>UTF-8</encoding>

                </configuration>

            </plugin>

            <plugin>

                <groupId>org.scala-tools</groupId>

                <artifactId>maven-scala-plugin</artifactId>

                <version>2.15.2</version>

                <executions>

                    <execution>

                        <goals>

                            <goal>compile</goal>

                            <goal>testCompile</goal>

                        </goals>

                    </execution>

                </executions>

            </plugin>

            <plugin>

                <groupId>org.apache.maven.plugins</groupId>

                <artifactId>maven-jar-plugin</artifactId>

                <version>2.3.1</version>

            </plugin>

        </plugins>

    </build>

    <pluginRepositories>

        <pluginRepository>

            <id>scala</id>

            <name>Scala Tools</name>

            <url>http://scala-tools.org/repo-releases/</url>

            <releases>

                <enabled>true</enabled>

            </releases>

            <snapshots>

                <enabled>false</enabled>

            </snapshots>

        </pluginRepository>

    </pluginRepositories>

</project>

spark2.0配置的更多相关文章

  1. Spark2.0集成Hive操作的相关配置与注意事项

    前言 已完成安装Apache Hive,具体安装步骤请参照,Linux基于Hadoop2.8.0集群安装配置Hive2.1.1及基础操作 补充说明 Hive中metastore(元数据存储)的三种方式 ...

  2. spark2.0.1 安装配置

    1. 官网下载 wget http://d3kbcqa49mib13.cloudfront.net/spark-2.0.1-bin-hadoop2.7.tgz 2. 解压 tar -zxvf spar ...

  3. Ubuntu14.04或16.04下安装JDK1.8+Scala+Hadoop2.7.3+Spark2.0.2

    为了将Hadoop和Spark的安装简单化,今日写下此帖. 首先,要看手头有多少机器,要安装伪分布式的Hadoop+Spark还是完全分布式的,这里分别记录. 1. 伪分布式安装 伪分布式的Hadoo ...

  4. Spark2.0编译

    Spark2.0编译 1 前言 Spark2.0正式版于今天正式发布,本文基于CDH5.0.2的Spark编译. 2 编译步骤 #2.1 下载源码 wget https://github.com/ap ...

  5. CentOS下SparkR安装部署：hadoop2.7.3+spark2.0.0+scale2.11.8+hive2.1.0

    注:之前本人写了一篇SparkR的安装部署文章:SparkR安装部署及数据分析实例,当时SparkR项目还没正式入主Spark,需要自己下载SparkR安装包,但现在spark已经支持R接口,so更新 ...

  6. windows 本地构建hadoop-spark运行环境（hadoop-2.6, spark2.0）

    下载hadoop http://hadoop.apache.org/releases.html --> http://mirrors.tuna.tsinghua.edu.cn/apache/ha ...

  7. 初识Spark2.0之Spark SQL

    内存计算平台spark在今年6月份的时候正式发布了spark2.0,相比上一版本的spark1.6版本,在内存优化,数据组织,流计算等方面都做出了较大的改变,同时更加注重基于DataFrame数据组织 ...

  8. 图文解析Spark2.0核心技术(转载)

    导语 Spark2.0于2016-07-27正式发布,伴随着更简单.更快速.更智慧的新特性,spark 已经逐步替代 hadoop 在大数据中的地位,成为大数据处理的主流标准.本文主要以代码和绘图的方 ...

  9. 在centos7上安装部署hadoop2.7.3和spark2.0.0

    一.安装装备 下载安装包: vmware workstations pro 12 三台centos7.1 mini 虚拟机 网络配置NAT网络如下: 二.创建hadoop用户和hadoop用户组 1. ...

随机推荐

  1. Robotium怎样判断测试结果

    Robotium判断测试结果的方法主要有三类:assert.is.search.assert方法除了Robotium API,还有Junit中的所有断言方法,Junit的断言方法下篇详解. void ...

  2. nodejs在同一台服务器上部署并同时运行两个或以上服务端时，一个服务用户登录后会挤掉另一个用户的问题

    问题描述:一台服务器,部署了两个或以上不同的Web服务,服务A的用户在登陆后,服务B的用户也登陆,此时服务A的用户在点击页面时,会返回登陆页面. 问题根源:浏览器保存的session相同,即cooki ...

  3. How to acquire an Android phone with locked bootloader?

    As we know that some devices come with locked bootloaders like Sony, HUAWEI, hTC...If you try to unl ...

  4. java io 流分类表

    Java输入/输出流体系中常用的流分类(表内容来自java疯狂讲义) 注:下表中带下划线的是抽象类,不能创建对象.粗体部分是节点流,其他就是常用的处理流. 流分类 使用分类 字节输入流 字节输出流 字 ...

  5. java学习第14天（集合的框架和基本遍历）

    今天主要是接触了集合的概念,集合简单意义上来说就是类对象的集合,我们一般用Collection 这个接口来表示,集合主要体系为: Collection |--List |--ArrayList |-- ...

  6. java学习第12天

    今天主要是看下正则表达式,其实在编译原理中就接触过.正则表达式就是符合一定规则的字符串. A:字符 x 字符 x.举例:'a'表示字符a \\ 反斜线字符. \n 新行(换行)符 ('\u000A') ...

  7. jqurey的跨域使用getjson(http://www.jb51.net/Special/894.htm)

    JS的跨域问题,我想很多程序员的脑海里面还认为JS是不能跨域的,其实这是一个错误的观点:有很多人在网上找其解决方法,教其用IFRAME去解决的文章很多,真有那么复杂吗?其实很简单的,如果你用JQUER ...

  8. 柯朗数(Courant number)研究

    在数值计算过程中,对于计算结果的准确性和效率有很高的要求,但是这两者之间往往互相矛盾:而使用柯朗数可用于平衡两者. 1.柯朗数的定义: C = sqrt(gh)*t/s 其中,t是时间步长,s是网格在 ...

  9. viewgroup用addview添加的view不显示问题

    先看代码: public class MyviewGroup extends ViewGroup { private final Context context; private View view; ...

  10. Python安装mysql-python错误提示python setup.py egg_info

    做python项目,需要用到mysql,一般用python-mysql,安装时遇到错误提示如下: Command "python setup.py egg_info" failed ...