<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"

         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">

    <modelVersion>4.0.0</modelVersion>

    <groupId>com.jd.ads</groupId>

    <modelVersion>4.0.0</modelVersion>
 
    <groupId>com.jd.ads</groupId>
    <artifactId>fun2spark</artifactId>
    <version>1.0.0</version>
    <packaging>jar</packaging>
    <properties>
        <spark.version>2.0.1</spark.version>
        <proto.path>${project.basedir}/src/main/proto</proto.path>
        <proto.file>*.proto</proto.file>
        <proto.gene>${project.basedir}/src/main/java/gen-java</proto.gene>
    </properties>
    <dependencies>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>2.2.0</version>
        </dependency>
        <dependency>
            <groupId>commons-codec</groupId>
            <artifactId>commons-codec</artifactId>
            <version>1.9</version>
        </dependency>
        <dependency>
            <groupId>org.scala-lang.modules</groupId>
            <artifactId>scala-xml_2.11</artifactId>
            <version>1.0.2</version>
        </dependency>
        <dependency>
            <groupId>org.scala-lang</groupId>
            <artifactId>scala-library</artifactId>
            <version>2.11.2</version>
        </dependency>
 
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-core_2.11</artifactId>
            <version>${spark.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-sql_2.11</artifactId>
            <version>${spark.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-hive_2.11</artifactId>
            <version>${spark.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-streaming_2.11</artifactId>
            <version>${spark.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-streaming-kafka-0-8_2.11</artifactId>
            <version>${spark.version}</version>
        </dependency>
 
        <dependency>
            <groupId>com.github.scopt</groupId>
            <artifactId>scopt_2.10</artifactId>
            <version>3.2.0</version>
        </dependency>
    </dependencies>
 
    <build>
        <resources>
            <resource>
                <directory>src/main/java</directory>
            </resource>
            <resource>
                <directory>src/main/scala</directory>
            </resource>
            <resource>
                <directory>src/main/resources</directory>
            </resource>
        </resources>
        <plugins>
 
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-clean-plugin</artifactId>
                <version>2.4.1</version>
                <configuration>
                    <verbose>true</verbose>
                    <filesets>
                        <fileset>
                            <directory>${proto.gene}</directory>
                        </fileset>
                    </filesets>
                </configuration>
            </plugin>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-compiler-plugin</artifactId>
                <version>2.3.2</version>
                <configuration>
                    <source>1.6</source>
                    <target>1.6</target>
                    <encoding>UTF-8</encoding>
                </configuration>
            </plugin>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-resources-plugin</artifactId>
                <version>2.4.3</version>
                <configuration>
                    <encoding>UTF-8</encoding>
                </configuration>
            </plugin>
            <plugin>
                <groupId>org.scala-tools</groupId>
                <artifactId>maven-scala-plugin</artifactId>
                <version>2.15.2</version>
                <executions>
                    <execution>
                        <goals>
                            <goal>compile</goal>
                            <goal>testCompile</goal>
                        </goals>
                    </execution>
                </executions>
            </plugin>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-jar-plugin</artifactId>
                <version>2.3.1</version>
            </plugin>
        </plugins>
    </build>
    <pluginRepositories>
        <pluginRepository>
            <id>scala</id>
            <name>Scala Tools</name>
            <url>http://scala-tools.org/repo-releases/</url>
            <releases>
                <enabled>true</enabled>
            </releases>
            <snapshots>
                <enabled>false</enabled>
            </snapshots>
        </pluginRepository>
    </pluginRepositories>
</project>

    <artifactId>fun2spark</artifactId>

    <version>1.0.0</version>

    <packaging>jar</packaging>

    <properties>

        <spark.version>2.0.1</spark.version>

        <proto.path>${project.basedir}/src/main/proto</proto.path>

        <proto.file>*.proto</proto.file>

        <proto.gene>${project.basedir}/src/main/java/gen-java</proto.gene>

    </properties>

    <dependencies>

        <dependency>

            <groupId>org.apache.hadoop</groupId>

            <artifactId>hadoop-client</artifactId>

            <version>2.2.0</version>

        </dependency>

        <dependency>

            <groupId>commons-codec</groupId>

            <artifactId>commons-codec</artifactId>

            <version>1.9</version>

        </dependency>

        <dependency>

            <groupId>org.scala-lang.modules</groupId>

            <artifactId>scala-xml_2.11</artifactId>

            <version>1.0.2</version>

        </dependency>

        <dependency>

            <groupId>org.scala-lang</groupId>

            <artifactId>scala-library</artifactId>

            <version>2.11.2</version>

        </dependency>

        <dependency>

            <groupId>org.apache.spark</groupId>

            <artifactId>spark-core_2.11</artifactId>

            <version>${spark.version}</version>

        </dependency>

        <dependency>

            <groupId>org.apache.spark</groupId>

            <artifactId>spark-sql_2.11</artifactId>

            <version>${spark.version}</version>

        </dependency>

        <dependency>

            <groupId>org.apache.spark</groupId>

            <artifactId>spark-hive_2.11</artifactId>

            <version>${spark.version}</version>

        </dependency>

        <dependency>

            <groupId>org.apache.spark</groupId>

            <artifactId>spark-streaming_2.11</artifactId>

            <version>${spark.version}</version>

        </dependency>

        <dependency>

            <groupId>org.apache.spark</groupId>

            <artifactId>spark-streaming-kafka-0-8_2.11</artifactId>

            <version>${spark.version}</version>

        </dependency>

        <dependency>

            <groupId>com.github.scopt</groupId>

            <artifactId>scopt_2.10</artifactId>

            <version>3.2.0</version>

        </dependency>

    </dependencies>

    <build>

        <resources>

            <resource>

                <directory>src/main/java</directory>

            </resource>

            <resource>

                <directory>src/main/scala</directory>

            </resource>

            <resource>

                <directory>src/main/resources</directory>

            </resource>

        </resources>

        <plugins>

            <plugin>

                <groupId>org.apache.maven.plugins</groupId>

                <artifactId>maven-clean-plugin</artifactId>

                <version>2.4.1</version>

                <configuration>

                    <verbose>true</verbose>

                    <filesets>

                        <fileset>

                            <directory>${proto.gene}</directory>

                        </fileset>

                    </filesets>

                </configuration>

            </plugin>

            <plugin>

                <groupId>org.apache.maven.plugins</groupId>

                <artifactId>maven-compiler-plugin</artifactId>

                <version>2.3.2</version>

                <configuration>

                    <source>1.6</source>

                    <target>1.6</target>

                    <encoding>UTF-8</encoding>

                </configuration>

            </plugin>

            <plugin>

                <groupId>org.apache.maven.plugins</groupId>

                <artifactId>maven-resources-plugin</artifactId>

                <version>2.4.3</version>

                <configuration>

                    <encoding>UTF-8</encoding>

                </configuration>

            </plugin>

            <plugin>

                <groupId>org.scala-tools</groupId>

                <artifactId>maven-scala-plugin</artifactId>

                <version>2.15.2</version>

                <executions>

                    <execution>

                        <goals>

                            <goal>compile</goal>

                            <goal>testCompile</goal>

                        </goals>

                    </execution>

                </executions>

            </plugin>

            <plugin>

                <groupId>org.apache.maven.plugins</groupId>

                <artifactId>maven-jar-plugin</artifactId>

                <version>2.3.1</version>

            </plugin>

        </plugins>

    </build>

    <pluginRepositories>

        <pluginRepository>

            <id>scala</id>

            <name>Scala Tools</name>

            <url>http://scala-tools.org/repo-releases/</url>

            <releases>

                <enabled>true</enabled>

            </releases>

            <snapshots>

                <enabled>false</enabled>

            </snapshots>

        </pluginRepository>

    </pluginRepositories>

</project>

spark2.0配置的更多相关文章

  1. Spark2.0集成Hive操作的相关配置与注意事项

    前言 已完成安装Apache Hive,具体安装步骤请参照,Linux基于Hadoop2.8.0集群安装配置Hive2.1.1及基础操作 补充说明 Hive中metastore(元数据存储)的三种方式 ...

  2. spark2.0.1 安装配置

    1. 官网下载 wget http://d3kbcqa49mib13.cloudfront.net/spark-2.0.1-bin-hadoop2.7.tgz 2. 解压 tar -zxvf spar ...

  3. Ubuntu14.04或16.04下安装JDK1.8+Scala+Hadoop2.7.3+Spark2.0.2

    为了将Hadoop和Spark的安装简单化,今日写下此帖. 首先,要看手头有多少机器,要安装伪分布式的Hadoop+Spark还是完全分布式的,这里分别记录. 1. 伪分布式安装 伪分布式的Hadoo ...

  4. Spark2.0编译

    Spark2.0编译 1 前言 Spark2.0正式版于今天正式发布,本文基于CDH5.0.2的Spark编译. 2 编译步骤 #2.1 下载源码 wget https://github.com/ap ...

  5. CentOS下SparkR安装部署：hadoop2.7.3+spark2.0.0+scale2.11.8+hive2.1.0

    注:之前本人写了一篇SparkR的安装部署文章:SparkR安装部署及数据分析实例,当时SparkR项目还没正式入主Spark,需要自己下载SparkR安装包,但现在spark已经支持R接口,so更新 ...

  6. windows 本地构建hadoop-spark运行环境（hadoop-2.6, spark2.0）

    下载hadoop http://hadoop.apache.org/releases.html --> http://mirrors.tuna.tsinghua.edu.cn/apache/ha ...

  7. 初识Spark2.0之Spark SQL

    内存计算平台spark在今年6月份的时候正式发布了spark2.0,相比上一版本的spark1.6版本,在内存优化,数据组织,流计算等方面都做出了较大的改变,同时更加注重基于DataFrame数据组织 ...

  8. 图文解析Spark2.0核心技术(转载)

    导语 Spark2.0于2016-07-27正式发布,伴随着更简单.更快速.更智慧的新特性,spark 已经逐步替代 hadoop 在大数据中的地位,成为大数据处理的主流标准.本文主要以代码和绘图的方 ...

  9. 在centos7上安装部署hadoop2.7.3和spark2.0.0

    一.安装装备 下载安装包: vmware workstations pro 12 三台centos7.1 mini 虚拟机 网络配置NAT网络如下: 二.创建hadoop用户和hadoop用户组 1. ...

随机推荐

  1. jsonp与JAVA的配合使用

    一.客户端 <!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w ...

  2. java反射，ReflectUtils

    public class ReflectUtils { /** * 通过构造函数实例化对象 * @param className 类的全路径名称 * @param parameterTypes 参数类 ...

  3. js table的笔记，实现添加 td，实现搜索功能

    <!DOCTYPE html> <html> <head> <meta charset="utf-8"> <meta http ...

  4. gdb 调试

    一.gdb 调试-源代码关联 在调试程序的过程中,可以自由地查看相关的源代码(如果有源代码的话)是一项最基本的特性.gdb 当然也提供了这项特性,虽然不如IDE直观,但在一定程度上要比IDE更加灵活和 ...

  5. python脚本基础总结

    1. 注释 ①单行注释:#单行注释 ②多行注释: ''' 三个单引号,多行注释符 ''' ③中文注释:#coding=utf-8 或者 #coding=gbk 2.输入输出 ① 输入:  3.0后的p ...

  6. 批量插入数据(基于Mybatis的实现-Oracle)

    前言:做一个数据同步项目,要求:同步数据不丢失的情况下,提高插入性能. 项目DB框架:Mybatis.DataBase:Oracle. -------------------------------- ...

  7. 2015-08-26: GCC编译选项（转载）

    gcc提供了大量的警告选项,对代码中可能存在的问题提出警告,通常可以使用-Wall来开启以下警告:           -Waddress -Warray-bounds (only with -O2) ...

  8. 用VBS实现公司自动打卡

    公司每天要求早晚打卡,方式是登录某一个页面,然后点击上面的圈圈,如下图: 每天都要打开书签,登录这个页面,然后打卡...这么重复性的工作,怎么能是程序员天天干的事情呢(其实是我太懒了),那么就尝试让程 ...

  9. redis常用总结

    1. 使用redis有哪些好处? (1) 速度快,因为数据存在内存中,类似于HashMap,HashMap的优势就是查找和操作的时间复杂度都是O(1) (2) 支持丰富数据类型,支持string,li ...

  10. javascript模式之模块模式

    使用模式来组织代码有很多优点:使代码的结构更清晰,逻辑性更强,更容易维护.还可以避免很多错误. 首先,在javascript主要分为两大类: 编程模式-- 一些专门为javascript语言开发出的最 ...