配置文件不是原始的 修改过，可以根据自己的配置文件来

jobmanager配置

flink-conf.yaml

################################################################################

#  Licensed to the Apache Software Foundation (ASF) under one

#  or more contributor license agreements.  See the NOTICE file

#  distributed with this work for additional information

#  regarding copyright ownership.  The ASF licenses this file

#  to you under the Apache License, Version 2.0 (the

#  "License"); you may not use this file except in compliance

#  with the License.  You may obtain a copy of the License at

#

#      http://www.apache.org/licenses/LICENSE-2.0

#

#  Unless required by applicable law or agreed to in writing, software

#  distributed under the License is distributed on an "AS IS" BASIS,

#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

#  See the License for the specific language governing permissions and

# limitations under the License.

################################################################################

#==============================================================================

# Common

#==============================================================================

# The external address of the host on which the JobManager runs and can be

# reached by the TaskManagers and any clients which want to connect. This setting

# is only used in Standalone mode and may be overwritten on the JobManager side

# by specifying the --host <hostname> parameter of the bin/jobmanager.sh executable.

# In high availability mode, if you use the bin/start-cluster.sh script and setup

# the conf/masters file, this will be taken care of automatically. Yarn/Mesos

# automatically configure the host name based on the hostname of the node where the

# JobManager runs.

jobmanager.rpc.address: jobmanager

# The RPC port where the JobManager is reachable.

jobmanager.rpc.port: 6123

# The total process memory size for the JobManager.

#

# Note this accounts for all memory usage within the JobManager process, including JVM metaspace and other overhead.

jobmanager.memory.process.size: 2g

# The total process memory size for the TaskManager.

#

# Note this accounts for all memory usage within the TaskManager process, including JVM metaspace and other overhead.

taskmanager.memory.process.size: 3g

taskmanager.memory.jvm-metaspace.size: 512m

# To exclude JVM metaspace and overhead, please, use total Flink memory size instead of 'taskmanager.memory.process.size'.

# It is not recommended to set both 'taskmanager.memory.process.size' and Flink memory.

#

# taskmanager.memory.flink.size: 1280m

# The number of task slots that each TaskManager offers. Each slot runs one parallel pipeline.

taskmanager.numberOfTaskSlots: 4

# The parallelism used for programs that did not specify and other parallelism.

parallelism.default: 1

# The default file system scheme and authority.

#

# By default file paths without scheme are interpreted relative to the local

# root file system 'file:///'. Use this to override the default and interpret

# relative paths relative to a different file system,

# for example 'hdfs://mynamenode:12345'

#

# fs.default-scheme

#==============================================================================

# High Availability

#==============================================================================

# The high-availability mode. Possible options are 'NONE' or 'zookeeper'.

#

# high-availability: zookeeper

# The path where metadata for master recovery is persisted. While ZooKeeper stores

# the small ground truth for checkpoint and leader election, this location stores

# the larger objects, like persisted dataflow graphs.

#

# Must be a durable file system that is accessible from all nodes

# (like HDFS, S3, Ceph, nfs, ...)

#

# high-availability.storageDir: hdfs:///flink/ha/

# The list of ZooKeeper quorum peers that coordinate the high-availability

# setup. This must be a list of the form:

# "host1:clientPort,host2:clientPort,..." (default clientPort: 2181)

#

# high-availability.zookeeper.quorum: localhost:2181

# ACL options are based on https://zookeeper.apache.org/doc/r3.1.2/zookeeperProgrammers.html#sc_BuiltinACLSchemes

# It can be either "creator" (ZOO_CREATE_ALL_ACL) or "open" (ZOO_OPEN_ACL_UNSAFE)

# The default value is "open" and it can be changed to "creator" if ZK security is enabled

#

# high-availability.zookeeper.client.acl: open

#==============================================================================

# Fault tolerance and checkpointing

#==============================================================================

# The backend that will be used to store operator state checkpoints if

# checkpointing is enabled.

#

# Supported backends are 'jobmanager', 'filesystem', 'rocksdb', or the

# <class-name-of-factory>.

#

# state.backend: filesystem

# Directory for checkpoints filesystem, when using any of the default bundled

# state backends.

#

# state.checkpoints.dir: hdfs://namenode-host:port/flink-checkpoints

# Default target directory for savepoints, optional.

#

#state.savepoints.dir: file:/tmp/savepoint

# Flag to enable/disable incremental checkpoints for backends that

# support incremental checkpoints (like the RocksDB state backend).

#

# state.backend.incremental: false

# The failover strategy, i.e., how the job computation recovers from task failures.

# Only restart tasks that may have been affected by the task failure, which typically includes

# downstream tasks and potentially upstream tasks if their produced data is no longer available for consumption.

jobmanager.execution.failover-strategy: region

#==============================================================================

# Rest & web frontend

#==============================================================================

# The port to which the REST client connects to. If rest.bind-port has

# not been specified, then the server will bind to this port as well.

#

#rest.port: 8081

# The address to which the REST client will connect to

#

#rest.address: 0.0.0.0

# Port range for the REST and web server to bind to.

#

#rest.bind-port: 8080-8090

# The address that the REST & web server binds to

#

#rest.bind-address: 0.0.0.0

# Flag to specify whether job submission is enabled from the web-based

# runtime monitor. Uncomment to disable.

#web.submit.enable: false

#==============================================================================

# Advanced

#==============================================================================

# Override the directories for temporary files. If not specified, the

# system-specific Java temporary directory (java.io.tmpdir property) is taken.

#

# For framework setups on Yarn or Mesos, Flink will automatically pick up the

# containers' temp directories without any need for configuration.

#

# Add a delimited list for multiple directories, using the system directory

# delimiter (colon ':' on unix) or a comma, e.g.:

#     /data1/tmp:/data2/tmp:/data3/tmp

#

# Note: Each directory entry is read from and written to by a different I/O

# thread. You can include the same directory multiple times in order to create

# multiple I/O threads against that directory. This is for example relevant for

# high-throughput RAIDs.

#

# io.tmp.dirs: /tmp

# The classloading resolve order. Possible values are 'child-first' (Flink's default)

# and 'parent-first' (Java's default).

#

# Child first classloading allows users to use different dependency/library

# versions in their application than those in the classpath. Switching back

# to 'parent-first' may help with debugging dependency issues.

#

# classloader.resolve-order: child-first

# The amount of memory going to the network stack. These numbers usually need

# no tuning. Adjusting them may be necessary in case of an "Insufficient number

# of network buffers" error. The default min is 64MB, the default max is 1GB.

#

# taskmanager.memory.network.fraction: 0.1

# taskmanager.memory.network.min: 64mb

# taskmanager.memory.network.max: 1gb

#==============================================================================

# Flink Cluster Security Configuration

#==============================================================================

# Kerberos authentication for various components - Hadoop, ZooKeeper, and connectors -

# may be enabled in four steps:

# 1. configure the local krb5.conf file

# 2. provide Kerberos credentials (either a keytab or a ticket cache w/ kinit)

# 3. make the credentials available to various JAAS login contexts

# 4. configure the connector to use JAAS/SASL

# The below configure how Kerberos credentials are provided. A keytab will be used instead of

# a ticket cache if the keytab path and principal are set.

# security.kerberos.login.use-ticket-cache: true

# security.kerberos.login.keytab: /path/to/kerberos/keytab

# security.kerberos.login.principal: flink-user

# The configuration below defines which JAAS login contexts

# security.kerberos.login.contexts: Client,KafkaClient

#==============================================================================

# ZK Security Configuration

#==============================================================================

# Below configurations are applicable if ZK ensemble is configured for security

# Override below configuration to provide custom ZK service name if configured

# zookeeper.sasl.service-name: zookeeper

# The configuration below must match one of the values set in "security.kerberos.login.contexts"

# zookeeper.sasl.login-context-name: Client

#==============================================================================

# HistoryServer

#==============================================================================

# The HistoryServer is started and stopped via bin/historyserver.sh (start|stop)

# Directory to upload completed jobs to. Add this directory to the list of

# monitored directories of the HistoryServer as well (see below).

#jobmanager.archive.fs.dir: hdfs:///completed-jobs/

# The address under which the web-based HistoryServer listens.

#historyserver.web.address: 0.0.0.0

# The port under which the web-based HistoryServer listens.

#historyserver.web.port: 8082

# Comma separated list of directories to monitor for completed jobs.

#historyserver.archive.fs.dir: hdfs:///completed-jobs/

# Interval in milliseconds for refreshing the monitored directories.

#historyserver.archive.fs.refresh-interval: 10000

blob.server.port: 6124

query.server.port: 6125

jobmanager.web.tmpdir: /tmp/flink-web

jobmanager.web.upload.dir: /tmp/flink-web

#任务失败后，重启时，每两次连续尝试之间间隔1 min，任务重启失败30次，则任务最终失败，配置默认全局生效。

restart-strategy: fixed-delay

restart-strategy.fixed-delay.attempts: 30

restart-strategy.fixed-delay.delay: 1 min

taskmanager

flink-conf.yaml 配置

################################################################################

#  Licensed to the Apache Software Foundation (ASF) under one

#  or more contributor license agreements.  See the NOTICE file

#  distributed with this work for additional information

#  regarding copyright ownership.  The ASF licenses this file

#  to you under the Apache License, Version 2.0 (the

#  "License"); you may not use this file except in compliance

#  with the License.  You may obtain a copy of the License at

#

#      http://www.apache.org/licenses/LICENSE-2.0

#

#  Unless required by applicable law or agreed to in writing, software

#  distributed under the License is distributed on an "AS IS" BASIS,

#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

#  See the License for the specific language governing permissions and

# limitations under the License.

################################################################################

#==============================================================================

# Common

#==============================================================================

# The external address of the host on which the JobManager runs and can be

# reached by the TaskManagers and any clients which want to connect. This setting

# is only used in Standalone mode and may be overwritten on the JobManager side

# by specifying the --host <hostname> parameter of the bin/jobmanager.sh executable.

# In high availability mode, if you use the bin/start-cluster.sh script and setup

# the conf/masters file, this will be taken care of automatically. Yarn/Mesos

# automatically configure the host name based on the hostname of the node where the

# JobManager runs.

jobmanager.rpc.address: jobmanager

# The RPC port where the JobManager is reachable.

jobmanager.rpc.port: 6123

# The total process memory size for the JobManager.

#

# Note this accounts for all memory usage within the JobManager process, including JVM metaspace and other overhead.

jobmanager.memory.process.size: 2g

# The total process memory size for the TaskManager.

#

# Note this accounts for all memory usage within the TaskManager process, including JVM metaspace and other overhead.

taskmanager.memory.process.size: 3g

taskmanager.memory.jvm-metaspace.size: 512m

# To exclude JVM metaspace and overhead, please, use total Flink memory size instead of 'taskmanager.memory.process.size'.

# It is not recommended to set both 'taskmanager.memory.process.size' and Flink memory.

#

# taskmanager.memory.flink.size: 1280m

# The number of task slots that each TaskManager offers. Each slot runs one parallel pipeline.

taskmanager.numberOfTaskSlots: 4

# The parallelism used for programs that did not specify and other parallelism.

parallelism.default: 1

# The default file system scheme and authority.

#

# By default file paths without scheme are interpreted relative to the local

# root file system 'file:///'. Use this to override the default and interpret

# relative paths relative to a different file system,

# for example 'hdfs://mynamenode:12345'

#

# fs.default-scheme

#==============================================================================

# High Availability

#==============================================================================

# The high-availability mode. Possible options are 'NONE' or 'zookeeper'.

#

# high-availability: zookeeper

# The path where metadata for master recovery is persisted. While ZooKeeper stores

# the small ground truth for checkpoint and leader election, this location stores

# the larger objects, like persisted dataflow graphs.

#

# Must be a durable file system that is accessible from all nodes

# (like HDFS, S3, Ceph, nfs, ...)

#

# high-availability.storageDir: hdfs:///flink/ha/

# The list of ZooKeeper quorum peers that coordinate the high-availability

# setup. This must be a list of the form:

# "host1:clientPort,host2:clientPort,..." (default clientPort: 2181)

#

# high-availability.zookeeper.quorum: localhost:2181

# ACL options are based on https://zookeeper.apache.org/doc/r3.1.2/zookeeperProgrammers.html#sc_BuiltinACLSchemes

# It can be either "creator" (ZOO_CREATE_ALL_ACL) or "open" (ZOO_OPEN_ACL_UNSAFE)

# The default value is "open" and it can be changed to "creator" if ZK security is enabled

#

# high-availability.zookeeper.client.acl: open

#==============================================================================

# Fault tolerance and checkpointing

#==============================================================================

# The backend that will be used to store operator state checkpoints if

# checkpointing is enabled.

#

# Supported backends are 'jobmanager', 'filesystem', 'rocksdb', or the

# <class-name-of-factory>.

#

# state.backend: filesystem

# Directory for checkpoints filesystem, when using any of the default bundled

# state backends.

#

# state.checkpoints.dir: hdfs://namenode-host:port/flink-checkpoints

# Default target directory for savepoints, optional.

#

#state.savepoints.dir: file:/tmp/savepoint

# Flag to enable/disable incremental checkpoints for backends that

# support incremental checkpoints (like the RocksDB state backend).

#

# state.backend.incremental: false

# The failover strategy, i.e., how the job computation recovers from task failures.

# Only restart tasks that may have been affected by the task failure, which typically includes

# downstream tasks and potentially upstream tasks if their produced data is no longer available for consumption.

jobmanager.execution.failover-strategy: region

#==============================================================================

# Rest & web frontend

#==============================================================================

# The port to which the REST client connects to. If rest.bind-port has

# not been specified, then the server will bind to this port as well.

#

#rest.port: 8081

# The address to which the REST client will connect to

#

#rest.address: 0.0.0.0

# Port range for the REST and web server to bind to.

#

#rest.bind-port: 8080-8090

# The address that the REST & web server binds to

#

#rest.bind-address: 0.0.0.0

# Flag to specify whether job submission is enabled from the web-based

# runtime monitor. Uncomment to disable.

#web.submit.enable: false

#==============================================================================

# Advanced

#==============================================================================

# Override the directories for temporary files. If not specified, the

# system-specific Java temporary directory (java.io.tmpdir property) is taken.

#

# For framework setups on Yarn or Mesos, Flink will automatically pick up the

# containers' temp directories without any need for configuration.

#

# Add a delimited list for multiple directories, using the system directory

# delimiter (colon ':' on unix) or a comma, e.g.:

#     /data1/tmp:/data2/tmp:/data3/tmp

#

# Note: Each directory entry is read from and written to by a different I/O

# thread. You can include the same directory multiple times in order to create

# multiple I/O threads against that directory. This is for example relevant for

# high-throughput RAIDs.

#

# io.tmp.dirs: /tmp

# The classloading resolve order. Possible values are 'child-first' (Flink's default)

# and 'parent-first' (Java's default).

#

# Child first classloading allows users to use different dependency/library

# versions in their application than those in the classpath. Switching back

# to 'parent-first' may help with debugging dependency issues.

#

# classloader.resolve-order: child-first

# The amount of memory going to the network stack. These numbers usually need

# no tuning. Adjusting them may be necessary in case of an "Insufficient number

# of network buffers" error. The default min is 64MB, the default max is 1GB.

#

# taskmanager.memory.network.fraction: 0.1

# taskmanager.memory.network.min: 64mb

# taskmanager.memory.network.max: 1gb

#==============================================================================

# Flink Cluster Security Configuration

#==============================================================================

# Kerberos authentication for various components - Hadoop, ZooKeeper, and connectors -

# may be enabled in four steps:

# 1. configure the local krb5.conf file

# 2. provide Kerberos credentials (either a keytab or a ticket cache w/ kinit)

# 3. make the credentials available to various JAAS login contexts

# 4. configure the connector to use JAAS/SASL

# The below configure how Kerberos credentials are provided. A keytab will be used instead of

# a ticket cache if the keytab path and principal are set.

# security.kerberos.login.use-ticket-cache: true

# security.kerberos.login.keytab: /path/to/kerberos/keytab

# security.kerberos.login.principal: flink-user

# The configuration below defines which JAAS login contexts

# security.kerberos.login.contexts: Client,KafkaClient

#==============================================================================

# ZK Security Configuration

#==============================================================================

# Below configurations are applicable if ZK ensemble is configured for security

# Override below configuration to provide custom ZK service name if configured

# zookeeper.sasl.service-name: zookeeper

# The configuration below must match one of the values set in "security.kerberos.login.contexts"

# zookeeper.sasl.login-context-name: Client

#==============================================================================

# HistoryServer

#==============================================================================

# The HistoryServer is started and stopped via bin/historyserver.sh (start|stop)

# Directory to upload completed jobs to. Add this directory to the list of

# monitored directories of the HistoryServer as well (see below).

#jobmanager.archive.fs.dir: hdfs:///completed-jobs/

# The address under which the web-based HistoryServer listens.

#historyserver.web.address: 0.0.0.0

# The port under which the web-based HistoryServer listens.

#historyserver.web.port: 8082

# Comma separated list of directories to monitor for completed jobs.

#historyserver.archive.fs.dir: hdfs:///completed-jobs/

# Interval in milliseconds for refreshing the monitored directories.

#historyserver.archive.fs.refresh-interval: 10000

blob.server.port: 6124

query.server.port: 6125

#flink重启后，不会丢失jar

jobmanager.web.tmpdir: /tmp/flink-web

jobmanager.web.upload.dir: /tmp/flink-web

#任务失败后，重启时，每两次连续尝试之间间隔1 min，任务重启失败30次，则任务最终失败，配置默认全局生效。

restart-strategy: fixed-delay

restart-strategy.fixed-delay.attempts: 30

restart-strategy.fixed-delay.delay: 1 min

docker-compose.yml

version: '3'

services:

  jobmanager:

    image: flink:1.13.0

    privileged: true

    expose:

    - "6123"

    ports:

    - "8081:8081"

    command: jobmanager

    environment:

    - JOB_MANAGER_RPC_ADDRESS=jobmanager

    - TZ=Asia/Shanghai

    volumes:

    - /data/flink/conf/job/flink-conf.yaml:/opt/flink/conf/flink-conf.yaml

    restart: unless-stopped

  taskmanager:

    image: flink:1.13.0

    privileged: true

    expose:

    - "6121"

    - "6122"

    depends_on:

    - jobmanager

    command: taskmanager

    environment:

    - JOB_MANAGER_RPC_ADDRESS=jobmanager

    - TZ=Asia/Shanghai

    volumes:

    - /data/flink/conf/task/flink-conf.yaml:/opt/flink/conf/flink-conf.yaml

    restart: unless-stopped

访问http://ip:8081/

这种方法部署有个问题没有解决就是

这里看不到日志，但是 logs日志可以看到

Linux使用docker安装flink的更多相关文章

  1. Linux通过docker安装运行酷Q--用QQ骰子君进行跑团

    Linux通过docker安装运行酷Q 文:铁乐与猫 需求:和小伙伴周末进行愉快的TRPG跑团,需要在QQ讨论组上加了qq小号后,将qq小号用酷Q配合投骰的应用变成骰子君. 限制:我个人的云计算服务器 ...

  2. Docker安装flink及避坑指南

    Docker安装flink 导航 无处不在的大数据 安装flink 拉取flink镜像 编写docker-compose.yml 生成启动 查看安装效果 常见坑及解决方案 问题1 问题2 参考   本 ...

  3. 【Linux】【自学笔记】Linux下面docker安装mysql

    写在前面: 捣腾继续,之前把一个SpringBoot的程序安装在docker上面,参考链接:https://www.cnblogs.com/aki-stones/p/2019-11-01-note.h ...

  4. Linux下docker安装教程

    目前最新版本的docker19.03支持nvidia显卡与容器的无缝对接,从而摆脱了对nvidia-docker的依赖.因此毫不犹豫安装19.03版本的docker,安装教程可参考官方教程Centos ...

  5. linux上Docker安装gogs私服亲测(详解)

    一.前言 有网友问我为什么要使用私服,可能大部分人都不是太懂,网上那么多存储仓库而且好用方便,但是你想过没有如果企业中的项目,放在人家的仓库上这个安全性不是太好,所以说一般企业都会有自己的私服.本章教 ...

  6. Linux、Docker安装Nginx

    Docker安装Nginx #docker images nginx #docker search nginx #docker pull nginx #docker run -it -p 8084:8 ...

  7. 使用 Linux Centos Docker 安装 2Bizbix

    在 Docker 安装 2Bizbix 安装 Centos 7 安装 mysql5.5 镜像 映射好数据库的配置文件和数据库目录 在 Windows 安装 2Bizbox 安装 jboss/base- ...

  8. Linux下Docker安装

    1 在 CentOS 6.4 上安装 docker   docker当前官方只支持Ubuntu,所以在 CentOS 安装Docker比较麻烦(Issue #172).   docker官方文档说要求 ...

  9. oracle linux 6 docker 安装

    docker对安装系统的内核版本有严格的要求,本文针对oracle linux 6.5进行讲解,其它系统参见: https://docs.docker.com/v1.5/installation/ 下 ...

随机推荐

  1. java 单例模式实现代码

    目录 1.使用静态内部类实现 2.使用枚举实现 3.序列化与反序列化 1.使用静态内部类实现 使用静态内部类实现单例模式,线程安全 class SingletonStaticInner { priva ...

  2. 【原】MDC日志链路设计

    背景 我们项目中现有日志系统,采用的是slf4j+logback这套日志组件,也是Java生态里面比较常用的一个日志组件,但是随着分布式的演进,这套组件明显存在以下几个问题: 1.各种无关日志穿行其中 ...

  3. MySQL全面瓦解28：分库分表

    1 为什么要分库分表 物理服务机的CPU.内存.存储设备.连接数等资源有限,某个时段大量连接同时执行操作,会导致数据库在处理上遇到性能瓶颈.为了解决这个问题,行业先驱门充分发扬了分而治之的思想,对大库 ...

  4. JDK 动态代理与 CGLIB 动态代理，它俩真的不一样

    摘要:一文带你搞懂JDK 动态代理与 CGLIB 动态代理 本文分享自华为云社区<一文带你搞懂JDK 动态代理与 CGLIB 动态代理>,作者: Code皮皮虾 . 两者有何区别 1.Jd ...

  5. wget 命令用法

    wget 命令用法 1. 用法/命令格式 wget [OPTION]... [URL]... wget [参数列表] [目标软件.网页的网址] 长选项所必须的参数在使用短选项时也是必须的 2. 常用参 ...

  6. 【基因组注释】同源注释比对软件tblastn、gamp和exonerate比较

    基因结构预测中同源注释策略,将mRNA.cDNA.蛋白.EST等序列比对到组装的基因组中,在文章中通常使用以下比对软件: tblastn gamp exonerate blat 根据我的实测,以上软件 ...

  7. PPT插件——iSlide全功能解析

    做幻灯展示是我们日常工作中不可缺少的一部分,有的人喜欢用代码如Latex, markdown+pandoc+revealjs 或 bookdown.这些都是自动化做幻灯的好工具.我也都有过体会,确实简 ...

  8. 数据分析体系 — 用户粘性的两个计算指标（DAU/MAU和月人均活跃天数）

    很多运营都了解DAU(日活跃用户数)和MAU(月活跃用户数)的重要性,但在某些情况下这两个数值本身并不能反映出太多问题,这个时候就要引用到[DAU/MAU]的概念,即[日活/月活] 用户粘性的两个计算 ...

  9. IDEA+maven+javafx(java 1.8)入坑记录

    序 好久没写博客了,主要是因为懒,写博客真的是个难坚持的事.但今天登上来看了看,之前记录ctf写的wp竟然点击量这么多了,突然让我有了继续写下去的动力. 这段时间遇到了好多事,中间也有想过写几篇文章记 ...

  10. go 函数进阶

    目录 回调函数和闭包 高阶函数示例 回调函数(sort.SliceStable) 闭包 最佳闭包实例 回调函数和闭包 当函数具备以下两种特性的时候,就可以称之为高阶函数(high order func ...