Docker Compose 一键部署多节点爬虫程序

Docker Compose 一键部署多节点爬虫程序

目录结构

[root@localhost ~]# tree compose_crawler/
compose_crawler/
├── centos
│   ├── curl.sh
│   └── Dockerfile
└── docker-compose.yml

---

1、创建DockerCompose项目目录

mkdir compose_crawler
cd compose_crawler/

2、创建centos管理目录

mkdir centos
cd centos

3、创建脚本模拟爬虫文件

vim curl.sh

for i in {1..100}; do
curl http://192.168.1.72 -o /dev/null
sleep 1
done

# 添加执行权限
chmod +x curl.sh

4、创建Dockerfile文件

vim Dockerfile

# 指定镜像
FROM centos:6
# 指定管理员
MAINTAINER xiangsikai
# 复制本地文件到容器内
COPY curl.sh /
# 运行脚本文件
CMD ["/bin/bash", "curl.sh"]

5、项目目录下创建docker-compose.yml文件

vim docker-compose.yml

# 指定服务版本号
version: ''
# 服务
services:
# 服务名称
  curl-test:
# 构建
    build:
# 指定目录上下文构建镜像
      context: ./centos
# 指定dockerfile文件名称
      dockerfile: Dockerfile

6、启动dockercompose构建镜像容器

# 后台执行容器
docker-compose up -d

# 扩展添加容器数量 curl-test为容器名
docker-compose scale curl-test=20

# 查看启动容器数量
docker-compose ps

            Name                     Command        State    Ports
------------------------------------------------------------------
compose_crawler_curl-test_1     /bin/bash curl.sh   Exit 0
compose_crawler_curl-test_10    /bin/bash curl.sh   Up
compose_crawler_curl-test_100   /bin/bash curl.sh   Up
compose_crawler_curl-test_11    /bin/bash curl.sh   Up
compose_crawler_curl-test_12    /bin/bash curl.sh   Up
compose_crawler_curl-test_13    /bin/bash curl.sh   Up
compose_crawler_curl-test_14    /bin/bash curl.sh   Up
compose_crawler_curl-test_15    /bin/bash curl.sh   Up
compose_crawler_curl-test_16    /bin/bash curl.sh   Up
compose_crawler_curl-test_17    /bin/bash curl.sh   Up
compose_crawler_curl-test_18    /bin/bash curl.sh   Up
compose_crawler_curl-test_19    /bin/bash curl.sh   Up
compose_crawler_curl-test_2     /bin/bash curl.sh   Up
compose_crawler_curl-test_20    /bin/bash curl.sh   Up