SaltStack项目实战（二）

架构图：

配置思路

(1).系统初始化

Base环境下存放所有系统都要执行的状态，调整内核参数，dns，装zabbix-agent等

(2).功能模块（如：上面的haproxy）

如上面的haproxy nginx php memcached等服务，每一个服务都建一个目录，把每一个服务要执行的状态都放在这个目录下.

(3).业务模块

以业务为单位，一个业务里可能包含haproxy，nginx，php等，业务需要什么服务就把功能模块里对应的服务include

1.编辑配置文件修改file_roots,并且建立对应目录

[root@node1 ~]# vim /etc/salt/master
file_roots:
  base:
    - /srv/salt/base
  test:
    - /srv/salt/test
  prod:
- /srv/salt/prod
[root@node1 ~]# /etc/init.d/salt-master restart
注：top.sls必须放在base环境下
[root@node1 ~]# mkdir /srv/salt/test    测试环境目录
[root@node1 ~]# mkdir /srv/salt/base    基础环境目录
[root@node1 ~]# mkdir /srv/salt/prod    生产环境目录

2.系统初始化模块

[root@node1 ~]# mkdir /srv/salt/base/init/           #创建一个系统初始化的目录
[root@node1 ~]# mkdir /srv/salt/base/init/files/     #创建一个文件目录，存放一些初始化需要的文件
[root@node1 ~]# cd /srv/salt/base/
[root@node1 base]# tree
├── init
│   ├── files
└── top.sls
[root@node1 base]# cd init/
(1).配置dns
[root@node1 init]# cat dns.sls
/etc/resolv.conf:                              #这里是指定name，这里没有指定ID
  file.managed:                                #文件管理方法
    - source: salt://init/files/resolv.conf    #这个路径式相当与配置文件中/srv/salt/base/
    - user: root
    - group: root
    - mode : 644
[root@node1 init]# cat files/resolv.conf
nameserver 192.168.10.1

(2)历史命令显示时间

[root@node1 init]# cat history.sls
/etc/profile:
  file.append:                                    #文件追加的方法
    - text:
      - export HISTTIMEFORMAT="%F %T `whoami`"    #执行之后该语句会被追加到/etc/profile

(3).log日志记录谁在什么时间使用了什么命令

[root@node1 init]# cat audit.sls
/etc/bashrc:
  file.append:
    - text:
      - export PROMPT_COMMAND='{ msg=$(history 1|{ read x y;echo $y; });logger "[euid=$(whoami)]":$(who am i):[`pwd`]"$msg";}'
 

(4)内核调优

[root@node1 init]# cat sysctl.sls
vm.swappiness:
  sysctl.present:
    - value: 0
net.ipv4.ip_local_port_range:
  sysctl.present:
    - value: 10000 65000
fs.file-max:
  sysctl.present:
    - value: 100000

注：上面的路径
/proc/sys/net/ipv4/ip_local_port_range     #监听端口
/proc/sys/fs/file-max                      #打开最大文件数
/proc/sys/vm/swappiness                    #交换分区

(5)将上面的状态include到 env_init.sls

[root@node1 init]# cat env_init.sls
include:
  - init.dns
  - init.history
  - init.audit
  - init.sysctl

(6).编写top.sls执行以上状态

[root@node1 init]# vim /srv/salt/base/top.sls
[root@node1 init]# cat /srv/salt/base/top.sls
base:
  '*':
- init.env_init    #这里只需要执行init目录下的env_init.sls即可,

(7)注：以上环境中用到的一些命令

export HISTTIMEFORMAT="%F %T `whoami`"    #该命令是将%F %T `whoami`命令执行的结果赋给变量HISTTIMEFORMAT，用export将其变成环境变量
export PROMPT_COMMAND='{ msg=$(history 1|{ read x y;echo $y; });logger "[euid=$(whoami)]":$(who am i):[`pwd`]"$msg";}'
执行该命令之后会在log日志里记录用户使用命令的情况，如：
[root@node1 base]# uptime
 05:17:38 up  4:08,  4 users,  load average: 0.00, 0.00, 0.00
[root@node1 base]# tail -1 /var/log/messages
Aug 11 05:17:38 node1 root: [euid=root]:root pts/3 2017-08-11 04:07 (192.168.10.1):[/srv/salt/base]uptime

[root@node1 init]# salt "*" state.highstate test=True    #这里可以测试一下前面所配置的是否成功

3.功能模块-------基础包模块

[root@node1 ~]# mkdir /srv/salt/prod/pkg     #基础包目录
[root@node1 ~]# cd /srv/salt/prod/pkg/
[root@node1 pkg]# vim pkg-init.sls           #安装一些基础的包
[root@node1 pkg]# cat pkg-init.sls
pkg-init:                                    #这里就是ID号，唯一性
  pkg.installed:                             #安装包的模块.方法
    - names:                                 #安装包的名字
      - gcc
      - gcc-c++
      - glibc
      - make
      - autoconf
      - openssl
      - openssl-devel

4.功能模块--------haproxy模块

#安装haproxy说明：该模块采用编译安装,用模块安装之前先用一台主机测试一下
#cd /usr/local/src/
#tar xf haproxy-1.6.2.tar.gz
#cd haproxy-1.6.2
#make TARGET=linux26 PREFIX=/usr/local/haproxy &&make install PREFIX=/usr/local/haproxy
#vim haproxy.init 启动脚步路径修改
#BIN=/usr/local/haproxy/sbin/$BASENAME
#修改之后
#cp haproxy.init /srv/salt/prod/haproxy/files/

[root@node1 ~]# mkdir /srv/salt/prod/haproxy                                #haprox模块目录
[root@node1 ~]# mkdir /srv/salt/prod/haproxy/files                          #存放haprox的一些文件
[root@node1 ~]# ll /srv/salt/prod/haproxy/files/
-rw-r--r--. 1 root root 1538976 Aug 11  2017 haproxy-1.6.2.tar.gzhaproxy    #源码包
-rw-r--r--. 1 root root    2395 Aug 11 08:31 haproxy.inithaproxy            #启动脚本
[root@node1 ~]# cd /srv/salt/prod/haproxy/
[root@node1 haproxy]# vim install.sls
include:
  - pkg.pkg-init                                           #这是个相对目录，意思是调用/srv/salt/prod/目录下的pkg目录下的pkg-init.sls

haproxy-install:                                           #ID
  file.managed:
    - name: /usr/local/src/haproxy-1.6.2.tar.gz            #name 声明，没有ID可以把name声明放在ID位置
    - source: salt://haproxy/files/haproxy-1.6.2.tar.gz    #这里也是相对路径/srv/salt/prod/,源码包所在
    - user: root
    - group: root
    - mode: 755
  cmd.run:                                                 #将要执行的命令放在这个模块下
    - name: cd /usr/local/src/ && tar xf haproxy-1.6.2.tar.gz && cd haproxy-1.6.2 && make TARGET=linux26 PREFIX=/usr/local/haproxy &&make install PREFIX=/usr/local/haproxy
    - unless: test -d /usr/local/haproxy                   #/usr/local/haproxy 目录不存在才执行cmd命令
    - require:                                             #指定依赖
      - pkg: pkg-init                                      #依赖pkg-init这个ID的pkg模块,这个模块必须执行成功才执行本模块cmd.run
      - file: haproxy-install                              #依赖haproxy-install这个ID的file模块

haproxy-init:
  file.managed:
    - name: /etc/init.d/haproxy
    - source: salt://haproxy/files/haproxy.init
    - user: root
    - group: root
    - mode: 755
    - require:
      - cmd: haproxy-install
  cmd.run:
    - name: chkconfig --add haproxy
    - unless: chkconfig --list|grep haproxy
    - require:
      - file: haproxy-init

net.ipv4.ip_nonlocal_bind:            #/proc/sys/net/ipv4/ip_nonlocal_bind，这里只需要写相对路径
  sysctl.present:#sysctl              #模块，管理内核模块
    - value: 1                        #默认不让监听非本地ip，改为1后可以监听

haproxy-config-dir:                   #ID
  file.directory:
    - name: /etc/haproxy              #创建配置文件目录
    - user: root
    - group: root
- mode: 755

[root@node1 haproxy]# salt 'node1' state.sls haproxy.install env=prod     #手动测试一下

5.业务模块-------haproxy模块

[root@node1 ~]# mkdir /srv/salt/prod/cluster
[root@node1 ~]# mkdir /srv/salt/prod/cluster/files

[root@node1 ~]# cd /srv/salt/prod/cluster/files
[root@node1 files]# vim haproxy-outside.cfg        #负载均衡文件
global
maxconn 100000
chroot /usr/local/haproxy
uid 99
gid 99
daemon
nbproc 1
pidfile /usr/local/haproxy/logs/haproxy.pid
log 127.0.0.1 local3 info

defaults
option http-keep-alive
maxconn 100000
mode http
timeout connect 5000ms
timeout client 5000ms
timeout server 5000ms

listen stats
mode http
bind 0.0.0.0:8888
stats enable
stats uri /haproxy-status
stats auth haproxy:saltstack

frontend frontend_www_example_com
bind 192.168.10.150:80
mode http
option httplog
log global
    default_backend backend_www_example_com
backend backend_www_example_com
option forwardfor header X-REAL-IP
option httpchk HEAD / HTTP/1.0
balance roundrobin
server web-node1  192.168.10.129:8080 check inter 2000 rise 30 fall 15
server web-node2  192.168.10.128:8080 check inter 2000 rise 30 fall 15

[root@node1 cluster]# cd /srv/salt/prod/cluster/
[root@node1 cluster]# cat haproxy-outside.sls
include:
  - haproxy.install                                      #执行haproxy目录下的install.sls
haproxy-service:#ID
  file.managed:
    - name: /etc/haproxy/haproxy.cfg                      #安装之后配置文件的名字
    - source: salt://cluster/files/haproxy-outside.cfg    #源配置文件，前面已经写好
    - user: root
    - group: root
    - mode: 644
  service.running:                           #service模块下的running方法，作用：启动服务
    - name: haproxy                          #服务名字
- enable: True                               #是否开机启动
    - reload: True                           #是否reload，如果不加，配置文件变了会restart
    - require:
      - cmd: haproxy-init                    #依赖haproxy-init下的cmd，意思是启动脚本那步必须执行成功
    - watch:                                 #关注某个文件状态
      - file: haproxy-service                #关注haproxy-service ID下的file模块里的文件，文件改变会reload
 

 

6.编辑top.sls

[root@node1 base]# cd /srv/salt/base/
[root@node1 base]# vim top.sls
base:
  '*':
    - init.env_init                                      #所有主机都执行init目录下的env_init.sls
prod:
  'node1':
    - cluster.haproxy-outside                            #node1执行cluster目录下的haproxy-outside.sls
  'node2':
    - cluster.haproxy-outside

[root@node1 base]# salt '*' state.highstate test=True    #测试执行一下
[root@node1 prod]# salt '*' state.highstate

使用httpd测试一下

[root@node1 prod]# vim /etc/httpd/conf/httpd.conf
Listen 8080
[root@node1 prod]# cat /var/www/html/index.html
node1
[root@node1 prod]# /etc/init.d/httpd restart
[root@node2 ~]# vim /etc/httpd/conf/httpd.conf
[root@node2 prod]# cat /var/www/html/index.html
Node2

Listen 8080
[root@node2 ~]# /etc/init.d/httpd restart

在网页输入下面的地址登录：
192.168.10.129:8888/haproxy-status
192.168.10.128:8888/haproxy-status
用户haproxy密码saltstack

功能模块-----keepalived模块

写之前先找一台主机源码安装测试

http://www.keepalived.org/software/keepalived-1.2.19.tar.gz
[root@node1 tools]# tar xf keepalived-1.2.19.tar.gz
[root@node1 tools]# cd keepalived-1.2.19
[root@node1 keepalived-1.2.19]# ./configure --prefix=/usr/local/keepalived --disable-fwmark
[root@node1 keepalived-1.2.19]# make && make install
keepalived-1.2.19/keepalived/etc/init.d/keepalived.init        #启动脚本
keepalived-1.2.19/keepalived/etc/keepalived/keepalived.conf    #配置文件

配置keepalived模块路径及相关文件

[root@node1 ~]# mkdir /srv/salt/prod/keepalived
[root@node1 ~]# mkdir /srv/salt/prod/keepalived/files
[root@node1 keepalived]# cp ~/tools/keepalived-1.2.19.tar.gz /srv/salt/prod/keepalived/
files/
[root@node1 tools]#cp keepalived-1.2.19/keepalived/etc/init.d/keepalived.init /srv/salt/prod/keepalived/files/          #复制启动脚本
[root@node1 tools]#cp keepalived-1.2.19/keepalived/etc/keepalived/keepalived.conf /srv/salt/prod/keepalived/files/      #复制配置文件
[root@node1 tools]# cp keepalived-1.2.19/keepalived/etc/init.d/keepalived.sysconfig /srv/salt/prod/keepalived/files/
[root@node1 tools]# cd /srv/salt/prod/keepalived/files/
[root@node1 files]# vim keepalived.init                               #修改启动脚本路径
daemon /usr/local/keepalived/sbin/keepalived ${KEEPALIVED_OPTIONS}

1.keepalived功能模块

[root@node1 keepalived]# cd /srv/salt/prod/keepalived/
[root@node1 keepalived]# cat install.sls
include:
  - pkg.pkg-init
keepalived-install:
  file.managed:
    - name: /usr/local/src/keepalived-1.2.19.tar.gz
    - source: salt://keepalived/files/keepalived-1.2.19.tar.gz
    - user: root
    - group: root
    - mode: 755
  cmd.run:
    - name: cd /usr/local/src/ && tar xf keepalived-1.2.19.tar.gz && cd keepalived-1.2.19 && ./configure --prefix=/usr/local/keepalived --disable-fwmark && make &&make install
    - unless: test -d /usr/local/keepalived
    - require:
      - pkg: pkg-init
      - file: keepalived-install
keepalived-init:
  file.managed:
    - name: /etc/init.d/keepalived
    - source: salt://keepalived/files/keepalived.init
    - user: root
    - group: root
    - mode: 755
  cmd.run:
    - name: chkconfig --add keepalived
    - unless: chkconfig --list |grep keepalived
    - require:
      - file: keepalived-init
/etc/sysconfig/keepalived:
  file.managed:
    - source: salt://keepalived/files/keepalived.sysconfig
    - user: root
    - group: root
    - mode: 644
/etc/keepalived:
  file.directory:
    - user: root
    - group: root
    - mode: 755

[root@node1 files]# salt '*' state.sls keepalived.install env=prod     #手动测试一下

2.keepalived业务模块

[root@node1 ~]# cd /srv/salt/prod/cluster/files/
[root@node1 files]# cat haproxy-outside-keepalived.cfg     #keepalived配置文件,里面用到了jinja变量
#configutation file for keepalive
globlal_defs {
    notification_email {
      saltstack@example.com
}
    notification_email_from keepalived@example.com
    smtp_server 127.0.0.1
    smtp_connect_timeout 30
    router_id `ROUTEID`
}
vrrp_instance haproxy_ha {
state `STATEID`
interface eth2
    virtual_router_id 36
priority `PRIORITYID`
    advert_int 1
authentication {
auth_type PASS
    auth_pass 1111
  }
  virtual_ipaddress {
    192.168.10.130
  }
}

[root@node1 ~]# cd /srv/salt/prod/cluster/
[root@node1 cluster]# cat haproxy-outside-keepalived.sls
include:
 - keepalived.install

keepalived-service:
  file.managed:
    - name: /etc/keepalived/keepalived.conf
    - source: salt://cluster/files/haproxy-outside-keepalived.cfg
    - user: root
    - group: root
    - mode: 644
    - template: jinja
    {% if grains['fqdn'] == 'node1' %}
    - ROUTEID: haproxy_ha
    - STATEID: MASTER
    - PRIORITYID: 150
    {% elif grains['fqdn'] == 'node2' %}
    - ROUTEID: haproxy_ha
    - STATEID: BACKUP
    - PRIORITYID: 100
    {% endif %}
  service.running:
    - name: keepalived
    - enable: True
    - watch:
      - file: keepalived-service

[root@node1 cluster]# salt '*' state.sls cluster.haproxy-outside-keepalived env=prod  #测试一下

指定服务器执行keepalived模块

[root@node1 salt]# cat /srv/salt/base/top.sls
base:
  '*':
    - init.env_init
prod:
  'node1':
    - cluster.haproxy-outside
    - cluster.haproxy-outside-keepalived
  'node2':
    - cluster.haproxy-outside
    - cluster.haproxy-outside-keepalived

[root@node1 salt]# salt '*' state.highstate    #到这步执行成功的话就实现了keepalived+haproxy

遇到问题：发现keepalived 虚拟vip写不上去

查看日志 cat /var/log/messages，发现下面一句

Aug 11 15:10:12 node1 Keepalived_vrrp[29442]: VRRP_Instance(haproxy_ha{) sending 0 priority

haproxy_ha后面打了个空格解决

vrrp_instance haproxy_ha {

系统初始化模块--------------zabbix-agent

在配置文件里设置pillar路径

[root@node1 init]# vim /etc/salt/master
pillar_roots:
  base:
- /srv/pillar/base
[root@node1 init]# /etc/init.d/salt-master restart

在pillar里建立top.sls和zabbix.sls

[root@node1 init]# mkdir /srv/pillar/base
[root@node1 pillar]# cd base/
[root@node1 base]# cat top.sls
base:
  '*':
- zabbix

[root@node1 base]# cat zabbix.sls
zabbix-agent:
  Zabbix_Server: 192.168.10.129

[root@node1 init]# cd /srv/salt/base/init/
[root@node1 init]# cat zabbix_agent.sls
zabbix-agent-install:
  pkg.installed:
    - name: zabbix-agent
  file.managed:
    - name: /etc/zabbix/zabbix_agentd.conf
    - source: salt://init/files/zabbix_agentd.conf
    - template: jinja
    - defaults:
      Server: {{ pillar['zabbix-agent']['Zabbix_Server'] }}  #这里将pillar里ID为zabbix-agent，Zabbix_Server的值赋给变量Server
    - require:
      - pkg: zabbix-agent-install
  service.running:
    - name: zabbix-agent
    - enable: True
    - watch:
      - pkg: zabbix-agent-install
      - file: zabbix-agent-install

编写配置文件利用jinja将Server变量的值传给Server，也就是指定zabbix-Server地址

cp /etc/zabbix/zabbix_agentd.conf /srv/salt/base/init/files/
[root@node1 base]#vim /srv/salt/base/init/files/zabbix_agent.conf
Server=`Server`

将zabbix_agent.sls include到env_init.sls

[root@node1 init]# cat env_init.sls
include:
  - init.dns
  - init.history
  - init.audit
  - init.sysctl
  - init.zabbix_agent

[root@node1 init]# salt '*' state.highstate