用shell实现linux系统应用文件清理工具

用shell实现linux系统文件清理工具

1：原始需求

在系统运维中，会产生大量应用备份文件、落地文件等，这些文件需要定时清理。一般来说，都是使用crontab 拉起一个脚本来清理。类似这样：

30 0,12 * * * find /data02/rating_bak/loading -type f -name "gprs_ol*gz" -atime +1 -exec rm {} \;

但是如果机器很多，目录多，这种方式非常不利于管理，而且也没有一个清晰的配置，来知道什么机器上哪个目录的文件留存时间。这个工具用于在数据库中存一个配置，然后在每台机器上，配置一个定时脚本执行即可。

实现的基本方式为：用一个shell从数据库中读取配置，根据文件的最后修改时间，大于X的mv到带日期的子目录中，大于Y的压缩，大于Z的删除。用crontab拉起这个脚本。

2：数据库表配置

create table WH_BAKFILE_CFG

(

BAKID   NUMBER,    --主键ID

IP      VARCHAR2(30),  --IP地址

SCANPWD VARCHAR2(300), --扫描目录

BAKPWD  VARCHAR2(300),  --备份目录

FILECH  VARCHAR2(100),  --文件特征

MVTIME  NUMBER,    --文件移动时间 单位分

ZIPTIME NUMBER,      --文件压缩时间 单位分

DELTIME number,     --文件删除时间 单位分

Needcheck  number  --是否需要校验文件被占用 1需要校验

)

配置举例：

BAKID	IP	SCANPWD	BAKPWD	FILECH	MVTIME	ZIPTIME	DELTIME	NEEDCHECK
91	172.20.31.24	/data02/rating_bak/upcheck/migrate_sd		*	1440	2880	46080
10	172.20.31.24	/app/billapp/user/wxk/mybak		*	1440	2880	8640
11	172.20.31.24	/data02/rating_bak/upfile/d		*	1440	5760	46080
13	172.20.31.24	/data02/rating_bak/upfile/q		*	1440	5760	46080
14	172.20.31.24	/data02/rating_bak/upfile/vpmn		*	1440	5760	46080

3：相关代码

A：读取配置，拉起处理shell。

从数据库中获取配置，写入${IP}.cfg，生成$(bakid).lock，避免处理shell被多次拉起，然后拉起处理shell。

#!/bin/sh

CONNSTR="conn dz/dz_xxxxx@nbilldb"

LOGFILE=./getbakinfo.log

##得到本机IP

IP=""

getip()

{

if [[ x"${IP}" = x ]];then

IP=`/sbin/ifconfig   |grep "inet addr"| cut -f 2 -d ":" |sed -n '1p' |awk '{print $1}'`

fi

}

##记录日志

writelog()

{

echo  `date +%Y-%m-%d" "%X` "$1" >>${LOGFILE}

}

##SQL

selectsql()

{

sqlplus -S /nolog   <<EOF

set heading off feedback off pagesize 0 verify off echo off linesize 3000

${CONNSTR}

${1};

commit;

exit

EOF

}

MYWORKSHELL=$0

if [[ ${MYWORKSHELL} = "getbakinfo.sh" ]] ; then

MYWORKPATH=.

else

MYWORKPATH=${MYWORKSHELL%\/*}

fi

cd $MYWORKPATH

##判断连接

sql="select  'HELLOSMK' from dual"

selectresult=`selectsql "${sql}"`

if [[ ! "${selectresult}" = "HELLOSMK" ]] ; then

writelog "error  连接数据库失败!:${selectresult}"

else

##写配置文件

getip

rm ${IP}.cfg

sql="select trim(bakid)||'##'||trim(t.scanpwd)||'##'||trim(t.bakpwd)||'##'||trim(t.filech)||'##'||t.mvtime||'##'||t.ziptime||'##'||t.deltime||'##'||nvl(t.needcheck,0) \

from wh_bakfile_cfg t  where t.ip ='${IP}' "

#selectsql "${sql}"

#IFS=$'\x0A'

selectresult=`selectsql "${sql}"`

##逐条处理

for result in ${selectresult}

do

bakid=`echo  ${result}  |awk -F "##" '{print $1}'`

scanpwd=`echo ${result}  |awk -F "##" '{print $2}'`

bakpwd=`echo  ${result}  |awk -F "##" '{print $3}'`

filech=`echo  ${result}  |awk -F "##" '{print $4}'`

mvtime=`echo  ${result}  |awk -F "##" '{print $5}'`

ziptime=`echo  ${result}  |awk -F "##" '{print $6}'`

deltime=`echo  ${result}  |awk -F "##" '{print $7}'`

needcheck=`echo  ${result}  |awk -F "##" '{print $8}'`

echo ${bakid},${scanpwd},${bakpwd},${filech},${mvtime},${ziptime},${deltime},${needcheck}>>${IP}.cfg

done

fi

cat ${IP}.cfg  |while read filestr

do

bakid=`echo  ${filestr}  |awk -F "," '{print $1}'`

scanpwd=`echo ${filestr}  |awk -F "," '{print $2}'`

bakpwd=`echo  ${filestr}  |awk -F "," '{print $3}'`

filech=`echo  ${filestr}  |awk -F "," '{print $4}'`

mvtime=`echo  ${filestr}  |awk -F "," '{print $5}'`

ziptime=`echo  ${filestr}  |awk -F "," '{print $6}'`

deltime=`echo  ${filestr}  |awk -F "," '{print $7}'`

needcheck=`echo  ${filestr}  |awk -F "," '{print $8}'`

##目录检查

if [ -d "${scanpwd}" ] ; then

##锁检查

if [ ! -f "${bakid}.lock" ]; then

touch "${bakid}.lock"

nohup ${MYWORKPATH}/whbakfile.sh "${bakid}" "${scanpwd}" "${bakpwd}" "${filech}" "${mvtime}" "${ziptime}" "${needcheck}" "${MYWORKPATH}" "${deltime}" >> ${LOGFILE} 2>&1 &

fi

else

writelog "error目录不存在!:${scanpwd}"

fi

done

B:文件处理shell

#!/bin/sh

ddtime=`date +%Y%m%d`

LOGFILE=$HOME/log/bakfile/bakfile${ddtime}.log

bakid=$1

scanpwd=$2

bakpwd=$3

filech=$4

mvtime=$5

ziptime=$6

needcheck=$7

MYWORKPATH=$8

deltime=$9

cd $MYWORKPATH

##记录日志

writelog()

{

echo  `date +%Y-%m-%d\ %H:%M:%S` "$1" >>${LOGFILE}

}

cd $MYWORKPATH

if [[ x"${bakpwd}" = x ]];then

bakpwd=${scanpwd}

fi

if [[ x"${deltime}" = x ]];then

deltime=9999999

fi

if [[ x"${ziptime}" = x ]];then

ziptime=9999999

fi

if [[ x"${filech}" = x ]];then

filech="*"

fi

###mv

find ${scanpwd} -maxdepth 1 -type f -name "${filech}" -mmin +${mvtime} |while read filename

do

filetime=`ls -l ${filename} --time-style '+%Y%m%d'|awk '{print $6}'`

if [ ! -d "${bakpwd}/${filetime}" ]; then

mkdir -p ${bakpwd}/${filetime}

fi

##check

checkstat=0

if [[ ${needcheck} -eq 1 ]] ; then

checkstat=`lsof|grep filename |wc -l`

fi

if [[ ${checkstat} -eq 0 ]] ; then

mv ${filename} ${bakpwd}/${filetime}

writelog "mv ${filename} ${bakpwd}/${filetime}"

fi

done

###gzip

if [[ ! ${ziptime} -eq 9999999 ]] ; then

find ${bakpwd} -type f -name "${filech}"  -mmin +${ziptime}|grep -v .gz|grep ${bakpwd}/20|while read filename

do

gzip ${filename}

writelog "gzip ${filename}"

done

fi

###rm

if [[ ! ${deltime} -eq 9999999 ]] ; then

find ${bakpwd} -type f -name "${filech}"  -mmin +${deltime}|grep ${bakpwd}/20|while read filename

do

rm ${filename}

writelog "rm ${filename}"

done

fi

##rmdir

if [[ ! ${deltime} -eq 9999999 ]] ; then

find ${bakpwd} -type d  |grep ${bakpwd}/20|while read pathname

do

pathnum=`ls ${pathname} | wc -l`

if [[  ${pathnum} -eq 0 ]] ; then

rmdir ${pathname}

writelog "rmdir ${pathname}"

fi

done

fi

rm ${bakid}.lock