shell脚本选择LOG里面特定的行，生成新文件并rsync上传

rsync.sh

#!/bin/bash

tool_path=$(cd `dirname $`; pwd)
eval `cat ${tool_path}/conf.properties`

rsync_include=${tool_path}/include/rsync_include.lst

export RSYNC_PASSWORD=密码

echo `date '+%Y-%m-%d %T'` >> ${tool_path}/rsync.log
for((i=;i<;i++));
do
   dstDate=`date -d "-$i day" +"%Y%m%d"`
   localPath=${tool_path}"/tlog/"$dstDate"/"
  echo $localPath
   if [[ -d "$localPath" ]] ; then
      startTime=`date -d "-$i day" +"%s"`
      echo "start rsync file for $localPath ---------------------" >> ${tool_path}/rsync.log
      rsync -rvzPL --append --port  --include-from=${rsync_include} ${localPath} gameserverlog@rsync2.com::game_server_log_rsync/${game_id}/$dstDate
      endTime=`date -d "-$i day" +"%s"`
      echo "finish rsync file for $localPath, used time: $[((endTime - startTime))] seconds --------------------" >> ${tool_path}/rsync.log
     fi
done

killrsync.sh

#!/bin/bash
tool_path=$(cd `dirname $`; pwd)
eval `cat ${tool_path}/conf.properties`

source ~/.bash_profile
SYSTEM_TIME=`date '+%Y-%m-%d %T'`

pid=`ps -e -o 'pid,args' |grep ${tool_path}/include|awk '{print $1}'`
echo "$SYSTEM_TIME kill rsync $pid" >> ${tool_path}/rsync.log

ps -e -o 'pid,args' |grep ${tool_path}/include|grep -v grep|awk '{print $1}'|xargs kill -

配置文件conf.properties

log_root=/data/log
file_name_prefix=aa
file_name_suffix=.log
game_id=games2

start.sh

#!/bin/bash
source /etc/profile

tool_path=$(cd `dirname $`; pwd)
eval `cat ${tool_path}/conf.properties`
echo ${tool_path}
echo ${log_root} 

###
#yyyy-mm-dd
date_today=`date +"%Y-%m-%d"`
date_yesterday=`date -d "yesterday" +%Y-%m-%d`

#yyyymmdd
date_today_hadoop=`date +"%Y%m%d"`
date_yesterday_hadoop=`date -d "yesterday" +%Y%m%d`

#yyyymd
file_today=${date_today//-0/-}
file_yesterday=${date_yesterday//-0/-}

### kill rsync
${tool_path}/killrsync.sh

###
destFile_yesterday="${tool_path}/tlog/${date_yesterday_hadoop}"
if [[ ! -d ${destFile_yesterday} ]]; then
    mkdir -p ${destFile_yesterday}
fi

destFile_today="${tool_path}/tlog/${date_today_hadoop}"
if [[ ! -d ${destFile_today} ]]; then
    mkdir -p ${destFile_today}
fi

###
cd ${log_root}
list_alldir(){
    for file2 in `ls -a $`
    do
        if [ x"$file2" != x"." -a x"$file2" != x".." ]; then
            if [ -d "$1/$file2" ];then
                echo "$1/$file2"
                cd $/${file2}
                folder_name=${//${log_root}/}
                cat $/${file2}/*.${file_yesterday} | grep '20..-.*LOG:.*|.*|.*|.*' | awk '{$1="";$2="";$3="";$4="";sub("    LOG:", "");print}' > ${destFile_yesterday}/tlog_${date_yesterday_hadoop}_${folder_name//\//_}_${file2}.log
                cat $1/${file2}/*.${file_today}     | grep '20..-.*LOG:.*|.*|.*|.*' | awk '{$1="";$2="";$3="";$4="";sub("    LOG:", "");print}' > ${destFile_today}/tlog_${date_today_hadoop}_${folder_name//\//_}_${file2}.log

                list_alldir "$1/${file2}"
            fi
        fi
    done
}  

list_alldir ${log_root}  

cd ${destFile_yesterday}
find . -name "*" -type f -size 0c | xargs -n 1 rm -f
cd ${destFile_today}
find . -name "*" -type f -size 0c | xargs -n 1 rm -f

### start rsync
flock -xn ${tool_path}/rsync.lock -c "${tool_path}/rsync.sh"

创建软连接

#!/bin/bash
source /etc/profile
cd $(dirname $)
tool_path=`pwd`
eval `cat ${tool_path}/conf.properties`
root="$log_root"
for file in $root/*;
do
   fileName=${file##*/}
   datestr=`echo ${fileName} |cut -d '_' -f2`
   destFile="${tool_path}/tlog/${datestr}"
   if [[ ! -d ${destFile} ]]; then
      mkdir -p ${destFile}
   fi

   if [[ ! -L "${destFile}/${fileName}" ]]; then
     ln -s ${file} ${destFile}
   fi
done