[分享]ip地址爬取过滤的shell

http://www.hbbzy.me/分享ip地址爬取过滤的shell

#!/bin/base
#ip zhi地址匹配
#获取最新的ip地址
#author:haifeng
#wget ftp://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest
#ftp://ftp.ripe.net/ripe/stats/delegated-ripencc-latest
#ftp://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-latest
#ftp://ftp.apnic.net/pub/stats/apnic/delegated-apnic-latest
#ftp://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-latest

#改变字段分割符
IFSOLD=$IFS
IFS=","
#文件数组
FILES=(delegated-arin-extended-latest,delegated-ripencc-latest,delegated-afrinic-latest,delegated-apnic-latest,delegated-lacnic-latest)

#大洋洲 国家(地区)简称数组
oceania=(AU,CC,CX,FJ,KI,MH,FM,NR,NZ,PW,PG,WS,TO,TV,VU)
#亚洲除中国大陆 香港 澳门 台湾之外的国家(地区)简称数组
asia=(AF,AS,BH,BD,BT,BN,KH,CN,CK,IO,IN,ID,IR,IQ,IL,JP,JO,KZ,KP,KR,KW,KG,LA,LB,MY,MV,MM,MN,MP,NP,NC,NF,NU,OM,PK,PF,PH,QA,RU,SB,SA,SG,LK,TK,TJ,TH,TL,TR,TM,AE,UZ,VN,WF,YE)
#中国大陆
cn=(CN)
#香港 澳门 台湾
hk=(HK,MO,TW)
#欧盟区域
eu=(AL,AD,AM,AX,AT,AZ,BY,BE,BL,BA,BQ,BG,BV,CW,HR,EH,CY,CZ,DK,EE,FK,FO,FI,FR,GG,GI,GE,DE,GL,GR,GS,IM,HU,IS,IE,JE,IT,LV,LI,LT,LU,MK,MT,MD,MC,ME,MQ,NL,NO,PL,PT,PN,RO,PS,SH,SJ,SM,RS,SX,SK,SI,SY,ES,TF,SE,CH,VA,UA,UK,GB)
#美洲区域
namerica=(AI,AG,BS,BB,BZ,BM,CA,CR,CU,DM,DO,SV,GP,GD,GU,GT,HT,HN,JM,KY,MF,MX,MS,NI,PA,PM,PR,KN,LC,TC,UM,TT,US,VC,VG,VI)
#南美
samerica=(AW,AR,BO,BR,CL,CO,EC,GF,GY,PY,PE,SR,UY,VE)
#非洲
africa=(DZ,AO,BJ,BI,BW,BF,CM,CV,CF,TD,CI,KM,CG,CD,DJ,EG,GQ,ER,ET,GA,GM,GH,GN,GW,HM,KE,LS,LR,LY,MG,MW,ML,MR,MU,MA,MZ,NA,NE,NG,RE,RW,ST,SN,SC,SL,SO,ZA,SS,SD,SZ,TZ,TG,TN,UG,YT,ZM,ZW)
#创建文件保存目录
if [ ! -d "db" ];then
mkdir db
fi
#遍历文件
for FILE in ${FILES[@]}
do
num=1;

echo create $FILE;
echo create $FILE>>db/other.txt
#找到匹配的ipv行,获取国家(地区)，协议,ip，ip数
grep 'ipv' $FILE | cut -f 2,3,4,5 -d '|'|sed -e 's/|/,/g' | while read country agr ip cnt
do
num=$(($num+1));
echo "ip num:" $num;
if [ "${agr}" == "ipv4" ];then
#掩码
mask=`cat << EOF | bc | tail -1
pow=32;
define log2(x){
if (x<=1) return (pow);
pow--;
return(log2(x/2));
}
log2($cnt)
EOF`
elif [ "${agr}" == "ipv6" ];then
mask=$cnt;
fi
if [ -n "$country" ];then
if [[ "$cn[@]" =~ $country ]] ; then
echo $ip/$mask>>db/cn.txt
elif [[ "$oceania[@]" =~ $country ]] ; then
echo $ip/$mask>>db/oceania.txt
elif [[ "$asia[@]" =~ $country ]] ; then
echo $ip/$mask>>db/asia.txt
elif [[ "$hk[@]" =~ $country ]] ; then
echo $ip/$mask>>db/hk.txt
elif [[ "$eu[@]" =~ $country ]] ; then
echo $ip/$mask>>db/eu.txt
elif [[ "$namerica[@]" =~ $country ]] ; then
echo $ip/$mask>>db/namerica.txt
elif [[ "$samerica[@]" =~ $country ]] ; then
echo $ip/$mask>>db/samerica.txt
elif [[ "$africa[@]" =~ $country ]] ; then
echo $ip/$mask>>db/africa.txt
else
echo $country,$ip/$mask>>db/other.txt
fi
fi
echo $FILE,$country,$ip/$mask >> db/all.txt
done
done

#还原字段分割符
IFS=$IFSOLD