500万url的es 批删除

bash  循环 算术计算 读写文件

[root@hadoop2 ~]# sh looh.sh
1
2
3
4
LIZ
1 2 3 4
0
1
2
3
4
5
6
7
8
9
10
0
games:x:12:100:games:/usr/games:/sbin/nologin
12
ok
0
rpc:x:32:32:Rpcbind
24
ok
0
saslauth:x:499:76:Saslauthd
36
ok
0
OProfile:/home/oprofile:/sbin/nologin
48
ok
58

[root@hadoop2 ~]# cat looh.sh
for n in 1 2 3 4
do
echo $n
done

echo 'LIZ'

for n in "1 2 3 4"
do
echo $n
done

var0=0
limit=10

while [ "$var0" -le "$limit" ]
do
echo " $var0 "

var0=$((var0+1))
done

loop_c=0
loop_step=12
loop_tag=0
for LINE in `cat /etc/passwd`
do
loop_c=$((loop_c+1))
loop_tag=$((loop_c%loop_step))
if [ $loop_tag -eq 0 ]
then
echo $loop_tag
echo $LINE
echo $loop_c
echo 'ok'
else
continue
fi
done

echo $loop_c

echo
exit 0

[root@hadoop2 ~]#

loop_c=0
loop_step=10000
loop_tag=0
str_head='{"delete":{"_index":"direct_vote","_type":"kwaddress","_id":"'
str_foot='"}}'

for LINE in `cat /etc/passwd`
do
loop_c=$((loop_c+1))
loop_tag=$((loop_c%loop_step))
echo ${str_head}$LINE${str_foot} >> bash.url.txt
echo $loop_tag
echo $loop_c
echo ${str_head}$LINE${str_foot}
if [ $loop_tag -eq 0 ]
then
echo '' > bash.url.txt
#curl www.baidu.com
else
continue
fi
done

echo $loop_c

echo
exit 0

【执行计划】下午计划在线上A服务器执行该脚本（执行前可能会局部修改），对es中3100万URL中的490万URL进行删除，涉及文件为该脚本和490万URL的单个文本。

[root@hadoop2 ~]# cat  looh.sh
loop_c=0
loop_step=10001
loop_tag=0
str_head='{"delete":{"_index":"direct_vote","_type":"kwaddress","_id":"'
str_foot='"}}'
bulk_file=bulk.del.es.json
log_file=bulk.del.es.json.log

echo '' >  $bulk_file
for LINE in `cat /root/chk_url_404_pure_url_small_150000.txt`
do
 loop_c=$((loop_c+1))
 loop_tag=$((loop_c%loop_step))
 echo ${str_head}$LINE${str_foot} >> $bulk_file
 echo $loop_c
 if [ $loop_tag -eq 0 ]
  then
   echo $loop_c
   curl -XPOST hadoop2:9200/_bulk --data-binary @$bulk_file >> $log_file
   sleep 90s
   echo '' >  $bulk_file
 else
   continue
 fi
done

echo $loop_c

curl -XPOST hadoop2:9200/_bulk --data-binary @$bulk_file >> $log_file

echo
exit 0

[root@hadoop2 ~]#