AWK Demo

　　AWK is so poweful. AWK功能太强大了:

　　　　@1:处理格式化文本(感觉主要还是以这个为主)
　　　　@2:对文本逐行进行处理(过滤某些行,保留符合条件的)
　　　　@3:将文件根据不同的条件分成多个文件
　　　　@4:统计文件中的符合某些条件的文本行数

　　　　......

Demo1:

#!/bin/bash
#File: awkScript1.sh
#Author: lxw
#Time: --
#Usage: 'awk' demo.

main(){
    #$: The whole line.    $: The first column.    $: The second column.    ...
    echo -e "\nDEMO1:-----------------------------------------------------"
    head -n  ./demo.txt|awk 'BEGIN{FS="|"}{print $0}'

    echo -e "\nDEMO2: PRINTF FORMAT---------------------------------------"
    head -n  ./demo.txt|tail -n |awk 'BEGIN{FS="|"}{printf "%-8s%-5s%-10s\n", $1, $2, $4}'

    echo -e "\nDEMO3: FILTER-----------------------------------------------"
    head -n  ./demo.txt|tail -n |awk 'BEGIN{FS="|"} $2=="CN" && $3=="ipv4" {printf "%-8s%-5s%-10s\n", $1, $2, $4}'

    echo -e "\nDEMO4: FS-NR-NF-FILENAME------------------------------------"
    head -n  ./demo.txt|tail -n |awk 'BEGIN{FS="|"} $2=="CN" || NR==1 {printf "%-10s%-03d%-3d%-8s%-5s%-10s\n", FILENAME, NR, NF, $1, $2, $4}'

    echo -e "\nDEMO5: OFS--------------------------------------------------"
    head -n  ./demo.txt|tail -n |awk 'BEGIN{FS="|"} $2=="CN" || NR==1 {print FILENAME, NR, NF, $1, $2, $4}' OFS="\t"

    #String match
    #@:'~' means to match.    @:The string between // is the RE.
    echo -e "\nDEMO6: String Match------------------------------------------"
    head -n  ./demo.txt|tail -n |awk 'BEGIN{FS="|"} $3 ~ /ip.4/ || NR==1 {print NR, $1, $2, $3, $4}' OFS="\t"

    echo -e "\nDEMO7: The line which contains the specific string.----------"
    head -n  ./demo.txt|tail -n |awk '/.*12.*|.*14.*/'

    echo -e "\nDEMO8: The field which DOES NOT contain the specific string.--"
    head -n  ./demo.txt|tail -n |awk 'BEGIN{FS="|"} $6 !~ /.*0412.*|.*0414.*/'

    #Split the file.
    echo -e "\nDEMO9: Split the file.----------------------------------------"
    head -n  ./demo.txt|tail -n |awk 'BEGIN{FS="|"} $2 != "" {print > $2}'
    head -n  ./demo.txt|tail -n |awk 'BEGIN{FS="|"} $2 != "" {print NR, $4 > $2"-1"}'
    head -n  ./demo.txt|tail -n |awk 'BEGIN{FS="|"} {if($2 ~ /CN/) print > "cn.txt"; else if($2 ~ /TH/) print > "th.txt"; else print > "others.txt";}'

    #Count.
    echo -e "\nDEMO10: Count.------------------------------------------------"
    awk 'BEGIN{FS="|"} $2 != "" {arr[$2]++}; END{for(i in arr) print i, ", ", arr[i];}' ./demo.txt

    #DON'T MIND TOO MUCH about the following 4 cases.
    #The "assignment WITH -v" can be accessed in BEGIN & PROCESS & END.
    echo -e "\nDEMO11: \"-v\" ----------------------------------------"
    awk -v a= 'BEGIN{print "BEGIN:" a} NR==1 {print "PROCESS:" a}END{print "END:" a}' a= ./demo.txt
    #Case that "assignment WITHOU -v" only accessed in PROCESS & END.
    echo -e "\nDEMO12 ------------------------------------------------"
    awk 'BEGIN{print "BEGIN:" a} NR==1 {print "PROCESS:" a}END{print "END:" a}' a= ./demo.txt
    #Case that "assignment WITHOU -v" only accessed in END.
    echo -e "\nDEMO13 ------------------------------------------------"
    awk 'BEGIN{print "BEGIN:" a} NR==1 {print "PROCESS:" a}END{print "END:" a}' ./demo.txt a=
    echo -e "\nDEMO14 ------------------------------------------------"
    awk 'BEGIN{print "BEGIN:" a}a < 15{print "PROCESS:" a; ++a}END{print "END:" a}' a= ./demo.txt
}

main

　　关于赋值操作有以下几点说明:

　　@1:awk中变量值只有两种形式：字符串和数值

　　@2:如果变量赋值在第一个文件参数之前，在BEGIN动作之后执行，影响到正常处理和END动作；

　　@3:如果变量赋值在最后一个文件参数之后，在END动作之前执行，仅影响END动作；

　　@4:如果文件参数不存在，情况同1所述；

　　@5:如果变量赋值位于多个文件参数之间，在变量赋值前面的文件被处理后执行，影响到后续文件的处理和END动作；

Demo2:

　　几点说明:

　　@1:函数的参数默认是局部变量，无法在函数之外访问，而在函数中定义的变量为全局变量，可以在函数之外访问

　　@2:next语句会读取下一条记录，并重新回到脚本的最开始处执行(read next input record and process)

#!/bin/bash
#File: awkScript2.sh
#Author: lxw
#Time: --
#Usage: More about awk.

#Function
cat ./demo.txt|awk '
function func(a){
    b=a;
    print a;
}
{
    if(NR==){
        print b;        #nothing
        func("lxw");    #"lxw"
        print b;        #"lxw"
    }
}'

#Pattern
#The second one is more simple.
#seq  |awk '$0 ~ /3/{print $0}'
seq  |awk '/3/{print}'

seq  |awk '!/3/{print}'
seq  |awk '/1/ && /3/{print}'

#Array
echo "1 2 3"|awk '
{
    for(i=; i<NF; ++i)
        a[i]=i;
}
END{
if( in a)
    print "yes"
else
    print "no"
print  in a
for(i in a)
    printf "%s: %s\n", i, a[i];
}'

#next
#<echo -e ""> equals to <echo $''>
#echo -e "line1\nline2"|awk '{
echo $'line1\nline2'|awk '{
print "Before next.."
print $
next
print "After next.."
}'

#comma in print.
#The output of the following  lines are DIFFERENT.
echo $'1_2_3'|awk 'BEGIN{FS="_";OFS="!"}{print $1, $2, $3}' #comma is important
echo $'1_2_3'|awk 'BEGIN{FS="_";OFS="!"}{print $1 $2 $3}'    #OFS is NOT USEFUL here.

#Command
#NOTE that, in "sort -n", " is ESSENTIAL.
echo $'1\n3\n2\n9\n5'|awk '{print|"sort -n;date"}'

#String Operations.
#sub(ere, repl[,target]:ere->pattern; repl->new string; target->default value is $. The return value of the sub() is the times that replacement happens.
echo "hello, world!"|awk '{print sub(/ello/, "i"); print $0}'
#The following awk has no arguments to deal with(has no target to deal with by awk). BUT it's OK. BUT BEGIN is ESSENTIAL if there is no arguments for awk.
#NOTE & means the value that matches.
awk 'BEGIN{var="lxw"; sub(/lxw/, "hello, &", var); print var}'
#gsub: gsub is similar to sub, "g" means "global".
#index: the return value is counted from  not .
awk 'BEGIN{print index("lxw", "l")}'
awk 'BEGIN{print index("lxw", "o")}'
#length:
awk 'BEGIN{print length("lxw")}'
#The following  lines are equal.
echo "lxw"|awk '{print length()}'
echo "lxw"|awk '{print length($0)}'
#match:return value is the index that matched. and the index is counted from  not .
awk 'BEGIN{print match("lxw", /xw/);printf "Matched at: %d, Matched substr length: %d\n", RSTART, RLENGTH;}'
#split:split(s, a, fs)->split s by fs into a. The return value is the number of pieces.
awk 'BEGIN{
split("1;2;3;4;5", arr, ";");
for(i in arr)
    printf "arr[%d]=%d\n", i, arr[i];
}'

awk 'BEGIN{
n=split("1;2;3;4;5", arr, ";");
for(i=; i<=n; ++i)
    printf "arr[%d]=%d\n", i, arr[i];
}'
#sprintf: similar to printf, but won't output the result, instead return it.
awk 'BEGIN{var=sprintf("%s=%s", "hello", "lxw");print var;}'
#substr(s, m[,n]):the index is counted from  not .
awk 'BEGIN{print substr("hello", 2, 3)}'
#tolower(s),toupper(s)
awk 'BEGIN{print toupper("lxw"); print tolower("LXW")}'

#IO.
#getline:
#NOTE: " in "head -n  ./demo.txt" is ESSENTIAL.
awk 'BEGIN{while("head -n 10 ./demo.txt"|getline var) print var;}'
#The following  lines are equal.
head -n  ./demo.txt|awk '{while(getline) print NF, NR, FNR, $0;}'
head -n  ./demo.txt|awk '{while(getline $0) print NF, NR, FNR, $0;}'
#system:execute the outer command.
awk 'BEGIN{system("tail -n 10 /etc/passwd")}'

demo.txt内容如下(非apnic原始文件,其中的数据已经进行了随机的修改):

apnic|CN|ipv4|233.223.176.0|||allocated|A8294605
apnic|AD|ipv4|233.223.192.0|||allocated|A9237638
apnic|JP|ipv4|233.223.208.0|||allocated|A12B1B09
apnic|AK|ipv4|233.223.216.0|||allocated|A92AEBC2
apnic|BG|ipv4|233.223.220.0|||allocated|A9498FAD
apnic|CP|ipv4|233.223.224.0|||allocated|A914F5F3
apnic|IN|ipv4|233.224.0.0|||allocated|A1199197
apnic|AD|ipv4|233.240.0.0|||allocated|A2248097
apnic|US|ipv4|233.248.0.0|||allocated|A32869B9
apnic|AU|ipv4|223.252.0.0|||allocated|A91376C8
apnic|AB|ipv4|223.252.128.0|||allocated|A92A14F7
apnic|KR|ipv4|223.253.0.0|||allocated|A925D732
apnic|CT|ipv4|223.254.0.0|||allocated|A92FE265
apnic|CE|ipv4|223.255.0.0|||allocated|A91FEB6F
apnic|CK|ipv4|223.255.128.0|||allocated|A2132B1E
apnic|AR|ipv4|223.255.192.0|||allocated|A9239F52
apnic|ID|ipv4|223.255.224.0|||allocated|A12ED542
apnic|EU|ipv4|223.255.232.0|||allocated|A9290386
apnic|AN|ipv4|223.255.236.0|||allocated|A329E12C
apnic|HK|ipv4|223.255.240.0|||allocated|A95846F6
apnic|IN|ipv4|223.255.244.0|||allocated|A22523BF
apnic||ipv4|223.255.248.0|||reserved|
apnic|DN|ipv4|223.255.252.0|||allocated|A42E1062
apnic|SG|ipv4|223.255.254.0|||assigned|A93B8C6C
apnic|EU|ipv4|223.255.255.0|||assigned|A3173591
apnic|CP|ipv6|:::|||allocated|A917B6AA
apnic|DP|ipv6|::::|||allocated|A716B6AA
apnic|AF|ipv6|::::|||allocated|A816B6AA
apnic|AP|ipv6|::::|||allocated|A913B6AA
apnic||ipv6|:::|||available|
apnic||ipv6|:::|||available|
apnic||ipv6|:::|||available|
apnic|BG|ipv6|:::|||allocated|A519DB08
apnic|SG|ipv6|::::|||allocated|A919DB58
apnic|BG|ipv6|::::|||allocated|A919DB32
apnic|UN|ipv6|::::|||allocated|A919DB38
apnic||ipv6|:::|||available|
apnic||ipv6|:20a::|||available|
apnic||ipv6|:20c::|||available|
apnic||ipv6|:::|||reserved|
apnic||ipv6|::::|||available|
apnic||ipv6|::::|||available|
apnic||ipv6|::::|||available|
apnic||ipv6|:::|||available|
apnic||ipv6|:::|||available|
apnic||ipv6|:::|||available|
apnic|DE|ipv6|:::|||allocated|A91D8D8A
apnic|BP|ipv6|::::|||allocated|A91D8D8A
apnic|SC|ipv6|::::|||allocated|A91D8D8A
apnic|UP|ipv6|::::|||allocated|A91D8D8A
apnic||ipv6|:::|||available|
apnic||ipv6|:21a::|||available|
apnic||ipv6|:21c::|||available|
apnic|DR|ipv6|:::|||allocated|A92B4E0E
apnic|CR|ipv6|::::|||allocated|A92B4E0E
apnic|KE|ipv6|::::|||allocated|A92B4E0E
apnic|AR|ipv6|::::|||allocated|A92B4E0E
apnic||ipv6|:::|||available|
apnic||ipv6|:::|||available|
apnic||ipv6|:::|||available|
apnic||ipv6|:::|||reserved|
apnic||ipv6|::::|||available|
apnic||ipv6|::::|||available|
apnic||ipv6|::::|||available|
apnic||ipv6|:::|||available|
apnic||ipv6|:22a::|||available|
apnic||ipv6|:22c::|||available|
apnic|AR|ipv6|:::|||allocated|A9267E82
apnic|KB|ipv6|::::|||allocated|A9267E82
apnic|DR|ipv6|::::|||allocated|A9267E82
apnic|KF|ipv6|::::|||allocated|A9267E82
apnic||ipv6|:::|||available|
apnic||ipv6|:::|||available|
apnic||ipv6|:::|||available|
apnic|CE|ipv6|:::|||allocated|A912C007
apnic|SD|ipv6|::::|||allocated|A912C007
apnic|TA|ipv6|::::|||allocated|A912C007
apnic|CI|ipv6|::::|||allocated|A912C007
apnic||ipv6|:::|||available|
apnic||ipv6|:23a::|||available|
apnic||ipv6|:23c::|||available|

Reference:

AWK简明教程: http://coolshell.cn/articles/9070.html

AWK简明教程: http://xu020408.blog.163.com/blog/static/26548920132704942773/

《sed & awk》读书笔记之 awk 篇: http://blog.jobbole.com/31817/