日志分析-mime统计

提取日志中未落入标准字段的mime，分adx，adtype 统计mime的数量和包含js的数量占比

require 'date'
require 'net/http'
require 'uri'
require 'json'

def getmimes (  adx , bodyobj ,totalmimes, statics)
  if bodyobj.class  != Hash
        return
  end
  mimes = []
  bodyobj.keys.each  do |key|
        val = bodyobj[key]
          if val.class == Fixnum || val.class == Float  || val.class == Array
               if key == "imp"
                   if  val[0]['banner'] != nil  && val[0]['banner']['mimes'] != nil
                     statics['includmime'] +=1
                     mimes +=  val[0]['banner']['mimes']
                   end
                   if  val[0]['video'] != nil  && val[0]['video']['mimes'] != nil
                     statics['includmime'] +=1
                     mimes +=  val[0]['video']['mimes']
                   end
               end
          end
  end

  if mimes.length >0
     mimes.each do |mime|
        kk = adx.to_s + "_" + mime.to_s
        if mime.include?"javascript"
          statics['includejs'] +=1
        end
             totalmimes[ kk] +=1
     end
  end
end

filepath = "/data/mvdsp/log/request.log.2017-11-30-12"
puts filepath
i  = 0
totalmimes = Hash.new(0)
statics = Hash.new(0)
begin
  File.open("#{filepath}").each do |line|
    statics['total'] +=1
    if  line.length < 1000
      statics['invalidbody'] +=1
        next
    end
   if ! line.valid_encoding?
     s = line.encode("UTF-16be", :invalid=>:replace, :replace=>"?").encode('UTF-8')
     line = s.gsub(/dr/i,'med')
   end 

   fields = line.split("\t")
    if fields.length <10
        next
    end
    adx  = fields[3]
    ext10  = fields[45]
    adtype = ""
    if ext10.class ==Hash &&  ext10['reqtype']!= nil
      adtype =ext10['reqtype']
    end

    jsonstr = fields[6]
    bodyobj = {}
    begin
     bodyobj =  JSON.parse  jsonstr

    rescue JSON::ParserError
      i +=1
   end
     getmimes(adx,bodyobj,totalmimes ,  statics)
  end
rescue SystemCallError
  puts "warn:: #{filepath} not  exits!!"
end

puts "-----------totalmimes---------------------"
print   totalmimes
sorted = totalmimes.sort_by {|_key, value| value}
puts sorted

puts "-----------statics--------------"
print   statics
puts "--------------------------------"

ruby hash sort by value

hsh ={"a" => 1000, "b" => 10, "c" => 200000}
Hash[hsh.sort_by{|k,v| v}]
#or
hsh.sort_by{|k,v| v}.to_h
#or
hsh.sort_by(&:last)