hmm CDN检测
# -*- coding:utf-8 -*- import sys
import re
from hmmlearn import hmm
import numpy as np
from sklearn.externals import joblib
import matplotlib.pyplot as plt
import tldextract
import os def iterbrowse(path):
for home, dirs, files in os.walk(path):
for filename in files:
yield os.path.join(home, filename) def extract_domain(domain):
suffix = {'.com','.la','.io', '.co', '.cn','.info', '.net', '.org','.me', '.mobi', '.us', '.biz', '.xxx', '.ca', '.co.jp', '.com.cn', '.net.cn', '.org.cn', '.mx','.tv', '.ws', '.ag', '.com.ag', '.net.ag', '.org.ag','.am','.asia', '.at', '.be', '.com.br', '.net.br', '.name', '.live', '.news', '.bz', '.tech', '.pub', '.wang', '.space', '.top', '.xin', '.social', '.date', '.site', '.red', '.studio', '.link', '.online', '.help', '.kr', '.club', '.com.bz', '.net.bz', '.cc', '.band', '.market', '.com.co', '.net.co', '.nom.co', '.lawyer', '.de', '.es', '.com.es', '.nom.es', '.org.es', '.eu', '.wiki', '.design', '.software', '.fm', '.fr', '.gs', '.in', '.co.in', '.firm.in', '.gen.in', '.ind.in', '.net.in', '.org.in', '.it', '.jobs', '.jp', '.ms', '.com.mx', '.nl','.nu','.co.nz','.net.nz', '.org.nz', '.se', '.tc', '.tk', '.tw', '.com.tw', '.idv.tw', '.org.tw', '.hk', '.co.uk', '.me.uk', '.org.uk', '.vg'} domain = domain.lower()
names = domain.split(".")
if len(names) >= 3:
if ("."+".".join(names[-2:])) in suffix:
return ".".join(names[-3:]), ".".join(names[:-3])
elif ("."+names[-1]) in suffix:
return ".".join(names[-2:]), ".".join(names[:-2])
print "New domain suffix found. Use tld extract domain..." pos = domain.rfind("/")
if pos >= 0: # maybe subdomain contains /, for dns tunnel tool
ext = tldextract.extract(domain[pos+1:])
subdomain = domain[:pos+1] + ext.subdomain
else:
ext = tldextract.extract(domain)
subdomain = ext.subdomain
if ext.suffix:
mdomain = ext.domain + "." + ext.suffix
else:
mdomain = ext.domain
return mdomain, subdomain def parse(log):
data = log.split('^')
SRC_PORT_IDX = 5-1
DST_PORT_IDX = 6-1
PROTOCOL_IDX = 7-1
protol = data[PROTOCOL_IDX]
dstport = data[DST_PORT_IDX]
if '' == protol and ('' == dstport):
DNS_QUERY_NAME_IDX = 55-1 # domain
if (len(data) < 55):
print "error line:"
print log
return ("", "")
domain = data[DNS_QUERY_NAME_IDX]
mdomain, subdomain = extract_domain(domain)
return (mdomain, subdomain)
else:
print "error line not a DNS:"
print log
return ("", "") #处理域名的最小长度
MIN_LEN=3 #状态个数
N=5
#最大似然概率阈值
T=-50 #模型文件名
FILE_MODEL="hmm-cdn.m" def get_cdn_domains(dir_path):
domain_list=[]
for path in iterbrowse(dir_path):
with open(path) as f:
for line in f:
mdomain, sub_domain = parse(line)
if len(sub_domain) >= MIN_LEN:
domain_list.append(sub_domain)
if len(domain_list) >= 2000:
return domain_list
#else:
# print path, "pass line:", line
return domain_list def domain2ver(domain):
ver=[]
for i in range(0,len(domain)):
ver.append([ord(domain[i])])
return ver def train_hmm(domain_list):
X = [[0]]
X_lens = [1]
for domain in domain_list:
ver=domain2ver(domain)
np_ver = np.array(ver)
#print len(np_ver)
try:
X=np.concatenate([X,np_ver])
except ValueError:
print domain
print len(X), len(np_ver)
print X
print np_ver
raise
X_lens.append(len(np_ver)) remodel = hmm.GaussianHMM(n_components=N, covariance_type="full", n_iter=100)
remodel.fit(X,X_lens)
joblib.dump(remodel, FILE_MODEL) return remodel def test(remodel, domain_list):
x=[]
y=[]
for domain in domain_list:
domain_ver=domain2ver(domain)
np_ver = np.array(domain_ver)
pro = remodel.score(np_ver)
print "SCORE:(%d) DOMAIN:(%s) " % (pro, domain)
x.append(len(domain))
y.append(pro)
return x,y if __name__ == '__main__':
domain_list = get_cdn_domains("/home/bonelee/latest_metadata_sample/labeled_cdn")
remodel=train_hmm(domain_list)
remodel=joblib.load(FILE_MODEL) x_1,y_1=test(remodel, domain_list)
print x_1
print y_1
#sys.exit(0)
domain_list = get_cdn_domains("/home/bonelee/latest_metadata_sample/labeled_black")
x_2,y_2=test(remodel, domain_list)
print x_2
print y_2
domain_list = get_cdn_domains("/home/bonelee/latest_metadata_sample/labeled_white_like")
x_3,y_3=test(remodel, domain_list)
print x_3
print y_3
#%matplotlib inline
fig,ax=plt.subplots()
ax.set_xlabel('Domain Length')
ax.set_ylabel('HMM Score')
ax.scatter(x_3,y_3,color='b',label="WHITE")
ax.scatter(x_2, y_2, color='g', label="BLACK")
ax.scatter(x_1, y_1, color='r', label="CDN")
ax.legend(loc='right')
plt.show()
使用pickle保存和加载模型：
# -*- coding:utf-8 -*- import sys
import re
from hmmlearn import hmm
import numpy as np
#from sklearn.externals import joblib
import matplotlib.pyplot as plt
import tldextract
import os
import pickle def iterbrowse(path):
for home, dirs, files in os.walk(path):
for filename in files:
yield os.path.join(home, filename) def extract_domain(domain):
suffix = {'.com','.la','.io', '.co', '.cn','.info', '.net', '.org','.me', '.mobi', '.us', '.biz', '.xxx', '.ca', '.co.jp', '.com.cn', '.net.cn', '.org.cn', '.mx','.tv', '.ws', '.ag', '.com.ag', '.net.ag', '.org.ag','.am','.asia', '.at', '.be', '.com.br', '.net.br', '.name', '.live', '.news', '.bz', '.tech', '.pub', '.wang', '.space', '.top', '.xin', '.social', '.date', '.site', '.red', '.studio', '.link', '.online', '.help', '.kr', '.club', '.com.bz', '.net.bz', '.cc', '.band', '.market', '.com.co', '.net.co', '.nom.co', '.lawyer', '.de', '.es', '.com.es', '.nom.es', '.org.es', '.eu', '.wiki', '.design', '.software', '.fm', '.fr', '.gs', '.in', '.co.in', '.firm.in', '.gen.in', '.ind.in', '.net.in', '.org.in', '.it', '.jobs', '.jp', '.ms', '.com.mx', '.nl','.nu','.co.nz','.net.nz', '.org.nz', '.se', '.tc', '.tk', '.tw', '.com.tw', '.idv.tw', '.org.tw', '.hk', '.co.uk', '.me.uk', '.org.uk', '.vg'} domain = domain.lower()
names = domain.split(".")
if len(names) >= 3:
if ("."+".".join(names[-2:])) in suffix:
return ".".join(names[-3:]), ".".join(names[:-3])
elif ("."+names[-1]) in suffix:
return ".".join(names[-2:]), ".".join(names[:-2])
print "New domain suffix found. Use tld extract domain..." pos = domain.rfind("/")
if pos >= 0: # maybe subdomain contains /, for dns tunnel tool
ext = tldextract.extract(domain[pos+1:])
subdomain = domain[:pos+1] + ext.subdomain
else:
ext = tldextract.extract(domain)
subdomain = ext.subdomain
if ext.suffix:
mdomain = ext.domain + "." + ext.suffix
else:
mdomain = ext.domain
return mdomain, subdomain def parse(log):
data = log.split('^')
SRC_PORT_IDX = 5-1
DST_PORT_IDX = 6-1
PROTOCOL_IDX = 7-1
protol = data[PROTOCOL_IDX]
dstport = data[DST_PORT_IDX]
if '' == protol and ('' == dstport):
DNS_QUERY_NAME_IDX = 55-1 # domain
if (len(data) < 55):
print "error line:"
print log
return ("", "")
domain = data[DNS_QUERY_NAME_IDX]
mdomain, subdomain = extract_domain(domain)
return (mdomain, subdomain)
else:
print "error line not a DNS:"
print log
return ("", "") #处理域名的最小长度
MIN_LEN=1 #状态个数
N=8
#最大似然概率阈值
T=-50 #模型文件名
FILE_MODEL="hmm-cdn.m"
FILE_MODEL2 ="hmm-cdn-white.pkl" def get_cdn_domains(dir_path):
domain_list=[]
for path in iterbrowse(dir_path):
with open(path) as f:
for line in f:
mdomain, sub_domain = parse(line)
if len(sub_domain) >= MIN_LEN:
domain_list.append(sub_domain)
if len(domain_list) >= 3000:
return domain_list
#else:
# print path, "pass line:", line
return domain_list def domain2ver(domain):
ver=[]
for i in range(0,len(domain)):
ver.append([ord(domain[i])])
return ver def train_hmm(domain_list):
if os.path.exists(FILE_MODEL2):
print "found model file, use it..."
file_model = open(FILE_MODEL2, 'rb')
model = pickle.load(file_model)
file_model.close()
return model X = [[0]]
X_lens = [1]
for domain in domain_list:
ver=domain2ver(domain)
np_ver = np.array(ver)
#print len(np_ver)
try:
X=np.concatenate([X,np_ver])
except ValueError:
print domain
print len(X), len(np_ver)
print X
print np_ver
raise
X_lens.append(len(np_ver)) #remodel = hmm.GaussianHMM(n_components=N, covariance_type="spherical", n_iter=500) #spherical, diag full,tied
remodel = hmm.GaussianHMM(n_components=N, covariance_type="full", n_iter=500)
remodel.fit(X,X_lens)
#joblib.dump(remodel, FILE_MODEL) file_model = open(FILE_MODEL2, 'wb')
pickle.dump(remodel, file_model)
file_model.close() return remodel def test(remodel, domain_list):
x=[]
y=[]
for domain in domain_list:
domain_ver=domain2ver(domain)
np_ver = np.array(domain_ver)
pro = remodel.score(np_ver)
print "SCORE:(%d) DOMAIN:(%s) " % (pro, domain)
x.append(len(domain))
y.append(pro)
return x,y if __name__ == '__main__':
domain_list = get_cdn_domains("/home/bonelee/latest_metadata_sample/labeled_cdn")
domain_list2 = get_cdn_domains("/home/bonelee/latest_metadata_sample/labeled_white_like")
#remodel=train_hmm(domain_list)
remodel=train_hmm(domain_list+domain_list2)
#remodel=joblib.load(FILE_MODEL) x_1,y_1=test(remodel, domain_list)
print x_1
print y_1
#sys.exit(0)
domain_list = get_cdn_domains("/home/bonelee/latest_metadata_sample/labeled_black")
x_2,y_2=test(remodel, domain_list)
print x_2
print y_2
domain_list = get_cdn_domains("/home/bonelee/latest_metadata_sample/labeled_white_like")
x_3,y_3=test(remodel, domain_list)
print x_3
print y_3
#%matplotlib inline
fig,ax=plt.subplots()
ax.set_xlabel('Domain Length')
ax.set_ylabel('HMM Score')
#ax.scatter(x_3,y_3,color='b',label="WHITE")
ax.scatter(x_2, y_2, color='g', label="DNS tunnel")
ax.scatter(x_1, y_1, color='r', label="CDN")
ax.legend(loc='right')
plt.show()
其中：X = [[0]]，X_lens = [] 也可以按照下面方式进行读写。除去了冗余的初始化。
def train_hmm(domain_list):
if os.path.exists(FILE_MODEL2):
print "found model file, use it..."
file_model = open(FILE_MODEL2, 'rb')
model = pickle.load(file_model)
file_model.close()
return model #X = [[0]]
#X_lens = [1]
X = []
X_lens = []
#print X
for domain in domain_list:
ver=domain2ver(domain)
#np_ver = np.array(ver)
try:
#X=np.concatenate([X,np_ver])
X = X + ver
except ValueError:
print domain
print X
print ver
raise
X_lens.append(len(ver))
#remodel = hmm.GaussianHMM(n_components=N, covariance_type="spherical", n_iter=500) #spherical, diag full,tied
remodel = hmm.GaussianHMM(n_components=N, covariance_type="full", n_iter=500)
remodel.fit(X,X_lens)
#joblib.dump(remodel, FILE_MODEL) file_model = open(FILE_MODEL2, 'wb')
pickle.dump(remodel, file_model)
file_model.close() return remodel
hmm CDN检测的更多相关文章
- HMM XSS检测
HMM XSS检测 转自:http://www.freebuf.com/articles/web/133909.html 前言 上篇我们介绍了HMM的基本原理以及常见的基于参数的异常检测实现,这次我们 ...
- 绕过CDN查找真实IP方法总结
CDN的全称是Content Delivery Network,即内容分发网络.CDN是构建在现有网络基础之上的智能虚拟网络,依靠部署在各地的边缘服务器,通过中心平台的负载均衡.内容分发.调度等功能模 ...
- [转载]绕过CDN查找真实IP方法总结
前言 类似备忘录形式记录一下,这里结合了几篇绕过CDN寻找真实IP的文章,总结一下绕过CDN查找真实的IP的方法 介绍 CDN的全称是Content Delivery Network,即内容分发网络. ...
- web渗透测试
信息收集 网络搜索 目录遍历:site:域名 intitle:index.of 配置文件泄露:site:域名 ext:xml | ext:conf | ext:cnf | ext:reg | ext: ...
- 基于Python的渗透测试信息收集系统的设计和实现
信息收集系统的设计和实现 渗透测试是保卫网络安全的一种有效且必要的技术手段,而渗透测试的本质就是信息收集,信息搜集整理可为后续的情报跟进提供强大的保证,目标资产信息搜集的广度,决定渗透过程的复杂程度, ...
- 脚本检测CDN节点资源是否与源站资源一致
需求: 1.所有要检测的资源url放到一个单独文件中 2.检测cdn节点资源大小与源站文件大小是否一致 3.随机抽查几个资源,检查md5sum是否一致 4.使用多线程,可配置线程数 代码目录: hex ...
- 简单检测CDN链接是否有效
CDN链接经常是使用的.但是,CDN链接挂了怎么办,因此,就要调用使用本站点的库,那么怎么实现呢? 检测CDN的jquery链接是否有效(这种方法比较简单) <script src=" ...
- 大数据DDos检测——DDos攻击本质上是时间序列数据，t+1时刻的数据特点和t时刻强相关，因此用HMM或者CRF来做检测是必然！ 和一个句子的分词算法CRF没有区别！
DDos攻击本质上是时间序列数据,t+1时刻的数据特点和t时刻强相关,因此用HMM或者CRF来做检测是必然!——和一个句子的分词算法CRF没有区别!注:传统DDos检测直接基于IP数据发送流量来识别, ...
- 基于机器学习的web异常检测——基于HMM的状态序列建模，将原始数据转化为状态机表示，然后求解概率判断异常与否
基于机器学习的web异常检测 from: https://jaq.alibaba.com/community/art/show?articleid=746 Web防火墙是信息安全的第一道防线.随着网络 ...
随机推荐
- Android 通用Dialog中设置RecyclerView
通用Dialog public class IOSRecyclerViewDialog{ private Context context; private Dialog dialog; private ...
- Javascript阻止表单提交
Javascript阻止表单提交 Html 1.<form name="loginForm" action="login.aspx" method=&qu ...
- dw2018修改为中文
dw2018 英文版修改为中文, 把zh_CN文件夹内的内容复制到en_US文件夹内并替换, 或者重命名zh_CN文件夹为en_US
- python编写简单的html登陆页面（2）
1 在python编写简单的html登陆页面(1)的基础上在延伸一下: 可以将动态分配数据,实现页面跳转功能: 2 跳转到新的页面:return render_template('home1.ht ...
- 浅谈[^>]在正则中的2种用法
/^A/会匹配"An e"中的A,但是不会匹配"ab A"中的A,此时^A的意思是“匹配开头的A” /[^a-z\s]/会匹配"my 3 sister ...
- Bonjour/Zeroconf with Arduino
转自:http://gkaindl.com/software/arduino-ethernet/bonjour Bonjour/Zeroconf with Arduino DownloadVersio ...
- JavaScript JSON简单操作（增删改）
JavaScript 中对json处理: 声明;: var json={};或 json={"name":"asd","age":24}; ...
- Project Euler 38 Pandigital multiples
题意: 将192分别与1.2.3相乘: 192 × 1 = 192192 × 2 = 384192 × 3 = 576 连接这些乘积,我们得到一个1至9全数字的数192384576.我们称192384 ...
- Java包名称中通配符的含义
"com.abc 表示的意义为:系统从com.abc这个包及其子孙包扫描组件 "com.abc.* 表示的意义为:系统从com.abc这个包的子孙包扫描组件
- BZOJ 1197 [HNOI2006]花仙子的魔法 (数学题)
题面:洛谷传送门 BZOJ传送门 非常有意思的一道数学题,浓浓的$CF$风,然而我并没有想出来.. 我们想把一个$n$维空间用$n$维球分成尽可能多的块 而新增加一个$n$维球时,肯定要尽可能多地切割 ...