Spidering and Modeling Email Data（week4&5）

Mailing List - Gmane

  • Crawl the archive of a mailing list
  • Do some analysis / cleanup
  • Visualize the data as word cloud and lines

code segment

gmane.py

import sqlite3

import time

import ssl

import urllib.request, urllib.parse, urllib.error

from urllib.parse import urljoin

from urllib.parse import urlparse

import re

from datetime import datetime, timedelta

# Not all systems have this so conditionally define parser

try:

    import dateutil.parser as parser

except:

    pass

def parsemaildate(md) :

    # See if we have dateutil

    try:

        pdate = parser.parse(tdate)

        test_at = pdate.isoformat()

        return test_at

    except:

        pass

    # Non-dateutil version - we try our best

    pieces = md.split()

    notz = " ".join(pieces[:4]).strip()

    # Try a bunch of format variations - strptime() is *lame*

    dnotz = None

    for form in [ '%d %b %Y %H:%M:%S', '%d %b %Y %H:%M:%S',

        '%d %b %Y %H:%M', '%d %b %Y %H:%M', '%d %b %y %H:%M:%S',

        '%d %b %y %H:%M:%S', '%d %b %y %H:%M', '%d %b %y %H:%M' ] :

        try:

            dnotz = datetime.strptime(notz, form)

            break

        except:

            continue

    if dnotz is None :

        # print 'Bad Date:',md

        return None

    iso = dnotz.isoformat()

    tz = "+0000"

    try:

        tz = pieces[4]

        ival = int(tz) # Only want numeric timezone values

        if tz == '-0000' : tz = '+0000'

        tzh = tz[:3]

        tzm = tz[3:]

        tz = tzh+":"+tzm

    except:

        pass

    return iso+tz

# Ignore SSL certificate errors

ctx = ssl.create_default_context()

ctx.check_hostname = False

ctx.verify_mode = ssl.CERT_NONE

conn = sqlite3.connect('content.sqlite')

cur = conn.cursor()

baseurl = "http://mbox.dr-chuck.net/sakai.devel/"

cur.execute('''CREATE TABLE IF NOT EXISTS Messages

    (id INTEGER UNIQUE, email TEXT, sent_at TEXT,

     subject TEXT, headers TEXT, body TEXT)''')

# Pick up where we left off

start = None

cur.execute('SELECT max(id) FROM Messages' )

try:

    row = cur.fetchone()

    if row is None :

        start = 0

    else:

        start = row[0]

except:

    start = 0

if start is None : start = 0

many = 0

count = 0

fail = 0

while True:

    if ( many < 1 ) :

        sval = input('How many messages:')

        if ( len(sval) < 1 ) : break

        many = int(sval)

    start = start + 1

    cur.execute('SELECT id FROM Messages WHERE id=?', (start,) )

    try:

        row = cur.fetchone()

        if row is not None : continue

    except:

        row = None

    many = many - 1

    url = baseurl + str(start) + '/' + str(start + 1)

    text = "None"

    try:

        # Open with a timeout of 30 seconds

        document = urllib.request.urlopen(url, None, 30, context=ctx)

        text = document.read().decode()

        if document.getcode() != 200 :

            print("Error code=",document.getcode(), url)

            break

    except KeyboardInterrupt:

        print('')

        print('Program interrupted by user...')

        break

    except Exception as e:

        print("Unable to retrieve or parse page",url)

        print("Error",e)

        fail = fail + 1

        if fail > 5 : break

        continue

    print(url,len(text))

    count = count + 1

    if not text.startswith("From "):

        print(text)

        print("Did not find From ")

        fail = fail + 1

        if fail > 5 : break

        continue

    pos = text.find("\n\n")

    if pos > 0 :

        hdr = text[:pos]

        body = text[pos+2:]

    else:

        print(text)

        print("Could not find break between headers and body")

        fail = fail + 1

        if fail > 5 : break

        continue

    email = None

    x = re.findall('\nFrom: .* <(\S+@\S+)>\n', hdr)

    if len(x) == 1 :

        email = x[0];

        email = email.strip().lower()

        email = email.replace("<","")

    else:

        x = re.findall('\nFrom: (\S+@\S+)\n', hdr)

        if len(x) == 1 :

            email = x[0];

            email = email.strip().lower()

            email = email.replace("<","")

    date = None

    y = re.findall('\Date: .*, (.*)\n', hdr)

    if len(y) == 1 :

        tdate = y[0]

        tdate = tdate[:26]

        try:

            sent_at = parsemaildate(tdate)

        except:

            print(text)

            print("Parse fail",tdate)

            fail = fail + 1

            if fail > 5 : break

            continue

    subject = None

    z = re.findall('\Subject: (.*)\n', hdr)

    if len(z) == 1 : subject = z[0].strip().lower();

    # Reset the fail counter

    fail = 0

    print("   ",email,sent_at,subject)

    cur.execute('''INSERT OR IGNORE INTO Messages (id, email, sent_at, subject, headers, body)

        VALUES ( ?, ?, ?, ?, ?, ? )''', ( start, email, sent_at, subject, hdr, body))

    if count % 50 == 0 : conn.commit()

    if count % 100 == 0 : time.sleep(1)

conn.commit()

cur.close()

gmodel.py

import sqlite3

import time

import re

import zlib

from datetime import datetime, timedelta

# Not all systems have this

try:

    import dateutil.parser as parser

except:

    pass

dnsmapping = dict()

mapping = dict()

def fixsender(sender,allsenders=None) :

    global dnsmapping

    global mapping

    if sender is None : return None

    sender = sender.strip().lower()

    sender = sender.replace('<','').replace('>','')

    # Check if we have a hacked gmane.org from address

    if allsenders is not None and sender.endswith('gmane.org') :

        pieces = sender.split('-')

        realsender = None

        for s in allsenders:

            if s.startswith(pieces[0]) :

                realsender = sender

                sender = s

                # print(realsender, sender)

                break

        if realsender is None :

            for s in mapping:

                if s.startswith(pieces[0]) :

                    realsender = sender

                    sender = mapping[s]

                    # print(realsender, sender)

                    break

        if realsender is None : sender = pieces[0]

    mpieces = sender.split("@")

    if len(mpieces) != 2 : return sender

    dns = mpieces[1]

    x = dns

    pieces = dns.split(".")

    if dns.endswith(".edu") or dns.endswith(".com") or dns.endswith(".org") or dns.endswith(".net") :

        dns = ".".join(pieces[-2:])

    else:

        dns = ".".join(pieces[-3:])

    # if dns != x : print(x,dns)

    # if dns != dnsmapping.get(dns,dns) : print(dns,dnsmapping.get(dns,dns))

    dns = dnsmapping.get(dns,dns)

    return mpieces[0] + '@' + dns

def parsemaildate(md) :

    # See if we have dateutil

    try:

        pdate = parser.parse(md)

        test_at = pdate.isoformat()

        return test_at

    except:

        pass

    # Non-dateutil version - we try our best

    pieces = md.split()

    notz = " ".join(pieces[:4]).strip()

    # Try a bunch of format variations - strptime() is *lame*

    dnotz = None

    for form in [ '%d %b %Y %H:%M:%S', '%d %b %Y %H:%M:%S',

        '%d %b %Y %H:%M', '%d %b %Y %H:%M', '%d %b %y %H:%M:%S',

        '%d %b %y %H:%M:%S', '%d %b %y %H:%M', '%d %b %y %H:%M' ] :

        try:

            dnotz = datetime.strptime(notz, form)

            break

        except:

            continue

    if dnotz is None :

        # print('Bad Date:',md)

        return None

    iso = dnotz.isoformat()

    tz = "+0000"

    try:

        tz = pieces[4]

        ival = int(tz) # Only want numeric timezone values

        if tz == '-0000' : tz = '+0000'

        tzh = tz[:3]

        tzm = tz[3:]

        tz = tzh+":"+tzm

    except:

        pass

    return iso+tz

# Parse out the info...

def parseheader(hdr, allsenders=None):

    if hdr is None or len(hdr) < 1 : return None

    sender = None

    x = re.findall('\nFrom: .* <(\S+@\S+)>\n', hdr)

    if len(x) >= 1 :

        sender = x[0]

    else:

        x = re.findall('\nFrom: (\S+@\S+)\n', hdr)

        if len(x) >= 1 :

            sender = x[0]

    # normalize the domain name of Email addresses

    sender = fixsender(sender, allsenders)

    date = None

    y = re.findall('\nDate: .*, (.*)\n', hdr)

    sent_at = None

    if len(y) >= 1 :

        tdate = y[0]

        tdate = tdate[:26]

        try:

            sent_at = parsemaildate(tdate)

        except Exception as e:

            # print('Date ignored ',tdate, e)

            return None

    subject = None

    z = re.findall('\nSubject: (.*)\n', hdr)

    if len(z) >= 1 : subject = z[0].strip().lower()

    guid = None

    z = re.findall('\nMessage-ID: (.*)\n', hdr)

    if len(z) >= 1 : guid = z[0].strip().lower()

    if sender is None or sent_at is None or subject is None or guid is None :

        return None

    return (guid, sender, subject, sent_at)

conn = sqlite3.connect('index.sqlite')

cur = conn.cursor()

cur.execute('''DROP TABLE IF EXISTS Messages ''')

cur.execute('''DROP TABLE IF EXISTS Senders ''')

cur.execute('''DROP TABLE IF EXISTS Subjects ''')

cur.execute('''DROP TABLE IF EXISTS Replies ''')

cur.execute('''CREATE TABLE IF NOT EXISTS Messages

    (id INTEGER PRIMARY KEY, guid TEXT UNIQUE, sent_at INTEGER,

     sender_id INTEGER, subject_id INTEGER,

     headers BLOB, body BLOB)''')

cur.execute('''CREATE TABLE IF NOT EXISTS Senders

    (id INTEGER PRIMARY KEY, sender TEXT UNIQUE)''')

cur.execute('''CREATE TABLE IF NOT EXISTS Subjects

    (id INTEGER PRIMARY KEY, subject TEXT UNIQUE)''')

cur.execute('''CREATE TABLE IF NOT EXISTS Replies

    (from_id INTEGER, to_id INTEGER)''')

conn_1 = sqlite3.connect('mapping.sqlite')

cur_1 = conn_1.cursor()

cur_1.execute('''SELECT old,new FROM DNSMapping''')

for message_row in cur_1 :

    dnsmapping[message_row[0].strip().lower()] = message_row[1].strip().lower()

mapping = dict()

cur_1.execute('''SELECT old,new FROM Mapping''')

for message_row in cur_1 :

    old = fixsender(message_row[0])

    new = fixsender(message_row[1])

    mapping[old] = fixsender(new)

# Done with mapping.sqlite

conn_1.close()

# Open the main content (Read only)

conn_1 = sqlite3.connect('file:content.sqlite?mode=ro', uri=True)

cur_1 = conn_1.cursor()

allsenders = list()

cur_1.execute('''SELECT email FROM Messages''')

for message_row in cur_1 :

    sender = fixsender(message_row[0])

    if sender is None : continue

    if 'gmane.org' in sender : continue

    if sender in allsenders: continue

    allsenders.append(sender)

print("Loaded allsenders",len(allsenders),"and mapping",len(mapping),"dns mapping",len(dnsmapping))

cur_1.execute('''SELECT headers, body, sent_at

    FROM Messages ORDER BY sent_at''')

senders = dict()

subjects = dict()

guids = dict()

count = 0

for message_row in cur_1 :

    hdr = message_row[0]

    parsed = parseheader(hdr, allsenders)

    if parsed is None: continue

    (guid, sender, subject, sent_at) = parsed

    # Apply the sender mapping

    sender = mapping.get(sender,sender)

    count = count + 1

    if count % 250 == 1 : print(count,sent_at, sender)

    # print(guid, sender, subject, sent_at)

    if 'gmane.org' in sender:

        print("Error in sender ===", sender)

    sender_id = senders.get(sender,None)

    subject_id = subjects.get(subject,None)

    guid_id = guids.get(guid,None)

    if sender_id is None :

        cur.execute('INSERT OR IGNORE INTO Senders (sender) VALUES ( ? )', ( sender, ) )

        conn.commit()

        cur.execute('SELECT id FROM Senders WHERE sender=? LIMIT 1', ( sender, ))

        try:

            row = cur.fetchone()

            sender_id = row[0]

            senders[sender] = sender_id

        except:

            print('Could not retrieve sender id',sender)

            break

    if subject_id is None :

        cur.execute('INSERT OR IGNORE INTO Subjects (subject) VALUES ( ? )', ( subject, ) )

        conn.commit()

        cur.execute('SELECT id FROM Subjects WHERE subject=? LIMIT 1', ( subject, ))

        try:

            row = cur.fetchone()

            subject_id = row[0]

            subjects[subject] = subject_id

        except:

            print('Could not retrieve subject id',subject)

            break

    # print(sender_id, subject_id)

    cur.execute('INSERT OR IGNORE INTO Messages (guid,sender_id,subject_id,sent_at,headers,body) VALUES ( ?,?,?,datetime(?),?,? )',

            ( guid, sender_id, subject_id, sent_at,

            zlib.compress(message_row[0].encode()), zlib.compress(message_row[1].encode())) )

    conn.commit()

    cur.execute('SELECT id FROM Messages WHERE guid=? LIMIT 1', ( guid, ))

    try:

        row = cur.fetchone()

        message_id = row[0]

        guids[guid] = message_id

    except:

        print('Could not retrieve guid id',guid)

        break

cur.close()

cur_1.close()

gbasic.py

import sqlite3

import time

import zlib

howmany = int(input("How many to dump? "))

conn = sqlite3.connect('index.sqlite')

cur = conn.cursor()

cur.execute('SELECT id, sender FROM Senders')

senders = dict()

for message_row in cur :

    senders[message_row[0]] = message_row[1]

cur.execute('SELECT id, subject FROM Subjects')

subjects = dict()

for message_row in cur :

    subjects[message_row[0]] = message_row[1]

# cur.execute('SELECT id, guid,sender_id,subject_id,headers,body FROM Messages')

cur.execute('SELECT id, guid,sender_id,subject_id,sent_at FROM Messages')

messages = dict()

for message_row in cur :

    messages[message_row[0]] = (message_row[1],message_row[2],message_row[3],message_row[4])

print("Loaded messages=",len(messages),"subjects=",len(subjects),"senders=",len(senders))

sendcounts = dict()

sendorgs = dict()

for (message_id, message) in list(messages.items()):

    sender = message[1]

    sendcounts[sender] = sendcounts.get(sender,0) + 1

    pieces = senders[sender].split("@")

    if len(pieces) != 2 : continue

    dns = pieces[1]

    sendorgs[dns] = sendorgs.get(dns,0) + 1

print('')

print('Top',howmany,'Email list participants')

x = sorted(sendcounts, key=sendcounts.get, reverse=True)

for k in x[:howmany]:

    print(senders[k], sendcounts[k])

    if sendcounts[k] < 10 : break

print('')

print('Top',howmany,'Email list organizations')

x = sorted(sendorgs, key=sendorgs.get, reverse=True)

for k in x[:howmany]:

    print(k, sendorgs[k])

    if sendorgs[k] < 10 : break

Coursera课程笔记----P4E.Capstone----Week 4&5的更多相关文章

  1. Coursera课程笔记----P4E.Capstone----Week 6&7

    Visualizing Email Data(Week 6&7) code segment gword.py import sqlite3 import time import zlib im ...

  2. Coursera课程笔记----P4E.Capstone----Week 2&3

    Building a Search Engine(week 2&3) Search Engine Architecture Web Crawling Index Building Search ...

  3. 操作系统学习笔记----进程/线程模型----Coursera课程笔记

    操作系统学习笔记----进程/线程模型----Coursera课程笔记 进程/线程模型 0. 概述 0.1 进程模型 多道程序设计 进程的概念.进程控制块 进程状态及转换.进程队列 进程控制----进 ...

  4. Coursera课程笔记----C++程序设计----Week3

    类和对象(Week 3) 内联成员函数和重载成员函数 内联成员函数 inline + 成员函数 整个函数题出现在类定义内部 class B{ inline void func1(); //方式1 vo ...

  5. Coursera课程笔记----Write Professional Emails in English----Week 3

    Introduction and Announcement Emails (Week 3) Overview of Introduction & Announcement Emails Bas ...

  6. Coursera课程笔记----Write Professional Emails in English----Week 1

    Get to Know Basic Email Writing Structures(Week 1) Introduction to Course Email and Editing Basics S ...

  7. Coursera课程笔记----C程序设计进阶----Week 5

    指针(二) (Week 5) 字符串与指针 指向数组的指针 int a[10]; int *p; p = a; 指向字符串的指针 指向字符串的指针变量 char a[10]; char *p; p = ...

  8. Coursera课程笔记----Write Professional Emails in English----Week 5

    Culture Matters(Week 5) High/Low Context Communication High Context Communication The Middle East, A ...

  9. Coursera课程笔记----Write Professional Emails in English----Week 4

    Request and Apology Emails(Week 4) How to Write Request Emails Write more POLITELY & SINCERELUY ...

随机推荐

  1. c++ string类的一些使用

    初始化: string类的初始化是不可以用字符进行的,如; string str='c'; string str('c');必须传递字符串字面量作为参数:string本身是用模板类进行实例化的类. s ...

  2. stand up meeting 11/25/2015 暨sprint2总结

    今天在课堂上进行了小组项目的阶段性总结,这两天小组内也是频繁的开会,具体细节我们已经反复核查,具体不表~ sprint2个人工作总结: 冯晓云:完成了必应词典在线查词api的调用和网络状况的检测:完成 ...

  3. B - Raising Modulo Numbers

    People are different. Some secretly read magazines full of interesting girls' pictures, others creat ...

  4. BUU刷题01

    [安洵杯 2019]easy_serialize_php 直接给了源代码 <?php $function = @$_GET['f']; function filter($img){ $filte ...

  5. MySQL笔记总结-DDL语言

    DDL语言 数据类型 一.数值型 1.整型 tinyint.smallint.mediumint.int/integer.bigint 1 2 3 4 8 特点: ①都可以设置无符号和有符号,默认有符 ...

  6. 天池Docker学习赛笔记

    容器的基本概念 什么是容器? 容器就是一个视图隔离.资源可限制.独立文件系统的进程集合.所谓"视图隔离"就是能够看到部分进程以及具有独立的主机名等:控制资源使用率则是可以对于内存大 ...

  7. PHP--关于上传文件大小的问题

    参考:https://www.cnblogs.com/jianqingwang/p/5863960.html https://blog.csdn.net/u013168253/article/deta ...

  8. auth权限逻辑

    下面本人为大家讲解一下如何实现auth权限, 第一步,新建Auth.php,复制下面的代码,把注释中的表都创建一下.把文件放到extend新建文件夹org放进去即可, <?php // +--- ...

  9. 关于foreach总是报错invalid param等问题

    原因为被foreach的数组可能为空,则会报错,只需做好容错即可,例如 if ( !empty( $arr ) ) { foreach ( $arr as $k => $v ) { } }

  10. （第二篇）shell的简单了解

    Shell (类似开发工具) Shell 是一个用 C 语言编写的程序,它是用户使用 Linux 的桥梁.Shell 既是一种命令语言,又是一种程序设计语言. Shell 是指一种应用程序,这个应用程 ...