python查找并删除相同文件-UNIQ File-script版本

今天用wxPython做了一个GUI程序，实现查找指定目录内的相同文件，主要原理是计算文件的md5值（计算前先找出文件大小相同的文件，然后计算这些文件的md5值，而不是所有文件都计算，大大减少了md5的计算量），加入了多线程功能。

以下是其脚本版本(无需安装wxPython)

UNIQFile-script.py

 # -*- coding: gbk -*-

 '''
 Author:@DoNotSpyOnMe
 Blog: http://www.cnblogs.com/aaronhoo
 '''

 import hashlib
 import os
 import threading

 def getFileSize(filePath):
     return os.path.getsize(filePath)        

 ''' 一般文件的md5计算方法,一次读取文件的全部内容'''
 def CalcMD5(filepath):
     with open(filepath,'rb') as f:
         md5obj = hashlib.md5()
         md5obj.update(f.read())
         hash = md5obj.hexdigest()
         return hash
 '''大文件计算md5的方法，分批读取文件内容，防止内存爆掉'''
 def GetFileMd5(filename):
     if not os.path.isfile(filename):
         return
     myhash = hashlib.md5()
     f = open(filename,'rb')
     while True:
         b = f.read(8*1024)
         if not b :
             break
         myhash.update(b)
     f.close()
     return myhash.hexdigest()

 def GetAllFiles(directory):
     files=[]
     for dirpath, dirnames,filenames in os.walk(directory):
         if filenames!=[]:
             for file in filenames:
                 files.append(dirpath+'\\'+file)
　　　　 files.sort(key=len)#按照文件名的长度排序
     return files

 def findSameSizeFiles(files):
     dicSize={}
     for f in files:
         size=getFileSize(f)
         if not dicSize.has_key(size):
             dicSize[size]=f
         else:
             dicSize[size]=dicSize[size]+';'+f
     dicCopy=dicSize.copy()
     for k in dicSize.iterkeys():
         if dicSize[k].find(';')==-1:
            dicCopy.pop(k)
     del dicSize
     return dicCopy

 def findSameMD5Files(files):
     dicMD5={}
     for f in files:
         print 'calculating the md5 value of file %s'%f
         md5=GetFileMd5(f)
         if not dicMD5.has_key(md5):
             dicMD5[md5]=f
         else:
             dicMD5[md5]=dicMD5[md5]+';'+f
     dicCopy=dicMD5.copy()
     for k in dicMD5.iterkeys():
         if dicMD5[k].find(';')==-1:
            dicCopy.pop(k)
     del dicMD5
     return dicCopy

 def removeSameFile(mydir):
     msg=''
     msgUniq='Result:No file is removed since they are all uniq.'
     try:
         existsFlag=False
         files=GetAllFiles(mydir)
         print'%s files found in directory %s\n'%(len(files),mydir)
         dicFileOfSameSize=findSameSizeFiles(files)
         if dicFileOfSameSize=={}:
             print msgUniq
             return
         else:
             #list the duplicated files first:
             dicFiltered={}
             for k in dicFileOfSameSize.iterkeys():
                 filesOfSameSize=dicFileOfSameSize[k].split(';')
                 dicSameMD5file=findSameMD5Files(filesOfSameSize)
                 if dicSameMD5file!={}:
                     existsFlag=True
                     for k in dicSameMD5file.iterkeys():
                         msg=msg+'md5 %s: %s'%(k,dicSameMD5file[k])+'\n'
                         dicFiltered[k]=dicSameMD5file[k]
             if not existsFlag:
                 msg=msgUniq
                 return
             else:
                 msg='Duplicated files:\n'+msg+'\n'
                 #then remove the duplicated files:
                 removeCount=0
                 for k in dicFiltered.iterkeys():
                     sameFiles=dicFiltered[k].split(';')
                     flagRemove=False
                     for f in sameFiles:
                         if not flagRemove:
                             flagRemove=True
                         else:
                             msg=msg+'Removing file: %s'%f+'\n'
                             os.remove(f)
                             removeCount=removeCount+1
                 msg=msg+'%s files are removed.\n'%removeCount
     except Exception,e:
             print e
 #         msg='Exception occured.'
     finally:
         print msg+'\n'+'Operation finished.'

 def listSameFile(mydir):
     msg=''
     msgUniq='Result:All files are uniq.'
     try:
         existsFlag=False
         files=GetAllFiles(mydir)
         print '%s files found in directory %s\n'%(len(files),mydir)
         dicFileOfSameSize=findSameSizeFiles(files)
         if dicFileOfSameSize=={}:
             print msgUniq
             return
         else:
             for k in dicFileOfSameSize.iterkeys():
                 filesOfSameSize=dicFileOfSameSize[k].split(';')
                 dicSameMD5file=findSameMD5Files(filesOfSameSize)
                 if dicSameMD5file!={}:
                     existsFlag=True
                     for k in dicSameMD5file.iterkeys():
                         msg=msg+'md5 %s: %s'%(k,dicSameMD5file[k])+'\n'
             if not existsFlag:
                 msg=msgUniq
             else:
                 msg='Duplicated files:\n'+msg
     except Exception,e:
             print e
 #         msg='Exception occured.'
     finally:
         print msg+'\n'+'Operation finished.'

 if __name__=="__main__":
     print 'This program is designed for clearing the duplicated files and saving memory space.Select a directory and we will find or remove the duplicated files.'
     print 'All rights are reserved by @DoNotSpyOnMe'
     print '\n'

     print "You have three options:"
     print "'f' for finding the duplicated files in the directory that you're required to enter later,or"
     print "'r' for finding and the removing the duplicated file,or"
     print "'q' to quit"
     while True:
         option=raw_input('Please enter your option:\n')
         option=option.lower()
         while option!='f' and option!='r' and option!='q':
             option=raw_input('Please enter your option:\n')
         if option=='f' or option=='r':
             mydir=raw_input('Please enter the direcotry containing files:\n')
             mydir=mydir.lower()
             while mydir.find('\\')==-1 or not os.path.isdir(mydir):
                 mydir=raw_input('Please enter a valid direcotry containing files:\n')
             if option=='f':
                 listSameFile(mydir)
             else:
                 removeSameFile(mydir)
         elif option=='q':
             exit(0)
         print ''