Python 线程池的实现

1. import urllib2
2. import time
3. import socket
4. from datetime import datetime
5. from thread_pool import *
6. def main():
7. url_list = {"sina":"http://www.sina.com.cn",
8. "sohu":"http://www.sohu.com",
9. "yahoo":"http://www.yahoo.com",
10. "xiaonei":"http://www.xiaonei.com",
11. "qihoo":"http://www.qihoo.com",
12. "laohan":"http://www.laohan.org",
13. "eyou":"http://www.eyou.com",
14. "chinaren":"http://www.chinaren.com",
15. "douban":"http://www.douban.com",
16. "163":"http://www.163.com",
17. "daqi":"http://www.daqi.com",
18. "qq":"http://www.qq.com",
19. "baidu_1":"http://www.baidu.com/s?wd=asdfasdf",
20. "baidu_2":"http://www.baidu.com/s?wd=dddddddf",
21. "google_1":"http://www.baidu.com/s?wd=sadfas",
22. "google_2":"http://www.baidu.com/s?wd=sadflasd",
23. "hainei":"http://www.hainei.com",
24. "microsoft":"http://www.microsoft.com",
25. "wlzuojia":"http://www.wlzuojia.com"}
26. #使用线程池
27. socket.setdefaulttimeout(10)
28. print 'start testing'
29. wm = WorkerManager(50)
30. for url_name in url_list.keys():
31. wm.add_job(do_get_con, url_name, url_list[url_name])
32. wm.wait_for_complete()
33. print 'end testing'
34. def do_get_con(url_name,url_link):
35. try:
36. fd = urllib2.urlopen(url_link)
37. data = fd.read()
38. f_hand = open("/tmp/ttt/%s" % url_name,"w")
39. f_hand.write(data)
40. f_hand.close()
41. except Exception,e:
42. pass
43. if __name__ == "__main__":
44. main()
45. thread_pool的代码（非原创，转自：http://blog.daviesliu.net/2006/10/09/234822/）
46. import Queue, threading, sys
47. from threading import Thread
48. import time
49. import urllib
50. # working thread
51. class Worker(Thread):
52. worker_count = 0
53. timeout = 1
54. def __init__( self, workQueue, resultQueue, **kwds):
55. Thread.__init__( self, **kwds )
56. self.id = Worker.worker_count
57. Worker.worker_count += 1
58. self.setDaemon( True )
59. self.workQueue = workQueue
60. self.resultQueue = resultQueue
61. self.start( )
62. def run( self ):
63. ''''' the get-some-work, do-some-work main loop of worker threads '''
64. while True:
65. try:
66. callable, args, kwds = self.workQueue.get(timeout=Worker.timeout)
67. res = callable(*args, **kwds)
68. print "worker[%2d]: %s" % (self.id, str(res) )
69. self.resultQueue.put( res )
70. #time.sleep(Worker.sleep)
71. except Queue.Empty:
72. break
73. except :
74. print 'worker[%2d]' % self.id, sys.exc_info()[:2]
75. raise
76. class WorkerManager:
77. def __init__( self, num_of_workers=10, timeout = 2):
78. self.workQueue = Queue.Queue()
79. self.resultQueue = Queue.Queue()
80. self.workers = []
81. self.timeout = timeout
82. self._recruitThreads( num_of_workers )
83. def _recruitThreads( self, num_of_workers ):
84. for i in range( num_of_workers ):
85. worker = Worker( self.workQueue, self.resultQueue )
86. self.workers.append(worker)
87. def wait_for_complete( self):
88. # ...then, wait for each of them to terminate:
89. while len(self.workers):
90. worker = self.workers.pop()
91. worker.join( )
92. if worker.isAlive() and not self.workQueue.empty():
93. self.workers.append( worker )
94. print "All jobs are are completed."
95. def add_job( self, callable, *args, **kwds ):
96. self.workQueue.put( (callable, args, kwds) )
97. def get_result( self, *args, **kwds ):
98. return self.resultQueue.get( *args, **kwds )