Python多进程multiprocessing使用示例
mutilprocess简介
像线程一样管理进程,这个是mutilprocess的核心,他与threading很是相像,对多核CPU的利用率会比threading好的多。
import multiprocessing def worker(num):
"""thread worker function"""
print 'Worker:', num
return if __name__ == '__main__':
jobs = []
for i in range(5):
p = multiprocessing.Process(target=worker, args=(i,))
jobs.append(p)
p.start()
简单的创建进程
确定当前的进程,即是给进程命名,方便标识区分,跟踪
import multiprocessing
import time def worker():
name = multiprocessing.current_process().name
print name, 'Starting'
time.sleep(2)
print name, 'Exiting' def my_service():
name = multiprocessing.current_process().name
print name, 'Starting'
time.sleep(3)
print name, 'Exiting' if __name__ == '__main__':
service = multiprocessing.Process(name='my_service',
target=my_service)
worker_1 = multiprocessing.Process(name='worker 1',
target=worker)
worker_2 = multiprocessing.Process(target=worker) # default name worker_1.start()
worker_2.start()
service.start()
守护进程就是不阻挡主程序退出,自己干自己的 mutilprocess.setDaemon(True)
就这句等待守护进程退出,要加上join,join可以传入浮点数值,等待n久就不等了
import multiprocessing
import time
import sys def daemon():
name = multiprocessing.current_process().name
print 'Starting:', name
time.sleep(2)
print 'Exiting :', name def non_daemon():
name = multiprocessing.current_process().name
print 'Starting:', name
print 'Exiting :', name if __name__ == '__main__':
d = multiprocessing.Process(name='daemon',
target=daemon)
d.daemon = True n = multiprocessing.Process(name='non-daemon',
target=non_daemon)
n.daemon = False d.start()
n.start() d.join(1)
print 'd.is_alive()', d.is_alive()
n.join()
守护进程
最好使用 poison pill,强制的使用terminate()注意 terminate之后要join,使其可以更新状态
import multiprocessing
import time def slow_worker():
print 'Starting worker'
time.sleep(0.1)
print 'Finished worker' if __name__ == '__main__':
p = multiprocessing.Process(target=slow_worker)
print 'BEFORE:', p, p.is_alive() p.start()
print 'DURING:', p, p.is_alive() p.terminate()
print 'TERMINATED:', p, p.is_alive() p.join()
print 'JOINED:', p, p.is_alive()
终止进程
- == 0 未生成任何错误
- 0 进程有一个错误,并以该错误码退出
- < 0 进程由一个-1 * exitcode信号结束
import multiprocessing
import sys
import time def exit_error():
sys.exit(1) def exit_ok():
return def return_value():
return 1 def raises():
raise RuntimeError('There was an error!') def terminated():
time.sleep(3) if __name__ == '__main__':
jobs = []
for f in [exit_error, exit_ok, return_value, raises, terminated]:
print 'Starting process for', f.func_name
j = multiprocessing.Process(target=f, name=f.func_name)
jobs.append(j)
j.start() jobs[-1].terminate() for j in jobs:
j.join()
print '%15s.exitcode = %s' % (j.name, j.exitcode)
进程的退出状态
方便的调试,可以用logging
import multiprocessing
import logging
import sys def worker():
print 'Doing some work'
sys.stdout.flush() if __name__ == '__main__':
multiprocessing.log_to_stderr()
logger = multiprocessing.get_logger()
logger.setLevel(logging.INFO)
p = multiprocessing.Process(target=worker)
p.start()
p.join()
日志
利用class来创建进程,定制子类
import multiprocessing class Worker(multiprocessing.Process): def run(self):
print 'In %s' % self.name
return if __name__ == '__main__':
jobs = []
for i in range(5):
p = Worker()
jobs.append(p)
p.start()
for j in jobs:
j.join()
派生进程
import multiprocessing class MyFancyClass(object): def __init__(self, name):
self.name = name def do_something(self):
proc_name = multiprocessing.current_process().name
print 'Doing something fancy in %s for %s!' % \
(proc_name, self.name) def worker(q):
obj = q.get()
obj.do_something() if __name__ == '__main__':
queue = multiprocessing.Queue() p = multiprocessing.Process(target=worker, args=(queue,))
p.start() queue.put(MyFancyClass('Fancy Dan')) # Wait for the worker to finish
queue.close()
queue.join_thread()
p.join() import multiprocessing
import time class Consumer(multiprocessing.Process): def __init__(self, task_queue, result_queue):
multiprocessing.Process.__init__(self)
self.task_queue = task_queue
self.result_queue = result_queue def run(self):
proc_name = self.name
while True:
next_task = self.task_queue.get()
if next_task is None:
# Poison pill means shutdown
print '%s: Exiting' % proc_name
self.task_queue.task_done()
break
print '%s: %s' % (proc_name, next_task)
answer = next_task()
self.task_queue.task_done()
self.result_queue.put(answer)
return class Task(object):
def __init__(self, a, b):
self.a = a
self.b = b
def __call__(self):
time.sleep(0.1) # pretend to take some time to do the work
return '%s * %s = %s' % (self.a, self.b, self.a * self.b)
def __str__(self):
return '%s * %s' % (self.a, self.b) if __name__ == '__main__':
# Establish communication queues
tasks = multiprocessing.JoinableQueue()
results = multiprocessing.Queue() # Start consumers
num_consumers = multiprocessing.cpu_count() * 2
print 'Creating %d consumers' % num_consumers
consumers = [ Consumer(tasks, results)
for i in xrange(num_consumers) ]
for w in consumers:
w.start() # Enqueue jobs
num_jobs = 10
for i in xrange(num_jobs):
tasks.put(Task(i, i)) # Add a poison pill for each consumer
for i in xrange(num_consumers):
tasks.put(None) # Wait for all of the tasks to finish
tasks.join() # Start printing results
while num_jobs:
result = results.get()
print 'Result:', result
num_jobs -= 1
python进程间传递消息
Event提供一种简单的方法,可以在进程间传递状态信息。事件可以切换设置和未设置状态。通过使用一个可选的超时值,时间对象的用户可以等待其状态从未设置变为设置。
import multiprocessing
import time def wait_for_event(e):
"""Wait for the event to be set before doing anything"""
print 'wait_for_event: starting'
e.wait()
print 'wait_for_event: e.is_set()->', e.is_set() def wait_for_event_timeout(e, t):
"""Wait t seconds and then timeout"""
print 'wait_for_event_timeout: starting'
e.wait(t)
print 'wait_for_event_timeout: e.is_set()->', e.is_set() if __name__ == '__main__':
e = multiprocessing.Event()
w1 = multiprocessing.Process(name='block',
target=wait_for_event,
args=(e,))
w1.start() w2 = multiprocessing.Process(name='nonblock',
target=wait_for_event_timeout,
args=(e, 2))
w2.start() print 'main: waiting before calling Event.set()'
time.sleep(3)
e.set()
print 'main: event is set'
进程间信号传递
Python多进程,一般的情况是Queue来传递。
from multiprocessing import Process, Queue def f(q):
q.put([42, None, 'hello']) if __name__ == '__main__':
q = Queue()
p = Process(target=f, args=(q,))
p.start()
print q.get() # prints "[42, None, 'hello']"
p.join()
Queue
import Queue
import threading
import time exitFlag = 0 class myThread (threading.Thread):
def __init__(self, threadID, name, q):
threading.Thread.__init__(self)
self.threadID = threadID
self.name = name
self.q = q
def run(self):
print "Starting " + self.name
process_data(self.name, self.q)
print "Exiting " + self.name def process_data(threadName, q):
while not exitFlag:
queueLock.acquire()
if not workQueue.empty():
data = q.get()
queueLock.release()
print "%s processing %s" % (threadName, data)
else:
queueLock.release()
time.sleep(1) threadList = ["Thread-1", "Thread-2", "Thread-3"]
nameList = ["One", "Two", "Three", "Four", "Five"]
queueLock = threading.Lock()
workQueue = Queue.Queue(10)
threads = []
threadID = 1 # Create new threads
for tName in threadList:
thread = myThread(threadID, tName, workQueue)
thread.start()
threads.append(thread)
threadID += 1 # Fill the queue
queueLock.acquire()
for word in nameList:
workQueue.put(word)
queueLock.release() # Wait for queue to empty
while not workQueue.empty():
pass # Notify threads it's time to exit
exitFlag = 1 # Wait for all threads to complete
for t in threads:
t.join()
print "Exiting Main Thread"
多线程优先队列Queue
多进程使用Queue通信的例子
import time
from multiprocessing import Process,Queue MSG_QUEUE = Queue(5) def startA(msgQueue):
while True:
if msgQueue.empty() > 0:
print ('queue is empty %d' % (msgQueue.qsize()))
else:
msg = msgQueue.get()
print( 'get msg %s' % (msg,))
time.sleep(1) def startB(msgQueue):
while True:
msgQueue.put('hello world')
print( 'put hello world queue size is %d' % (msgQueue.qsize(),))
time.sleep(3) if __name__ == '__main__':
processA = Process(target=startA,args=(MSG_QUEUE,))
processB = Process(target=startB,args=(MSG_QUEUE,)) processA.start()
print( 'processA start..')
主进程定义了一个Queue类型的变量,并作为Process的args参数传给子进程processA和processB,两个进程一个向队列中写数据,一个读数据。
Python多进程multiprocessing使用示例的更多相关文章
- Python 多进程 multiprocessing.Pool类详解
Python 多进程 multiprocessing.Pool类详解 https://blog.csdn.net/SeeTheWorld518/article/details/49639651
- Python 多进程multiprocessing
一.python多线程其实在底层来说只是单线程,因此python多线程也称为假线程,之所以用多线程的意义是因为线程不停的切换这样比串行还是要快很多.python多线程中只要涉及到io或者sleep就会 ...
- python ---多进程 Multiprocessing
和 threading 的比较 多进程 Multiprocessing 和多线程 threading 类似, 他们都是在 python 中用来并行运算的. 不过既然有了 threading, 为什么 ...
- python多进程-----multiprocessing包
multiprocessing并非是python的一个模块,而是python中多进程管理的一个包,在学习的时候可以与threading这个模块作类比,正如我们在上一篇转载的文章中所提,python的多 ...
- python多进程multiprocessing Pool相关问题
python多进程想必大部分人都用到过,可以充分利用多核CPU让代码效率更高效. 我们看看multiprocessing.pool.Pool.map的官方用法 map(func, iterable[, ...
- 操作系统OS,Python - 多进程(multiprocessing)、多线程(multithreading)
多进程(multiprocessing) 参考: https://docs.python.org/3.6/library/multiprocessing.html 1. 多进程概念 multiproc ...
- python多进程(multiprocessing)
最近有个小课题,需要用到双进程,翻了些资料,还算圆满完成任务.记录一下~ 1.简单地双进程启动 同时的调用print1()和print2()两个打印函数,代码如下: #/usr/bin/python ...
- python多进程multiprocessing模块中Queue的妙用
最近的部门RPA项目中,小爬为了提升爬虫性能,使用了Python中的多进程(multiprocessing)技术,里面需要用到进程锁Lock,用到进程池Pool,同时利用map方法一次构造多个proc ...
- Python(多进程multiprocessing模块)
day31 http://www.cnblogs.com/yuanchenqi/articles/5745958.html 由于GIL的存在,python中的多线程其实并不是真正的多线程,如果想要充分 ...
随机推荐
- CODE[VS]-蛇形矩阵-模拟-天梯白银
题目描述 Description 小明玩一个数字游戏,取个n行n列数字矩阵(其中n为不超过100的奇数),数字的填补方法为:在矩阵中心从1开始以逆时针方向绕行,逐圈扩大,直到n行n列填满数字,请输出该 ...
- pack://application:,,,/
FrameworkElementFactory gridFactory = new FrameworkElementFactory(typeof(Grid)); gridFactory.SetValu ...
- excel 导入 与 导出
Excel导入 public ActionResult Excel(HttpPostedFileBase file) { HttpPostedFileBase fi ...
- 关于MongoDB数据库中文件唯一性的问题
※重要※——介绍一下我的环境:MongoDB的“win32-x86_64-2008plus-ssl-3.0.5”,MongoVUE版本是1.6.9,VS2010,dll是1.10版本. MongoDB ...
- Wise Registry Cleaner Pro(智能注册表清理) V9.31 绿色版
软件名称: Wise Registry Cleaner Pro(智能注册表清理)软件语言: 简体中文授权方式: 免费试用运行环境: Win7 / Vista / Win2003 / WinXP 软件大 ...
- [阿当视频]WEB组件学习笔记
— 视频看完了,自定义事件还不懂,等完全懂了再更新并完成整篇案例 1. JS分层和组件的种类浏览器底层包括HTML CSS JS(DOM/BOM/Style/Canvas 2D/WebGl/SVG) ...
- My Sql多表操作(转载)
DELETE 在Mysql4.0之后,mysql开始支持跨表delete. Mysql可以在一个sql语句中同时删除多表记录,也可以根据多个表之间的关系来删除某一个表中的记录. 假定我们有两张表:Pr ...
- .net中读取xml文件中节点的所有属性信息
功能描述: 将数据以xml的格式记录成配置文件,需要获取配置文件中的数据时,则获取对应的配置文件,读取配置文件里对应节点的所有属性. 逻辑实现: 1.将数据配置好在xml文件中. 2.获取xml文件中 ...
- jquery学习笔记3 jq遍历
遍历方式:向上(父级元素) 向下(子元素) 水平(同胞元素) 一.向上遍历 parent() 向上一级 放回被选元素的直接父元素 parents() 返回被选元 ...
- Android:关于Edittext的一些设置
1.自动弹出输入框. et_order_search.setFocusableInTouchMode(true); et_order_search.requestFocus(); CmzBossApp ...