python多进程执行任务

https://blog.csdn.net/qq_39694935/article/details/84552076

【Python】multiprocessing Pool 进程间通信共享

直接上代码：

from tqdm import tqdm
from multiprocessing import Pool
import functools
from pymongo import MongoClient
mdb = MongoClient('120.xx.26.xx:20002', username='xx', password='xxxxx')

# 三种main的写法只写一种即可

def create_data(image):

    # TODO 具体处理逻辑
    print(image)
    return str(image)

def main_deal():

    num_processor = 20
    p = Pool(num_processor)
    images = mdb['db_name']['image'].find(no_cursor_timeout=True).batch_size(200)

    fw = open('result.txt', 'w+')
    for result in tqdm(p.imap(create_data, images), total=images.count()):
        fw.write(result + '\n')
    fw.close()

    for _ in tqdm(p.imap_unordered(create_data, images)):
        pass
    p.close()
    p.join()

def main_deal():

    num_processor = 20
    p = Pool(num_processor)

    images = mdb['goodlook']['image_generated_data'].find(no_cursor_timeout=True).batch_size(200)
    fw = open('result.txt', 'w+')
    for result in tqdm(p.imap_unordered(create_data, images)):
        fw.write(result + '\n')
    fw.close()

    p.close()
    p.join()

def main_deal():
    num_processor = 20
    p = Pool(num_processor)

    images = mdb['goodlook']['image_generated_data'].find(no_cursor_timeout=True).batch_size(200)
    fw = open('result.txt', 'w+')

    pt = functools.partial(create_data)
    for result in tqdm(p.imap_unordered(pt, images)):
        fw.write(result + '\n')
    fw.close()

    p.close()
    p.join()

if __name__ == '__main__':
    main_deal()