[Python知识库]python 实现多核 CPU 并行计算

1. 使用原因：

通常现有的计算机都包含多个 CPU 内核，然而，现实中运行程序时，通常仅用到单核 CPU，导致 CPU资源无法充分利用。因此，我们可以通过多核 CPU 并行计算来加快程序的运行。

2. 使用方法

2.1. 需要用到的功能函数

• 获取 CPU的内核数量

cpu_num = multiprocessing.cpu_count()

• 并行计算函数

proc = multiprocessing.Process(target=single_run, args=(digits, "parallel"))
proc.start()
proc.join()

2.2 范例程序

import numpy as np
import multiprocessing
from sklearn.manifold import TSNE
import time

path = "E:\\blog\\data\\MNIST50m\\"


def single_run(digits, fold="1by1"):
    sum = 0
    for i in range(0,500000000):
        sum = sum+i
    print("sum:",sum)


def one_by_one():
    start_time = time.time()
    for i in range(0,12):
        single_run(digits=[], fold="1by1")
    end_time = time.time()
    print("one by one time:",end_time-start_time)

def parallel():
    begin_time = time.time()
    n = 10  # 10
    procs = []
    n_cpu = multiprocessing.cpu_count()
    chunk_size = int(n / n_cpu)

    for i in range(0, n_cpu):
        min_i = chunk_size * i

        if i < n_cpu - 1:
            max_i = chunk_size * (i + 1)
        else:
            max_i = n
        digits = []
        for digit in range(min_i, max_i):
            digits.append(digit)
        print("digits:",digits)
        print("CPU:",i)
        procs.append(multiprocessing.Process(target=single_run, args=(digits, "parallel")))

    for proc in procs:
        proc.start()
    for proc in procs:
        proc.join()

    end_time = time.time()
    print("parallel time: ", end_time - begin_time)


if __name__ == '__main__':

    parallel()

    one_by_one()