为了利用多核性能,使用多线程
import os import tqdm from multiprocessing import Pool def worker(line): # 对于每一个样本line进行处理,并保存 # 注意不能return结果,而是要将结果直接输出/写入文件 pass with open("file_list.txt" ,"r") as f_r: lines = f_r.readlines() with tqdm.tqdm(lines) as t, Pool(152) as p: for _ in p.imap_unordered(worker, lines): t.update()