import click @click.command() @click.option('-i', '--input', 'opt_fp_in', required=True, help='Input file') @click.option('-o', '--output', 'opt_fp_out', required=True, help='Output file') @click.option('-t', '--threads', 'opt_threads', default=4, help='Number of threads') @click.pass_context def cli(ctx, opt_fp_in, opt_fp_out, opt_threads): """Template for multithreading""" from functools import partial from multiprocessing.dummy import Pool as ThreadPool from tqdm import tqdm from app.utils.logger_utils import Logger log = Logger.getLogger() log.info('multithreaded template') # setup multithreading function def pool_process(data_obj): # threaded function global parse_yt_page results = [] try: # do something here with data_obj except Exception as e: log.debug(f'Error: {e}') pbar.update(1) return results # setup multithreading data holds items = [] # list of dicts to process results = [] num_items = len(items) # run the multithreading with progress bar pbar = tqdm(total=num_items) pool_process = partial(pool_process) pool = ThreadPool(opt_threads) with tqdm(total=num_items) as pbar: results = pool.map(pool_process, media_items) pbar.close()