import sys import multiprocessing _current = None _total = None def _init(current, total): global _current global _total _current = current _total = total def _wrapped_func(func_and_args): func, argument, should_print_progress, filter_ = func_and_args if should_print_progress: with _current.get_lock(): _current.value += 1 sys.stdout.write("\r\t{} of {}".format(_current.value, _total.value)) sys.stdout.flush() return func(argument, filter_) def pmap( func, iterable, processes, should_print_progress, filter_=None, *args, **kwargs ): """ A parallel map function that reports on its progress. Applies `func` to every item of `iterable` and return a list of the results. If `processes` is greater than one, a process pool is used to run the functions in parallel. `should_print_progress` is a boolean value that indicates whether a string 'N of M' should be printed to indicate how many of the functions have finished being run. """ global _current global _total _current = multiprocessing.Value("i", 0) _total = multiprocessing.Value("i", len(iterable)) func_and_args = [(func, arg, should_print_progress, filter_) for arg in iterable] if processes == 1: result = list(map(_wrapped_func, func_and_args, *args, **kwargs)) else: pool = multiprocessing.Pool( initializer=_init, initargs=( _current, _total, ), processes=processes, ) result = pool.map(_wrapped_func, func_and_args, *args, **kwargs) pool.close() pool.join() if should_print_progress: sys.stdout.write("\r") return result