diff --git a/.ci_support/environment.yml b/.ci_support/environment.yml index 3c00cc9..23f3a9d 100644 --- a/.ci_support/environment.yml +++ b/.ci_support/environment.yml @@ -10,5 +10,5 @@ dependencies: - jinja2 =3.1.2 - paramiko =3.3.1 - tqdm =4.66.1 - - pympipool =0.6.2 + - pympipool =0.7.0 - cloudpickle =2.2.1 diff --git a/pysqa/executor/backend.py b/pysqa/executor/backend.py index 20ab4a2..776e764 100644 --- a/pysqa/executor/backend.py +++ b/pysqa/executor/backend.py @@ -1,7 +1,7 @@ import os import sys -from pympipool import PoolExecutor +from pympipool.mpi import PyMPIExecutor from pysqa.executor.helper import ( read_from_file, deserialize, @@ -39,15 +39,16 @@ def execute_files_from_list(tasks_in_progress_dict, cache_directory, executor): def execute_tasks(cores, cache_directory): tasks_in_progress_dict = {} - with PoolExecutor( + with PyMPIExecutor( max_workers=cores, + cores_per_worker=1, + threads_per_core=1, + gpus_per_worker=0, oversubscribe=False, - enable_flux_backend=False, - enable_slurm_backend=False, + init_function=None, cwd=cache_directory, sleep_interval=0.1, - queue_adapter=None, - queue_adapter_kwargs=None, + enable_slurm_backend=False, ) as exe: while True: execute_files_from_list( diff --git a/pysqa/executor/executor.py b/pysqa/executor/executor.py index c986cf5..fd99bee 100644 --- a/pysqa/executor/executor.py +++ b/pysqa/executor/executor.py @@ -2,7 +2,7 @@ import queue from concurrent.futures import Future, Executor as FutureExecutor -from pympipool import cancel_items_in_queue, RaisingThread +from pympipool.shared import cancel_items_in_queue, RaisingThread from pysqa.executor.helper import ( reload_previous_futures, find_executed_tasks, diff --git a/setup.py b/setup.py index 368ccf3..f991c77 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ extras_require={ "sge": ['defusedxml==0.7.1'], "remote": ['paramiko==3.3.1', 'tqdm==4.66.1'], - "executor": ['pympipool==0.6.2', 'cloudpickle==2.2.1'], + "executor": ['pympipool==0.7.0', 'cloudpickle==2.2.1'], }, cmdclass=versioneer.get_cmdclass(), entry_points={