Coverage for src/bob/pipelines/config/distributed/sge_demanding.py: 0%

1from dask.distributed import Client 

2 

3from bob.pipelines.distributed.sge import SGEMultipleQueuesCluster, get_max_jobs 

4from bob.pipelines.distributed.sge_queues import QUEUE_MTH 

5 

6min_jobs = 1 

7max_jobs = get_max_jobs(QUEUE_MTH) 

8cluster = SGEMultipleQueuesCluster(min_jobs=min_jobs, sge_job_spec=QUEUE_MTH) 

9cluster.scale(max_jobs) 

10 

11# Adapting to minimim 1 job to maximum 48 jobs 

12# interval: Milliseconds between checks from the scheduler 

13# wait_count: Number of consecutive times that a worker should be suggested for 

14# removal before we remove it. 

15cluster.adapt( 

16 minimum=min_jobs, 

17 maximum=max_jobs, 

18 wait_count=5, 

19 interval=10, 

20 target_duration="10s", 

21) 

22 

23dask_client = Client(cluster)