topo/dask_test_debug.py at main · ZuzkaU/topo · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
from dask_jobqueue import SLURMCluster
from dask.distributed import Client, as_completed
from tqdm import tqdm
import time

def dummy_task(x, ijk_to_idx):
    from datetime import datetime
    import time
    print(datetime.now(), f"started a dummy task {x}", flush=True)
    time.sleep(2)
    print("Length of ijk_to_idx:", len(ijk_to_idx), flush=True)
    print(datetime.now(), f"ending a dummy task {x}", flush=True)
    return x

# Create a large dictionary
shape = (1, 27119, 27350, 15494)
patch_size = 512
xs = list(range(0, shape[1], patch_size))
ys = list(range(0, shape[2], patch_size))
zs = list(range(0, shape[3], patch_size))
ijk_to_idx = {
    (i, j, k): i * len(ys) * len(zs) + j * len(zs) + k
    for i in range(len(xs))
    for j in range(len(ys))
    for k in range(len(zs))
}

cluster = SLURMCluster(
    cores=16,
    memory="500GB",
    processes=1,
    worker_extra_args=["--resources", "processes=1"],
    log_directory="/cajal/scratch/projects/misc/zuzur/slurm_logs/relabel/",
)
cluster.adapt(minimum_jobs=1, maximum_jobs=32)

with Client(cluster) as client:
    print("Dask Client Dashboard:", client.dashboard_link)

    # Scatter the dictionary once
    ijk_to_idx_future = client.scatter(ijk_to_idx, broadcast=True)

    # Submit tasks, pass the Future as a real argument
    num_tasks = 10  # keep this small to test; change later
    futures = client.map(
        dummy_task,
        range(num_tasks),
        ijk_to_idx=ijk_to_idx_future,
        resources={"processes": 1},
    )

    for _ in tqdm(as_completed(futures), total=num_tasks):
        pass

exit(42)

cluster = SLURMCluster(
    cores=16,
    memory="4GB",
    processes=1,
    worker_extra_args=["--resources", "processes=1"],
    walltime="00:10:00",
    log_directory="/cajal/scratch/projects/misc/zuzur/slurm_logs/test_dask_resources",
)

cluster.adapt(minimum_jobs=1, maximum_jobs=5)

with Client(cluster) as client:
    print("Dashboard:", client.dashboard_link)

    time.sleep(5)  # Give scheduler time to spin up workers
    from pprint import pprint
    pprint(client.scheduler_info()["workers"])

    futures = client.map(dummy_task, range(10), resources={"processes": 1})

    pprint(client.scheduler_info()["workers"])
    results = client.gather(futures)
    print("Results:", results)