Chronicle/process_parallel.py at master · QTIM-Lab/Chronicle · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
import os, math, pdb, time
import subprocess
import multiprocessing


IMAGES_DB_and_DATAFOLDER_KEY={
    "axispacs": '/persist/PACS/DICOM',
    "forum": '/persist/PACS/forum',
    "forum_all": '/persist/PACS/forum_all',
    "imagepools": '/persist/PACS/imagepools',
}

IMAGES_DB = 'axispacs'
DATAFOLDER = IMAGES_DB_and_DATAFOLDER_KEY[IMAGES_DB]


# URL and dbname
COUCHDB_USER="admin"
COUCHDB_PASSWORD="password_qtim"
url=f"http://{COUCHDB_USER}:{COUCHDB_PASSWORD}@localhost:5984"

directories = os.listdir(DATAFOLDER)

def process_directory(dir=""):
    try:
        cmd = ['/persist/python_virtual_environments/bearceb/.pyenv/shims/python',
               './bin/record.py',
               os.path.join(DATAFOLDER, dir),
               '--url', url,
               '--dbName', IMAGES_DB,
               '--dontAttachOriginals']

        # print(f"Processing directory: {dir}")
        process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        stdout, stderr = process.communicate()

        if process.returncode != 0:
            print(f"ERROR in {dir}: {stderr.decode()}")
            return False, stderr.decode()

        # print(f"Completed directory: {dir}")
        return True, stdout.decode()

    except Exception as e:
        print(f"EXCEPTION in {dir}: {str(e)}")
        return False, str(e)


pool = multiprocessing.Pool(int(multiprocessing.cpu_count()*0.90))
try:
    start_time = time.perf_counter()
    processes = [pool.apply_async(process_directory, args=(x,)) for x in directories]
    result = [p.get() for p in processes]
    finish_time = time.perf_counter()
    print(f"Program finished in {finish_time-start_time} seconds")
    print(result)
finally:
    pool.close()
    pool.join()

# def process_directory(dir=""):
#     cmd = ['/home/bearceb/.local/share/pdm/venvs/Chronicle-FgmQ98C8-Chronicle/bin/python', './bin/record.py', os.path.join(DATAFOLDER, dir), '--url', url, '--dbName', IMAGES_DB]
#     print(cmd)
#     os.system(" ".join(cmd))
#     # process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
#     # stdout, stderr = process.communicate()

# section = math.floor(len(directories)/10)
# dir = directories[0]
# for dir in directories[0*section:1*section]:
#     cmd = ['/home/bearceb/.local/share/pdm/venvs/Chronicle-FgmQ98C8-Chronicle/bin/python', './bin/record.py', os.path.join(DATAFOLDER, dir), '--url', url, '--dbName', IMAGES_DB]
#     os.system(" ".join(cmd))
# directories[1*section:2*section]
# directories[2*section:3*section]
# directories[3*section:4*section]
# directories[4*section:5*section]
# directories[5*section:6*section]
# directories[6*section:7*section]
# directories[7*section:8*section]
# directories[8*section:9*section]
# directories[9*section:10*section]
# directories[10*section:11*section]