feat: wait compress to complete

README.md

@@ -9,4 +9,3 @@
 * 本程式可以在任意目錄執行
 * `/path/to/temp/directory` 暫存目錄，可以任意指定
 * `/path/to/output/directory` 輸出目錄，可以任意指定
-* **注意**：本程式結束時，壓縮尚未完成，請檢查壓縮檔大小不再變動時才是壓縮完畢。

cms2toj.py

@@ -3,137 +3,158 @@
 import logging
 import os
 import re
-import subprocess
+import asyncio
 from function import *
 
-logging.basicConfig(level=logging.INFO,
-                    format='%(asctime)s %(levelname)s %(message)s')
-
-parser = argparse.ArgumentParser(description='cms2toj')
-parser.add_argument('inputpath', type=str,
-                    help='使用cmsDumpExporter產出的路徑，不支援壓縮檔')
-parser.add_argument('outputpath', type=str, help='輸出的資料夾')
-args = parser.parse_args()
-inputpath = args.inputpath
-outputpath = args.outputpath
-
-with open(os.path.join(inputpath, 'contest.json')) as f:
-    data = json.load(f)
-
-contestids = []
-for idx in data:
-    if type(data[idx]) is not dict:
-        continue
-    if data[idx]['_class'] == 'Contest':
-        contestids.append(idx)
-
-contest = None
-while contest is None:
-    print('-' * 70)
-    print('請選擇競賽ID')
-    print('ID\tName')
-    for idx in contestids:
-        print('{}\t{}'.format(idx, data[idx]['description']))
-    idx = input('選擇>')
-    if idx in contestids:
-        contest = data[idx]
-
-print('-' * 70)
-print('正在處理 {}'.format(contest['description']))
-print(contest)
-for taskid in contest['tasks']:
-    task = data[taskid]
+async def main():
+    logging.basicConfig(level=logging.INFO,
+                        format='%(asctime)s %(levelname)s %(message)s')
+
+    parser = argparse.ArgumentParser(description='cms2toj')
+    parser.add_argument('inputpath', type=str,
+                        help='使用cmsDumpExporter產出的路徑，不支援壓縮檔')
+    parser.add_argument('outputpath', type=str, help='輸出的資料夾')
+    args = parser.parse_args()
+    inputpath = args.inputpath
+    outputpath = args.outputpath
+
+    with open(os.path.join(inputpath, 'contest.json')) as f:
+        data = json.load(f)
+
+    contestids = []
+    for idx in data:
+        if type(data[idx]) is not dict:
+            continue
+        if data[idx]['_class'] == 'Contest':
+            contestids.append(idx)
+
+    contest = None
+    while contest is None:
+        print('-' * 70)
+        print('請選擇競賽ID')
+        print('ID\tName')
+        for idx in contestids:
+            print('{}\t{}'.format(idx, data[idx]['description']))
+        idx = input('選擇>')
+        if idx in contestids:
+            contest = data[idx]
+
     print('-' * 70)
-    print(taskid, task['name'], task['title'])
-
-    taskpath = os.path.join(outputpath, taskid)
-    makedirs(taskpath)
-
-    # res/testdata / testcases
-    makedirs(taskpath, 'res/testdata')
-
-    datasetid = task['active_dataset']
-    dataset = data[datasetid]
-
-    datacasemap = {}
-    offset = 1
-    logging.info('Copying {} testdatas'.format(len(dataset['testcases'])))
-    for filename in dataset['testcases']:
-        testcaseid = dataset['testcases'][filename]
-        testcase = data[testcaseid]
-        datacasemap[testcase['codename']] = offset
-        copyfile(
-            (inputpath, 'files', testcase['input']),
-            (taskpath, 'res/testdata', '{}.in'.format(offset))
-        )
-        copyfile(
-            (inputpath, 'files', testcase['output']),
-            (taskpath, 'res/testdata', '{}.out'.format(offset))
-        )
-        offset += 1
-
-    # conf
-    conf = {
-        'timelimit': 0,
-        'memlimit': 0,
-        'compile': 'g++',
-        'score': 'rate',
-        'check': 'diff',
-        'test': [],
-        'metadata': {},
-    }
-    conf['timelimit'] = int(dataset['time_limit'] * 1000)
-    conf['memlimit'] = int(dataset['memory_limit'] * 1024)
-    if isinstance(dataset['score_type_parameters'][0][1], int):
-        # Case 1. See https://cms.readthedocs.io/en/v1.4/Score%20types.html#groupmin
+    print('正在處理 {}'.format(contest['description']))
+    print(contest)
+
+    compress_tasks = []
+    for taskid in contest['tasks']:
+        task = data[taskid]
+        print('-' * 70)
+        print(taskid, task['name'], task['title'])
+
+        taskpath = os.path.join(outputpath, taskid)
+        makedirs(taskpath)
+
+        # res/testdata / testcases
+        makedirs(taskpath, 'res/testdata')
+
+        datasetid = task['active_dataset']
+        dataset = data[datasetid]
+
+        datacasemap = {}
         offset = 1
-        for score in dataset['score_type_parameters']:
-            conf['test'].append({
-                'data': list(range(offset, offset + score[1])),
-                'weight': score[0]
-            })
-            offset += score[1]
-    elif isinstance(dataset['score_type_parameters'][0][1], str):
-        # Case 2.
-        for score in dataset['score_type_parameters']:
-            test = {
-                'data': [],
-                'weight': score[0]
-            }
-            for codename in datacasemap:
-                if re.match(score[1], codename):
-                    test['data'].append(datacasemap[codename])
-            conf['test'].append(test)
-    else:
-        raise Exception('Bad score_type_parameters type: {}'.format(dataset['score_type_parameters']))
+        logging.info('Copying {} testdatas'.format(len(dataset['testcases'])))
+        for filename in dataset['testcases']:
+            testcaseid = dataset['testcases'][filename]
+            testcase = data[testcaseid]
+            datacasemap[testcase['codename']] = offset
+            copyfile(
+                (inputpath, 'files', testcase['input']),
+                (taskpath, 'res/testdata', '{}.in'.format(offset))
+            )
+            copyfile(
+                (inputpath, 'files', testcase['output']),
+                (taskpath, 'res/testdata', '{}.out'.format(offset))
+            )
+            offset += 1
+
+        # conf
+        conf = {
+            'timelimit': 0,
+            'memlimit': 0,
+            'compile': 'g++',
+            'score': 'rate',
+            'check': 'diff',
+            'test': [],
+            'metadata': {},
+        }
+        conf['timelimit'] = int(dataset['time_limit'] * 1000)
+        conf['memlimit'] = int(dataset['memory_limit'] * 1024)
+        if isinstance(dataset['score_type_parameters'][0][1], int):
+            # Case 1. See https://cms.readthedocs.io/en/v1.4/Score%20types.html#groupmin
+            offset = 1
+            for score in dataset['score_type_parameters']:
+                conf['test'].append({
+                    'data': list(range(offset, offset + score[1])),
+                    'weight': score[0]
+                })
+                offset += score[1]
+        elif isinstance(dataset['score_type_parameters'][0][1], str):
+            # Case 2.
+            for score in dataset['score_type_parameters']:
+                test = {
+                    'data': [],
+                    'weight': score[0]
+                }
+                for codename in datacasemap:
+                    if re.match(score[1], codename):
+                        test['data'].append(datacasemap[codename])
+                conf['test'].append(test)
+        else:
+            raise Exception('Bad score_type_parameters type: {}'.format(dataset['score_type_parameters']))
+
+        logging.info('Creating config file')
+        with open(os.path.join(taskpath, 'conf.json'), 'w') as conffile:
+            json.dump(conf, conffile, indent=4)
+
+        # http / statements
+        makedirs(taskpath, 'http')
+
+        statements = task['statements']
+        if len(statements) == 0:
+            logging.info('No statements')
+            statement = None
+        else:
+            statementid = list(statements.values())[0]
+            statement = data[statementid]
+            logging.info('Copying statements')
+            copyfile(
+                (inputpath, 'files', statement['digest']),
+                (taskpath, 'http', 'cont.pdf')
+            )
+        async def _compress_task(taskpath, outputpath, taskid, task_name, task_title):
+            returncode = run_and_wait_process('tar', *[
+                '-C',
+                taskpath,
+                '-cJf',
+                os.path.join(outputpath, '{}.tar.xz'.format(taskid)),
+                'http',
+                'res',
+                'conf.json'
+            ])
+            if returncode != 0:
+                logging.info('{} {} {} Compress failed'.format(taskid, task_name, task_title))
+            else:
+                logging.info('{} {} {} Compress finished'.format(taskid, task_name, task_title))
 
-    logging.info('Creating config file')
-    with open(os.path.join(taskpath, 'conf.json'), 'w') as conffile:
-        json.dump(conf, conffile, indent=4)
+        compress_tasks.append(_compress_task(taskpath, outputpath, taskid, task['name'], task['title']))
 
-    # http / statements
-    makedirs(taskpath, 'http')
+    logging.info('Starting compress')
+    await asyncio.gather(*compress_tasks)
 
-    statements = task['statements']
-    if len(statements) == 0:
-        logging.info('No statements')
-        statement = None
+if __name__ == '__main__':
+    import sys
+    if sys.version_info.minor >= 7:
+        asyncio.run(main())
     else:
-        statementid = list(statements.values())[0]
-        statement = data[statementid]
-        logging.info('Copying statements')
-        copyfile(
-            (inputpath, 'files', statement['digest']),
-            (taskpath, 'http', 'cont.pdf')
-        )
-
-    p = subprocess.Popen([
-        'tar',
-        '-C',
-        taskpath,
-        '-cJf',
-        os.path.join(outputpath, '{}.tar.xz'.format(taskid)),
-        'http',
-        'res',
-        'conf.json'
-    ])
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+        loop.run_until_complete(main())
+    asyncio.run(main())

function.py

@@ -1,6 +1,7 @@
 import os
 import logging
 import shutil
+import asyncio
 
 
 def makedirs(*path):
@@ -17,3 +18,9 @@ def copyfile(source, target):
     if not os.path.exists(target):
         logging.debug('Copying {} to {}'.format(source, target))
         shutil.copyfile(source, target)
+
+async def run_and_wait_process(program, *args):
+    process = await asyncio.create_subprocess_exec(program, *args)
+    returncode = await process.wait()
+
+    return returncode