Add functionality for capturing error for later use

Author: akashdhruv

jobrunner/api/_commands.py

@@ -21,6 +21,9 @@ def setup(dirlist, verbose=False, exit_on_failure=False):
     # set variable to determine console separator
     separator = False
 
+    # create an empty list for output
+    output = []
+
     # loop over dirlist
     for workdir in dirlist:
 
@@ -51,14 +54,21 @@ def setup(dirlist, verbose=False, exit_on_failure=False):
                 print(f'{" "*4}- {value.replace(basedir,"<ROOT>")}')
 
         # run a bash process
-        lib.BashProcess(basedir, workdir, "job.setup", verbose, exit_on_failure)
+        process = lib.BashProcess(
+            basedir, workdir, "job.setup", verbose, exit_on_failure
+        )
+
+        # append process to output
+        output.append(process)
 
         # set separator value
         separator = True
 
         # Return to base directory
         os.chdir(basedir)
 
+    return output
+
 
 def submit(dirlist, verbose=False, exit_on_failure=False):
     """
@@ -70,6 +80,9 @@ def submit(dirlist, verbose=False, exit_on_failure=False):
     # set variable to determine console separator
     separator = False
 
+    # create an empty list for output
+    output = []
+
     # loop over dirlist
     for workdir in dirlist:
 
@@ -116,19 +129,26 @@ def submit(dirlist, verbose=False, exit_on_failure=False):
 
         # Submit job
         if config.schedular.command == "bash":
-            lib.BashProcess(basedir, workdir, "job.submit", verbose, exit_on_failure)
+            process = lib.BashProcess(
+                basedir, workdir, "job.submit", verbose, exit_on_failure
+            )
 
         else:
-            lib.SchedularProcess(
+            process = lib.SchedularProcess(
                 basedir, workdir, config.schedular.command, "job.submit"
             )
 
+        # append process to output
+        output.append(process)
+
         # set separator value
         separator = True
 
         # Return to base directory
         os.chdir(basedir)
 
+    return output
+
 
 def clean(dirlist):
     """

jobrunner/lib/_console.py

@@ -39,6 +39,8 @@ def SchedularProcess(basedir, workdir, command, script):
         check=True,
     )
 
+    return process
+
 
 def BashProcess(basedir, workdir, script, verbose=False, exit_on_failure=False):
     """
@@ -73,11 +75,11 @@ def BashProcess(basedir, workdir, script, verbose=False, exit_on_failure=False):
         #        output.write(line)
 
     else:
-        with open("job.output", "w") as output:
+        with open("job.output", "w") as output, open("job.error", "w") as error:
             process = subprocess.Popen(
                 f"bash {script}".split(),
                 stdout=output,
-                stderr=subprocess.STDOUT,
+                stderr=error,
                 text=True,
             )
 
@@ -87,8 +89,8 @@ def BashProcess(basedir, workdir, script, verbose=False, exit_on_failure=False):
 
     if process.returncode != 0:
         if not verbose:
-            with open("job.output", "r") as output:
-                print("".join(output.readlines()[-8:]))
+            with open("job.error", "r") as error:
+                print("".join(error.readlines()[-8:]))
 
         if exit_on_failure:
             raise ValueError(f"{lib.Color.red}FAILURE {lib.Color.end}")
@@ -101,3 +103,5 @@ def BashProcess(basedir, workdir, script, verbose=False, exit_on_failure=False):
     print(
         f'\n{lib.Color.purple}OUTPUT:{lib.Color.end} {workdir.replace(basedir,"<ROOT>")}/job.output'
     )
+
+    return process

jobrunner/lib/_filetools.py

@@ -232,6 +232,7 @@ def RemoveNodeFiles(config, nodedir):
         nodedir + os.sep + "job.submit",
         nodedir + os.sep + "job.target",
         nodedir + os.sep + "job.output",
+        nodedir + os.sep + "job.error",
     ]
 
     # loop over list of files in nodedir and append to